Dataset Captures test mostly work and BB3D tests work

com.unity.perception/Runtime/GroundTruth/Consumers/SoloConsumer.cs

 #endif
             var outRgb = ToSensorHeader(frame, sensor);
             outRgb["fileName"] = path;
-            outRgb["imageFormat"] = sensor.imageFormat;
+            outRgb["imageFormat"] = sensor.imageFormat.ToString();
             outRgb["dimension"] = FromVector2(sensor.dimension);
 
             var annotations = new JArray();

com.unity.perception/Runtime/GroundTruth/DataModel.cs

 using System;
 using System.Collections.Generic;
+using Unity.Mathematics;
 using UnityEngine.Perception.GroundTruth.Exporters.Solo;
 
 namespace UnityEngine.Perception.GroundTruth.DataModel
     [Serializable]
     public class Frame : IMessageProducer
     {
-        public Frame(int frame, int sequence, int step)
+        public Frame(int frame, int sequence, int step, float timestamp)
+            this.timestamp = timestamp;
-
         }
 
         /// <summary>
     [Serializable]
     public class RgbSensor : Sensor
     {
+        public enum ImageFormat
+        {
+            PNG
+        };
+
+        public float3x3 intrinsics;
+
-        public string imageFormat;
+        public ImageFormat imageFormat;
 
         // The dimensions (width, height) of the image
         public Vector2 dimension;
         public override void ToMessage(IMessageBuilder builder)
         {
             base.ToMessage(builder);
-            builder.AddString("image_format", imageFormat);
+            builder.AddString("image_format", imageFormat.ToString());
             builder.AddFloatVector("dimension", Utils.ToFloatVector(dimension));
             builder.AddPngImage("camera", buffer);
         }
     /// annotations. Concrete instances of this class will add
     /// data for their specific annotation type.
     /// </summary>
-    [Serializable]
+        public Annotation() {}
+
+        public Annotation(string id, string sensorId, string description, string annotationType)
+        {
+            this.Id = id;
+            this.sensorId = sensorId;
+            this.description = description;
+            this.annotationType = annotationType;
+        }
+
         /// <summary>
         /// The unique, human readable ID for the annotation.
         /// </summary>
         /// A human readable description of what this metric is for.
         /// </summary>
         public string description;
-        /// <summary>
-        /// Additional key/value pair metadata that can be associated with
-        /// any metric.
-        /// </summary>
-        public Dictionary<string, object> metadata;
+
         public virtual void ToMessage(IMessageBuilder builder)
         {
             builder.AddString("id", Id);
         public SimulationMetadata()
         {
             unityVersion = "not_set";
-            perceptionVersion = "0.8.0-preview.4";
+            perceptionVersion = "not_set";
 #if HDRP_PRESENT
             renderPipeline = "HDRP";
 #elif URP_PRESENT

com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs

 using UnityEngine;
 using UnityEngine.Perception.GroundTruth.Consumers;
 using UnityEngine.Perception.GroundTruth.DataModel;
+using UnityEngine.Rendering;
 
 #pragma warning disable 649
 namespace UnityEngine.Perception.GroundTruth
     {
         static DatasetCapture s_Instance;
 
-        List<SimulationState> m_Simulations = new List<SimulationState>();
+        SimulationState m_ActiveSimulation;
+        List<SimulationState> m_ShuttingDownSimulations = new List<SimulationState>();
 
         public static DatasetCapture Instance => s_Instance ?? (s_Instance = new DatasetCapture());
 
         Type m_ActiveConsumerType = typeof(SoloConsumer);
 
-        bool CanBeShutdown()
+        public bool CanBeShutdown()
-            if (m_ReadyToShutdown && m_Simulations.All(s => s.IsNotRunning()))
+            if (m_ReadyToShutdown && m_ActiveSimulation.IsNotRunning() && m_ShuttingDownSimulations.All(s => s.IsNotRunning()))
-            return m_ReadyToShutdown && m_Simulations.All(s => s.IsNotRunning());
+            return m_ReadyToShutdown && m_ActiveSimulation.IsNotRunning() && m_ShuttingDownSimulations.All(s => s.IsNotRunning());
+        }
+
+        public class WaitUntilComplete : CustomYieldInstruction
+        {
+            public override bool keepWaiting => !Instance.CanBeShutdown();
         }
 
         class AllCapturesCompleteShutdownCondition : ICondition
             }
         }
 
+        bool m_automaticShutdown = true;
+
+        public bool automaticShutdown
+        {
+            get => m_automaticShutdown;
+            set
+            {
+                switch (value)
+                {
+                    case true when !m_automaticShutdown:
+                        Manager.Instance.ShutdownCondition = new AllCapturesCompleteShutdownCondition();
+                        break;
+                    case false when m_automaticShutdown:
+                        Manager.Instance.ShutdownCondition = null;
+                        break;
+                }
+
+                m_automaticShutdown = value;
+            }
+
+        }
+
-            Manager.Instance.ShutdownCondition = new AllCapturesCompleteShutdownCondition();
+            if (automaticShutdown)
+                Manager.Instance.ShutdownCondition = new AllCapturesCompleteShutdownCondition();
+        internal SimulationState currentSimulation
+        {
+            get => m_ActiveSimulation ?? (m_ActiveSimulation = CreateSimulationData());
+        }
+#if false
         internal SimulationState simulationState
         {
             get
             }
             private set => m_Simulations.Add(value);
         }
-
+#endif
         /// <summary>
         /// The json metadata schema version the DatasetCapture's output conforms to.
         /// </summary>
 
         public SensorHandle RegisterSensor(SensorDefinition sensor)
         {
-            return simulationState.AddSensor(sensor, sensor.simulationDeltaTime);
+            return currentSimulation.AddSensor(sensor, sensor.simulationDeltaTime);
-            simulationState.RegisterMetric(metricDefinition);
+            currentSimulation.RegisterMetric(metricDefinition);
-            simulationState.RegisterAnnotationDefinition(definition);
+            currentSimulation.RegisterAnnotationDefinition(definition);
-        public void StartNewSequence() => simulationState.StartNewSequence();
+        public void StartNewSequence() => currentSimulation.StartNewSequence();
-        internal bool IsValid(string id) => simulationState.Contains(id);
+        internal bool IsValid(string id) => currentSimulation.Contains(id);
-        SimulationState CreateSimulationData()
+        static ConsumerEndpoint s_Endpoint;
+        static Type s_EndpointType = typeof(SoloConsumer);
+
+        public static void SetEndpoint(ConsumerEndpoint endpoint)
-            return new SimulationState(typeof(SoloConsumer));
+            // TODO I think that we need to do some checking to make sure we're not running
+
+            s_Endpoint = endpoint;
+            Instance.currentSimulation.consumerEndpoint = endpoint;
+        }
+
+        public static bool SetEndpointType(Type inType)
+        {
+            if (inType.IsSubclassOf(typeof(ConsumerEndpoint)))
+            {
+                Debug.Log("Setting endpoint type");
+                s_EndpointType = inType;
+                return true;
+            }
+
+            Debug.Log("Not setting endpoint type");
+            return false;
+        }
+
+        static SimulationState CreateSimulationData()
+        {
+            if (s_Endpoint != null) return new SimulationState(s_Endpoint.GetType());
+            if (s_EndpointType != null) return new SimulationState(s_EndpointType);
+            throw new InvalidOperationException("Dataset capture cannot create a new simulation state without either a valid consumer endpoint or an endpoint type to instantiate.");
         }
 
         [RuntimeInitializeOnLoadMethod]
 
         public void Update()
         {
-            foreach (var simulation in m_Simulations)
+            currentSimulation.Update();
+
+            foreach (var simulation in m_ShuttingDownSimulations)
-            m_Simulations.RemoveAll(sim => sim.ExecutionState == SimulationState.ExecutionStateType.Complete);
+            m_ShuttingDownSimulations.RemoveAll(sim => sim.ExecutionState == SimulationState.ExecutionStateType.Complete);
-            simulationState.End();
-            simulationState = CreateSimulationData();
-            m_ReadyToShutdown = true;
+
+            if (m_ActiveSimulation.IsRunning())
+            {
+                var old = m_ActiveSimulation;
+                m_ShuttingDownSimulations.Add(old);
+                old.End();
+                m_ReadyToShutdown = true;
+            }
+
+            m_ActiveSimulation = CreateSimulationData();
         }
     }
 
         T GetId();
 
         FutureType GetFutureType();
+
+        bool IsValid();
+
         bool IsPending();
     }
 
         public FutureType GetFutureType()
         {
             return FutureType.Sensor;
+        }
+
+        public bool IsValid()
+        {
+            return m_SimulationState.IsRunning();
         }
 
         public bool IsPending()
             return FutureType.Annotation;
         }
 
+        public bool IsValid()
+        {
+            return m_SimulationState.IsRunning();
+        }
+
         public bool IsPending()
         {
             return m_SimulationState.IsPending(this);
             return FutureType.Metric;
         }
 
+        public bool IsValid()
+        {
+            return m_SimulationState.IsRunning();
+        }
+
         public bool IsPending()
         {
             return m_SimulationState.IsPending(this);
         /// </summary>
         public bool Enabled
         {
-            get => DatasetCapture.Instance.simulationState.IsEnabled(this);
+            get => DatasetCapture.Instance.currentSimulation.IsEnabled(this);
-                DatasetCapture.Instance.simulationState.SetEnabled(this, value);
+                DatasetCapture.Instance.currentSimulation.SetEnabled(this, value);
             }
         }
 
             if (!definition.IsValid())
                 throw new ArgumentException("The given annotationDefinition is invalid", nameof(definition));
 
-            DatasetCapture.Instance.simulationState.ReportAnnotation(this, definition, annotation);
+            DatasetCapture.Instance.currentSimulation.ReportAnnotation(this, definition, annotation);
         }
 
         /// <summary>
             if (!annotationDefinition.IsValid())
                 throw new ArgumentException("The given annotationDefinition is invalid", nameof(annotationDefinition));
 
-            return DatasetCapture.Instance.simulationState.ReportAnnotationAsync(annotationDefinition, this);
+            return DatasetCapture.Instance.currentSimulation.ReportAnnotationAsync(annotationDefinition, this);
         }
 
         public AsyncSensorFuture ReportSensorAsync()
             if (!IsValid)
                 throw new ArgumentException($"The given annotationDefinition is invalid {Id}");
 
-            return DatasetCapture.Instance.simulationState.ReportSensorAsync(this);
+            return DatasetCapture.Instance.currentSimulation.ReportSensorAsync(this);
         }
 
         public void ReportSensor(Sensor sensor)
             if (!IsValid)
                 throw new ArgumentException("The given annotationDefinition is invalid", Id);
 
-           DatasetCapture.Instance.simulationState.ReportSensor(this, sensor);
+           DatasetCapture.Instance.currentSimulation.ReportSensor(this, sensor);
         }
 
         /// <summary>
-        public bool ShouldCaptureThisFrame => DatasetCapture.Instance.simulationState.ShouldCaptureThisFrame(this);
+        public bool ShouldCaptureThisFrame => DatasetCapture.Instance.currentSimulation.ShouldCaptureThisFrame(this);
 
         /// <summary>
         /// Requests a capture from this sensor on the next rendered frame. Can only be used with manual capture mode (<see cref="CaptureTriggerMode.Manual"/>).
-            DatasetCapture.Instance.simulationState.SetNextCaptureTimeToNowForSensor(this);
+            DatasetCapture.Instance.currentSimulation.SetNextCaptureTimeToNowForSensor(this);
+        }
+
+        public void ReportMetric(MetricDefinition definition, Metric metric)
+        {
+            if (!ShouldCaptureThisFrame)
+                throw new InvalidOperationException("Annotation reported on SensorHandle in frame when its ShouldCaptureThisFrame is false.");
+            if (!IsValid)
+                throw new ArgumentException("The given annotationDefinition is invalid", Id);
+
+            DatasetCapture.Instance.currentSimulation.ReportMetric(this, definition, metric);
+
+
 #if false
         public MetricHandle ReportMetric(MetricDefinition definition, Metric metric)
         {
             if (!metricDefinition.IsValid())
                 throw new ArgumentException("The passed in metric definition is invalid", nameof(metricDefinition));
 
-            return DatasetCapture.Instance.simulationState.CreateAsyncMetric(metricDefinition, this);
+            return DatasetCapture.Instance.currentSimulation.CreateAsyncMetric(metricDefinition, this);
         }
 
         /// <summary>

com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs

                     sensorId = "",
                     annotationId = default,
                     description = m_Definition.description,
-                    metadata = new Dictionary<string, object>(),
                     objectCounts = m_ClassCountValues
                 };
                 classCountAsyncMetric.Report(payload);

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

         bool m_IsGroundTruthRendererFeaturePresent;
         internal List<ScriptableRenderPass> passes = new List<ScriptableRenderPass>();
 #endif
-        public string ID;
+        public string ID = "camera";
 
         /// <summary>
         /// A human-readable description of the camera.
                 rotation = transform.eulerAngles,
                 velocity = Vector3.zero,
                 acceleration = Vector3.zero,
-                imageFormat = ".png",
+                imageFormat = RgbSensor.ImageFormat.PNG,
                 dimension = new Vector2(cam.pixelWidth, cam.pixelHeight),
                 buffer = new byte[0]
             };

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

 {
     public class SimulationState
     {
+        public static int TimeOutFrameCount = 60;
+
+            Starting,
-        internal bool IsCurrentlyRunning()
+        internal bool IsRunning()
-            return ExecutionState == ExecutionStateType.Running || ExecutionState == ExecutionStateType.ShuttingDown;
+            return !IsNotRunning();
         }
 
         internal bool IsNotRunning()
 
-        internal ExecutionStateType ExecutionState { get; private set; } = ExecutionStateType.NotStarted;
+        internal ExecutionStateType ExecutionState { get; private set; }
 
         HashSet<SensorHandle> m_ActiveSensors = new HashSet<SensorHandle>();
         Dictionary<SensorHandle, SensorData> m_Sensors = new Dictionary<SensorHandle, SensorData>();
         List<AdditionalInfoTypeData> m_AdditionalInfoTypeData = new List<AdditionalInfoTypeData>();
 
         Dictionary<PendingFrameId, int> m_PendingIdToFrameMap = new Dictionary<PendingFrameId, int>();
-        Dictionary<PendingFrameId, PendingFrame> m_PendingFrames = new Dictionary<PendingFrameId, PendingFrame>();
+        //Dictionary<PendingFrameId, PendingFrame> m_PendingFrames = new Dictionary<PendingFrameId, PendingFrame>();
+        SortedDictionary<PendingFrameId, PendingFrame> m_PendingFrames = new SortedDictionary<PendingFrameId, PendingFrame>();
 
         CustomSampler m_SerializeCapturesSampler = CustomSampler.Create("SerializeCaptures");
         CustomSampler m_SerializeCapturesAsyncSampler = CustomSampler.Create("SerializeCapturesAsync");
         CustomSampler m_GetOrCreatePendingCaptureForThisFrameSampler = CustomSampler.Create("GetOrCreatePendingCaptureForThisFrame");
         float m_LastTimeScale;
 
-        public bool IsValid(Guid guid) => true; // TODO obvs we need to do this for realz
-
-//        public bool IsRunning { get; private set; }
-
         //A sensor will be triggered if sequenceTime is within includeThreshold seconds of the next trigger
         const float k_SimulationTimingAccuracy = 0.01f;
 
 
+            m_SimulationMetadata = new SimulationMetadata()
+            {
+                unityVersion = Application.unityVersion,
+                perceptionVersion = DatasetCapture.PerceptionVersion,
+            };
+
-        bool ReadyToShutdown => !m_PendingFrames.Any();
+        bool readyToShutdown => !m_PendingFrames.Any();
-
-            bool IsValid();
-        public readonly struct PendingFrameId : IPendingId, IEquatable<PendingFrameId>, IEquatable<PendingSensorId>, IEquatable<PendingCaptureId>
+        public readonly struct PendingFrameId : IPendingId, IEquatable<PendingFrameId>, IEquatable<PendingSensorId>, IEquatable<PendingCaptureId>, IComparable<PendingFrameId>
         {
             public PendingFrameId(int sequence, int step)
             {
             {
                 var otherId = other.AsFrameId();
                 return Sequence == otherId.Sequence && Step == otherId.Step;
+            }
+
+            public int CompareTo(PendingFrameId other)
+            {
+                if (Sequence == other.Sequence) return Step - other.Step;
+                return Sequence - other.Sequence;
             }
 
             public override bool Equals(object obj)
             }
         }
 
-        bool m_FirstNewSequence = true;
+            m_SequenceId++;
             m_Step = -1;
             foreach (var kvp in m_Sensors.ToArray())
             {
                 m_Sensors[kvp.Key] = sensorData;
             }
-
-            if (m_FirstNewSequence)
-                m_FirstNewSequence = false;
-            else
-                m_SequenceId++;
         }
 
         void ResetTimings()
             m_Sensors.Add(sensorHandle, sensorData);
 
             consumerEndpoint.OnSensorRegistered(sensor);
+
+            if (ExecutionState == ExecutionStateType.NotStarted)
+            {
+                ExecutionState = ExecutionStateType.Starting;
+            }
 
             return sensorHandle;
         }
 #endif
         public void Update()
         {
-            if (m_ActiveSensors.Count == 0)
+            // If there aren't any sensors then we are currently stateless?
+            if (ExecutionState == ExecutionStateType.NotStarted)
+            {
+            }
-            if (ExecutionState == ExecutionStateType.NotStarted)
+            if (ExecutionState == ExecutionStateType.Starting)
-                UpdateNotStarted();
+                UpdateStarting();
             }
 
             if (ExecutionState == ExecutionStateType.Running)
                 UpdateShuttdingDown();
             }
 
-            if (ExecutionState == ExecutionStateType.Complete)
+            if (ExecutionState == ExecutionStateType.NotStarted)
-        void UpdateNotStarted()
+        void UpdateStarting()
         {
             m_SimulationMetadata = new SimulationMetadata()
             {
         {
             if (_Ids.Count == 0)
             {
-                ExecutionState = ExecutionStateType.Complete;
+                ExecutionState = ExecutionStateType.NotStarted;
                 return;
             }
 
         void OnComplete()
         {
             if (_Ids.Count == 0)
-            {
-            }
-            if (ReadyToShutdown)
+            if (readyToShutdown)
             {
                 var metadata = new CompletionMetadata()
                 {
 
                 consumerEndpoint.OnSimulationCompleted(metadata);
 
-                ExecutionState = ExecutionStateType.Complete;
+                ExecutionState = ExecutionStateType.NotStarted;
 
                 VerifyNoMorePendingFrames();
             }
         void VerifyNoMorePendingFrames()
         {
             if (m_PendingFrames.Count > 0)
-                Debug.LogError($"Simulation ended with pending annotations: {string.Join(", ", m_PendingFrames.Select(c => $"id:{c.Key}"))}");
+                Debug.LogError($"Simulation ended with pending {m_PendingFrames.Count} annotations (final id): ({m_PendingFrames.Last().Key.Sequence},{m_PendingFrames.Last().Key.Step})");
         }
 
         public void SetNextCaptureTimeToNowForSensor(SensorHandle sensorHandle)
 
         public void End()
         {
+// This is just here for debug reasons, this here will always report errors, but they get cleaned up in shutdown
+//            VerifyNoMorePendingFrames();
+
+
             ExecutionState = ExecutionStateType.ShuttingDown;
         }
 #endif
             return new AsyncMetricFuture(pendingId, this);
         }
 
+        public PendingCaptureId ReportMetric(SensorHandle sensor, MetricDefinition definition, Metric metric)
+        {
+            if (definition == null)
+                throw new ArgumentNullException(nameof(metric));
+
+            var pendingId = new PendingCaptureId(sensor.Id, definition.id, m_SequenceId, AcquireStep());
+            var pendingFrame = GetOrCreatePendingFrame(pendingId.AsFrameId());
+
+            if (pendingFrame == null)
+                throw new InvalidOperationException($"Could not get or create a pending frame for {pendingId}");
+
+            var pendingSensor = pendingFrame.GetOrCreatePendingSensor(pendingId.SensorId);
+
+            pendingSensor.Metrics[pendingId] = metric;
+            return pendingId;
+        }
+
         public PendingCaptureId ReportMetric(SensorHandle sensor, MetricDefinition definition, Metric metric, AnnotationHandle annotation)
         {
             if (definition == null)
                 rotation = Vector3.zero,
                 velocity = Vector3.zero,
                 acceleration = Vector3.zero,
-                imageFormat = "png",
+                imageFormat = RgbSensor.ImageFormat.PNG,
                 dimension = Vector2.zero,
                 buffer = null
             };
 
-        // TODO rename this to 'WritePendingFrames'
+        // TODO rename this to 'ReportPendingFrames'
-            var pendingFramesToWrite = new List<KeyValuePair<PendingFrameId,PendingFrame>>(m_PendingFrames.Count);
+            // TODO do not new these each frame
+            var pendingFramesToWrite = new Queue<KeyValuePair<PendingFrameId,PendingFrame>>(m_PendingFrames.Count);
+            var timedOutFrames = new List<KeyValuePair<PendingFrameId, PendingFrame>>(m_PendingFrames.Count);
+
+            // Write out each frame until we reach one that is not ready to write yet, this is in order to
+            // assure that all reports happen in sequential order
+
-                    pendingFramesToWrite.Add(frame);
+                    pendingFramesToWrite.Enqueue(frame);
+                }
+                else if (currentFrame > recordedFrame + TimeOutFrameCount)
+                {
+                    timedOutFrames.Add(frame);
+                }
+                else
+                {
+                    break;
                 }
             }
 
             }
 
+            foreach (var pf in timedOutFrames)
+            {
+                Debug.LogError($"Frame {pf.Key} timed out");
+                m_PendingFrames.Remove(pf.Key);
+            }
+
             IEnumerable<Sensor> ConvertToSensors(PendingFrame frame, SimulationState simulationState)
             {
                 return frame.sensors.Values.Where(s => s.IsReadyToReport()).Select(s => s.ToSensor());
             {
                 var frameId = m_PendingIdToFrameMap[pendingFrame.PendingId];
-                var frame = new Frame(frameId, pendingFrame.PendingId.Sequence, pendingFrame.PendingId.Step);
+                var frame = new Frame(frameId, pendingFrame.PendingId.Sequence, pendingFrame.PendingId.Step, pendingFrame.Timestamp);
 
                 frame.sensors = ConvertToSensors(pendingFrame, simState);
 #if false
                 return frame;
             }
 
-            void Write(List<KeyValuePair<PendingFrameId, PendingFrame>> frames, SimulationState simulationState)
+            void Write(Queue<KeyValuePair<PendingFrameId, PendingFrame>> frames, SimulationState simulationState)
             {
 #if true
                 // TODO this needs to be done properly, we need to wait on all of the frames to come back so we
                 {
-                    var frame = frames.First();
-                    if (frame.Value.IsReadyToReport())
+                    var converted = ConvertToFrameData(frames.Dequeue().Value, simulationState);
+                    m_TotalFrames++;
+
+                    if (converted == null)
-                        frames.Remove(frame);
-                        var converted = ConvertToFrameData(frame.Value, simulationState);
-                        m_TotalFrames++;
-                        consumerEndpoint.OnFrameGenerated(converted);
+                        Debug.LogError("Could not convert frame data");
+
+                    if (consumerEndpoint == null)
+                    {
+                        Debug.LogError("Consumer endpoint is null");
+                    }
+
+                    consumerEndpoint.OnFrameGenerated(converted);
                 }
 #else
                 foreach (var pendingFrame in frames)
                 }
 #endif
             }
-
+#if false
+#endif
-            }
+#if false
+        }
             else
             {
                 var req = Manager.Instance.CreateRequest<AsyncRequest<WritePendingCaptureRequestData>>();
                 });
                 req.Execute(AsyncRequest.ExecutionContext.JobSystem);
             }
-
+#endif
-
+#if false
-            public List<KeyValuePair<PendingFrameId, PendingFrame>> PendingFrames;
+            public Queue<KeyValuePair<PendingFrameId, PendingFrame>> PendingFrames;
+#endif
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox3dTests.cs

 using Unity.Mathematics;
 using UnityEngine;
 using UnityEngine.Perception.GroundTruth;
+using UnityEngine.Perception.GroundTruth.DataModel;
 using UnityEngine.TestTools;
 
 namespace GroundTruthTests

com.unity.perception/Tests/Runtime/GroundTruthTests/DatasetCaptureSensorSchedulingTests.cs

 
         internal SimulationStateTestHelper()
         {
-            m_State = DatasetCapture.Instance.simulationState;
+            m_State = DatasetCapture.Instance.currentSimulation;
             var bindingFlags = BindingFlags.NonPublic | BindingFlags.Instance;
             m_SequenceTimeOfNextCaptureMethod = m_State.GetType().GetMethod("GetSequenceTimeOfNextCapture", bindingFlags);
             Debug.Assert(m_SequenceTimeOfNextCaptureMethod != null, "Couldn't find sequence time method.");

com.unity.perception/Tests/Runtime/GroundTruthTests/GroundTruthTestBase.cs

                 SceneManager.UnloadSceneAsync(s);
 
             m_ScenesToUnload.Clear();
+
+            DatasetCapture.Instance.ResetSimulation();
+            Time.timeScale = 1;
-            DatasetCapture.ResetSimulation();
-            Time.timeScale = 1;
             if (Directory.Exists(DatasetCapture.OutputDirectory))
                 Directory.Delete(DatasetCapture.OutputDirectory, true);
 #endif

com.unity.perception/Tests/Runtime/GroundTruthTests/PerceptionCameraIntegrationTests.cs

         [UnityPlatform(RuntimePlatform.LinuxPlayer, RuntimePlatform.WindowsPlayer)]
         public IEnumerator EnableBoundingBoxes_GeneratesCorrectDataset()
         {
-            #if false
-            var jsonExpected = $@"[
-            {{
-              ""label_id"": 100,
-              ""label_name"": ""label"",
-              ""instance_id"": 1,
-              ""x"": 0.0,
-              ""y"": {Screen.height / 4:F1},
-              ""width"": {Screen.width:F1},
-              ""height"": {Screen.height / 2:F1}
-            }}
-          ]";
+          //   var jsonExpected = $@"[
+          //   {{
+          //     ""label_id"": 100,
+          //     ""label_name"": ""label"",
+          //     ""instance_id"": 1,
+          //     ""x"": 0.0,
+          //     ""y"": {Screen.height / 4:F1},
+          //     ""width"": {Screen.width:F1},
+          //     ""height"": {Screen.height / 2:F1}
+          //   }}
+          // ]";
+
             var labelingConfiguration = CreateLabelingConfiguration();
             SetupCamera(pc =>
             {
             plane2.transform.localScale = new Vector3(.1f, -1f, .1f);
             plane2.transform.localPosition = new Vector3(0, 0, 5);
             yield return null;
-            DatasetCapture.ResetSimulation();
+
+            var collector = new CollectEndpoint();
+            DatasetCapture.SetEndpoint(collector);
+
+            DatasetCapture.Instance.automaticShutdown = false;
+
+            DatasetCapture.Instance.ResetSimulation();
+
+            var dcWatcher = new DatasetCapture.WaitUntilComplete();
+            yield return dcWatcher;
-            var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
-            var capturesJson = File.ReadAllText(capturesPath);
-            StringAssert.Contains(TestHelper.NormalizeJson(jsonExpected, true), TestHelper.NormalizeJson(capturesJson, true));
-#endif
-            yield return null;
+            // TODO what do I check here
+            Assert.NotZero(0);
+
+
+            // var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
+            // var capturesJson = File.ReadAllText(capturesPath);
+            // StringAssert.Contains(TestHelper.NormalizeJson(jsonExpected, true), TestHelper.NormalizeJson(capturesJson, true));
-#if false
+
+            var collector = new CollectEndpoint();
+            DatasetCapture.SetEndpoint(collector);
+            DatasetCapture.Instance.automaticShutdown = false;
 
             SemanticSegmentationLabeler semanticSegmentationLabeler = null;
             SetupCamera(pc =>
 
             this.AddTestObjectForCleanup(TestHelper.CreateLabeledPlane());
             yield return null;
-            DatasetCapture.ResetSimulation();
+            DatasetCapture.Instance.ResetSimulation();
-            if (enabled)
-            {
-                var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
-                var capturesJson = File.ReadAllText(capturesPath);
-                var imagePath = $"{semanticSegmentationLabeler.semanticSegmentationDirectory}/{expectedImageFilename}";
-                StringAssert.Contains(imagePath, capturesJson);
-            }
-            else
-            {
-                DirectoryAssert.DoesNotExist(DatasetCapture.OutputDirectory);
-            }
+            var dcWatcher = new DatasetCapture.WaitUntilComplete();
+            yield return dcWatcher;
+
+            // What should I test for here...
+
+            Assert.NotZero(0);
+
+            // if (enabled)
+            // {
+            //     var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
+            //     var capturesJson = File.ReadAllText(capturesPath);
+            //     var imagePath = $"{semanticSegmentationLabeler.semanticSegmentationDirectory}/{expectedImageFilename}";
+            //     StringAssert.Contains(imagePath, capturesJson);
+            // }
+            // else
+            // {
+            //     DirectoryAssert.DoesNotExist(DatasetCapture.OutputDirectory);
+            // }
+            var collector = new CollectEndpoint();
+            DatasetCapture.SetEndpoint(collector);
+            DatasetCapture.Instance.automaticShutdown = false;
+
             SemanticSegmentationLabeler semanticSegmentationLabeler = null;
             SetupCamera(pc =>
             {
 
             this.AddTestObjectForCleanup(TestHelper.CreateLabeledPlane());
             yield return null;
-            DatasetCapture.ResetSimulation();
+            DatasetCapture.Instance.ResetSimulation();
-            var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
-            var capturesJson = File.ReadAllText(capturesPath);
-            var imagePath = $"{semanticSegmentationLabeler.semanticSegmentationDirectory}/{expectedImageFilename}";
-            StringAssert.Contains(imagePath, capturesJson);
+            var dcWatcher = new DatasetCapture.WaitUntilComplete();
+            yield return dcWatcher;
+
+            // What should I test for here...
+
+            Assert.NotZero(0);
+
+            // var capturesPath = Path.Combine(DatasetCapture.OutputDirectory, "captures_000.json");
+            // var capturesJson = File.ReadAllText(capturesPath);
+            // var imagePath = $"{semanticSegmentationLabeler.semanticSegmentationDirectory}/{expectedImageFilename}";
+            // StringAssert.Contains(imagePath, capturesJson);
         }
 
         static IdLabelConfig CreateLabelingConfiguration()
             });
             return labelingConfiguration;
         }
-#endif
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs

             boundingBoxes.Dispose();
             cache.Dispose();
         }
-#if false
 
         [UnityTest]
         public IEnumerator LabelsCorrectWhenIdsReset()
                 Debug.Log($"Bounding boxes received. FrameCount: {eventArgs.frameCount}");
 
                 Assert.AreEqual(1, eventArgs.data.Count());
-                Assert.AreEqual(expectedLabelIdAtFrame[eventArgs.frameCount], eventArgs.data.First().label_id);
+                Assert.AreEqual(expectedLabelIdAtFrame[eventArgs.frameCount], eventArgs.data.First().labelId);
             }
 
             var idLabelConfig = ScriptableObject.CreateInstance<IdLabelConfig>();
 
             Assert.AreEqual(3, timesInfoReceived);
         }
-#endif
+
         private GameObject SetupCameraBoundingBox2D(Action<BoundingBox2DLabeler.BoundingBoxesCalculatedEventArgs> onBoundingBoxesCalculated, IdLabelConfig idLabelConfig)
         {
             var cameraObject = SetupCamera(camera =>

com.unity.perception/Tests/Runtime/GroundTruthTests/VisualizationTests.cs

             });
 
             var labeler1 = new ObjectCountLabeler(cfg);
-//            labeler1.objectCountMetricId = "a1da3c27-369d-4929-aea6-d01614635ce2";
-//            labeler2.objectCountMetricId = "b1da3c27-369d-4929-aea6-d01614635ce2";
 
             perceptionCamera.AddLabeler(labeler1);
             perceptionCamera.AddLabeler(labeler2);

com.unity.perception/Tests/Runtime/GroundTruthTests/CollectEndpoint.cs

+using System.Collections.Generic;
+using UnityEngine;
+using UnityEngine.Perception.GroundTruth;
+using UnityEngine.Perception.GroundTruth.DataModel;
+
+namespace GroundTruthTests
+{
+    public class CollectEndpoint : ConsumerEndpoint
+    {
+        public List<SensorDefinition> sensors = new List<SensorDefinition>();
+        public List<AnnotationDefinition> annotationDefinitions = new List<AnnotationDefinition>();
+        public List<MetricDefinition> metricDefinitions = new List<MetricDefinition>();
+
+        public struct SimulationRun
+        {
+            public int TotalFrames => frames.Count;
+            public List<Frame> frames;
+            public SimulationMetadata metadata;
+        }
+
+        public List<SimulationRun> collectedRuns = new List<SimulationRun>();
+        public SimulationRun currentRun;
+
+        public override void OnSensorRegistered(SensorDefinition sensor)
+        {
+            sensors.Add(sensor);
+        }
+
+        public override void OnAnnotationRegistered(AnnotationDefinition annotationDefinition)
+        {
+            annotationDefinitions.Add(annotationDefinition);
+        }
+
+        public override void OnMetricRegistered(MetricDefinition metricDefinition)
+        {
+            metricDefinitions.Add(metricDefinition);
+        }
+
+        protected override bool IsComplete()
+        {
+            return true;
+        }
+
+        public override void OnSimulationStarted(SimulationMetadata metadata)
+        {
+            currentRun = new SimulationRun
+            {
+                frames = new List<Frame>()
+            };
+            Debug.Log("Collect Enpoint OnSimulationStarted");
+        }
+
+        public override void OnFrameGenerated(Frame frame)
+        {
+            if (currentRun.frames == null)
+            {
+                Debug.LogError("Current run frames is null, probably means that OnSimulationStarted was never called");
+            }
+
+            currentRun.frames.Add(frame);
+            Debug.Log("Collect Enpoint OnFrameGenerted");
+        }
+
+        public override void OnSimulationCompleted(CompletionMetadata metadata)
+        {
+            currentRun.metadata = metadata;
+            collectedRuns.Add(currentRun);
+            Debug.Log("Collect Enpoint OnSimulationCompleted");
+        }
+    }
+}

com.unity.perception/Tests/Runtime/GroundTruthTests/CollectEndpoint.cs.meta

+fileFormatVersion: 2
+guid: a8f768cefda95fe4a9a52e35cc744d4d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: