Merge pull request #164 from Unity-Technologies/manual_capture_2

New capture options added to Perception Camera

LICENSE.md

-com.unity.perception copyright © 2020 Unity Technologies ApS
+com.unity.perception copyright © 2021 Unity Technologies ApS
 
                                  Apache License
                            Version 2.0, January 2004

com.unity.perception/CHANGELOG.md

 
 Added ParameterUIElementsEditor class to allow custom ScriptableObjects and MonoBehaviours to render Parameter and Sampler typed public fields correctly in their inspector windows.
 
+Added new capture options to Perception Camera:
+* Can now render intermediate frames between captures.
+* Capture can now be triggered manually using a function call, instead of automatic capturing on a schedule.
+
 ### Changed
 
 Randomizers now access their parent scenario through the static activeScenario property

com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs

             serializedObject.ApplyModifiedProperties();
         }
 
+        const string k_FrametimeTitle = "Simulation Delta Time";
+
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.description)), new GUIContent("Description", "Provide a description for this perception camera (optional)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.period)), new GUIContent("Capture Interval", "The interval at which the perception camera should render and capture (seconds)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.startTime)), new GUIContent("Start Time","Time at which this perception camera starts rendering and capturing (seconds)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.showVisualizations)), new GUIContent("Show Labeler Visualizations", "Display realtime visualizations for labelers that are currently active on this perception camera."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureRgbImages)),new GUIContent("Save Camera Output to Disk", "For each captured frame, save an RGB image of the perception camera's output to disk."));
+                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.description)), new GUIContent("Description", "Provide a description for this camera (optional)."));
+                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.showVisualizations)), new GUIContent("Show Labeler Visualizations", "Display realtime visualizations for labelers that are currently active on this camera."));
+                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureRgbImages)),new GUIContent("Save Camera RGB Output to Disk", "For each captured frame, save an RGB image of the camera's output to disk."));
+                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureTriggerMode)),new GUIContent("Capture Trigger Mode", $"The method of triggering captures for this camera. In {nameof(CaptureTriggerMode.Scheduled)} mode, captures happen automatically based on a start frame and frame delta time. In {nameof(CaptureTriggerMode.Manual)} mode, captures should be triggered manually through calling the {nameof(perceptionCamera.RequestCapture)} method of {nameof(PerceptionCamera)}."));
+
+                GUILayout.Space(5);
+                if (perceptionCamera.captureTriggerMode.Equals(CaptureTriggerMode.Scheduled))
+                {
+                    GUILayout.BeginVertical("TextArea");
+                    EditorGUILayout.LabelField("Scheduled Capture Properties", EditorStyles.boldLabel);
+
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.simulationDeltaTime)),new GUIContent(k_FrametimeTitle, $"Sets Unity's Time.{nameof(Time.captureDeltaTime)} to the specified number, causing a fixed number of frames to be simulated for each second of elapsed simulation time regardless of the capabilities of the underlying hardware. Thus, simulation time and real time will not be synchronized."));
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.firstCaptureFrame)), new GUIContent("Start at Frame",$"Frame number at which this camera starts capturing."));
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.framesBetweenCaptures)),new GUIContent("Frames Between Captures", "The number of frames to simulate and render between the camera's scheduled captures. Setting this to 0 makes the camera capture every frame."));
+
+                    var interval = (perceptionCamera.framesBetweenCaptures + 1) * perceptionCamera.simulationDeltaTime;
+                    var startTime = perceptionCamera.simulationDeltaTime * perceptionCamera.firstCaptureFrame;
+                    EditorGUILayout.HelpBox($"First capture at {startTime} seconds and consecutive captures every {interval} seconds of simulation time.", MessageType.None);
+
+                    GUILayout.EndVertical();
+                }
+                else
+                {
+                    GUILayout.BeginVertical("TextArea");
+                    EditorGUILayout.LabelField("Manual Capture Properties", EditorStyles.boldLabel);
+
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.manualSensorAffectSimulationTiming)),new GUIContent("Affect Simulation Timing", $"Have this camera affect simulation timings (similar to a scheduled camera) by requesting a specific frame delta time."));
+
+                    if (perceptionCamera.manualSensorAffectSimulationTiming)
+                    {
+                        EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.simulationDeltaTime)),new GUIContent(k_FrametimeTitle, $"Sets Unity's Time.{nameof(Time.captureDeltaTime)} to the specified number, causing a fixed number of frames to be generated for each second of elapsed simulation time regardless of the capabilities of the underlying hardware. Thus, simulation time and real time will not be synchronized."));
+                    }
+
+                    EditorGUILayout.HelpBox($"Captures should be triggered manually through calling the {nameof(perceptionCamera.RequestCapture)} method of {nameof(PerceptionCamera)}.", MessageType.None);
+                    GUILayout.EndVertical();
+                }
+
+
-                //EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(PerceptionCamera.labelers)));
+                GUILayout.Space(15);
+
                 m_LabelersList.DoLayoutList();
             }
 

com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs

         /// <param name="egoHandle">The ego container for the sensor. Sensor orientation will be reported in the context of the given ego.</param>
         /// <param name="modality">The kind of the sensor (ex. "camera", "lidar")</param>
         /// <param name="description">A human-readable description of the sensor (ex. "front-left rgb camera")</param>
-        /// <param name="period">The period, in seconds, on which the sensor should capture. Frames will be scheduled in the simulation such that each sensor is triggered every _period_ seconds.</param>
-        /// <param name="firstCaptureTime">The time, in seconds, from the start of the sequence on which this sensor should first be scheduled.</param>
+        /// <param name="firstCaptureFrame">The time, in seconds, from the start of the sequence on which this sensor should first be scheduled.</param>
+        /// <param name="captureTriggerMode">The method of triggering captures for this sensor.</param>
+        /// <param name="simulationDeltaTime">The simulation frame time (seconds) requested by this sensor.</param>
+        /// <param name="framesBetweenCaptures">The number of frames to simulate and render between the camera's scheduled captures. Setting this to 0 makes the camera capture every frame.</param>
+        /// <param name="manualSensorAffectSimulationTiming">Have this unscheduled (manual capture) camera affect simulation timings (similar to a scheduled camera) by requesting a specific frame delta time</param>
-        public static SensorHandle RegisterSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime)
+        public static SensorHandle RegisterSensor(EgoHandle egoHandle, string modality, string description, float firstCaptureFrame, CaptureTriggerMode captureTriggerMode, float simulationDeltaTime, int framesBetweenCaptures, bool manualSensorAffectSimulationTiming = false)
-            SimulationState.AddSensor(egoHandle, modality, description, period, firstCaptureTime, sensor);
+            SimulationState.AddSensor(egoHandle, modality, description, firstCaptureFrame, captureTriggerMode, simulationDeltaTime, framesBetweenCaptures, manualSensorAffectSimulationTiming, sensor);
             return sensor;
         }
 
         }
     }
 
+
+    /// <summary>
+    /// Capture trigger modes for sensors.
+    /// </summary>
+    public enum CaptureTriggerMode
+    {
+        /// <summary>
+        /// Captures happen automatically based on a start frame and frame delta time.
+        /// </summary>
+        Scheduled,
+        /// <summary>
+        /// Captures should be triggered manually through calling the manual capture method of the sensor using this trigger mode.
+        /// </summary>
+        Manual
+    }
+
     /// <summary>
     /// A handle to a sensor managed by the <see cref="DatasetCapture"/>. It can be used to check whether the sensor
     /// is expected to capture this frame and report captures, annotations, and metrics regarding the sensor.
         /// they should capture during the frame. Captures should only be reported when this is true.
         /// </summary>
         public bool ShouldCaptureThisFrame => DatasetCapture.SimulationState.ShouldCaptureThisFrame(this);
+
+        /// <summary>
+        /// Requests a capture from this sensor on the next rendered frame. Can only be used with manual capture mode (<see cref="PerceptionCamera.CaptureTriggerMode.Manual"/>).
+        /// </summary>
+        public void RequestCapture()
+        {
+            DatasetCapture.SimulationState.SetNextCaptureTimeToNowForSensor(this);
+        }
 
         /// <summary>
         /// Report a metric regarding this sensor in the current frame.

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

         /// A human-readable description of the camera.
         /// </summary>
         public string description;
-        /// <summary>
-        /// The interval in seconds at which the camera should render and capture.
-        /// </summary>
-        public float period = .0166f;
-        /// <summary>
-        /// The start time in seconds of the first frame in the simulation.
-        /// </summary>
-        public float startTime;
+
         /// <summary>
         /// Whether camera output should be captured to disk
         /// </summary>
         public Camera attachedCamera => m_AttachedCamera;
 
         /// <summary>
+        /// Frame number at which this camera starts capturing.
+        /// </summary>
+        public int firstCaptureFrame = 0;
+
+        /// <summary>
+        /// The method of triggering captures for this camera.
+        /// </summary>
+        public CaptureTriggerMode captureTriggerMode = CaptureTriggerMode.Scheduled;
+
+        /// <summary>
+        /// Have this unscheduled (manual capture) camera affect simulation timings (similar to a scheduled camera) by requesting a specific frame delta time
+        /// </summary>
+        public bool manualSensorAffectSimulationTiming = false;
+
+        /// <summary>
+        /// The simulation frame time (seconds) for this camera. E.g. 0.0166 translates to 60 frames per second. This will be used as Unity's <see cref="Time.captureDeltaTime"/>, causing a fixed number of frames to be generated for each second of elapsed simulation time regardless of the capabilities of the underlying hardware.
+        /// </summary>
+        public float simulationDeltaTime = 0.0166f;
+
+        /// <summary>
+        /// The number of frames to simulate and render between the camera's scheduled captures. Setting this to 0 makes the camera capture every frame.
+        /// </summary>
+        public int framesBetweenCaptures = 0;
+
+        /// <summary>
+        /// Requests a capture from this camera on the next rendered frame. Can only be used when using <see cref="PerceptionCamera.CaptureTriggerMode.Manual"/> capture mode.
+        /// </summary>
+        public void RequestCapture()
+        {
+            if (captureTriggerMode.Equals(CaptureTriggerMode.Manual))
+            {
+                SensorHandle.RequestCapture();
+            }
+            else
+            {
+                Debug.LogError($"{nameof(RequestCapture)} can only be used if the camera is in {nameof(CaptureTriggerMode.Manual)} capture mode.");
+            }
+        }
+
+        /// <summary>
         /// Event invoked after the camera finishes rendering during a frame.
         /// </summary>
 
             {
                 m_EgoMarker = GetComponentInParent<Ego>();
                 var ego = m_EgoMarker == null ? DatasetCapture.RegisterEgo("") : m_EgoMarker.EgoHandle;
-                SensorHandle = DatasetCapture.RegisterSensor(ego, "camera", description, period, startTime);
+                SensorHandle = DatasetCapture.RegisterSensor(ego, "camera", description, firstCaptureFrame, captureTriggerMode, simulationDeltaTime, framesBetweenCaptures, manualSensorAffectSimulationTiming);
             }
         }
 
             EnsureSensorRegistered();
             if (!SensorHandle.IsValid)
                 return;
-
-            m_AttachedCamera.enabled = SensorHandle.ShouldCaptureThisFrame;
 
             bool anyVisualizing = false;
             foreach (var labeler in m_Labelers)

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

         }
 
         //A sensor will be triggered if sequenceTime is within includeThreshold seconds of the next trigger
-        const float k_IncludeInFrameThreshold = .01f;
+        const float k_SimulationTimingAccuracy = 0.01f;
-        const int k_MaxDeltaTime = 10;
+        const float k_MaxDeltaTime = 100f;
 
         public SimulationState(string outputDirectory)
         {
         {
             public string modality;
             public string description;
-            public float period;
+            public CaptureTriggerMode captureTriggerMode;
+            public float renderingDeltaTime;
+            public int framesBetweenCaptures;
+            public bool manualSensorAffectSimulationTiming;
-            public float sequenceTimeNextCapture;
+            public float sequenceTimeOfNextCapture;
+            public float sequenceTimeOfNextRender;
             public int lastCaptureFrameCount;
             public EgoHandle egoHandle;
         }
             foreach (var kvp in m_Sensors.ToArray())
             {
                 var sensorData = kvp.Value;
-                sensorData.sequenceTimeNextCapture = SequenceTimeOfNextCapture(sensorData);
+                sensorData.sequenceTimeOfNextCapture = GetSequenceTimeOfNextCapture(sensorData);
+                sensorData.sequenceTimeOfNextRender = 0;
                 m_Sensors[kvp.Key] = sensorData;
             }
 
             m_LastTimeScale = Time.timeScale;
         }
 
-        public void AddSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime, SensorHandle sensor)
+        public void AddSensor(EgoHandle egoHandle, string modality, string description, float firstCaptureFrame, CaptureTriggerMode captureTriggerMode, float renderingDeltaTime, int framesBetweenCaptures, bool manualSensorAffectSimulationTiming, SensorHandle sensor)
-                period = period,
-                firstCaptureTime = firstCaptureTime,
+                firstCaptureTime = firstCaptureFrame * renderingDeltaTime,
+                captureTriggerMode = captureTriggerMode,
+                renderingDeltaTime = renderingDeltaTime,
+                framesBetweenCaptures = framesBetweenCaptures,
+                manualSensorAffectSimulationTiming = manualSensorAffectSimulationTiming,
-            sensorData.sequenceTimeNextCapture = SequenceTimeOfNextCapture(sensorData);
+            sensorData.sequenceTimeOfNextCapture = GetSequenceTimeOfNextCapture(sensorData);
+            sensorData.sequenceTimeOfNextRender = 0;
-        float SequenceTimeOfNextCapture(SensorData sensorData)
+        float GetSequenceTimeOfNextCapture(SensorData sensorData)
-                return sensorData.firstCaptureTime;
-            return sensorData.sequenceTimeNextCapture;
+            {
+                return sensorData.captureTriggerMode == CaptureTriggerMode.Scheduled? sensorData.firstCaptureTime : float.MaxValue;
+            }
+
+            return sensorData.sequenceTimeOfNextCapture;
         }
 
         public bool Contains(Guid id) => m_Ids.Contains(id);
                 m_HasStarted = true;
             }
 
+            EnsureSequenceTimingsUpdated();
+
-                if (!activeSensor.ShouldCaptureThisFrame)
-                    continue;
+
+                if (Mathf.Abs(sensorData.sequenceTimeOfNextRender - UnscaledSequenceTime) < k_SimulationTimingAccuracy)
+                {
+                    //means this frame fulfills this sensor's simulation time requirements, we can move target to next frame.
+                    sensorData.sequenceTimeOfNextRender += sensorData.renderingDeltaTime;
+                }
+
+                if (activeSensor.ShouldCaptureThisFrame)
+                {
+                    if (sensorData.captureTriggerMode.Equals(CaptureTriggerMode.Scheduled))
+                    {
+                        sensorData.sequenceTimeOfNextCapture += sensorData.renderingDeltaTime * (sensorData.framesBetweenCaptures + 1);
+                        Debug.Assert(sensorData.sequenceTimeOfNextCapture > UnscaledSequenceTime,
+                            $"Next scheduled capture should be after {UnscaledSequenceTime} but is {sensorData.sequenceTimeOfNextCapture}");
+                    }
+                    else if (sensorData.captureTriggerMode.Equals(CaptureTriggerMode.Manual))
+                    {
+                        sensorData.sequenceTimeOfNextCapture = float.MaxValue;
+                    }
+
+                    sensorData.lastCaptureFrameCount = Time.frameCount;
+                }
-                // TODO: AISV-845 This is an errant modification of this record that can lead to undefined behavior
-                // Leaving as-is for now because too many components depend on this logic
-                sensorData.sequenceTimeNextCapture += sensorData.period;
-                Debug.Assert(sensorData.sequenceTimeNextCapture > UnscaledSequenceTime,
-                    $"Next scheduled capture should be after {UnscaledSequenceTime} but is {sensorData.sequenceTimeNextCapture}");
-                // sensorData.sequenceTimeNextCapture = SequenceTimeOfNextCapture(sensorData);
-                sensorData.lastCaptureFrameCount = Time.frameCount;
                 m_Sensors[activeSensor] = sensorData;
             }
 
             {
+                float thisSensorNextFrameDt = -1;
+
-                var thisSensorNextFrameDt = sensorData.sequenceTimeNextCapture - UnscaledSequenceTime;
+                if (sensorData.captureTriggerMode.Equals(CaptureTriggerMode.Scheduled))
+                {
+                    thisSensorNextFrameDt = sensorData.sequenceTimeOfNextRender - UnscaledSequenceTime;
-                Debug.Assert(thisSensorNextFrameDt > 0f, "Sensor was scheduled to run in the past but got skipped over.");
+                    Debug.Assert(thisSensorNextFrameDt > 0f, "Sensor was scheduled to capture in the past but got skipped over.");
+                }
+                else if (sensorData.captureTriggerMode.Equals(CaptureTriggerMode.Manual) && sensorData.manualSensorAffectSimulationTiming)
+                {
+                    thisSensorNextFrameDt = sensorData.sequenceTimeOfNextRender - UnscaledSequenceTime;
+                }
+
+            }
+
+            if (Math.Abs(nextFrameDt - k_MaxDeltaTime) < 0.0001)
+            {
+                //means no sensor is controlling simulation timing, so we set Time.captureDeltaTime to 0 (default) which means the setting does not do anything
+                nextFrameDt = 0;
             }
 
             WritePendingCaptures();
         }
 
+        public void SetNextCaptureTimeToNowForSensor(SensorHandle sensorHandle)
+        {
+            if (!m_Sensors.ContainsKey(sensorHandle))
+                return;
+
+            var data = m_Sensors[sensorHandle];
+            data.sequenceTimeOfNextCapture = UnscaledSequenceTime;
+            m_Sensors[sensorHandle] = data;
+        }
+
         public bool ShouldCaptureThisFrame(SensorHandle sensorHandle)
         {
             if (!m_Sensors.ContainsKey(sensorHandle))
             if (data.lastCaptureFrameCount == Time.frameCount)
                 return true;
 
-            return (data.sequenceTimeNextCapture - UnscaledSequenceTime) < k_IncludeInFrameThreshold;
+            return data.sequenceTimeOfNextCapture - UnscaledSequenceTime < k_SimulationTimingAccuracy;
         }
 
         public void End()

com.unity.perception/Tests/Editor/DatasetCaptureEditorTests.cs

             yield return new EnterPlayMode();
             DatasetCapture.ResetSimulation();
             var ego = DatasetCapture.RegisterEgo("ego");
-            var sensor = DatasetCapture.RegisterSensor(ego, "camera", "", 0.1f, 0);
+            var sensor = DatasetCapture.RegisterSensor(ego, "camera", "", 0, CaptureTriggerMode.Scheduled, 0.1f, 0);
             sensor.ReportCapture("file.txt", new SensorSpatialData());
             expectedDatasetPath = DatasetCapture.OutputDirectory;
             yield return new ExitPlayMode();

com.unity.perception/Tests/Runtime/GroundTruthTests/DatasetCaptureSensorSchedulingTests.cs

 using System.Collections;
 using System.Collections.Generic;
+using System.Linq;
+using Unity.Mathematics;
+using Random = UnityEngine.Random;
 
 namespace GroundTruthTests
 {
         internal SimulationStateTestHelper()
         {
             var bindingFlags = BindingFlags.NonPublic | BindingFlags.Instance;
-            m_SequenceTimeOfNextCaptureMethod = m_State.GetType().GetMethod("SequenceTimeOfNextCapture", bindingFlags);
+            m_SequenceTimeOfNextCaptureMethod = m_State.GetType().GetMethod("GetSequenceTimeOfNextCapture", bindingFlags);
             Debug.Assert(m_SequenceTimeOfNextCaptureMethod != null, "Couldn't find sequence time method.");
             var sensorsField = m_State.GetType().GetField("m_Sensors", bindingFlags);
             Debug.Assert(sensorsField != null, "Couldn't find internal sensors field");
         public IEnumerator SequenceTimeOfNextCapture_ReportsCorrectTime()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            var firstCaptureTime = 1.5f;
-            var period = .4f;
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", period, firstCaptureTime);
+            var firstCaptureFrame = 2f;
+            var simulationDeltaTime = .4f;
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, simulationDeltaTime, 0);
+            var startTime = firstCaptureFrame * simulationDeltaTime;
-                firstCaptureTime,
-                period + firstCaptureTime,
-                period * 2 + firstCaptureTime,
-                period * 3 + firstCaptureTime
+                startTime,
+                simulationDeltaTime + startTime,
+                simulationDeltaTime * 2 + startTime,
+                simulationDeltaTime * 3 + startTime
-            for (var i = 0; i < sequenceTimesExpected.Length; i++)
+
+            for (var i = 0; i < firstCaptureFrame; i++)
+                //render the non-captured frames before firstCaptureFrame
+            }
+
+            for (var i = 0; i < sequenceTimesExpected.Length; i++)
+            {
+                yield return null;
+            }
+        }
+
+        [UnityTest]
+        public IEnumerator SequenceTimeOfNextCapture_WithInBetweenFrames_ReportsCorrectTime()
+        {
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var firstCaptureFrame = 2;
+            var simulationDeltaTime = .4f;
+            var framesBetweenCaptures = 2;
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, simulationDeltaTime, framesBetweenCaptures);
+            var startingFrame = Time.frameCount;
+
+            var startTime = firstCaptureFrame * simulationDeltaTime;
+            var interval = (framesBetweenCaptures + 1) * simulationDeltaTime;
+            float[] sequenceTimesExpected =
+            {
+                startTime,
+                interval + startTime,
+                interval * 2 + startTime,
+                interval * 3 + startTime
+            };
+
+            int[] simulationFramesToCheck =
+            {
+                firstCaptureFrame,
+                firstCaptureFrame + (framesBetweenCaptures + 1),
+                firstCaptureFrame + (framesBetweenCaptures + 1) * 2,
+                firstCaptureFrame + (framesBetweenCaptures + 1) * 3,
+            };
+
+            int checkedFrame = 0;
+            var currentSimFrame = Time.frameCount - startingFrame;
+            while (currentSimFrame <= simulationFramesToCheck[simulationFramesToCheck.Length - 1] && checkedFrame < simulationFramesToCheck.Length)
+            {
+                currentSimFrame = Time.frameCount - startingFrame;
+                if (currentSimFrame == simulationFramesToCheck[checkedFrame])
+                {
+                    var sensorData = m_TestHelper.GetSensorData(sensorHandle);
+                    var sequenceTimeActual = m_TestHelper.CallSequenceTimeOfNextCapture(sensorData);
+                    Assert.AreEqual(sequenceTimesExpected[checkedFrame], sequenceTimeActual, 0.0001f);
+                    checkedFrame++;
+                }
+                else
+                {
+                    yield return null;
+                }
             }
         }
 
             var ego = DatasetCapture.RegisterEgo("ego");
-            var firstCaptureTime = 1.5f;
-            var period = .4f;
-            DatasetCapture.RegisterSensor(ego, "cam", "", period, firstCaptureTime);
+            var firstCaptureFrame = 2f;
+            var simulationDeltaTime = .4f;
+            DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, simulationDeltaTime, 0);
-                firstCaptureTime,
-                period,
-                period,
-                period
+                simulationDeltaTime,
+                simulationDeltaTime,
+                simulationDeltaTime,
+                simulationDeltaTime
             };
             float[] deltaTimeSamples = new float[deltaTimeSamplesExpected.Length];
             for (int i = 0; i < deltaTimeSamples.Length; i++)
         public IEnumerator FramesScheduled_WithTimeScale_ResultsInProperDeltaTime()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            var firstCaptureTime = 2f;
-            var period = 1f;
+            var firstCaptureFrame = 2f;
+            var simulationDeltaTime = 1f;
-            DatasetCapture.RegisterSensor(ego, "cam", "", period, firstCaptureTime);
+            DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, simulationDeltaTime, 0);
-                timeScale * firstCaptureTime,
-                timeScale * period,
-                timeScale * period,
-                timeScale * period
+                timeScale * simulationDeltaTime,
+                timeScale * simulationDeltaTime,
+                timeScale * simulationDeltaTime,
+                timeScale * simulationDeltaTime
             };
             float[] deltaTimeSamples = new float[deltaTimeSamplesExpected.Length];
             for (int i = 0; i < deltaTimeSamples.Length; i++)
         public IEnumerator ChangingTimeScale_CausesDebugError()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            DatasetCapture.RegisterSensor(ego, "cam", "", 1f, 2f);
+            DatasetCapture.RegisterSensor(ego, "cam", "", 2f, CaptureTriggerMode.Scheduled, 1, 0);
 
             yield return null;
             Time.timeScale = 5;
         public IEnumerator ChangingTimeScale_DuringStartNewSequence_Succeeds()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            DatasetCapture.RegisterSensor(ego, "cam", "", 1f, 2f);
+            DatasetCapture.RegisterSensor(ego, "cam", "", 2f, CaptureTriggerMode.Scheduled, 1, 0);
 
             yield return null;
             Time.timeScale = 1;
         public IEnumerator FramesScheduled_WithChangingTimeScale_ResultsInProperDeltaTime()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            var firstCaptureTime = 2f;
-            var period = 1f;
+            var firstCaptureFrame = 2f;
+            var simulationDeltaTime = 1f;
             float[] newTimeScalesPerFrame =
             {
                 2f,
             };
-            DatasetCapture.RegisterSensor(ego, "cam", "", period, firstCaptureTime);
+            DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, 1, 0);
-                newTimeScalesPerFrame[0] * firstCaptureTime,
-                newTimeScalesPerFrame[1] * period,
-                newTimeScalesPerFrame[2] * period,
-                newTimeScalesPerFrame[3] * period
+                newTimeScalesPerFrame[0] * simulationDeltaTime,
+                newTimeScalesPerFrame[1] * simulationDeltaTime,
+                newTimeScalesPerFrame[2] * simulationDeltaTime,
+                newTimeScalesPerFrame[3] * simulationDeltaTime
             };
             float[] deltaTimeSamples = new float[deltaTimeSamplesExpected.Length];
             for (int i = 0; i < deltaTimeSamples.Length; i++)
         public IEnumerator ResetSimulation_ResetsCaptureDeltaTime()
         {
             var ego = DatasetCapture.RegisterEgo("ego");
-            DatasetCapture.RegisterSensor(ego, "cam", "", 4, 10);
+            DatasetCapture.RegisterSensor(ego, "cam", "", 0, CaptureTriggerMode.Scheduled, 5, 0);
-            Assert.AreEqual(10, Time.captureDeltaTime);
+            Assert.AreEqual(5, Time.captureDeltaTime);
-        public IEnumerator ShouldCaptureThisFrame_ReturnsTrueOnProperFrames()
+        public IEnumerator ShouldCaptureFlagsAndRenderTimesAreCorrectWithMultipleSensors()
-            var firstCaptureTime1 = 10;
-            var frequencyInMs1 = 4;
-            var sensor1 = DatasetCapture.RegisterSensor(ego, "cam", "1", frequencyInMs1, firstCaptureTime1);
+            var firstCaptureFrame1 = 2;
+            var simDeltaTime1 = 4;
+            var framesBetweenCaptures1 = 2;
+            var sensor1 = DatasetCapture.RegisterSensor(ego, "cam", "1", firstCaptureFrame1, CaptureTriggerMode.Scheduled, simDeltaTime1, framesBetweenCaptures1);
-            var firstCaptureTime2 = 10;
-            var frequencyInMs2 = 6;
-            var sensor2 = DatasetCapture.RegisterSensor(ego, "cam", "2", frequencyInMs2, firstCaptureTime2);
+            var firstCaptureFrame2 = 1;
+            var simDeltaTime2 = 6;
+            var framesBetweenCaptures2 = 1;
+            var sensor2 = DatasetCapture.RegisterSensor(ego, "cam", "2", firstCaptureFrame2, CaptureTriggerMode.Scheduled, simDeltaTime2, framesBetweenCaptures2);
-            var sensor3 = DatasetCapture.RegisterSensor(ego, "cam", "3", 1, 1);
-            sensor3.Enabled = false;
+            //Third sensor is a manually triggered one. All it does in this test is affect delta times.
+            var simDeltaTime3 = 5;
+            var sensor3 = DatasetCapture.RegisterSensor(ego, "cam", "3", 0, CaptureTriggerMode.Manual, simDeltaTime3, 0, true);
-            (float deltaTime, bool sensor1ShouldCapture, bool sensor2ShouldCapture)[] samplesExpected =
+            (float deltaTime, bool sensor1ShouldCapture, bool sensor2ShouldCapture, bool sensor3ShouldCapture)[] samplesExpected =
-                ((float)firstCaptureTime1, true, true),
-                (4, true, false),
-                (2, false, true),
-                (2, true, false),
-                (4, true, true)
+                (4, false, false, false), //Simulation time since sensors created: 4
+                (1, false, false, false), //5
+                (1, false, true, false), //6
+                (2, true, false, false), //8
+                (2, false, false, false), //10
+                (2, false, false, false), //12
+                (3, false, false, false), //15
+                (1, false, false, false), //16
+                (2, false, true, false), //18
+                (2, true, false, false), //20
+                (4, false, false, false), //24
+                (1, false, false, false), //25
-            var samplesActual = new(float deltaTime, bool sensor1ShouldCapture, bool sensor2ShouldCapture)[samplesExpected.Length];
+            var samplesActual = new (float deltaTime, bool sensor1ShouldCapture, bool sensor2ShouldCapture, bool sensor3ShouldCapture)[samplesExpected.Length];
-                samplesActual[i] = (Time.deltaTime, sensor1.ShouldCaptureThisFrame, sensor2.ShouldCaptureThisFrame);
+                samplesActual[i] = (Time.deltaTime, sensor1.ShouldCaptureThisFrame, sensor2.ShouldCaptureThisFrame, sensor3.ShouldCaptureThisFrame);
-        [Test]
-        public void Enabled_StartsTrue()
+        [UnityTest]
+        public IEnumerator SequenceTimeOfManualCapture_ReportsCorrectTime_ManualSensorDoesNotAffectTimings()
+        {
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", 0, CaptureTriggerMode.Manual, 0, 0, false);
+
+            var framesToCaptureOn = new List<int>();
+
+            var startFrame = Time.frameCount;
+            var startTime = Time.time;
+
+            while (framesToCaptureOn.Count < 10)
+            {
+                var randomFrame = Random.Range(startFrame, startFrame + 100);
+                if(!framesToCaptureOn.Contains(randomFrame))
+                    framesToCaptureOn.Add(randomFrame);
+            }
+
+            framesToCaptureOn.Sort();
+
+            var frameIndex = 0;
+            for (var i = 0; i < framesToCaptureOn.Max(); i++)
+            {
+                if (frameIndex == framesToCaptureOn.Count)
+                    break;
+
+                if (Time.frameCount == framesToCaptureOn[frameIndex])
+                {
+                    frameIndex++;
+                    sensorHandle.RequestCapture();
+                    var sensorData = m_TestHelper.GetSensorData(sensorHandle);
+                    var sequenceTimeActual = m_TestHelper.CallSequenceTimeOfNextCapture(sensorData);
+                    var elapsed = Time.time - startTime;
+                    Assert.AreEqual(elapsed, sequenceTimeActual, 0.0001f);
+                }
+                yield return null;
+            }
+
+            Assert.AreEqual(frameIndex, framesToCaptureOn.Count, 0.0001f);
+        }
+
+        [UnityTest]
+        public IEnumerator SequenceTimeOfManualCapture_ReportsCorrectTime_ManualSensorAffectsTimings()
-            var sensor1 = DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "cam", "1", 1, 1);
-            Assert.IsTrue(sensor1.Enabled);
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var simulationDeltaTime = 0.05f;
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", 0, CaptureTriggerMode.Manual, simulationDeltaTime, 0, true);
+
+            var framesToCaptureOn = new List<int>();
+
+            var startFrame = Time.frameCount;
+            var startTime = Time.time;
+
+            while (framesToCaptureOn.Count < 10)
+            {
+                var randomFrame = Random.Range(startFrame, startFrame + 100);
+                if(!framesToCaptureOn.Contains(randomFrame))
+                    framesToCaptureOn.Add(randomFrame);
+            }
+
+            framesToCaptureOn.Sort();
+
+            float[] sequenceTimesExpected = new float[framesToCaptureOn.Count];
+
+            for (int i = 0; i < sequenceTimesExpected.Length; i++)
+            {
+                sequenceTimesExpected[i] = (framesToCaptureOn[i] - startFrame) * simulationDeltaTime;
+            }
+
+            var frameIndex = 0;
+            for (var i = 0; i < framesToCaptureOn.Max(); i++)
+            {
+                if (frameIndex == framesToCaptureOn.Count)
+                    break;
+
+                if (Time.frameCount == framesToCaptureOn[frameIndex])
+                {
+                    sensorHandle.RequestCapture();
+                    var sensorData = m_TestHelper.GetSensorData(sensorHandle);
+                    var sequenceTimeActual = m_TestHelper.CallSequenceTimeOfNextCapture(sensorData);
+                    Assert.AreEqual(sequenceTimesExpected[frameIndex], sequenceTimeActual, 0.0001f);
+                    frameIndex++;
+                }
+                yield return null;
+            }
+            Assert.AreEqual(frameIndex, framesToCaptureOn.Count, 0.0001f);
         }
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/DatasetCaptureTests.cs

 }}";
 
             var ego = DatasetCapture.RegisterEgo(egoDescription);
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, modality, sensorDescription, 1, 1);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, modality, sensorDescription, 1, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.IsTrue(sensorHandle.IsValid);
             DatasetCapture.ResetSimulation();
             Assert.IsFalse(sensorHandle.IsValid);
 }}";
 
             var ego = DatasetCapture.RegisterEgo("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "camera", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "camera", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var sensorSpatialData = new SensorSpatialData(new Pose(egoPosition, egoRotation), new Pose(position, rotation), egoVelocity, null);
             sensorHandle.ReportCapture(filename, sensorSpatialData, ("camera_intrinsic", intrinsics));
 
             };
 
             var ego = DatasetCapture.RegisterEgo("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 2, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 2, 0);
             var sensorSpatialData = new SensorSpatialData(default, default, null, null);
             Assert.IsTrue(sensorHandle.ShouldCaptureThisFrame);
             sensorHandle.ReportCapture("f", sensorSpatialData);
       ]";
 
             var ego = DatasetCapture.RegisterEgo("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             sensorHandle.ReportCapture(filename, default);
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("semantic segmentation", "pixel-wise semantic segmentation label", "PNG", annotationDefinitionGuid);
             sensorHandle.ReportAnnotationFile(annotationDefinition, "annotations/semantic_segmentation_000.png");
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
 
             sensorHandle.ReportAnnotationValues(annotationDefinition, values);
             DatasetCapture.ResetSimulation();
         {
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 100);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 100, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.Throws<InvalidOperationException>(() => sensorHandle.ReportAnnotationFile(annotationDefinition, ""));
         }
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 100);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 100, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.Throws<InvalidOperationException>(() => sensorHandle.ReportAnnotationValues(annotationDefinition, new int[0]));
         }
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 100);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 100, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.Throws<InvalidOperationException>(() => sensorHandle.ReportAnnotationAsync(annotationDefinition));
         }
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             sensorHandle.ReportAnnotationAsync(annotationDefinition);
             DatasetCapture.ResetSimulation();
             LogAssert.Expect(LogType.Error, new Regex("Simulation ended with pending .*"));
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var asyncAnnotation = sensorHandle.ReportAnnotationAsync(annotationDefinition);
 
             Assert.IsTrue(asyncAnnotation.IsValid);
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var asyncAnnotation = sensorHandle.ReportAnnotationAsync(annotationDefinition);
 
             Assert.IsTrue(asyncAnnotation.IsPending);
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var asyncAnnotation = sensorHandle.ReportAnnotationAsync(annotationDefinition);
 
             Assert.IsTrue(asyncAnnotation.IsPending);
 
             var ego = DatasetCapture.RegisterEgo("");
             var annotationDefinition = DatasetCapture.RegisterAnnotationDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
 
             // Record one capture for this frame
             sensorHandle.ReportCapture(fileName, default);
         {
             var ego = DatasetCapture.RegisterEgo("");
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 100);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 100, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.Throws<InvalidOperationException>(() => sensorHandle.ReportMetric(metricDefinition, new int[0]));
         }
 
             var ego = DatasetCapture.RegisterEgo("");
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 100);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 100, CaptureTriggerMode.Scheduled, 1, 0);
             Assert.Throws<InvalidOperationException>(() => sensorHandle.ReportMetricAsync(metricDefinition));
         }
 
             var ego = DatasetCapture.RegisterEgo("");
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             sensorHandle.ReportMetricAsync(metricDefinition);
             DatasetCapture.ResetSimulation();
             LogAssert.Expect(LogType.Error, new Regex("Simulation ended with pending .*"));
 
             var ego = DatasetCapture.RegisterEgo("");
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 1, 0);
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var asyncMetric = sensorHandle.ReportMetricAsync(metricDefinition);
 
             Assert.IsTrue(asyncMetric.IsValid);
             var expectedLine = @"""step"": 0";
 
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 1, 0);
+            DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
 
             yield return null;
             yield return null;
             var expectedLine = @"""step"": 0";
 
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensor = DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 1, 0);
+            var sensor = DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
 
             yield return null;
             sensor.ReportMetric(metricDefinition, values);
 }}";
 
             var metricDefinition = DatasetCapture.RegisterMetricDefinition("");
-            var sensor = DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 1, 0);
+            var sensor = DatasetCapture.RegisterSensor(DatasetCapture.RegisterEgo(""), "", "", 0, CaptureTriggerMode.Scheduled, 1, 0);
             var annotation = sensor.ReportAnnotationFile(DatasetCapture.RegisterAnnotationDefinition(""), "");
             var valuesJsonArray = JArray.FromObject(values).ToString(Formatting.Indented);
             if (async)

com.unity.perception/Tests/Runtime/Randomization/ScenarioTests.cs

 
             // Second frame, first iteration
             yield return null;
-            Assert.AreEqual(DatasetCapture.SimulationState.SequenceTime, perceptionCamera.period);
+            Assert.AreEqual(DatasetCapture.SimulationState.SequenceTime, perceptionCamera.simulationDeltaTime);
 
             // Third frame, second iteration, SequenceTime has been reset
             yield return null;