more progress

com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs

             using(new EditorGUI.DisabledScope(EditorApplication.isPlaying))
             {
                 EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.description)), new GUIContent("Description", "Provide a description for this perception camera (optional)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.period)), new GUIContent("Capture Interval", "The interval at which the perception camera should render and capture (seconds)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.startTime)), new GUIContent("Start Time","Time at which this perception camera starts rendering and capturing (seconds)."));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.scheduledCapture)),new GUIContent("Scheduled Capture", ""));
-                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.affectSimulationTiming)),new GUIContent("Affect Simulation Timings", ""));
+                EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureTriggerMode)),new GUIContent("Scheduled Capture", ""));
+
+                if (perceptionCamera.captureTriggerMode.Equals(PerceptionCamera.CaptureTriggerMode.Scheduled))
+                {
+                    GUILayout.BeginVertical("Box");
+                    GUILayout.Label("Scheduled Capture Properties");
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.startTime)), new GUIContent("Start Time","Time at which this perception camera starts rendering and capturing (seconds)."));
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.period)), new GUIContent("Capture Interval", "The interval at which the perception camera should render and capture (seconds)."));
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.onlyRenderCapturedFrames)),new GUIContent("Only Render Captured Frames", ""));
+                    GUILayout.EndVertical();
+                }
+                else
+                {
+                    perceptionCamera.onlyRenderCapturedFrames = false;
+                }
+
+                if (perceptionCamera.onlyRenderCapturedFrames)
+                {
+
+                }
+                else
+                {
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.renderingDeltaTime)),new GUIContent("Rendering Frame Time", ""));
+                }
+
+
             if (EditorSettings.asyncShaderCompilation)
             {
                 EditorGUILayout.HelpBox("Asynchronous shader compilation may result in invalid data in beginning frames. " +

com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs

         /// <param name="description">A human-readable description of the sensor (ex. "front-left rgb camera")</param>
         /// <param name="period">The period, in seconds, on which the sensor should capture. Frames will be scheduled in the simulation such that each sensor is triggered every _period_ seconds.</param>
         /// <param name="firstCaptureTime">The time, in seconds, from the start of the sequence on which this sensor should first be scheduled.</param>
-        /// <param name="autoCapture"></param>
-        /// <param name="controlSimulationTiming"></param>
+        /// <param name="captureTriggerMode"></param>
+        /// <param name="onlyRenderCapturedFrames"></param>
+        /// <param name="renderingDeltaTime"></param>
-        public static SensorHandle RegisterSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime, bool autoCapture, bool controlSimulationTiming)
+        public static SensorHandle RegisterSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime, PerceptionCamera.CaptureTriggerMode captureTriggerMode, bool onlyRenderCapturedFrames, float renderingDeltaTime = -1)
-            SimulationState.AddSensor(egoHandle, modality, description, period, firstCaptureTime, autoCapture, controlSimulationTiming, sensor);
+            SimulationState.AddSensor(egoHandle, modality, description, period, firstCaptureTime, captureTriggerMode, onlyRenderCapturedFrames, renderingDeltaTime, sensor);
             return sensor;
         }
 

com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/OverlayPanel.cs

 
         internal void OnDrawGUI(float x, float y, float width, float height)
         {
+
+            if (GUILayout.Button("Capture"))
+            {
+                perceptionCamera.SensorHandle.CaptureOnNextUpdate();
+            }
+
+
             var any = perceptionCamera.labelers.Any(l => l is IOverlayPanelProvider && l.enabled);
 
             // If there used to be active providers, but they have been turned off, remove
             GUILayout.Label("Background Alpha:", m_LabelStyle);
             m_BackgroundTransparency = GUILayout.HorizontalSlider(m_BackgroundTransparency, 0.0f, 1.0f, m_SliderStyle, GUI.skin.horizontalSliderThumb);
             GUI.skin.label.padding.left = 0;
-
-            if (GUILayout.Button("Capture"))
-            {
-                perceptionCamera.SensorHandle.CaptureOnNextUpdate();
-            }
 
             // Grab the overlay image from the active provider
             m_OverlayImage.texture = m_ActiveProvider?.overlayImage;

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

         /// </summary>
         public Camera attachedCamera => m_AttachedCamera;
 
-        public bool scheduledCapture;
+        public enum CaptureTriggerMode
+        {
+            Scheduled,
+            Manual
+        }
+        public CaptureTriggerMode captureTriggerMode;
+
+        public bool onlyRenderCapturedFrames;
-        public bool affectSimulationTiming;
+        public float renderingDeltaTime = 0.0166f;
 
         /// <summary>
         /// Event invoked after the camera finishes rendering during a frame.
             {
                 m_EgoMarker = GetComponentInParent<Ego>();
                 var ego = m_EgoMarker == null ? DatasetCapture.RegisterEgo("") : m_EgoMarker.EgoHandle;
-                SensorHandle = DatasetCapture.RegisterSensor(ego, "camera", description, period, startTime, scheduledCapture, affectSimulationTiming);
+                SensorHandle = DatasetCapture.RegisterSensor(ego, "camera", description, period, startTime, captureTriggerMode, onlyRenderCapturedFrames, renderingDeltaTime);
             }
         }
 

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

             public string description;
             public float period;
             public float firstCaptureTime;
-            public bool autoCapture;
-            public bool controlSimulationTiming;
+            public PerceptionCamera.CaptureTriggerMode captureTriggerMode;
+            public bool onlyRenderCapturedFrames;
+            public float renderingDeltaTime;
 
             public bool externalCaptureRequestPending;
             public float sequenceTimeOfNextCapture;
             m_LastTimeScale = Time.timeScale;
         }
 
-        public void AddSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime, bool autoCapture, bool controlSimulationTiming, SensorHandle sensor)
+        public void AddSensor(EgoHandle egoHandle, string modality, string description, float period, float firstCaptureTime, PerceptionCamera.CaptureTriggerMode captureTriggerMode, bool controlSimulationTiming, float renderingDeltaTime, SensorHandle sensor)
         {
             var sensorData = new SensorData()
             {
                 firstCaptureTime = firstCaptureTime,
-                autoCapture = autoCapture,
-                controlSimulationTiming = controlSimulationTiming,
+                captureTriggerMode = captureTriggerMode,
+                onlyRenderCapturedFrames = controlSimulationTiming,
+                renderingDeltaTime = renderingDeltaTime,
                 egoHandle = egoHandle,
                 lastCaptureFrameCount = -1
             };
             foreach (var activeSensor in m_ActiveSensors)
             {
                 var sensorData = m_Sensors[activeSensor];
-                if (sensorData.autoCapture)
+                if (sensorData.captureTriggerMode.Equals(PerceptionCamera.CaptureTriggerMode.Scheduled))
                 {
                     if (!activeSensor.ShouldCaptureThisFrame)
                         continue;
                     sensorData.sequenceTimeOfNextCapture += sensorData.period;
-                    Debug.Assert(sensorData.sequenceTimeOfNextCapture > UnscaledSequenceTime,
+                    //Debug.Log($"unscaled : {UnscaledSequenceTime}, seqNext {sensorData.sequenceTimeOfNextCapture}");
+                    Debug.Assert(!sensorData.onlyRenderCapturedFrames || sensorData.sequenceTimeOfNextCapture > UnscaledSequenceTime,
                         $"Next scheduled capture should be after {UnscaledSequenceTime} but is {sensorData.sequenceTimeOfNextCapture}");
                     // sensorData.sequenceTimeNextCapture = SequenceTimeOfNextCapture(sensorData);
                     sensorData.lastCaptureFrameCount = Time.frameCount;
             foreach (var activeSensor in m_ActiveSensors)
             {
                 var sensorData = m_Sensors[activeSensor];
-                if (sensorData.controlSimulationTiming)
+                if (sensorData.onlyRenderCapturedFrames)
                 {
                     var thisSensorNextFrameDt = sensorData.sequenceTimeOfNextCapture - UnscaledSequenceTime;
 
                 }
-            }
+                else
+                {
+                    var thisSensorNextFrameDt = sensorData.renderingDeltaTime;
-            if (Math.Abs(nextFrameDt - k_MaxDeltaTime) < 0.00001)
-            {
-                //means no sensor is controlling simulation timing, default to 60fps
-                nextFrameDt = 1f/60;
+                    if (thisSensorNextFrameDt > 0f && thisSensorNextFrameDt < nextFrameDt)
+                        nextFrameDt = thisSensorNextFrameDt;
+                }
+
+            // if (Math.Abs(nextFrameDt - k_MaxDeltaTime) < 0.00001)
+            // {
+            //     //means no sensor is controlling simulation timing, default to 60fps
+            //     nextFrameDt = 1f/60;
+            // }
 
             WritePendingCaptures();
             WritePendingMetrics();
                 m_Sensors[sensorHandle] = data;
             }
 
-            return (data.autoCapture && (data.sequenceTimeOfNextCapture - UnscaledSequenceTime) < k_IncludeInFrameThreshold)
+            return (data.captureTriggerMode.Equals(PerceptionCamera.CaptureTriggerMode.Scheduled) && (data.sequenceTimeOfNextCapture - UnscaledSequenceTime) < k_IncludeInFrameThreshold)
                 || captureRequested;
         }