Merge remote-tracking branch 'origin/master' into keypoint_self_occlusion

com.unity.perception/CHANGELOG.md

 ### Known Issues
 
 ### Added
+Added support for 'step' button in editor.
 
 ### Changed
 Increased color variety in instance segmentation images
 ### Removed
 
 ### Fixed
+Fixed keypoint labeling bug when visualizations are disabled.
+
+Fixed an issue where Simulation Delta Time values larger than 100 seconds (in Perception Camera) would cause incorrect capture scheduling behavior.
 
 ## [0.8.0-preview.3] - 2021-03-24
 ### Changed

com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs

         }
 
         const string k_FrametimeTitle = "Simulation Delta Time";
-
+        const float k_DeltaTimeTooLarge = 200;
         public override void OnInspectorGUI()
         {
             using(new EditorGUI.DisabledScope(EditorApplication.isPlaying))
                     GUILayout.BeginVertical("TextArea");
                     EditorGUILayout.LabelField("Scheduled Capture Properties", EditorStyles.boldLabel);
 
-                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.simulationDeltaTime)),new GUIContent(k_FrametimeTitle, $"Sets Unity's Time.{nameof(Time.captureDeltaTime)} to the specified number, causing a fixed number of frames to be simulated for each second of elapsed simulation time regardless of the capabilities of the underlying hardware. Thus, simulation time and real time will not be synchronized."));
+                    EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.simulationDeltaTime)),new GUIContent(k_FrametimeTitle, $"Sets Unity's Time.{nameof(Time.captureDeltaTime)} to the specified number, causing a fixed number of frames to be simulated for each second of elapsed simulation time regardless of the capabilities of the underlying hardware. Thus, simulation time and real time will not be synchronized. Note that large {k_FrametimeTitle} values will lead to lower performance as the engine will need to simulate longer periods of elapsed time for each rendered frame."));
+
+                    if (perceptionCamera.simulationDeltaTime > k_DeltaTimeTooLarge)
+                    {
+                        EditorGUILayout.HelpBox($"Large {k_FrametimeTitle} values can lead to significantly lower simulation performance.", MessageType.Warning);
+                    }
 
                     var interval = (perceptionCamera.framesBetweenCaptures + 1) * perceptionCamera.simulationDeltaTime;
                     var startTime = perceptionCamera.simulationDeltaTime * perceptionCamera.firstCaptureFrame;

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

             // Ignore the subsequent calls.
             if (lastFrameCalledThisCallback == Time.frameCount)
                 return false;
-#if UNITY_EDITOR
-            if (UnityEditor.EditorApplication.isPaused)
-                return false;
-#endif
+
             return true;
         }
 

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

         const float k_SimulationTimingAccuracy = 0.01f;
         const int k_MinPendingCapturesBeforeWrite = 150;
         const int k_MinPendingMetricsBeforeWrite = 150;
-        const float k_MaxDeltaTime = 100f;
 
         public SimulationState(string outputDirectory)
         {
             {
                 var sensorData = m_Sensors[activeSensor];
 
+#if UNITY_EDITOR
+                if (UnityEditor.EditorApplication.isPaused)
+                {
+                    //When the user clicks the 'step' button in the editor, frames will always progress at .02 seconds per step.
+                    //In this case, just run all sensors each frame to allow for debugging
+                    Debug.Log($"Frame step forced all sensors to synchronize, changing frame timings.");
+
+                    sensorData.sequenceTimeOfNextRender = UnscaledSequenceTime;
+                    sensorData.sequenceTimeOfNextCapture = UnscaledSequenceTime;
+                }
+#endif
+
                 if (Mathf.Abs(sensorData.sequenceTimeOfNextRender - UnscaledSequenceTime) < k_SimulationTimingAccuracy)
                 {
                     //means this frame fulfills this sensor's simulation time requirements, we can move target to next frame.
                         sensorData.sequenceTimeOfNextCapture += sensorData.renderingDeltaTime * (sensorData.framesBetweenCaptures + 1);
                         Debug.Assert(sensorData.sequenceTimeOfNextCapture > UnscaledSequenceTime,
                             $"Next scheduled capture should be after {UnscaledSequenceTime} but is {sensorData.sequenceTimeOfNextCapture}");
+                        while (sensorData.sequenceTimeOfNextCapture <= UnscaledSequenceTime)
+                            sensorData.sequenceTimeOfNextCapture += sensorData.renderingDeltaTime * (sensorData.framesBetweenCaptures + 1);
                     }
                     else if (sensorData.captureTriggerMode.Equals(CaptureTriggerMode.Manual))
                     {
             }
 
             //find the deltatime required to land on the next active sensor that needs simulation
-            float nextFrameDt = k_MaxDeltaTime;
+            var nextFrameDt = float.PositiveInfinity;
             foreach (var activeSensor in m_ActiveSensors)
             {
                 float thisSensorNextFrameDt = -1;
                 }
 
                 if (thisSensorNextFrameDt > 0f && thisSensorNextFrameDt < nextFrameDt)
+                {
+                }
-            if (Math.Abs(nextFrameDt - k_MaxDeltaTime) < 0.0001)
+            if (float.IsPositiveInfinity(nextFrameDt))
             {
                 //means no sensor is controlling simulation timing, so we set Time.captureDeltaTime to 0 (default) which means the setting does not do anything
                 nextFrameDt = 0;

com.unity.perception/Tests/Editor/DatasetCaptureEditorTests.cs

 using System.Collections;
 using System.IO;
 using NUnit.Framework;
+using UnityEditor;
 using UnityEngine;
 using UnityEngine.Perception.GroundTruth;
 using UnityEngine.TestTools;
             expectedDatasetPath = DatasetCapture.OutputDirectory;
             yield return new ExitPlayMode();
             FileAssert.Exists(Path.Combine(expectedDatasetPath, "sensors.json"));
+        }
+        [UnityTest]
+        public IEnumerator StepFunction_OverridesSimulationDeltaTime_AndRunsSensors()
+        {
+            yield return new EnterPlayMode();
+            DatasetCapture.ResetSimulation();
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var sensor = DatasetCapture.RegisterSensor(ego, "camera", "", 0, CaptureTriggerMode.Scheduled, 2f, 0);
+            yield return null;
+            var timeBeforeStep = Time.time;
+            EditorApplication.isPaused = true;
+            EditorApplication.Step();
+            Assert.True(Time.time - timeBeforeStep < .3f);
+            Assert.True(sensor.ShouldCaptureThisFrame);
+            yield return new ExitPlayMode();
         }
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/DatasetCaptureSensorSchedulingTests.cs

         }
 
         [UnityTest]
+        [TestCase(1, 0, 0, 1, 2, 3, ExpectedResult = (IEnumerator)null)]
+        [TestCase(10, 5, 50, 60, 70, 80, ExpectedResult = (IEnumerator)null)]
+        [TestCase(55, 0, 0, 55, 110, 165, ExpectedResult = (IEnumerator)null)]
+        [TestCase(235, 10, 2350, 2585, 2820, 3055, ExpectedResult = (IEnumerator)null)]
+        public IEnumerator SequenceTimeOfNextCapture_ReportsCorrectTime_VariedDeltaTimesAndStartFrames(float simulationDeltaTime, int firstCaptureFrame, float firstCaptureTime, float secondCaptureTime, float thirdCaptureTime, float fourthCaptureTime)
+        {
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var sensorHandle = DatasetCapture.RegisterSensor(ego, "cam", "", firstCaptureFrame, CaptureTriggerMode.Scheduled, simulationDeltaTime, 0);
+
+            float[] sequenceTimesExpected =
+            {
+                firstCaptureTime,
+                secondCaptureTime,
+                thirdCaptureTime,
+                fourthCaptureTime
+            };
+
+            for (var i = 0; i < firstCaptureFrame; i++)
+            {
+                //render the non-captured frames before firstCaptureFrame
+                yield return null;
+            }
+
+            for (var i = 0; i < sequenceTimesExpected.Length; i++)
+            {
+                var sensorData = m_TestHelper.GetSensorData(sensorHandle);
+                var sequenceTimeActual = m_TestHelper.CallSequenceTimeOfNextCapture(sensorData);
+                Assert.AreEqual(sequenceTimesExpected[i], sequenceTimeActual, 0.0001f);
+                yield return null;
+            }
+        }
+
+        [UnityTest]
         public IEnumerator SequenceTimeOfManualCapture_ReportsCorrectTime_ManualSensorDoesNotAffectTimings()
         {
             var ego = DatasetCapture.RegisterEgo("ego");