Merge pull request #253 from Unity-Technologies/disable_labeling_fixes

Addressing feedback on a previous PR regarding disabling labeling components

com.unity.perception/CHANGELOG.md

 
 `ScenarioBase`'s `Awake()`, `Start()`, and `Update()` functions are now private. If you previously used these, replace the usages with `OnAwake()`, `OnStart()`, and `OnUpdate()`.
 
+The interface `IGroundTruthGenerator` now contains a new method named `ClearMaterialProperties` for disabling ground truth generation on a `Labeling` component or its associated `MaterialPropertyBlock`. Update your implementing classes to including this method.
+
 ### Known Issues
 
 ### Added
 
 Improved UI for `KeypointTemplate` and added useful default colors for keypoint and skeleton definitions.
 
-Added the ability to switch ground-truth labeling on or off for an object at runtime by enabling or disabling its `Labeling` component.
+Added the ability to switch ground truth generation on or off for an object at runtime by enabling or disabling its `Labeling` component. A new method named `ClearMaterialProperties()` in `IGroundTruthGenerator` handles this functionality.
 
 ### Changed
 

com.unity.perception/Runtime/GroundTruth/IGroundTruthGenerator.cs

     public interface IGroundTruthGenerator
     {
         /// <summary>
-        /// Called by <see cref="LabelManager"/> when first registered or when a Labeling is created at runtime.
+        /// Enables ground truth generation for a <see cref="Labeling"/> component or its associated <see cref="MaterialPropertyBlock"/>. This function is called by <see cref="LabelManager"/> when a <see cref="Labeling"/> component is registered, created, or enabled.
-        /// <param name="mpb">The MaterialPropertyBlock for the given meshRenderer. Can be used to set properties for custom rendering.</param>
-        /// <param name="renderer">The Renderer under the given Labeling.</param>
-        /// <param name="labeling">The Labeling component created</param>
-        /// <param name="instanceId">The instanceId assigned to the given Labeling instance.</param>
+        /// <param name="mpb">The <see cref="MaterialPropertyBlock"/> for the given <see cref="MeshRenderer"/>. Can be used to set properties for custom rendering.</param>
+        /// <param name="renderer">The <see cref="Renderer"/> under the given <see cref="LabelManager"/>.</param>
+        /// <param name="labeling">The <see cref="LabelManager"/> component that was registered, created, or enabled</param>
+        /// <param name="instanceId">The instanceId assigned to the given <see cref="LabelManager"/> instance.</param>
+
+        /// <summary>
+        /// Disables ground truth generation for a <see cref="Labeling"/> component or its associated <see cref="MaterialPropertyBlock"/>. This function is called by <see cref="LabelManager"/> when a <see cref="Labeling"/> component is disabled.
+        /// </summary>
+        /// <param name="mpb">The <see cref="MaterialPropertyBlock"/> for the given <see cref="MeshRenderer"/>. Can be used to set properties for custom rendering.</param>
+        /// <param name="renderer">The <see cref="Renderer"/> under the given <see cref="LabelManager"/>.</param>
+        /// <param name="labeling">The <see cref="LabelManager"/> component for which ground-truth generation should stop.</param>
+        /// <param name="instanceId">The instanceId assigned to the given <see cref="LabelManager"/> instance.</param>
+        void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId);
     }
 }

com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs

         const uint k_HslCount = 64;
         const uint k_ColorsPerAlpha = 256 * 256 * 256;
         const uint k_InvalidPackedColor = 255; // packed uint for color (0, 0, 0, 255);
-        public static readonly Color32 invalidColor = new Color(0, 0, 0, 255);
+        /// <summary>
+        /// The color returned when an instanceId is not mapped to any color, and for clearing ground truth material properties on a <see cref="MaterialPropertyBlock"/>.
+        /// </summary>
+        public static readonly Color32 invalidColor = new Color(0, 0, 0, 255);
         static void InitializeMaps()
         {
 

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

 
         void ProcessLabel(Labeling labeledEntity)
         {
-            if (idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
+            if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
+                return;
+
+            // Cache out the data of a labeled game object the first time we see it, this will
+            // save performance each frame. Also checks to see if a labeled game object can be annotated.
+            if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId))
-                // Cache out the data of a labeled game object the first time we see it, this will
-                // save performance each frame. Also checks to see if a labeled game object can be annotated.
-                if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId))
+                var cached = new CachedData()
-                    var cached = new CachedData()
-                    {
-                        status = false,
-                        animator = null,
-                        keypoints = new KeypointEntry(),
-                        overrides = new List<(JointLabel, int)>()
-                    };
+                    status = false,
+                    animator = null,
+                    keypoints = new KeypointEntry(),
+                    overrides = new List<(JointLabel, int)>()
+                };
+                var entityGameObject = labeledEntity.gameObject;
-                    var entityGameObject = labeledEntity.gameObject;
+                cached.keypoints.instance_id = labeledEntity.instanceId;
+                cached.keypoints.label_id = labelEntry.id;
+                cached.keypoints.template_guid = activeTemplate.templateID;
-                    cached.keypoints.instance_id = labeledEntity.instanceId;
-                    cached.keypoints.label_id = labelEntry.id;
-                    cached.keypoints.template_guid = activeTemplate.templateID;
+                cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];
+                for (var i = 0; i < cached.keypoints.keypoints.Length; i++)
+                {
+                    cached.keypoints.keypoints[i] = new Keypoint { index = i, state = 0 };
+                }
-                    cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];
-                    for (var i = 0; i < cached.keypoints.keypoints.Length; i++)
-                    {
-                        cached.keypoints.keypoints[i] = new Keypoint { index = i, state = 0 };
-                    }
+                var animator = entityGameObject.transform.GetComponentInChildren<Animator>();
+                if (animator != null)
+                {
+                    cached.animator = animator;
+                    cached.status = true;
+                }
-                    var animator = entityGameObject.transform.GetComponentInChildren<Animator>();
-                    if (animator != null)
+                foreach (var joint in entityGameObject.transform.GetComponentsInChildren<JointLabel>())
+                {
+                    if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx))
-                        cached.animator = animator;
+                        cached.overrides.Add((joint, idx));
+                }
-                    foreach (var joint in entityGameObject.transform.GetComponentsInChildren<JointLabel>())
-                    {
-                        if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx))
-                        {
-                            cached.overrides.Add((joint, idx));
-                            cached.status = true;
-                        }
-                    }
+                m_KnownStatus[labeledEntity.instanceId] = cached;
+            }
-                    m_KnownStatus[labeledEntity.instanceId] = cached;
-                }
+            var cachedData = m_KnownStatus[labeledEntity.instanceId];
-                var cachedData = m_KnownStatus[labeledEntity.instanceId];
+            if (cachedData.status)
+            {
+                var animator = cachedData.animator;
+                var keypoints = cachedData.keypoints.keypoints;
-                if (cachedData.status)
+                // Go through all of the rig keypoints and get their location
+                for (var i = 0; i < activeTemplate.keypoints.Length; i++)
-                    var animator = cachedData.animator;
-                    var keypoints = cachedData.keypoints.keypoints;
-
-                    // Go through all of the rig keypoints and get their location
-                    for (var i = 0; i < activeTemplate.keypoints.Length; i++)
+                    var pt = activeTemplate.keypoints[i];
+                    if (pt.associateToRig)
-                        var pt = activeTemplate.keypoints[i];
-                        if (pt.associateToRig)
+                        var bone = animator.GetBoneTransform(pt.rigLabel);
+                        if (bone != null)
-                            var bone = animator.GetBoneTransform(pt.rigLabel);
-                            if (bone != null)
-                            {
-                                var loc = ConvertToScreenSpace(bone.position);
-                                keypoints[i].index = i;
-                                keypoints[i].x = loc.x;
-                                keypoints[i].y = loc.y;
-                                keypoints[i].state = 2;
-                            }
+                            var loc = ConvertToScreenSpace(bone.position);
+                            keypoints[i].index = i;
+                            keypoints[i].x = loc.x;
+                            keypoints[i].y = loc.y;
+                            keypoints[i].state = 2;
+                }
-                    // Go through all of the additional or override points defined by joint labels and get
-                    // their locations
-                    foreach (var (joint, idx) in cachedData.overrides)
-                    {
-                        var loc = ConvertToScreenSpace(joint.transform.position);
-                        keypoints[idx].index = idx;
-                        keypoints[idx].x = loc.x;
-                        keypoints[idx].y = loc.y;
-                        keypoints[idx].state = 2;
-                    }
+                // Go through all of the additional or override points defined by joint labels and get
+                // their locations
+                foreach (var (joint, idx) in cachedData.overrides)
+                {
+                    var loc = ConvertToScreenSpace(joint.transform.position);
+                    keypoints[idx].index = idx;
+                    keypoints[idx].x = loc.x;
+                    keypoints[idx].y = loc.y;
+                    keypoints[idx].state = 2;
+                }
-                    cachedData.keypoints.pose = "unset";
-
-                    if (cachedData.animator != null)
-                    {
-                        cachedData.keypoints.pose = GetPose(cachedData.animator);
-                    }
+                cachedData.keypoints.pose = "unset";
-                    m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = cachedData.keypoints;
+                if (cachedData.animator != null)
+                {
+                    cachedData.keypoints.pose = GetPose(cachedData.animator);
+
+                m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = cachedData.keypoints;
             }
         }
 

com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs

             if (m_IdLabelConfig.TryGetMatchingConfigurationEntry(labeling, out _, out var index))
             {
                 Debug.Assert(index < k_DefaultValue, "Too many entries in the label config");
-                if (labeling.enabled)
-                {
-                    if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
-                    {
-                        var oldLength = m_InstanceIdToLabelEntryIndexLookup.Length;
-                        m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
-                        for (var i = oldLength; i < instanceId; i++)
-                            m_InstanceIdToLabelEntryIndexLookup[i] = k_DefaultValue;
-                    }
-                    m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = (ushort)index;
-                }
-                else if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
+                if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
-                    m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = k_DefaultValue;
+                    var oldLength = m_InstanceIdToLabelEntryIndexLookup.Length;
+                    m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
+
+                    for (var i = oldLength; i < instanceId; i++)
+                        m_InstanceIdToLabelEntryIndexLookup[i] = k_DefaultValue;
+                m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = (ushort)index;
             }
             else if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
             {
+
+        /// <inheritdoc/>
+        void IGroundTruthGenerator.ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
+            {
+                m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = k_DefaultValue;
+            }
+        }
+
 
         /// <inheritdoc/>
         public void Dispose()

com.unity.perception/Runtime/GroundTruth/Labeling/LabelManager.cs

             {
                 terrain.GetSplatMaterialPropertyBlock(mpb);
                 foreach (var pass in m_ActiveGenerators)
-                    pass.SetupMaterialProperties(mpb, null, labeling, instanceId);
+                {
+                    if (labeling.enabled)
+                    {
+                        pass.SetupMaterialProperties(mpb, null, labeling, instanceId);
+                    }
+                    else
+                    {
+                        pass.ClearMaterialProperties(mpb, null, labeling, instanceId);
+                    }
+                }
 
                 terrain.SetSplatMaterialPropertyBlock(mpb);
             }
             {
                 renderer.GetPropertyBlock(mpb);
                 foreach (var pass in m_ActiveGenerators)
-                    pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
+                {
+                    if (labeling.enabled)
+                    {
+                        pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
+                    }
+                    else
+                    {
+                        pass.ClearMaterialProperties(mpb, renderer, labeling, instanceId);
+                    }
+                }
 
                 renderer.SetPropertyBlock(mpb);
 
                     if (!mpb.isEmpty)
                     {
                         foreach (var pass in m_ActiveGenerators)
-                            pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
+                        {
+                            if (labeling.enabled)
+                            {
+                                pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
+                            }
+                            else
+                            {
+                                pass.ClearMaterialProperties(mpb, renderer, labeling, instanceId);
+                            }
+                        }
 
                         renderer.SetPropertyBlock(mpb, i);
                     }

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/GroundTruthCrossPipelinePass.cs

 
         public abstract void SetupMaterialProperties(
             MaterialPropertyBlock mpb, Renderer meshRenderer, Labeling labeling, uint instanceId);
+
+        public abstract void ClearMaterialProperties(
+            MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId);
     }
 }

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/InstanceSegmentationCrossPipelinePass.cs

             if (!found)
                 Debug.LogError($"Could not get a unique color for {instanceId}");
 
-            if (labeling.enabled)
-                mpb.SetVector(k_SegmentationIdProperty, (Color)color);
-            else
-                mpb.SetVector(k_SegmentationIdProperty, (Color) InstanceIdToColorMapping.invalidColor);
+            mpb.SetVector(k_SegmentationIdProperty, (Color)color);
+        }
+
+        public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            mpb.SetVector(k_SegmentationIdProperty, (Color) InstanceIdToColorMapping.invalidColor);
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LensDistortionCrossPipelinePass.cs

         {
             SetLensDistortionShaderParameters();
         }
+
+        public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            SetLensDistortionShaderParameters();
+        }
     }
 }
 

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs

         {
             var entry = new SemanticSegmentationLabelEntry();
             var found = false;
-            if (labeling.enabled)
+
+            foreach (var l in m_LabelConfig.labelEntries)
-                foreach (var l in m_LabelConfig.labelEntries)
+                if (labeling.labels.Contains(l.label))
-                    if (labeling.labels.Contains(l.label))
-                    {
-                        entry = l;
-                        found = true;
-                        break;
-                    }
+                    entry = l;
+                    found = true;
+                    break;
-            if (found)
-                mpb.SetVector(k_LabelingId, entry.color);
-            else
-                mpb.SetVector(k_LabelingId, Color.black);
+            mpb.SetVector(k_LabelingId, found ? entry.color : Color.black);
+        }
+        public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            mpb.SetVector(k_LabelingId, Color.black);
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/RenderPasses/HdrpPasses/GroundTruthPass.cs

         public abstract void SetupMaterialProperties(
             MaterialPropertyBlock mpb, Renderer meshRenderer, Labeling labeling, uint instanceId);
 
+        public abstract void ClearMaterialProperties(
+            MaterialPropertyBlock mpb, Renderer meshRenderer, Labeling labeling, uint instanceId);
+
         protected GroundTruthPass(Camera targetCamera)
         {
             this.targetCamera = targetCamera;

com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

             texture.Create();
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
         }
 
         [UnityTest]
+        public IEnumerator Keypoint_TestStaticLabeledCube_WithDisabledLabeling_AndSwitchingLabelingState()
+        {
+            var incoming = new List<List<KeypointLabeler.KeypointEntry>>();
+            var template = CreateTestTemplate(Guid.NewGuid(), "TestTemplate");
+            var texture = new RenderTexture(1024, 768, 16);
+            texture.Create();
+            var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
+            {
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
+            }, texture);
+
+            var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
+            SetupCubeJoints(cube, template);
+            var labeling = cube.GetComponent<Labeling>();
+
+            cube.SetActive(true);
+            cam.SetActive(true);
+
+            AddTestObjectForCleanup(cam);
+            AddTestObjectForCleanup(cube);
+
+            labeling.enabled = false;
+            yield return null;
+
+            labeling.enabled = true;
+            yield return null;
+
+            labeling.enabled = false;
+            yield return null;
+
+            //force all async readbacks to complete
+            DestroyTestObject(cam);
+            texture.Release();
+
+            var testCase = incoming[0];
+            Assert.AreEqual(0, testCase.Count);
+
+            testCase = incoming[1];
+            Assert.AreEqual(1, testCase.Count);
+            var t = testCase.First();
+            Assert.NotNull(t);
+            Assert.AreEqual(1, t.instance_id);
+            Assert.AreEqual(1, t.label_id);
+            Assert.AreEqual(template.templateID.ToString(), t.template_guid);
+            Assert.AreEqual(9, t.keypoints.Length);
+
+            Assert.AreEqual(t.keypoints[0].x, t.keypoints[1].x);
+            Assert.AreEqual(t.keypoints[2].x, t.keypoints[3].x);
+            Assert.AreEqual(t.keypoints[4].x, t.keypoints[5].x);
+            Assert.AreEqual(t.keypoints[6].x, t.keypoints[7].x);
+
+            Assert.AreEqual(t.keypoints[0].y, t.keypoints[3].y);
+            Assert.AreEqual(t.keypoints[1].y, t.keypoints[2].y);
+            Assert.AreEqual(t.keypoints[4].y, t.keypoints[7].y);
+            Assert.AreEqual(t.keypoints[5].y, t.keypoints[6].y);
+
+            for (var i = 0; i < 9; i++) Assert.AreEqual(i, t.keypoints[i].index);
+            for (var i = 0; i < 8; i++) Assert.AreEqual(2, t.keypoints[i].state);
+            Assert.Zero(t.keypoints[8].state);
+            Assert.Zero(t.keypoints[8].x);
+            Assert.Zero(t.keypoints[8].y);
+
+            testCase = incoming[2];
+            Assert.AreEqual(0, testCase.Count);
+        }
+
+        [UnityTest]
         public IEnumerator Keypoint_TestAllOffScreen()
         {
             var incoming = new List<List<KeypointLabeler.KeypointEntry>>();
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             });
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
             texture.Create();
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
             texture.Create();
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
             texture.Create();
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
             var texture = new RenderTexture(1024, 768, 16);
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
             texture.Create();
             var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
             {
-                incoming.Add(data);
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
             }, texture);
 
             var cameraComponent = cam.GetComponent<Camera>();