Adding some tests for self-occlusion. Some fixes in KeypointLabeler

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

         /// <summary>
         /// The max distance a keypoint can be from the front of an object before it is considered occluded
         /// </summary>
-        public float distanceThreshold = 0.1f;
+        public float distanceThreshold = 0.15f;
         // ReSharper restore MemberCanBePrivate.Global
 
         AnnotationDefinition m_AnnotationDefinition;
 
             var annotation = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
             var keypointEntries = new List<KeypointEntry>();
-            //uint4 so that we can fill a 4 channel texture with this data
-            var positions = new NativeList<uint4>(512, Allocator.Persistent);
+            var positions = new NativeList<float3>(512, Allocator.Persistent);
 
             foreach (var label in LabelManager.singleton.registeredLabels)
                 ProcessLabel(label, keypointEntries, positions);
             positions.Dispose();
         }
 
-        private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<uint4> positions)
+        private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
         {
             var keypointCount = keypointEntries.Count * activeTemplate.keypoints.Length;
 
-            var keypointDepthTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
+            var keypointCheckDepthTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
 
             var positionsPixeldata = new NativeArray<half>(positions.Length * 2, Allocator.Temp);
             var depthPixeldata = new NativeArray<half>(positions.Length, Allocator.Temp);
             {
                 var pos = positions[i];
-                positionsPixeldata[i * 2] = new half(pos.x/* / (float)depthTexture.width*/);
-                positionsPixeldata[i * 2 + 1] = new half(pos.y/* / (float)depthTexture.height*/);
-                depthPixeldata[i] = new half(pos.z / 100.0f);
+                positionsPixeldata[i * 2] = new half(pos.x);
+                positionsPixeldata[i * 2 + 1] = new half(pos.y);
+                depthPixeldata[i] = new half(pos.z - this.distanceThreshold);
-            keypointDepthTexture.SetPixelData(depthPixeldata, 0);
-            keypointDepthTexture.Apply();
+            keypointCheckDepthTexture.SetPixelData(depthPixeldata, 0);
+            keypointCheckDepthTexture.Apply();
-            m_MaterialDepthCheck.SetFloat("_DistanceThreshold", distanceThreshold);
-            m_MaterialDepthCheck.SetTexture("_KeypointDepth", keypointDepthTexture);
+            m_MaterialDepthCheck.SetTexture("_KeypointCheckDepth", keypointCheckDepthTexture);
             m_MaterialDepthCheck.SetTexture("_DepthTexture", depthTexture);
 
             commandBuffer.Blit(null, m_ResultsBuffer, m_MaterialDepthCheck);
             return false;
         }
 
-        void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<uint4> positions)
+        void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
         {
             if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
                 return;
                 var listStart = positions.Length;
                 positions.Resize(positions.Length + activeTemplate.keypoints.Length, NativeArrayOptions.ClearMemory);
                 //grab the slice of the list for the current object to assign positions in
-                var positionsSlice = new NativeSlice<uint4>(positions, listStart);
+                var positionsSlice = new NativeSlice<float3>(positions, listStart);
 
                 // Go through all of the rig keypoints and get their location
                 for (var i = 0; i < activeTemplate.keypoints.Length; i++)
             }
         }
 
-        private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<uint4> positions, int idx)
+        private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<float3> positions, int idx)
         {
             var loc = ConvertToScreenSpace(position);
 
                 pixelLocation = new int2(int.MaxValue, int.MaxValue);
             }
 
-            //TODO: Fix up the z computation
-            positions[idx] = new uint4((uint)pixelLocation.x, (uint)pixelLocation.y, (uint)(loc.z * 100), 1);
+            positions[idx] = new float3((uint)pixelLocation.x, (uint)pixelLocation.y, loc.z);
         }
 
         string GetPose(Animator animator)

com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader

     Properties
     {
         _Positions("Positions", 2D) = "defaultTexture" {}
-        _KeypointDepth("KeypointDepth", 2D) = "defaultTexture" {}
+        _KeypointCheckDepth("KeypointCheckDepth", 2D) = "defaultTexture" {}
-        _DistanceThreshold("Distance Threshold", float) = .1
     }
 
     HLSLINCLUDE
 
             Texture2D _Positions;
             SamplerState my_point_clamp_sampler;
-            Texture2D _KeypointDepth;
-
-            float _DistanceThreshold;
+            Texture2D _KeypointCheckDepth;
 
 #if HDRP_ENABLED
 
                 UNITY_SETUP_STEREO_EYE_INDEX_POST_VERTEX(varyings);
 
                 float4 checkPosition = _Positions.Load(float3(varyings.positionCS.xy, 0));
-                float4 checkDepth = _KeypointDepth.Load(float3(varyings.positionCS.xy, 0));
+                float4 checkDepth = _KeypointCheckDepth.Load(float3(varyings.positionCS.xy, 0));
 
                 float depth = LoadCameraDepth(float2(checkPosition.x, _ScreenSize.y - checkPosition.y));
                 depth = LinearEyeDepth(depth, _ZBufferParams);
-                uint result = depth < checkDepth.x - _DistanceThreshold ? 0 : 1;
+                uint result = depth > checkDepth.x ? 1 : 0;
                 return float4(result, result, result, 1);
             }
 #else

com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

 #endif
         }
 
-        static GameObject SetupCamera(IdLabelConfig config, KeypointTemplate template, Action<int, List<KeypointLabeler.KeypointEntry>> computeListener, RenderTexture renderTexture = null, KeypointObjectFilter keypointObjectFilter = KeypointObjectFilter.Visible)
+        static GameObject SetupCamera(IdLabelConfig config, KeypointTemplate template, Action<int, List<KeypointLabeler.KeypointEntry>> computeListener, RenderTexture renderTexture = null, KeypointObjectFilter keypointObjectFilter = KeypointObjectFilter.Visible, float defaultSelfOcclusionDistance = 0.15f)
         {
             var cameraObject = new GameObject();
             cameraObject.SetActive(false);
             perceptionCamera.captureRgbImages = false;
             var keyPointLabeler = new KeypointLabeler(config, template);
             keyPointLabeler.objectFilter = keypointObjectFilter;
+            keyPointLabeler.distanceThreshold = defaultSelfOcclusionDistance;
             if (computeListener != null)
                 keyPointLabeler.KeypointsComputed += computeListener;
 
 
             Assert.AreEqual(args.expectedBottomRight.x, t.keypoints[3].x, k_Delta);
             Assert.AreEqual(args.expectedBottomRight.y, t.keypoints[3].y, k_Delta);
+        }
+
+        /* Tests to write
+         Point inside mesh < threshold is labeled properly (with multiple thresholds)
+         Point inside mesh > threshold is labeled properly (with multiple thresholds)
+         Point up against other occluding mesh
+         Points close to the far plane are labeled properly
+         Perspective and orthographic
+        */
+
+        public enum CheckDistanceType
+        {
+            Global,
+            JointLabel
+        }
+
+        public static IEnumerable<(Vector3 origin, float checkDistance, float pointDistance, float cameraFieldOfView, bool expectOccluded)>
+            Keypoint_InsideBox_RespectsThreshold_TestCases()
+        {
+            yield return (
+                Vector3.zero,
+                0.1f,
+                0.2f,
+                60f,
+                true);
+            yield return (
+                Vector3.zero,
+                0.2f,
+                0.005f,
+                60f,
+                false);
+            yield return (
+                Vector3.zero,
+                0.01f,
+                0.005f,
+                60f,
+                false);
+            yield return (
+                new Vector3(0, 0, 950),
+                0.01f,
+                0.005f,
+                1f,
+                false);
+            yield return (
+                new Vector3(0, 0, 950),
+                0.1f,
+                0.2f,
+                1f,
+                true);
+        }
+
+        [UnityTest]
+        public IEnumerator Keypoint_InsideBox_RespectsThreshold(
+            [ValueSource(nameof(Keypoint_InsideBox_RespectsThreshold_TestCases))]
+            (Vector3 origin, float checkDistance, float pointDistance, float cameraFieldOfView, bool expectOccluded) args,
+            [Values(CheckDistanceType.Global, CheckDistanceType.JointLabel)] CheckDistanceType checkDistanceType)
+        {
+            if (checkDistanceType == CheckDistanceType.JointLabel)
+                Assert.Inconclusive("Not yet implemented");
+
+            var incoming = new List<List<KeypointLabeler.KeypointEntry>>();
+            var template = CreateTestTemplate(Guid.NewGuid(), "TestTemplate");
+            var frameSize = 1024;
+            var texture = new RenderTexture(frameSize, frameSize, 16);
+            var defaultSelfOcclusionDistance =
+                checkDistanceType == CheckDistanceType.Global ? args.checkDistance : 0.15f;
+            var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
+            {
+                incoming.Add(data);
+            }, texture, defaultSelfOcclusionDistance: defaultSelfOcclusionDistance);
+            var camComponent = cam.GetComponent<Camera>();
+            camComponent.fieldOfView = args.cameraFieldOfView;
+
+            // camComponent.orthographic = true;
+            // camComponent.orthographicSize = .5f;
+
+            var cube = TestHelper.CreateLabeledCube(scale: 1f, x: args.origin.x, y: args.origin.y, z: args.origin.z);
+            SetupCubeJoint(cube, template, "Center", 0f, 0f, -.5f + args.pointDistance);
+
+            cube.SetActive(true);
+            cam.SetActive(true);
+
+            AddTestObjectForCleanup(cam);
+            AddTestObjectForCleanup(cube);
+
+            // while (true)
+             yield return null;
+
+            //force all async readbacks to complete
+            DestroyTestObject(cam);
+            texture.Release();
+
+            var testCase = incoming.Last();
+            Assert.AreEqual(1, testCase.Count);
+            var t = testCase.First();
+            Assert.AreEqual(args.expectOccluded ? 1 : 2, t.keypoints[8].state);
         }
     }
 }