Ok, officially working now

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

         /// <see cref="KeypointObjectFilter"/>
         /// </summary>
         public KeypointObjectFilter objectFilter;
+        /// <summary>
+        /// The max distance a keypoint can be from the front of an object before it is considered occluded
+        /// </summary>
+        public float distanceThreshold = 0.1f;
         // ReSharper restore MemberCanBePrivate.Global
 
         AnnotationDefinition m_AnnotationDefinition;
             positionsPixeldata.Dispose();
             depthPixeldata.Dispose();
 
+            m_MaterialDepthCheck.SetFloat("_DistanceThreshold", distanceThreshold);
             m_MaterialDepthCheck.SetTexture("_Positions", keypointPositionsTexture);
             m_MaterialDepthCheck.SetTexture("_KeypointDepth", keypointDepthTexture);
             m_MaterialDepthCheck.SetTexture("_DepthTexture", depthTexture);
             for (var i = 0; i < totalLength; i++)
             {
                 var value = data[i];
-                if (value.a == 0)
+                if (value.r == 0)
                 {
                     var keypoints = frameKeypointData.keypoints[i / frameKeypointData.pointsPerEntry];
                     var indexInObject = i % frameKeypointData.pointsPerEntry;

com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader

         _Positions("Positions", 2D) = "defaultTexture" {}
         _KeypointDepth("KeypointDepth", 2D) = "defaultTexture" {}
         _DepthTexture("Depth", 2DArray) = "defaultTexture" {}
+        _DistanceThreshold("Distance Threshold", float) = .1
     }
 
     HLSLINCLUDE
             SamplerState my_point_clamp_sampler;
             Texture2D _KeypointDepth;
 
+            float _DistanceThreshold;
+
 #if HDRP_ENABLED
 
             #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/RenderPass/CustomPass/CustomPassCommon.hlsl"
                 float4 checkPosition = _Positions.Load(float3(varyings.positionCS.xy, 0));
                 float4 checkDepth = _KeypointDepth.Load(float3(varyings.positionCS.xy, 0));
 
-                float depth = LoadCameraDepth(checkPosition.xy);
+                float depth = LoadCameraDepth(float2(checkPosition.x, _ScreenSize.y - checkPosition.y));
-                uint result = depth < checkDepth.x - .001 ? 0 : 1;
-                return float4(result, result, result, result);
+                uint result = depth < checkDepth.x - _DistanceThreshold ? 0 : 1;
+                return float4(result, result, result, 1);
             }
 #else
             /// Dummy Implementation for non HDRP_ENABLED variants