浏览代码

Kind of working!

/keypoint_self_occlusion
Jon Hogins 3 年前
当前提交
904ff3db
共有 9 个文件被更改,包括 377 次插入482 次删除
  1. 2
      TestProjects/PerceptionHDRP/Assets/Prefabs/KeypointCube.prefab
  2. 481
      TestProjects/PerceptionHDRP/Assets/Scenes/SampleScene.unity
  3. 160
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  4. 83
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheck.shader
  5. 3
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheck.shader.meta
  6. 102
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader
  7. 3
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader.meta
  8. 17
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthTest.compute
  9. 8
      com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthTest.compute.meta

2
TestProjects/PerceptionHDRP/Assets/Prefabs/KeypointCube.prefab


m_Name:
m_EditorClassIdentifier:
labels:
- Crate
- Crate24
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!1 &3710685715225028277

481
TestProjects/PerceptionHDRP/Assets/Scenes/SampleScene.unity


m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 6
m_RootOrder: 2
--- !u!1 &411238276
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 411238281}
- component: {fileID: 411238280}
- component: {fileID: 411238279}
- component: {fileID: 411238278}
- component: {fileID: 411238277}
- component: {fileID: 411238282}
m_Layer: 0
m_Name: Crate
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &411238277
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Name:
m_EditorClassIdentifier:
labels:
- Crate
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!65 &411238278
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &411238279
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1
m_DynamicOccludee: 1
m_MotionVectors: 1
m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RayTracingMode: 2
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: ddfd18df3e5ef3043b7889c5a070d8ca, type: 2}
m_StaticBatchInfo:
firstSubMesh: 0
subMeshCount: 0
m_StaticBatchRoot: {fileID: 0}
m_ProbeAnchor: {fileID: 0}
m_LightProbeVolumeOverride: {fileID: 0}
m_ScaleInLightmap: 1
m_ReceiveGI: 1
m_PreserveUVs: 0
m_IgnoreNormalsForChartDetection: 0
m_ImportantGI: 0
m_StitchLightmapSeams: 1
m_SelectedEditorRenderState: 3
m_MinimumChartSize: 4
m_AutoUVMaxDistance: 0.5
m_AutoUVMaxAngle: 89
m_LightmapParameters: {fileID: 0}
m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &411238280
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!4 &411238281
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 121.42, y: 72.4, z: -161.73}
m_LocalScale: {x: 15, y: 15, z: 15}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &411238282
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 411238276}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 673a227032a8e4940b9828c5b6f852ab, type: 3}
m_Name:
m_EditorClassIdentifier:
yDegreesPerSecond: 180
--- !u!1 &705507993
GameObject:
m_ObjectHideFlags: 0

m_LocalScale: {x: 36.249973, y: 36.249977, z: 36.249973}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 3
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0}
--- !u!114 &705507996
MonoBehaviour:

showAdditionalSettings: 0
m_AreaLightEmissiveMeshShadowCastingMode: 0
m_AreaLightEmissiveMeshMotionVectorGenerationMode: 0
--- !u!1 &934158981
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 934158987}
- component: {fileID: 934158986}
- component: {fileID: 934158985}
- component: {fileID: 934158984}
- component: {fileID: 934158983}
- component: {fileID: 934158982}
m_Layer: 0
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &934158982
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 673a227032a8e4940b9828c5b6f852ab, type: 3}
m_Name:
m_EditorClassIdentifier:
yDegreesPerSecond: 180
--- !u!114 &934158983
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Name:
m_EditorClassIdentifier:
labels:
- Cube
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!65 &934158984
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &934158985
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1
m_DynamicOccludee: 1
m_MotionVectors: 1
m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RayTracingMode: 2
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 73c176f402d2c2f4d929aa5da7585d17, type: 2}
m_StaticBatchInfo:
firstSubMesh: 0
subMeshCount: 0
m_StaticBatchRoot: {fileID: 0}
m_ProbeAnchor: {fileID: 0}
m_LightProbeVolumeOverride: {fileID: 0}
m_ScaleInLightmap: 1
m_ReceiveGI: 1
m_PreserveUVs: 0
m_IgnoreNormalsForChartDetection: 0
m_ImportantGI: 0
m_StitchLightmapSeams: 1
m_SelectedEditorRenderState: 3
m_MinimumChartSize: 4
m_AutoUVMaxDistance: 0.5
m_AutoUVMaxAngle: 89
m_LightmapParameters: {fileID: 0}
m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &934158986
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!4 &934158987
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 934158981}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 107.21, y: 73.4, z: -144.12}
m_LocalScale: {x: 15, y: 15, z: 15}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &963194225
GameObject:
m_ObjectHideFlags: 0

m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_LocalRotation: {x: -0.22882307, y: 0.124303445, z: -0.029468497, w: -0.96504945}
m_LocalPosition: {x: 116.52855, y: 91.11087, z: -194.85445}
m_LocalRotation: {x: 0.10991156, y: 0.06852541, z: -0.0075960113, w: 0.99154735}
m_LocalPosition: {x: -0.67697793, y: 1.4851692, z: -3.2593393}
m_RootOrder: 4
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &963194229
MonoBehaviour:

m_Script: {fileID: 11500000, guid: 4467405dbcbd3d64ab4363e9ae8bb813, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Labelers:
- id: 0
- id: 1
- id: 2
- id: 3
- id: 4
simulationDeltaTime: 0.0166
simulationDeltaTime: 0.2
m_Labelers:
- id: 0
- id: 1
- id: 2
- id: 3
showVisualizations: 1
references:
version: 1

objectInfoMetricId: 5ba92024-b3b7-41a7-9d3f-c03a6a8ddd01
idLabelConfig: {fileID: 11400000, guid: 258de5b48703743468d34fc5bbdfa3aa,
type: 2}
--- !u!1 &1321518866
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1321518869}
- component: {fileID: 1321518868}
- component: {fileID: 1321518867}
- component: {fileID: 1321518870}
m_Layer: 0
m_Name: Terrain
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 4294967295
m_IsActive: 1
--- !u!154 &1321518867
TerrainCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1321518866}
m_Material: {fileID: 0}
m_Enabled: 1
m_TerrainData: {fileID: 15600000, guid: 15ded0116bd9f864f80b9813d4f4477f, type: 2}
m_EnableTreeColliders: 1
--- !u!218 &1321518868
Terrain:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1321518866}
m_Enabled: 1
serializedVersion: 6
m_TerrainData: {fileID: 15600000, guid: 15ded0116bd9f864f80b9813d4f4477f, type: 2}
m_TreeDistance: 5000
m_TreeBillboardDistance: 50
m_TreeCrossFadeLength: 5
m_TreeMaximumFullLODCount: 50
m_DetailObjectDistance: 80
m_DetailObjectDensity: 1
m_HeightmapPixelError: 5
m_SplatMapDistance: 1000
m_HeightmapMaximumLOD: 0
m_ShadowCastingMode: 2
m_DrawHeightmap: 1
m_DrawInstanced: 0
m_DrawTreesAndFoliage: 1
m_ReflectionProbeUsage: 1
m_MaterialTemplate: {fileID: 2100000, guid: 22ff8771d87ef27429e670136399094b, type: 2}
m_BakeLightProbesForTrees: 1
m_PreserveTreePrototypeLayers: 0
m_DeringLightProbesForTrees: 1
m_ScaleInLightmap: 0.0256
m_LightmapParameters: {fileID: 15203, guid: 0000000000000000f000000000000000, type: 0}
m_GroupingID: 0
m_RenderingLayerMask: 1
m_AllowAutoConnect: 1
--- !u!4 &1321518869
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1321518866}
m_LocalRotation: {x: -0, y: -0.07853227, z: -0, w: 0.9969116}
m_LocalPosition: {x: -604.5, y: 65.1, z: -902.3}
m_LocalScale: {x: 0.95858, y: 0.95858, z: 0.95858}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: -9.008, z: 0}
--- !u!114 &1321518870
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1321518866}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Name:
m_EditorClassIdentifier:
labels:
- Terrain
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!1 &1640252278
00000004:
type: {class: BoundingBox3DLabeler, ns: UnityEngine.Perception.GroundTruth,
asm: Unity.Perception.Runtime}
data:
enabled: 1
annotationId: 0bfbe00d-00fa-4555-88d1-471b58449f5c
idLabelConfig: {fileID: 11400000, guid: 258de5b48703743468d34fc5bbdfa3aa,
type: 2}
visualizationColor: {r: 0, g: 1, b: 0, a: 1}
--- !u!1 &1698223694
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}

m_Component:
- component: {fileID: 1640252283}
- component: {fileID: 1640252282}
- component: {fileID: 1640252281}
- component: {fileID: 1640252280}
- component: {fileID: 1640252279}
- component: {fileID: 1640252284}
- component: {fileID: 1698223696}
- component: {fileID: 1698223695}
m_Name: Box
m_Name: GameObject
--- !u!114 &1640252279
--- !u!114 &1698223695
m_GameObject: {fileID: 1640252278}
m_GameObject: {fileID: 1698223694}
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Script: {fileID: 11500000, guid: a79cddcc0c4fb6f4e8e2df3e7c260ba7, type: 3}
labels:
- Box
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!65 &1640252280
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1640252278}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &1640252281
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1640252278}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1
m_DynamicOccludee: 1
m_MotionVectors: 1
m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RayTracingMode: 2
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 73c176f402d2c2f4d929aa5da7585d17, type: 2}
m_StaticBatchInfo:
firstSubMesh: 0
subMeshCount: 0
m_StaticBatchRoot: {fileID: 0}
m_ProbeAnchor: {fileID: 0}
m_LightProbeVolumeOverride: {fileID: 0}
m_ScaleInLightmap: 1
m_ReceiveGI: 1
m_PreserveUVs: 0
m_IgnoreNormalsForChartDetection: 0
m_ImportantGI: 0
m_StitchLightmapSeams: 1
m_SelectedEditorRenderState: 3
m_MinimumChartSize: 4
m_AutoUVMaxDistance: 0.5
m_AutoUVMaxAngle: 89
m_LightmapParameters: {fileID: 0}
m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &1640252282
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1640252278}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!4 &1640252283
prefab: {fileID: 3710685714995308732, guid: da3b4cd41011c1b488592ed975cbb467, type: 3}
--- !u!4 &1698223696
m_GameObject: {fileID: 1640252278}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 95.88, y: 73.33, z: -165.54}
m_LocalScale: {x: 15, y: 15, z: 15}
m_GameObject: {fileID: 1698223694}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.03735147, y: 0.4415038, z: 1.3462234}
m_LocalScale: {x: 1, y: 1, z: 1}
m_RootOrder: 2
m_RootOrder: 3
--- !u!114 &1640252284
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1640252278}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 673a227032a8e4940b9828c5b6f852ab, type: 3}
m_Name:
m_EditorClassIdentifier:
yDegreesPerSecond: 180

160
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs


using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Mathematics;
using Unity.Simulation;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;
namespace UnityEngine.Perception.GroundTruth

AnnotationDefinition m_AnnotationDefinition;
Texture2D m_MissingTexture;
Material m_MaterialDepthCheck;
struct FrameKeypointData
{

public List<AnimationPoseConfig> animationPoseConfigs;
private ComputeShader m_KeypointDepthTestShader;
private RenderTexture m_ResultsBuffer;
private RenderTextureReader<Color32> m_DepthCheckReader;
/// <inheritdoc/>

m_KeypointEntriesToReport = new List<KeypointEntry>();
m_CurrentFrame = 0;
m_KeypointDepthTestShader = (ComputeShader) Resources.Load("KeypointDepthTest");
var depthCheckShader = Shader.Find("Perception/KeypointDepthCheck");
var shaderVariantCollection = new ShaderVariantCollection();
shaderVariantCollection.Add(
new ShaderVariantCollection.ShaderVariant(depthCheckShader, PassType.ScriptableRenderPipeline, "HDRP_ENABLED"));
shaderVariantCollection.WarmUp();
m_MaterialDepthCheck = new Material(depthCheckShader);
//TODO: URP support
m_MaterialDepthCheck.EnableKeyword("HDRP_ENABLED");
//TODO: Proper resizing
m_ResultsBuffer = new RenderTexture(1024, 1, 0, GraphicsFormat.R8G8B8A8_UNorm);
m_DepthCheckReader = new RenderTextureReader<Color32>(m_ResultsBuffer);
perceptionCamera.attachedCamera.depthTextureMode = DepthTextureMode.Depth;
perceptionCamera.InstanceSegmentationImageReadback += OnInstanceSegmentationImageReadback;
perceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfoReadback;

return lhs.r == rhs.r && lhs.g == rhs.g && lhs.b == rhs.b && lhs.a == rhs.a;
}
bool PixelOnScreen(int x, int y, (int x, int y) dimensions)
bool PixelOnScreen(int2 pixelLocation, (int x, int y) dimensions)
return x >= 0 && x < dimensions.x && y >= 0 && y < dimensions.y;
return pixelLocation.x >= 0 && pixelLocation.x < dimensions.x && pixelLocation.y >= 0 && pixelLocation.y < dimensions.y;
}
bool PixelsMatch(int x, int y, Color32 idColor, (int x, int y) dimensions, NativeArray<Color32> data)

static int s_PixelTolerance = 1;
// Determine the state of a keypoint. A keypoint is considered visible (state = 2) if it is on screen and not occluded
// by another object. The way that we determine if a point is occluded is by checking the pixel location of the keypoint
// by itself or another object. Self-occlusion has already been checked, so the input keypoint may be state 2, 1, or 0.
// We determine if a point is occluded by other objects is by checking the pixel location of the keypoint
// against the instance segmentation mask for the frame. The instance segmentation mask provides the instance id of the
// visible object at a pixel location. Which means, if the keypoint does not match the visible pixel, then another
// object is in front of the keypoint occluding it from view. An important note here is that the keypoint is an infintely small

{
if (keypoint.state == 0) return 0;
var centerX = Mathf.FloorToInt(keypoint.x);
var centerY = Mathf.FloorToInt(keypoint.y);
var pixelLocation = PixelLocationFromScreenPoint(keypoint);
if (!PixelOnScreen(centerX, centerY, dimensions))
if (!PixelOnScreen(pixelLocation, dimensions))
for (var y = centerY - s_PixelTolerance; y <= centerY + s_PixelTolerance; y++)
for (var y = pixelLocation.y - s_PixelTolerance; y <= pixelLocation.y + s_PixelTolerance; y++)
for (var x = centerX - s_PixelTolerance; x <= centerX + s_PixelTolerance; x++)
for (var x = pixelLocation.x - s_PixelTolerance; x <= pixelLocation.x + s_PixelTolerance; x++)
if (!PixelOnScreen(x, y, dimensions)) continue;
if (!PixelOnScreen(new int2(x, y), dimensions)) continue;
return 2;
return keypoint.state;
}
}
}

private static int2 PixelLocationFromScreenPoint(Keypoint keypoint)
{
var centerX = Mathf.FloorToInt(keypoint.x);
var centerY = Mathf.FloorToInt(keypoint.y);
int2 pixelLocation = new int2(centerX, centerY);
return pixelLocation;
}
void OnInstanceSegmentationImageReadback(int frameCount, NativeArray<Color32> data, RenderTexture renderTexture)
{
if (!m_FrameKeypointData.TryGetValue(frameCount, out var frameKeypointData))

var annotation = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
var keypointEntries = new List<KeypointEntry>();
var positions = new NativeList<float3>(512, Allocator.Persistent);
//uint4 so that we can fill a 4 channel texture with this data
var positions = new NativeList<uint4>(512, Allocator.Persistent);
if (keypointEntries.Count != 0)
DoDepthCheck(scriptableRenderContext, keypointEntries, positions);
m_FrameKeypointData[m_CurrentFrame] = new FrameKeypointData
{

};
if (keypointEntries.Count != 0)
DoDepthCheck(scriptableRenderContext, keypointEntries, positions);
positions.Dispose();
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<uint4> positions)
var commandBuffer = CommandBufferPool.Get();
var commandBuffer = CommandBufferPool.Get("KeypointDepthCheck");
var keypointPositionsTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16G16_SFloat, TextureCreationFlags.None);
var keypointDepthTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
var positionsPixeldata = new NativeArray<half>(positions.Length * 2, Allocator.Temp);
var depthPixeldata = new NativeArray<half>(positions.Length, Allocator.Temp);
commandBuffer.SetComputeTextureParam(m_KeypointDepthTestShader, 0, "DepthBuffer", depthTexture);
var positionSize = UnsafeUtility.SizeOf<float3>();
var keypointPositionsBuffer = new ComputeBuffer(keypointCount * positionSize, positionSize,
ComputeBufferType.Default, ComputeBufferMode.Dynamic);
keypointPositionsBuffer.SetData(positions.AsArray());
for (int i = 0; i < positions.Length; i++)
{
var pos = positions[i];
positionsPixeldata[i * 2] = new half(pos.x/* / (float)depthTexture.width*/);
positionsPixeldata[i * 2 + 1] = new half(pos.y/* / (float)depthTexture.height*/);
depthPixeldata[i] = new half(pos.z / 100.0f);
}
var resultsComputeBuffer =
new ComputeBuffer(keypointCount, 4, ComputeBufferType.Default, ComputeBufferMode.Dynamic);
keypointPositionsTexture.SetPixelData(positionsPixeldata, 0);
keypointPositionsTexture.Apply();
keypointDepthTexture.SetPixelData(depthPixeldata, 0);
keypointDepthTexture.Apply();
commandBuffer.SetComputeBufferParam(m_KeypointDepthTestShader, 0, "CheckPositions", keypointPositionsBuffer);
commandBuffer.SetComputeBufferParam(m_KeypointDepthTestShader, 0, "CheckResults", resultsComputeBuffer);
commandBuffer.DispatchCompute(m_KeypointDepthTestShader, 0, keypointCount, 1, 1);
positionsPixeldata.Dispose();
depthPixeldata.Dispose();
var currentFrame = Time.frameCount;
commandBuffer.RequestAsyncReadback(resultsComputeBuffer, request => OnDepthCheckReadback(currentFrame, request));
m_MaterialDepthCheck.SetTexture("_Positions", keypointPositionsTexture);
m_MaterialDepthCheck.SetTexture("_KeypointDepth", keypointDepthTexture);
m_MaterialDepthCheck.SetTexture("_DepthTexture", depthTexture);
commandBuffer.Blit(null, m_ResultsBuffer, m_MaterialDepthCheck);
// var resultsComputeBuffer =
// new ComputeBuffer(keypointCount, 4, ComputeBufferType.Default, ComputeBufferMode.Dynamic);
// var depthTexture = Shader.GetGlobalTexture("_CameraDepthTexture");
// var positionSize = UnsafeUtility.SizeOf<float3>();
// commandBuffer.SetComputeTextureParam(m_KeypointDepthTestShader, 0, "DepthBuffer", depthTexture);
//
// var positionSize = UnsafeUtility.SizeOf<float3>();
// var keypointPositionsBuffer = new ComputeBuffer(keypointCount * positionSize, positionSize,
// ComputeBufferType.Default, ComputeBufferMode.Dynamic);
// keypointPositionsBuffer.SetData(positions.AsArray());
//
// var resultsComputeBuffer =
// new ComputeBuffer(keypointCount, 4, ComputeBufferType.Default, ComputeBufferMode.Dynamic);
//
// commandBuffer.SetComputeBufferParam(m_KeypointDepthTestShader, 0, "CheckPositions", keypointPositionsBuffer);
// commandBuffer.SetComputeBufferParam(m_KeypointDepthTestShader, 0, "CheckResults", resultsComputeBuffer);
// commandBuffer.DispatchCompute(m_KeypointDepthTestShader, 0, keypointCount, 1, 1);
//
// commandBuffer.RequestAsyncReadback(resultsComputeBuffer, request => OnDepthCheckReadback(currentFrame, request));
m_DepthCheckReader.Capture(scriptableRenderContext, OnDepthCheckReadback);
}
private void OnDepthCheckReadback(int frame, NativeArray<Color32> data, RenderTexture renderTexture)
{
DoDepthCheckReadback(frame, data);
var frameKeypointData = m_FrameKeypointData[frameCount];
for (var i = 0; i < data.Length; i++)
//DoDepthCheckReadback(frameCount, data);
}
private void DoDepthCheckReadback(int frameCount, NativeArray<Color32> data)
{
var frameKeypointData = m_FrameKeypointData[frameCount];
var totalLength = frameKeypointData.pointsPerEntry * frameKeypointData.keypoints.Count;
Debug.Assert(totalLength < data.Length);
for (var i = 0; i < totalLength; i++)
if (value == 0)
if (value.a == 0)
{
var keypoints = frameKeypointData.keypoints[i / frameKeypointData.pointsPerEntry];
var indexInObject = i % frameKeypointData.pointsPerEntry;

}
}
frameKeypointData.isDepthCheckComplete = true;
m_FrameKeypointData[frameCount] = frameKeypointData;
ReportIfComplete(frameCount, frameKeypointData);

return false;
}
void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<uint4> positions)
{
if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
return;

var listStart = positions.Length;
positions.Resize(positions.Length + activeTemplate.keypoints.Length, NativeArrayOptions.ClearMemory);
//grab the slice of the list for the current object to assign positions in
var positionsSlice = new NativeSlice<float3>(positions, listStart);
var positionsSlice = new NativeSlice<uint4>(positions, listStart);
// Go through all of the rig keypoints and get their location
for (var i = 0; i < activeTemplate.keypoints.Length; i++)

}
}
private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<float3> positions, int idx)
private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<uint4> positions, int idx)
positions[idx] = position;
var keypoints = cachedData.keypoints.keypoints;
keypoints[idx].index = idx;
if (loc.z < 0)

keypoints[idx].y = loc.y;
keypoints[idx].state = 2;
}
//TODO: move this code
uint2 dimensions = new uint2((uint)perceptionCamera.attachedCamera.pixelWidth, (uint)perceptionCamera.attachedCamera.pixelHeight);
var pixelLocation = PixelLocationFromScreenPoint(keypoints[idx]);
if (pixelLocation.x < 0 || pixelLocation.y < 0)
{
pixelLocation = new int2(int.MaxValue, int.MaxValue);
}
//TODO: Fix up the z computation
positions[idx] = new uint4((uint)pixelLocation.x, (uint)pixelLocation.y, (uint)(loc.z * 100), 1);
}
string GetPose(Animator animator)

83
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheck.shader


Shader "Perception/KeypointDepthCheck"
{
Properties
{
_Positions("Positions", 2D) = "defaultTexture" {}
_KeypointDepth("KeypointDepth", 2D) = "defaultTexture" {}
_DepthTexture("Depth", 2DArray) = "defaultTexture" {}
}
HLSLINCLUDE
#pragma target 4.5
#pragma only_renderers d3d11 ps4 xboxone vulkan metal switch
//enable GPU instancing support
#pragma multi_compile_instancing
ENDHLSL
SubShader
{
Tags { "RenderType"="Opaque" }
LOD 100
Pass
{
Tags { "LightMode" = "SRP" }
Blend Off
ZWrite On
ZTest LEqual
Cull Back
CGPROGRAM
#pragma vertex depthCheckVertexStage
#pragma fragment depthCheckFragmentStage
#include "UnityCG.cginc"
UNITY_DECLARE_TEX2DARRAY(_DepthTexture);
//sampler2D _CameraDepthTexture;
Texture2D _Positions;
SamplerState my_point_clamp_sampler;
Texture2D _KeypointDepth;
struct in_vert
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct vertexToFragment
{
float4 vertex : SV_POSITION;
float2 uv : TEXCOORD0;
};
vertexToFragment depthCheckVertexStage (in_vert vertWorldSpace)
{
vertexToFragment vertScreenSpace;
vertScreenSpace.vertex = UnityObjectToClipPos(vertWorldSpace.vertex);
vertScreenSpace.uv = vertWorldSpace.uv;
return vertScreenSpace;
}
fixed4 depthCheckFragmentStage (vertexToFragment vertScreenSpace) : SV_Target
{
float4 checkPosition = _Positions.Sample(my_point_clamp_sampler, vertScreenSpace.uv);
float4 checkDepth = _KeypointDepth.Sample(my_point_clamp_sampler, vertScreenSpace.uv);
float depth = UNITY_SAMPLE_TEX2DARRAY(_DepthTexture, float3(checkPosition.xy, 0)).r; //SAMPLE_DEPTH_TEXTURE(_DepthTexture, checkPosition.xy);
//float depth_decoded = LinearEyeDepth(depth);
float depth_decoded = Linear01Depth(depth);
uint result = depth_decoded < checkDepth.x - .001 ? 0 : 1;
return fixed4(result, result, result, result);
}
ENDCG
}
}
}

3
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheck.shader.meta


fileFormatVersion: 2
guid: 5fd132f5a09c4301b4c9004b241e2659
timeCreated: 1618585131

102
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader


Shader "Perception/KeypointDepthCheck"
{
Properties
{
_Positions("Positions", 2D) = "defaultTexture" {}
_KeypointDepth("KeypointDepth", 2D) = "defaultTexture" {}
_DepthTexture("Depth", 2DArray) = "defaultTexture" {}
}
HLSLINCLUDE
#pragma target 4.5
#pragma only_renderers d3d11 ps4 xboxone vulkan metal switch
//enable GPU instancing support
#pragma multi_compile_instancing
ENDHLSL
SubShader
{
Tags{ "RenderPipeline" = "HDRenderPipeline" }
Pass
{
Tags { "LightMode" = "SRP" }
Name "KeypointDepthCheck"
ZWrite Off
ZTest Always
Blend SrcAlpha OneMinusSrcAlpha
Cull Off
HLSLPROGRAM
#pragma multi_compile HDRP_DISABLED HDRP_ENABLED
#pragma only_renderers d3d11 vulkan metal
#pragma target 4.5
#pragma vertex Vert
#pragma fragment FullScreenPass
//UNITY_DECLARE_TEX2DARRAY(_DepthTexture);
//sampler2D _CameraDepthTexture;
Texture2D _Positions;
SamplerState my_point_clamp_sampler;
Texture2D _KeypointDepth;
#if HDRP_ENABLED
#include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/RenderPass/CustomPass/CustomPassCommon.hlsl"
float4 FullScreenPass(Varyings varyings) : SV_Target
{
UNITY_SETUP_STEREO_EYE_INDEX_POST_VERTEX(varyings);
float4 checkPosition = _Positions.Load(float3(varyings.positionCS.xy, 0));
float4 checkDepth = _KeypointDepth.Load(float3(varyings.positionCS.xy, 0));
float depth = LoadCameraDepth(checkPosition.xy);
depth = LinearEyeDepth(depth, _ZBufferParams);
// float depth = UNITY_SAMPLE_TEX2DARRAY(_DepthTexture, float3(checkPosition.xy, 0)).r; //SAMPLE_DEPTH_TEXTURE(_DepthTexture, checkPosition.xy);
//float depth_decoded = LinearEyeDepth(depth);
// float depth_decoded = Linear01Depth(depth);
uint result = depth < checkDepth.x - .001 ? 0 : 1;
return float4(result, result, result, result);
}
#else
/// Dummy Implementation for non HDRP_ENABLED variants
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
float2 uv : TEXCOORD0;
float4 vertex : SV_POSITION;
};
v2f Vert(appdata v)
{
v2f o;
o.uv = float2(0, 0);
o.vertex = float4(0, 0, 0, 0);
return o;
}
float4 FullScreenPass(v2f i) : SV_Target
{
return float4(0, 0, 0, 1);
}
#endif
ENDHLSL
}
}
Fallback Off
}

3
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthCheckHDRP.shader.meta


fileFormatVersion: 2
guid: 9940b8b9f40d48b5bf942d679dbd9d5d
timeCreated: 1618629386

17
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthTest.compute


// Each #kernel tells which function to compile; you can have many kernels
#pragma kernel CSMain
// Create a RenderTexture with enableRandomWrite flag and set it
// with cs.SetTexture
StructuredBuffer<float3> CheckPositions;
Texture3D<float4> DepthBuffer;
RWStructuredBuffer<uint> CheckResults;
[numthreads(16,1,1)]
void CSMain (uint3 id : SV_DispatchThreadID)
{
float3 checkPosition = CheckPositions[id.x];
uint result = DepthBuffer[float3(checkPosition.x, checkPosition.y, 0)] < checkPosition.z - .001 ? 0 : 1;
CheckResults[id.x] = result;
}

8
com.unity.perception/Runtime/GroundTruth/Resources/KeypointDepthTest.compute.meta


fileFormatVersion: 2
guid: c78354f07f15bac48977fc60cc5082c9
ComputeShaderImporter:
externalObjects: {}
currentAPIMask: 4
userData:
assetBundleName:
assetBundleVariant:
正在加载...
取消
保存