unity-tech-cn
/
com.unity.perception
29
0
派生 7
代码 合并请求 版本发布
浏览代码

Updating JointLabel to include self-occlusion distance and drop the reference to the keypoint label config

/keypoint_self_occlusion
Jon Hogins 3 年前
当前提交
e19809d3
共有 5 个文件被更改，包括 122 次插入 和 149 次删除
  1. 71
      TestProjects/PerceptionURP/Assets/Human Pose Labeling and Randomization/Models and Animations/LabeledAndRandomized.prefab
  2. 104
      TestProjects/PerceptionURP/Assets/Scenes/PoseSample.unity
  3. 7
      com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs
  4. 46
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  5. 43
      com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

71
TestProjects/PerceptionURP/Assets/Human Pose Labeling and Randomization/Models and Animations/LabeledAndRandomized.prefab
查看文件


%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!114 &4295273093489125888
--- !u!1 &6018108664084516484
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 6798856831628976836}
- component: {fileID: 787957711928879280}
m_Layer: 0
m_Name: LeftEar
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &6798856831628976836
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 6018108664084516484}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.047, y: -0.095, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 460271976872290021}
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &787957711928879280
m_GameObject: {fileID: 8917933090883489511}
m_GameObject: {fileID: 6018108664084516484}
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Script: {fileID: 11500000, guid: 8cf4fa374b134b1680755f8280ae8e7d, type: 3}
labels:
- Character
useAutoLabeling: 0
autoLabelingSchemeType:
templateInformation:
- template: {fileID: 11400000, guid: a29b79d8ce98945a0855b1addec08d86, type: 2}
label: left_ear
--- !u!114 &1543322299
MonoBehaviour:
m_ObjectHideFlags: 0

range:
minimum: 0
maximum: 1
--- !u!114 &4295273093489125888
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 8917933090883489511}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8b33f0bc2b78db642a758f07826d0dd0, type: 3}
m_Name:
m_EditorClassIdentifier:
labels:
- Character
useAutoLabeling: 0
autoLabelingSchemeType:
--- !u!1001 &8575897352412525494
PrefabInstance:
m_ObjectHideFlags: 0

propertyPath: m_Name
value: LabeledAndRandomized
objectReference: {fileID: 0}
- target: {fileID: 5866666021909216657, guid: 1f279db2ebaa049d4aff8b82b6c0a1e7,
type: 3}
propertyPath: m_Controller
value:
objectReference: {fileID: 9100000, guid: 5a4ece82c4ce4894babb282606853fc6, type: 2}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 1f279db2ebaa049d4aff8b82b6c0a1e7, type: 3}
--- !u!1 &8917933090883489511 stripped

m_PrefabInstance: {fileID: 8575897352412525494}
m_PrefabAsset: {fileID: 0}
--- !u!4 &460271976872290021 stripped
Transform:
m_CorrespondingSourceObject: {fileID: 8169674691431683411, guid: 1f279db2ebaa049d4aff8b82b6c0a1e7,
type: 3}
m_PrefabInstance: {fileID: 8575897352412525494}
m_PrefabAsset: {fileID: 0}

104
TestProjects/PerceptionURP/Assets/Scenes/PoseSample.unity
查看文件


m_TrainingDataDestination: TrainingData
m_LightProbeSampleCountMultiplier: 4
m_LightingDataAsset: {fileID: 0}
m_UseShadowmask: 0
m_UseShadowmask: 1
--- !u!196 &4
NavMeshSettings:
serializedVersion: 2

objectReference: {fileID: 0}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 09c632a4eeff8204c87fa56b8632d9ba, type: 3}
--- !u!4 &1667437135 stripped
Transform:
m_CorrespondingSourceObject: {fileID: 460271976872290021, guid: 09c632a4eeff8204c87fa56b8632d9ba,
type: 3}
m_PrefabInstance: {fileID: 1667437134}
m_PrefabAsset: {fileID: 0}
--- !u!1 &1875122032
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1875122033}
- component: {fileID: 1875122034}
m_Layer: 0
m_Name: LeftEar
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &1875122033
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1875122032}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.059, y: -0.099, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 1667437135}
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &1875122034
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1875122032}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8cf4fa374b134b1680755f8280ae8e7d, type: 3}
m_Name:
m_EditorClassIdentifier:
templateInformation: []
labels:
- left_ear
useLocalSelfOcclusionDistance: 0
selfOcclusionDistance: 0.15
--- !u!1 &2094194684
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 2094194685}
- component: {fileID: 2094194686}
m_Layer: 0
m_Name: RightEar
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &2094194685
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 2094194684}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.059, y: 0.095, z: -0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 1667437135}
m_RootOrder: 6
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &2094194686
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 2094194684}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8cf4fa374b134b1680755f8280ae8e7d, type: 3}
m_Name:
m_EditorClassIdentifier:
templateInformation: []
labels:
- right_ear
useLocalSelfOcclusionDistance: 0
selfOcclusionDistance: 0.15
--- !u!1001 &3710685714535070024
PrefabInstance:
m_ObjectHideFlags: 0

7
com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs
查看文件


namespace UnityEngine.Perception.GroundTruth
{
public enum SelfOcclusionDistanceSource
{
JointLabel,
KeypointLabeler
}
/// <summary>
/// Label to designate a custom joint/keypoint. These are needed to add body
/// parts to a humanoid model that are not contained in its <see cref="Animator"/> <see cref="Avatar"/>

[SerializeField]
public List<string> labels = new List<string>();
public bool useLocalSelfOcclusionDistance = false;
public SelfOcclusionDistanceSource selfOcclusionDistanceSource = SelfOcclusionDistanceSource.JointLabel;
public float selfOcclusionDistance = .15f;
public void OnBeforeSerialize()

46
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
查看文件


using Unity.Simulation;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;
using UnityEngine.Serialization;
namespace UnityEngine.Perception.GroundTruth
{

/// <summary>
/// The max distance a keypoint can be from the front of an object before it is considered occluded
/// </summary>
public float distanceThreshold = 0.15f;
public float selfOcclusionDistance = 0.15f;
// ReSharper restore MemberCanBePrivate.Global
AnnotationDefinition m_AnnotationDefinition;

var annotation = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
var keypointEntries = new List<KeypointEntry>();
var positions = new NativeList<float3>(512, Allocator.Persistent);
var checkLocations = new NativeList<float3>(512, Allocator.Persistent);
ProcessLabel(label, keypointEntries, positions);
ProcessLabel(label, keypointEntries, checkLocations);
m_FrameKeypointData[m_CurrentFrame] = new FrameKeypointData
{

};
if (keypointEntries.Count != 0)
DoDepthCheck(scriptableRenderContext, keypointEntries, positions);
DoDepthCheck(scriptableRenderContext, keypointEntries, checkLocations);
else
{
var frameKeypointData = m_FrameKeypointData[m_CurrentFrame];

positions.Dispose();
checkLocations.Dispose();
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> checkLocations)
{
var keypointCount = keypointEntries.Count * activeTemplate.keypoints.Length;

var keypointCheckDepthTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
var positionsPixeldata = new NativeArray<half>(positions.Length * 2, Allocator.Temp);
var depthPixeldata = new NativeArray<half>(positions.Length, Allocator.Temp);
var positionsPixeldata = new NativeArray<half>(checkLocations.Length * 2, Allocator.Temp);
var depthPixeldata = new NativeArray<half>(checkLocations.Length, Allocator.Temp);
for (int i = 0; i < positions.Length; i++)
for (int i = 0; i < checkLocations.Length; i++)
var pos = positions[i];
var pos = checkLocations[i];
depthPixeldata[i] = new half(pos.z - this.distanceThreshold);
depthPixeldata[i] = new half(pos.z);
}
keypointPositionsTexture.SetPixelData(positionsPixeldata, 0);

return TryToGetTemplateIndexForJoint(activeTemplate, jointLabel, out _);
}
void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<float3> positions)
void ProcessLabel(Labeling labeledEntity, List<KeypointEntry> keypointEntries, NativeList<float3> checkLocations)
{
if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
return;

{
var animator = cachedData.animator;
var listStart = positions.Length;
positions.Resize(positions.Length + activeTemplate.keypoints.Length, NativeArrayOptions.ClearMemory);
var listStart = checkLocations.Length;
checkLocations.Resize(checkLocations.Length + activeTemplate.keypoints.Length, NativeArrayOptions.ClearMemory);
var positionsSlice = new NativeSlice<float3>(positions, listStart);
var checkLocationsSlice = new NativeSlice<float3>(checkLocations, listStart);
// Go through all of the rig keypoints and get their location
for (var i = 0; i < activeTemplate.keypoints.Length; i++)

var bone = animator.GetBoneTransform(pt.rigLabel);
if (bone != null)
{
InitKeypoint(bone.position, cachedData, positionsSlice, i);
InitKeypoint(bone.position, cachedData, checkLocationsSlice, i, this.selfOcclusionDistance);
}
}
}

foreach (var (joint, idx) in cachedData.overrides)
{
InitKeypoint(joint.transform.position, cachedData, positionsSlice, idx);
float jointSelfOcclusionDistance;
if (joint.selfOcclusionDistanceSource == SelfOcclusionDistanceSource.JointLabel)
jointSelfOcclusionDistance = joint.selfOcclusionDistance;
else
jointSelfOcclusionDistance = selfOcclusionDistance;
InitKeypoint(joint.transform.position, cachedData, checkLocationsSlice, idx, jointSelfOcclusionDistance);
}
cachedData.keypoints.pose = "unset";

}
}
private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<float3> positions, int idx)
private void InitKeypoint(Vector3 position, CachedData cachedData, NativeSlice<float3> checkLocations, int idx,
float occlusionDistance)
{
var loc = ConvertToScreenSpace(position);

pixelLocation = new int2(int.MaxValue, int.MaxValue);
}
positions[idx] = new float3(pixelLocation.x + .5f, pixelLocation.y + .5f, loc.z);
checkLocations[idx] = new float3(pixelLocation.x + .5f, pixelLocation.y + .5f, loc.z - occlusionDistance);
}
string GetPose(Animator animator)

43
com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs
查看文件


perceptionCamera.captureRgbImages = false;
var keyPointLabeler = new KeypointLabeler(config, template);
keyPointLabeler.objectFilter = keypointObjectFilter;
keyPointLabeler.distanceThreshold = defaultSelfOcclusionDistance;
keyPointLabeler.selfOcclusionDistance = defaultSelfOcclusionDistance;
if (computeListener != null)
keyPointLabeler.KeypointsComputed += computeListener;

return cfg;
}
static void SetupCubeJoint(GameObject cube, KeypointTemplate template, string label, float x, float y, float z)
static void SetupCubeJoint(GameObject cube, string label, float x, float y, float z, float? selfOcclusionDistance = null)
{
var joint = new GameObject();
joint.transform.parent = cube.transform;

if (selfOcclusionDistance.HasValue)
{
jointLabel.selfOcclusionDistanceSource = SelfOcclusionDistanceSource.JointLabel;
jointLabel.selfOcclusionDistance = selfOcclusionDistance.Value;
}
else
jointLabel.selfOcclusionDistanceSource = SelfOcclusionDistanceSource.KeypointLabeler;
SetupCubeJoint(cube, template, "FrontLowerLeft", -dim, -dim, -dim);
SetupCubeJoint(cube, template, "FrontUpperLeft", -dim, dim, -dim);
SetupCubeJoint(cube, template, "FrontUpperRight", dim, dim, -dim);
SetupCubeJoint(cube, template, "FrontLowerRight", dim, -dim, -dim);
SetupCubeJoint(cube, template, "BackLowerLeft", -dim, -dim, dim);
SetupCubeJoint(cube, template, "BackUpperLeft", -dim, dim, dim);
SetupCubeJoint(cube, template, "BackUpperRight", dim, dim, dim);
SetupCubeJoint(cube, template, "BackLowerRight", dim, -dim, dim);
SetupCubeJoint(cube, "FrontLowerLeft", -dim, -dim, -dim);
SetupCubeJoint(cube, "FrontUpperLeft", -dim, dim, -dim);
SetupCubeJoint(cube, "FrontUpperRight", dim, dim, -dim);
SetupCubeJoint(cube, "FrontLowerRight", dim, -dim, -dim);
SetupCubeJoint(cube, "BackLowerLeft", -dim, -dim, dim);
SetupCubeJoint(cube, "BackUpperLeft", -dim, dim, dim);
SetupCubeJoint(cube, "BackUpperRight", dim, dim, dim);
SetupCubeJoint(cube, "BackLowerRight", dim, -dim, dim);
}
[UnityTest]

cam.GetComponent<PerceptionCamera>().showVisualizations = false;
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
SetupCubeJoint(cube, template, "Center", 0, 0, -.5f);
SetupCubeJoint(cube, "Center", 0, 0, -.5f);
cube.SetActive(true);
cam.SetActive(true);

var labeling = cube.AddComponent<Labeling>();
labeling.labels.Add("label");
SetupCubeJoint(cube, template, "Center",0f, 0f, -.5f);
SetupCubeJoint(cube, "Center",0f, 0f, -.5f);
cube.SetActive(true);
cam.SetActive(true);

[Values(CheckDistanceType.Global, CheckDistanceType.JointLabel)] CheckDistanceType checkDistanceType,
[Values(ProjectionKind.Orthographic, ProjectionKind.Projection)] ProjectionKind projectionKind)
{
if (checkDistanceType == CheckDistanceType.JointLabel)
Assert.Inconclusive("Not yet implemented");
var defaultSelfOcclusionDistance =
checkDistanceType == CheckDistanceType.Global ? args.checkDistance : 0.15f;
var labelerSelfOcclusionDistance =
checkDistanceType == CheckDistanceType.Global ? args.checkDistance : 0.5f;
}, texture, defaultSelfOcclusionDistance: defaultSelfOcclusionDistance);
}, texture, defaultSelfOcclusionDistance: labelerSelfOcclusionDistance);
var camComponent = cam.GetComponent<Camera>();
camComponent.fieldOfView = args.cameraFieldOfView;
camComponent.farClipPlane = 100f;

}
var cube = TestHelper.CreateLabeledCube(scale: 1f, x: args.origin.x, y: args.origin.y, z: args.origin.z);
SetupCubeJoint(cube, template, "Center", 0f, 0f, -.5f + args.pointDistance);
var localSelfOcclusionDistance = checkDistanceType == CheckDistanceType.JointLabel ? (float?)args.checkDistance : null;
SetupCubeJoint(cube, "Center", 0f, 0f, -.5f + args.pointDistance, localSelfOcclusionDistance);
cube.SetActive(true);
cam.SetActive(true);

撰写 预览
正在加载...
取消
保存