unity-tech-cn
/
com.unity.perception
29
0
派生 7
代码 合并请求 版本发布
浏览代码

Fixing issues with large numbers of keypoints.

/keypoint_self_occlusion
Jon Hogins 3 年前
当前提交
d92bbc94
共有 3 个文件被更改，包括 100 次插入 和 68 次删除
  1. 1
      com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs
  2. 138
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  3. 29
      com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

1
com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs
查看文件


using System;
using Unity.Profiling;
using Unity.Simulation;
using UnityEngine.EventSystems;
using UnityEngine.Rendering;

138
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
查看文件


using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Mathematics;
using Unity.Profiling;
using Unity.Simulation;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;

public sealed class KeypointLabeler : CameraLabeler
{
internal const float defaultSelfOcclusionDistance = 0.15f;
const int k_MinTextureWidth = 4;
static ProfilerMarker k_OnEndRenderingMarker = new ProfilerMarker($"KeypointLabeler OnEndRendering");
static ProfilerMarker k_OnVisualizeMarker = new ProfilerMarker($"KeypointLabeler OnVisualize");
/// <summary>
/// The active keypoint template. Required to annotate keypoint data.

AnnotationDefinition m_AnnotationDefinition;
Texture2D m_MissingTexture;
Material m_MaterialDepthCheck;
Texture2D m_KeypointPositionsTexture;
Texture2D m_KeypointCheckDepthTexture;
struct FrameKeypointData
{

shaderVariantCollection.Add(
new ShaderVariantCollection.ShaderVariant(depthCheckShader, PassType.ScriptableRenderPipeline, keyword));
shaderVariantCollection.WarmUp();
SetupResultsBuffer(1024);
perceptionCamera.attachedCamera.depthTextureMode = DepthTextureMode.Depth;
#if URP_PRESENT

perceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfoReadback;
}
private void SetupResultsBuffer(int size)
private void SetupDepthCheckBuffers(int size)
if (m_ResultsBuffer != null && m_ResultsBuffer.width >= size)
var textureDimensions = TextureDimensions(size);
if (m_ResultsBuffer != null && textureDimensions.x * textureDimensions.y >= size)
// Object.Destroy(m_KeypointPositionsTexture);
// Object.Destroy(m_KeypointCheckDepthTexture);
m_ResultsBuffer = new RenderTexture(size, 1, 0, GraphicsFormat.R8G8B8A8_UNorm);
m_KeypointPositionsTexture = new Texture2D(textureDimensions.x, textureDimensions.y, GraphicsFormat.R16G16_SFloat, TextureCreationFlags.None);
m_KeypointCheckDepthTexture = new Texture2D(textureDimensions.x, textureDimensions.y, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
m_ResultsBuffer = new RenderTexture(textureDimensions.x, textureDimensions.y, 0, GraphicsFormat.R8G8B8A8_UNorm);
m_DepthCheckReader = new RenderTextureReader<Color32>(m_ResultsBuffer);
}

/// <inheritdoc/>
protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
{
m_CurrentFrame = Time.frameCount;
using (k_OnEndRenderingMarker.Auto())
{
m_CurrentFrame = Time.frameCount;
var annotation = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
var keypointEntries = new List<KeypointEntry>();
var checkLocations = new NativeList<float3>(512, Allocator.Persistent);
var annotation = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
var keypointEntries = new List<KeypointEntry>();
var checkLocations = new NativeList<float3>(512, Allocator.Persistent);
foreach (var label in LabelManager.singleton.registeredLabels)
ProcessLabel(label, keypointEntries, checkLocations);
foreach (var label in LabelManager.singleton.registeredLabels)
ProcessLabel(label, keypointEntries, checkLocations);
m_FrameKeypointData[m_CurrentFrame] = new FrameKeypointData
{
annotation = annotation,
keypoints = keypointEntries,
pointsPerEntry = activeTemplate.keypoints.Length
};
m_FrameKeypointData[m_CurrentFrame] = new FrameKeypointData
{
annotation = annotation,
keypoints = keypointEntries,
pointsPerEntry = activeTemplate.keypoints.Length
};
if (keypointEntries.Count != 0)
DoDepthCheck(scriptableRenderContext, keypointEntries, checkLocations);
else
{
var frameKeypointData = m_FrameKeypointData[m_CurrentFrame];
frameKeypointData.isDepthCheckComplete = true;
m_FrameKeypointData[m_CurrentFrame] = frameKeypointData;
}
if (keypointEntries.Count != 0)
DoDepthCheck(scriptableRenderContext, keypointEntries, checkLocations);
else
{
var frameKeypointData = m_FrameKeypointData[m_CurrentFrame];
frameKeypointData.isDepthCheckComplete = true;
m_FrameKeypointData[m_CurrentFrame] = frameKeypointData;
checkLocations.Dispose();
checkLocations.Dispose();
}
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> checkLocations)

var commandBuffer = CommandBufferPool.Get("KeypointDepthCheck");
var keypointPositionsTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16G16_SFloat, TextureCreationFlags.None);
var keypointCheckDepthTexture = new Texture2D(keypointCount, 1, GraphicsFormat.R16_SFloat, TextureCreationFlags.None);
var textureDimensions = TextureDimensions(keypointCount);
var positionsPixeldata = new NativeArray<half>(checkLocations.Length * 2, Allocator.Temp);
var depthPixeldata = new NativeArray<half>(checkLocations.Length, Allocator.Temp);
SetupDepthCheckBuffers(checkLocations.Length);
var positionsPixeldata = new NativeArray<half>(textureDimensions.x * textureDimensions.y * 2, Allocator.Temp);
var depthPixeldata = new NativeArray<half>(textureDimensions.x * textureDimensions.y, Allocator.Temp);
var depthTexture = Shader.GetGlobalTexture("_CameraDepthTexture");
for (int i = 0; i < checkLocations.Length; i++)

depthPixeldata[i] = new half(pos.z);
}
keypointPositionsTexture.SetPixelData(positionsPixeldata, 0);
keypointPositionsTexture.Apply();
keypointCheckDepthTexture.SetPixelData(depthPixeldata, 0);
keypointCheckDepthTexture.Apply();
m_KeypointPositionsTexture.SetPixelData(positionsPixeldata, 0);
m_KeypointPositionsTexture.Apply();
m_KeypointCheckDepthTexture.SetPixelData(depthPixeldata, 0);
m_KeypointCheckDepthTexture.Apply();
m_MaterialDepthCheck.SetTexture("_Positions", keypointPositionsTexture);
m_MaterialDepthCheck.SetTexture("_KeypointCheckDepth", keypointCheckDepthTexture);
m_MaterialDepthCheck.SetTexture("_Positions", m_KeypointPositionsTexture);
m_MaterialDepthCheck.SetTexture("_KeypointCheckDepth", m_KeypointCheckDepthTexture);
SetupResultsBuffer(checkLocations.Length);
commandBuffer.Blit(null, m_ResultsBuffer, m_MaterialDepthCheck);
scriptableRenderContext.ExecuteCommandBuffer(commandBuffer);

}
private static Vector2Int TextureDimensions(int keypointCount)
{
var width = Math.Max(k_MinTextureWidth, Mathf.NextPowerOfTwo((int)Math.Sqrt(keypointCount)));
var height = width;
var textureDimensions = new Vector2Int(width, height);
return textureDimensions;
}
private void OnDepthCheckReadback(int frame, NativeArray<Color32> data, RenderTexture renderTexture)

protected override void OnVisualize()
{
if (m_KeypointEntriesToReport == null) return;
using (k_OnVisualizeMarker.Auto())
{
var jointTexture = activeTemplate.jointTexture;
if (jointTexture == null) jointTexture = m_MissingTexture;
var jointTexture = activeTemplate.jointTexture;
if (jointTexture == null) jointTexture = m_MissingTexture;
var skeletonTexture = activeTemplate.skeletonTexture;
if (skeletonTexture == null) skeletonTexture = m_MissingTexture;
var skeletonTexture = activeTemplate.skeletonTexture;
if (skeletonTexture == null) skeletonTexture = m_MissingTexture;
foreach (var entry in m_KeypointEntriesToReport)
{
foreach (var bone in activeTemplate.skeleton)
foreach (var entry in m_KeypointEntriesToReport)
var joint1 = GetKeypointForJoint(entry, bone.joint1);
var joint2 = GetKeypointForJoint(entry, bone.joint2);
if (joint1 != null && joint1.Value.state == 2 && joint2 != null && joint2.Value.state == 2)
foreach (var bone in activeTemplate.skeleton)
VisualizationHelper.DrawLine(joint1.Value.x, joint1.Value.y, joint2.Value.x, joint2.Value.y, bone.color, 8, skeletonTexture);
var joint1 = GetKeypointForJoint(entry, bone.joint1);
var joint2 = GetKeypointForJoint(entry, bone.joint2);
if (joint1 != null && joint1.Value.state == 2 && joint2 != null && joint2.Value.state == 2)
{
VisualizationHelper.DrawLine(joint1.Value.x, joint1.Value.y, joint2.Value.x, joint2.Value.y, bone.color, 8, skeletonTexture);
}
}
foreach (var keypoint in entry.keypoints)
{
if (keypoint.state == 2)
VisualizationHelper.DrawPoint(keypoint.x, keypoint.y, activeTemplate.keypoints[keypoint.index].color, 8, jointTexture);
foreach (var keypoint in entry.keypoints)
{
if (keypoint.state == 2)
VisualizationHelper.DrawPoint(keypoint.x, keypoint.y, activeTemplate.keypoints[keypoint.index].color, 8, jointTexture);
}
}
}
}

29
com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs
查看文件


jointLabel.selfOcclusionDistanceSource = SelfOcclusionDistanceSource.KeypointLabeler;
}
static void SetupCubeJoints(GameObject cube, KeypointTemplate template)
static void SetupCubeJoints(GameObject cube, KeypointTemplate template, float? selfOcclusionDistance = null)
SetupCubeJoint(cube, "FrontLowerLeft", -dim, -dim, -dim);
SetupCubeJoint(cube, "FrontUpperLeft", -dim, dim, -dim);
SetupCubeJoint(cube, "FrontUpperRight", dim, dim, -dim);
SetupCubeJoint(cube, "FrontLowerRight", dim, -dim, -dim);
SetupCubeJoint(cube, "BackLowerLeft", -dim, -dim, dim);
SetupCubeJoint(cube, "BackUpperLeft", -dim, dim, dim);
SetupCubeJoint(cube, "BackUpperRight", dim, dim, dim);
SetupCubeJoint(cube, "BackLowerRight", dim, -dim, dim);
SetupCubeJoint(cube, "FrontLowerLeft", -dim, -dim, -dim, selfOcclusionDistance);
SetupCubeJoint(cube, "FrontUpperLeft", -dim, dim, -dim, selfOcclusionDistance);
SetupCubeJoint(cube, "FrontUpperRight", dim, dim, -dim, selfOcclusionDistance);
SetupCubeJoint(cube, "FrontLowerRight", dim, -dim, -dim, selfOcclusionDistance);
SetupCubeJoint(cube, "BackLowerLeft", -dim, -dim, dim, selfOcclusionDistance);
SetupCubeJoint(cube, "BackUpperLeft", -dim, dim, dim, selfOcclusionDistance);
SetupCubeJoint(cube, "BackUpperRight", dim, dim, dim, selfOcclusionDistance);
SetupCubeJoint(cube, "BackLowerRight", dim, -dim, dim, selfOcclusionDistance);
}
[UnityTest]

{
incoming.Add(data);
}, texture, defaultSelfOcclusionDistance: labelerSelfOcclusionDistance);
var count = new Vector2Int(100, 100);
var count = new Vector2Int(50, 50);
Rect placementRect = new Rect(-2, -2, 4, 4);
var cube = TestHelper.CreateLabeledCube(scale: 1f / count.x - .01f, x: 1f / count.x * x * 2 - 1, y: 1f / count.y * y * 2 - 1);
SetupCubeJoints(cube, template);
var cube = TestHelper.CreateLabeledCube(
scale: placementRect.width / count.x - .001f,
x: placementRect.width / count.x * x + placementRect.xMin,
y: placementRect.height / count.y * y + placementRect.yMin);
SetupCubeJoints(cube, template, .1f);
cube.SetActive(true);
AddTestObjectForCleanup(cube);
}

撰写 预览
正在加载...
取消
保存