浏览代码

Mostly updated comments

/keypoint_self_occlusion
Steve Borkman 3 年前
当前提交
b089f728
共有 3 个文件被更改,包括 21 次插入27 次删除
  1. 8
      com.unity.perception/Documentation~/GroundTruth/KeypointLabeler.md
  2. 18
      com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs
  3. 22
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

8
com.unity.perception/Documentation~/GroundTruth/KeypointLabeler.md


another object is occluding it (in between the camera and the keypoint), or another part of its own model is occluding it (for
example a human model's arm is raised and blocking its face from view). The keypoint is a point in space generally internal
to the model, for example the elbow joint is in the center of the arm volume. A self occlusion distance value has
been added to each keypoint allowing the keypoint to have depth. If a keypoint is occluded by itself, then this value is added to keypoint location to see
if that location is now closer than the mesh which is occluding the object. If it is then the keypoint will be marked as visible,
if it is not, then the keypoint will be marked as not visible. In the case of the elbow, the distance should be enough that
it is not blocked by the volume of the arm.
been added to each keypoint allowing the keypoint to have depth. If a keypoint is occluded by the mesh that it is residing in,
then this value is added to keypoint location to see if that location is now closer than the mesh which is occluding the object.
If it is then the keypoint will be marked as visible, if it is not, then the keypoint will be marked as not visible. In the
case of the elbow, the distance should be enough that it is not blocked by the volume of the arm.
## Keypoint Template

18
com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs


namespace UnityEngine.Perception.GroundTruth
{
public enum SelfOcclusionDistanceSource
{
JointLabel,
KeypointLabeler
}
/// <summary>
/// Label to designate a custom joint/keypoint. These are needed to add body
/// parts to a humanoid model that are not contained in its <see cref="Animator"/> <see cref="Avatar"/>

[Serializable]
public class JointLabel : MonoBehaviour, ISerializationCallbackReceiver
{
private static PerceptionCamera singlePerceptionCamera = null;
static PerceptionCamera s_SinglePerceptionCamera;
/// <summary>
/// Maps this joint to a joint in a <see cref="KeypointTemplate"/>

/// </summary>
[SerializeField]
[HideInInspector]
private List<TemplateData> templateInformation;
List<TemplateData> templateInformation;
/// <summary>
/// List of all of the templates that this joint can be mapped to.

private void OnDrawGizmosSelected()
{
if (singlePerceptionCamera == null)
if (s_SinglePerceptionCamera == null)
singlePerceptionCamera = FindObjectOfType<PerceptionCamera>();
s_SinglePerceptionCamera = FindObjectOfType<PerceptionCamera>();
}
Mesh sphereMesh = null;

}
else
{
if (singlePerceptionCamera == null)
if (s_SinglePerceptionCamera == null)
var keypointLabeler = (KeypointLabeler) singlePerceptionCamera.labelers.FirstOrDefault(l => l is KeypointLabeler);
var keypointLabeler = (KeypointLabeler) s_SinglePerceptionCamera.labelers.FirstOrDefault(l => l is KeypointLabeler);
var template = keypointLabeler?.activeTemplate;
if (template == null)
occlusionDistance = KeypointDefinition.defaultSelfOcclusionDistance;

22
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs


using System.Collections.Generic;
using System.Linq;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using UnityEngine.Serialization;
namespace UnityEngine.Perception.GroundTruth
{

// Smaller texture sizes produce assertion failures in the engine
const int k_MinTextureWidth = 8;
static ProfilerMarker k_OnEndRenderingMarker = new ProfilerMarker($"KeypointLabeler OnEndRendering");
static ProfilerMarker k_OnVisualizeMarker = new ProfilerMarker($"KeypointLabeler OnVisualize");
static ProfilerMarker s_OnEndRenderingMarker = new ProfilerMarker($"KeypointLabeler OnEndRendering");
static ProfilerMarker s_OnVisualizeMarker = new ProfilerMarker($"KeypointLabeler OnVisualize");
/// <summary>
/// The active keypoint template. Required to annotate keypoint data.

perceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfoReadback;
}
private void SetupDepthCheckBuffers(int size)
void SetupDepthCheckBuffers(int size)
{
var textureDimensions = TextureDimensions(size);
if (m_ResultsBuffer != null &&

{
m_ResultsBuffer.Release();
m_DepthCheckReader.Dispose(false);
// Object.Destroy(m_KeypointPositionsTexture);
// Object.Destroy(m_KeypointCheckDepthTexture);
}
m_KeypointPositionsTexture = new Texture2D(textureDimensions.x, textureDimensions.y, GraphicsFormat.R16G16_SFloat, TextureCreationFlags.None);

// We determine if a point is occluded by other objects is by checking the pixel location of the keypoint
// against the instance segmentation mask for the frame. The instance segmentation mask provides the instance id of the
// visible object at a pixel location. Which means, if the keypoint does not match the visible pixel, then another
// object is in front of the keypoint occluding it from view. An important note here is that the keypoint is an infintely small
// object is in front of the keypoint occluding it from view. An important note here is that the keypoint is an infinitely small
// point in space, which can lead to false negatives due to rounding issues if the keypoint is on the edge of an object or very
// close to the edge of the screen. Because of this we will test not only the keypoint pixel, but also the immediate surrounding
// pixels to determine if the pixel is really visible. This method returns 1 if the pixel is not visible but on screen, and 0

frameKeypointData.objectInfos.Dispose();
}
/// <param name="scriptableRenderContext"></param>
using (k_OnEndRenderingMarker.Auto())
using (s_OnEndRenderingMarker.Auto())
{
m_CurrentFrame = Time.frameCount;

}
}
/// Check self occlusion of each keypoint by passing keypoint location (x & y in one texture) and modified distance from camera (keypoint distance - keypoint threshold distance)
/// in an additional texture. The computer shader checks the depth buffer at each passed in location, converts the depth at the pixel to linear space, and then compares it to
/// the passed in modified keypoint distance. If the modified keypoint distance is less than the depth buffer distance, the keypoint is visible, else it is blocked by itself.
private void DoDepthCheck(ScriptableRenderContext scriptableRenderContext, List<KeypointEntry> keypointEntries, NativeList<float3> checkLocations)
{
var keypointCount = keypointEntries.Count * activeTemplate.keypoints.Length;

//DoDepthCheckReadback(frameCount, data);
}
// Go through each keypoint and check if the depth compute shader has determined if it is visible (depth texture
// value of 1.
private void DoDepthCheckReadback(int frameCount, NativeArray<Color32> data)
{
var frameKeypointData = m_FrameKeypointData[frameCount];

protected override void OnVisualize()
{
if (m_KeypointEntriesToReport == null) return;
using (k_OnVisualizeMarker.Auto())
using (s_OnVisualizeMarker.Auto())
{
var jointTexture = activeTemplate.jointTexture;
if (jointTexture == null) jointTexture = m_MissingTexture;

正在加载...
取消
保存