浏览代码

added LocalPositionLabeler

/local-position-labeler
sleal-unity 4 年前
当前提交
740e0853
共有 8 个文件被更改,包括 481 次插入0 次删除
  1. 260
      com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs
  2. 11
      com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs.meta
  3. 77
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs
  4. 11
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs.meta
  5. 31
      com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs
  6. 11
      com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs.meta
  7. 70
      com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader
  8. 10
      com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader.meta

260
com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs


using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Unity.Collections;
using Unity.Simulation;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;
#if HDRP_PRESENT
using UnityEngine.Rendering.HighDefinition;
#endif
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Labeler which generates a local position image each frame.
/// </summary>
[Serializable]
public sealed class LocalPositionLabeler : CameraLabeler, IOverlayPanelProvider
{
///<inheritdoc/>
public override string description
{
get => "Generates a local position image for each captured frame.";
protected set {}
}
const string k_LocalPositionDirectory = "LocalPosition";
const string k_LocalPositionFilePrefix = "localPosition_";
internal string LocalPositionDirectory;
/// <summary>
/// The id to associate with semantic segmentation annotations in the dataset.
/// </summary>
[Tooltip("The id to associate with local position annotations in the dataset.")]
public string annotationId = "12f94d8d-5425-4deb-9b21-5e53ad957d66";
/// <summary>
/// The IdLabelConfig which maps labels to pixel values.
/// </summary>
public IdLabelConfig labelConfig;
/// <summary>
/// Event information for <see cref="LocalPositionLabeler.imageReadback"/>
/// </summary>
public struct ImageReadbackEventArgs
{
/// <summary>
/// The <see cref="Time.frameCount"/> on which the image was rendered. This may be multiple frames in the past.
/// </summary>
public int frameCount;
/// <summary>
/// Color pixel data.
/// </summary>
public NativeArray<Color32> data;
/// <summary>
/// The source image texture.
/// </summary>
public RenderTexture sourceTexture;
}
/// <summary>
/// Event which is called each frame a local position image is read back from the GPU.
/// </summary>
public event Action<ImageReadbackEventArgs> imageReadback;
/// <summary>
/// The RenderTexture on which local position images are drawn. Will be resized on startup to match
/// the camera resolution.
/// </summary>
public RenderTexture targetTexture => m_TargetTextureOverride;
/// <inheritdoc cref="IOverlayPanelProvider"/>
public Texture overlayImage=> targetTexture;
/// <inheritdoc cref="IOverlayPanelProvider"/>
public string label => "LocalPosition";
[Tooltip("(Optional) The RenderTexture on which local position images will be drawn. Will be reformatted on startup.")]
[SerializeField]
RenderTexture m_TargetTextureOverride;
AnnotationDefinition m_LocalPositionAnnotationDefinition;
RenderTextureReader<Color32> m_LocalPositionTextureReader;
#if HDRP_PRESENT
LocalPositionPass m_LocalPositionPass;
LensDistortionPass m_LensDistortionPass;
#elif URP_PRESENT
LocalPositionUrpPass m_LocalPositionPass;
LensDistortionUrpPass m_LensDistortionPass;
#endif
Dictionary<int, AsyncAnnotation> m_AsyncAnnotations;
/// <summary>
/// Creates a new LocalPositionLabeler. Be sure to assign <see cref="labelConfig"/> before adding to a <see cref="PerceptionCamera"/>.
/// </summary>
public LocalPositionLabeler() { }
/// <summary>
/// Creates a new LocalPositionLabeler with the given <see cref="IdLabelConfig"/>.
/// </summary>
/// <param name="labelConfig">The label config associating labels with colors.</param>
/// <param name="targetTextureOverride">Override the target texture of the labeler. Will be reformatted on startup.</param>
public LocalPositionLabeler(IdLabelConfig labelConfig, RenderTexture targetTextureOverride = null)
{
this.labelConfig = labelConfig;
m_TargetTextureOverride = targetTextureOverride;
}
struct LocalPositionSpec
{
// ReSharper disable once InconsistentNaming
// ReSharper disable once NotAccessedField.Local
public int label_id;
}
struct AsyncLocalPositionWrite
{
public NativeArray<Color32> data;
public int width;
public int height;
public string path;
}
/// <inheritdoc/>
protected override bool supportsVisualization => true;
/// <inheritdoc/>
protected override void Setup()
{
var myCamera = perceptionCamera.GetComponent<Camera>();
var camWidth = myCamera.pixelWidth;
var camHeight = myCamera.pixelHeight;
if (labelConfig == null)
{
throw new InvalidOperationException(
"LocalPositionLabeler's LabelConfig must be assigned");
}
m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
if (targetTexture != null)
{
if (targetTexture.sRGB)
{
Debug.LogError("targetTexture supplied to LocalPositionLabeler must be in Linear mode. Disabling labeler.");
enabled = false;
}
var renderTextureDescriptor = new RenderTextureDescriptor(camWidth, camHeight, GraphicsFormat.R8G8B8A8_UNorm, 8);
targetTexture.descriptor = renderTextureDescriptor;
}
else
m_TargetTextureOverride = new RenderTexture(camWidth, camHeight, 8, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Linear);
targetTexture.Create();
targetTexture.name = "Labeling";
LocalPositionDirectory = k_LocalPositionDirectory + Guid.NewGuid();
#if HDRP_PRESENT
var gameObject = perceptionCamera.gameObject;
var customPassVolume = gameObject.GetComponent<CustomPassVolume>() ?? gameObject.AddComponent<CustomPassVolume>();
customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
customPassVolume.isGlobal = true;
m_LocalPositionPass = new LocalPositionPass(myCamera, targetTexture, labelConfig)
{
name = "Labeling Pass"
};
customPassVolume.customPasses.Add(m_LocalPositionPass);
m_LensDistortionPass = new LensDistortionPass(myCamera, targetTexture)
{
name = "Lens Distortion Pass"
};
customPassVolume.customPasses.Add(m_LensDistortionPass);
#elif URP_PRESENT
// Local Position Pass
m_LocalPositionPass = new LocalPositionUrpPass(myCamera, targetTexture, labelConfig);
perceptionCamera.AddScriptableRenderPass(m_LocalPositionPass);
// Lens Distortion Pass
m_LensDistortionPass = new LensDistortionUrpPass(myCamera, targetTexture);
perceptionCamera.AddScriptableRenderPass(m_LensDistortionPass);
#endif
var specs = labelConfig.labelEntries.Select(l => new LocalPositionSpec { label_id = l.id });
m_LocalPositionAnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition(
"local position",
specs.ToArray(),
"pixel-wise local position label",
"PNG",
Guid.Parse(annotationId));
m_LocalPositionTextureReader = new RenderTextureReader<Color32>(targetTexture);
visualizationEnabled = supportsVisualization;
}
void OnLocalPositionImageRead(int frameCount, NativeArray<Color32> data)
{
if (!m_AsyncAnnotations.TryGetValue(frameCount, out var annotation))
return;
var datasetRelativePath = $"{LocalPositionDirectory}/{k_LocalPositionFilePrefix}{frameCount}.png";
var localPath = $"{Manager.Instance.GetDirectoryFor(LocalPositionDirectory)}/{k_LocalPositionFilePrefix}{frameCount}.png";
annotation.ReportFile(datasetRelativePath);
var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncLocalPositionWrite>>();
imageReadback?.Invoke(new ImageReadbackEventArgs
{
data = data,
frameCount = frameCount,
sourceTexture = targetTexture
});
asyncRequest.data = new AsyncLocalPositionWrite
{
data = new NativeArray<Color32>(data, Allocator.Persistent),
width = targetTexture.width,
height = targetTexture.height,
path = localPath
};
asyncRequest.Enqueue(r =>
{
var pngBytes = ImageConversion.EncodeArrayToPNG(
r.data.data.ToArray(), GraphicsFormat.R8G8B8A8_UNorm, (uint)r.data.width, (uint)r.data.height);
File.WriteAllBytes(r.data.path, pngBytes);
Manager.Instance.ConsumerFileProduced(r.data.path);
r.data.data.Dispose();
return AsyncRequest.Result.Completed;
});
asyncRequest.Execute();
}
/// <inheritdoc/>
protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
{
m_AsyncAnnotations[Time.frameCount] = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_LocalPositionAnnotationDefinition);
m_LocalPositionTextureReader.Capture(scriptableRenderContext,
(frameCount, data, renderTexture) => OnLocalPositionImageRead(frameCount, data));
}
/// <inheritdoc/>
protected override void Cleanup()
{
m_LocalPositionTextureReader?.WaitForAllImages();
m_LocalPositionTextureReader?.Dispose();
m_LocalPositionTextureReader = null;
if (m_TargetTextureOverride != null)
m_TargetTextureOverride.Release();
m_TargetTextureOverride = null;
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs.meta


fileFormatVersion: 2
guid: d6752aa34eb59f84ca1e429ef00f2a6b
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

77
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs


using System;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Custom Pass which renders labeled images where each object labeled with a Labeling component is drawn with the
/// value specified by the given LabelingConfiguration.
/// </summary>
class LocalPositionCrossPipelinePass : GroundTruthCrossPipelinePass
{
const string k_ShaderName = "Perception/LocalPosition";
static readonly int k_Center = Shader.PropertyToID("_Center");
static readonly int k_Size = Shader.PropertyToID("_Size");
static int s_LastFrameExecuted = -1;
IdLabelConfig m_LabelConfig;
// NOTICE: Serialize the shader so that the shader asset is included in player builds when the SemanticSegmentationPass is used.
// Currently commented out and shaders moved to Resources folder due to serialization crashes when it is enabled.
// See https://fogbugz.unity3d.com/f/cases/1187378/
// [SerializeField]
Shader m_LocalPositionShader;
Material m_OverrideMaterial;
public LocalPositionCrossPipelinePass(
Camera targetCamera, IdLabelConfig labelConfig) : base(targetCamera)
{
m_LabelConfig = labelConfig;
}
public override void Setup()
{
base.Setup();
m_LocalPositionShader = Shader.Find(k_ShaderName);
var shaderVariantCollection = new ShaderVariantCollection();
if (shaderVariantCollection != null)
{
shaderVariantCollection.Add(
new ShaderVariantCollection.ShaderVariant(m_LocalPositionShader, PassType.ScriptableRenderPipeline));
}
m_OverrideMaterial = new Material(m_LocalPositionShader);
shaderVariantCollection.WarmUp();
}
protected override void ExecutePass(
ScriptableRenderContext renderContext, CommandBuffer cmd, Camera camera, CullingResults cullingResult)
{
if (s_LastFrameExecuted == Time.frameCount)
return;
s_LastFrameExecuted = Time.frameCount;
var renderList = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, -1);
cmd.ClearRenderTarget(true, true, Color.black);
DrawRendererList(renderContext, cmd, RendererList.Create(renderList));
}
public override void SetupMaterialProperties(
MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
var bounds = renderer.GetComponentInChildren<MeshFilter>().sharedMesh.bounds;
mpb.SetVector(k_Center, bounds.center);
mpb.SetVector(k_Size, bounds.size);
}
public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
// mpb.SetVector(k_Center, Color.black);
}
}
}

11
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs.meta


fileFormatVersion: 2
guid: 03d5f59306ff7384b9c0a18e8d2d4402
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

31
com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs


#if URP_PRESENT
using UnityEngine.Rendering;
using UnityEngine.Rendering.Universal;
namespace UnityEngine.Perception.GroundTruth
{
class LocalPositionUrpPass : ScriptableRenderPass
{
public LocalPositionCrossPipelinePass localPositionCrossPipelinePass;
public LocalPositionUrpPass(Camera camera, RenderTexture targetTexture, IdLabelConfig labelConfig)
{
localPositionCrossPipelinePass = new LocalPositionCrossPipelinePass(camera, labelConfig);
ConfigureTarget(targetTexture, targetTexture.depthBuffer);
localPositionCrossPipelinePass.Setup();
}
public override void Execute(ScriptableRenderContext context, ref RenderingData renderingData)
{
var commandBuffer = CommandBufferPool.Get(nameof(SemanticSegmentationUrpPass));
localPositionCrossPipelinePass.Execute(context, commandBuffer, renderingData.cameraData.camera, renderingData.cullResults);
CommandBufferPool.Release(commandBuffer);
}
public void Cleanup()
{
localPositionCrossPipelinePass.Cleanup();
}
}
}
#endif

11
com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs.meta


fileFormatVersion: 2
guid: 560eab869b74a79438b543fec20f847e
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

70
com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader


Shader "Perception/LocalPosition"
{
Properties
{
_Center ("Center", Vector) = (0, 0, 0, 0)
_Size ("Size", Vector) = (1, 1, 1, 0)
}
HLSLINCLUDE
#pragma target 4.5
#pragma only_renderers d3d11 ps4 xboxone vulkan metal switch
//enable GPU instancing support
#pragma multi_compile_instancing
ENDHLSL
SubShader
{
Tags { "RenderType"="Opaque" }
LOD 100
Pass
{
Tags { "LightMode" = "SRP" }
Blend Off
ZWrite On
ZTest LEqual
Cull Back
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
float4 _Center;
float4 _Size;
struct app_to_vertex
{
float4 vertex : POSITION;
};
struct vertex_to_fragment
{
float4 vertex : SV_POSITION;
float4 vertex_object_space : COLOR;
};
vertex_to_fragment vert (app_to_vertex input)
{
vertex_to_fragment o;
o.vertex = UnityObjectToClipPos(input.vertex);
o.vertex_object_space = (input.vertex - _Center) / _Size + float4(0.5, 0.5, 0.5, 1);
o.vertex_object_space.w = 1;
return o;
}
float4 frag (vertex_to_fragment input) : SV_Target
{
return input.vertex_object_space;
}
ENDCG
}
}
}

10
com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader.meta


fileFormatVersion: 2
guid: ef272c80b160cd247988b3e0723ed71e
ShaderImporter:
externalObjects: {}
defaultTextures: []
nonModifiableTextures: []
preprocessorOverride: 0
userData:
assetBundleName:
assetBundleVariant:
正在加载...
取消
保存