added LocalPositionLabeler

com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Unity.Collections;
+using Unity.Simulation;
+using UnityEngine.Experimental.Rendering;
+using UnityEngine.Rendering;
+
+#if HDRP_PRESENT
+using UnityEngine.Rendering.HighDefinition;
+#endif
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    /// <summary>
+    /// Labeler which generates a local position image each frame.
+    /// </summary>
+    [Serializable]
+    public sealed class LocalPositionLabeler : CameraLabeler, IOverlayPanelProvider
+    {
+        ///<inheritdoc/>
+        public override string description
+        {
+            get => "Generates a local position image for each captured frame.";
+            protected set {}
+        }
+
+        const string k_LocalPositionDirectory = "LocalPosition";
+        const string k_LocalPositionFilePrefix = "localPosition_";
+        internal string LocalPositionDirectory;
+
+        /// <summary>
+        /// The id to associate with semantic segmentation annotations in the dataset.
+        /// </summary>
+        [Tooltip("The id to associate with local position annotations in the dataset.")]
+        public string annotationId = "12f94d8d-5425-4deb-9b21-5e53ad957d66";
+
+        /// <summary>
+        /// The IdLabelConfig which maps labels to pixel values.
+        /// </summary>
+        public IdLabelConfig labelConfig;
+
+        /// <summary>
+        /// Event information for <see cref="LocalPositionLabeler.imageReadback"/>
+        /// </summary>
+        public struct ImageReadbackEventArgs
+        {
+            /// <summary>
+            /// The <see cref="Time.frameCount"/> on which the image was rendered. This may be multiple frames in the past.
+            /// </summary>
+            public int frameCount;
+            /// <summary>
+            /// Color pixel data.
+            /// </summary>
+            public NativeArray<Color32> data;
+            /// <summary>
+            /// The source image texture.
+            /// </summary>
+            public RenderTexture sourceTexture;
+        }
+
+        /// <summary>
+        /// Event which is called each frame a local position image is read back from the GPU.
+        /// </summary>
+        public event Action<ImageReadbackEventArgs> imageReadback;
+
+        /// <summary>
+        /// The RenderTexture on which local position images are drawn. Will be resized on startup to match
+        /// the camera resolution.
+        /// </summary>
+        public RenderTexture targetTexture => m_TargetTextureOverride;
+
+        /// <inheritdoc cref="IOverlayPanelProvider"/>
+        public Texture overlayImage=> targetTexture;
+
+        /// <inheritdoc cref="IOverlayPanelProvider"/>
+        public string label => "LocalPosition";
+
+        [Tooltip("(Optional) The RenderTexture on which local position images will be drawn. Will be reformatted on startup.")]
+        [SerializeField]
+        RenderTexture m_TargetTextureOverride;
+
+        AnnotationDefinition m_LocalPositionAnnotationDefinition;
+        RenderTextureReader<Color32> m_LocalPositionTextureReader;
+
+#if HDRP_PRESENT
+        LocalPositionPass m_LocalPositionPass;
+        LensDistortionPass m_LensDistortionPass;
+    #elif URP_PRESENT
+        LocalPositionUrpPass m_LocalPositionPass;
+        LensDistortionUrpPass m_LensDistortionPass;
+    #endif
+
+        Dictionary<int, AsyncAnnotation> m_AsyncAnnotations;
+
+        /// <summary>
+        /// Creates a new LocalPositionLabeler. Be sure to assign <see cref="labelConfig"/> before adding to a <see cref="PerceptionCamera"/>.
+        /// </summary>
+        public LocalPositionLabeler() { }
+
+        /// <summary>
+        /// Creates a new LocalPositionLabeler with the given <see cref="IdLabelConfig"/>.
+        /// </summary>
+        /// <param name="labelConfig">The label config associating labels with colors.</param>
+        /// <param name="targetTextureOverride">Override the target texture of the labeler. Will be reformatted on startup.</param>
+        public LocalPositionLabeler(IdLabelConfig labelConfig, RenderTexture targetTextureOverride = null)
+        {
+            this.labelConfig = labelConfig;
+            m_TargetTextureOverride = targetTextureOverride;
+        }
+
+        struct LocalPositionSpec
+        {
+            // ReSharper disable once InconsistentNaming
+            // ReSharper disable once NotAccessedField.Local
+            public int label_id;
+        }
+
+        struct AsyncLocalPositionWrite
+        {
+            public NativeArray<Color32> data;
+            public int width;
+            public int height;
+            public string path;
+        }
+
+        /// <inheritdoc/>
+        protected override bool supportsVisualization => true;
+
+        /// <inheritdoc/>
+        protected override void Setup()
+        {
+            var myCamera = perceptionCamera.GetComponent<Camera>();
+            var camWidth = myCamera.pixelWidth;
+            var camHeight = myCamera.pixelHeight;
+
+            if (labelConfig == null)
+            {
+                throw new InvalidOperationException(
+                    "LocalPositionLabeler's LabelConfig must be assigned");
+            }
+
+            m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
+
+            if (targetTexture != null)
+            {
+                if (targetTexture.sRGB)
+                {
+                    Debug.LogError("targetTexture supplied to LocalPositionLabeler must be in Linear mode. Disabling labeler.");
+                    enabled = false;
+                }
+                var renderTextureDescriptor = new RenderTextureDescriptor(camWidth, camHeight, GraphicsFormat.R8G8B8A8_UNorm, 8);
+                targetTexture.descriptor = renderTextureDescriptor;
+            }
+            else
+                m_TargetTextureOverride = new RenderTexture(camWidth, camHeight, 8, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Linear);
+
+            targetTexture.Create();
+            targetTexture.name = "Labeling";
+            LocalPositionDirectory = k_LocalPositionDirectory + Guid.NewGuid();
+
+#if HDRP_PRESENT
+            var gameObject = perceptionCamera.gameObject;
+            var customPassVolume = gameObject.GetComponent<CustomPassVolume>() ?? gameObject.AddComponent<CustomPassVolume>();
+            customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
+            customPassVolume.isGlobal = true;
+            m_LocalPositionPass = new LocalPositionPass(myCamera, targetTexture, labelConfig)
+            {
+                name = "Labeling Pass"
+            };
+            customPassVolume.customPasses.Add(m_LocalPositionPass);
+
+            m_LensDistortionPass = new LensDistortionPass(myCamera, targetTexture)
+            {
+                name = "Lens Distortion Pass"
+            };
+            customPassVolume.customPasses.Add(m_LensDistortionPass);
+#elif URP_PRESENT
+            // Local Position Pass
+            m_LocalPositionPass = new LocalPositionUrpPass(myCamera, targetTexture, labelConfig);
+            perceptionCamera.AddScriptableRenderPass(m_LocalPositionPass);
+
+            // Lens Distortion Pass
+            m_LensDistortionPass = new LensDistortionUrpPass(myCamera, targetTexture);
+            perceptionCamera.AddScriptableRenderPass(m_LensDistortionPass);
+#endif
+
+            var specs = labelConfig.labelEntries.Select(l => new LocalPositionSpec { label_id = l.id });
+
+            m_LocalPositionAnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition(
+                "local position",
+                specs.ToArray(),
+                "pixel-wise local position label",
+                "PNG",
+                Guid.Parse(annotationId));
+
+            m_LocalPositionTextureReader = new RenderTextureReader<Color32>(targetTexture);
+            visualizationEnabled = supportsVisualization;
+        }
+
+        void OnLocalPositionImageRead(int frameCount, NativeArray<Color32> data)
+        {
+            if (!m_AsyncAnnotations.TryGetValue(frameCount, out var annotation))
+                return;
+
+            var datasetRelativePath = $"{LocalPositionDirectory}/{k_LocalPositionFilePrefix}{frameCount}.png";
+            var localPath = $"{Manager.Instance.GetDirectoryFor(LocalPositionDirectory)}/{k_LocalPositionFilePrefix}{frameCount}.png";
+
+            annotation.ReportFile(datasetRelativePath);
+
+            var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncLocalPositionWrite>>();
+
+            imageReadback?.Invoke(new ImageReadbackEventArgs
+            {
+                data = data,
+                frameCount = frameCount,
+                sourceTexture = targetTexture
+            });
+            asyncRequest.data = new AsyncLocalPositionWrite
+            {
+                data = new NativeArray<Color32>(data, Allocator.Persistent),
+                width = targetTexture.width,
+                height = targetTexture.height,
+                path = localPath
+            };
+            asyncRequest.Enqueue(r =>
+            {
+                var pngBytes = ImageConversion.EncodeArrayToPNG(
+                    r.data.data.ToArray(), GraphicsFormat.R8G8B8A8_UNorm, (uint)r.data.width, (uint)r.data.height);
+                File.WriteAllBytes(r.data.path, pngBytes);
+                Manager.Instance.ConsumerFileProduced(r.data.path);
+                r.data.data.Dispose();
+                return AsyncRequest.Result.Completed;
+            });
+            asyncRequest.Execute();
+        }
+
+        /// <inheritdoc/>
+        protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
+        {
+            m_AsyncAnnotations[Time.frameCount] = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_LocalPositionAnnotationDefinition);
+            m_LocalPositionTextureReader.Capture(scriptableRenderContext,
+                (frameCount, data, renderTexture) => OnLocalPositionImageRead(frameCount, data));
+        }
+
+        /// <inheritdoc/>
+        protected override void Cleanup()
+        {
+            m_LocalPositionTextureReader?.WaitForAllImages();
+            m_LocalPositionTextureReader?.Dispose();
+            m_LocalPositionTextureReader = null;
+
+            if (m_TargetTextureOverride != null)
+                m_TargetTextureOverride.Release();
+
+            m_TargetTextureOverride = null;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/LocalPositionLabeler.cs.meta

+fileFormatVersion: 2
+guid: d6752aa34eb59f84ca1e429ef00f2a6b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs

+using System;
+using UnityEngine.Experimental.Rendering;
+using UnityEngine.Rendering;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    /// <summary>
+    /// Custom Pass which renders labeled images where each object labeled with a Labeling component is drawn with the
+    /// value specified by the given LabelingConfiguration.
+    /// </summary>
+    class LocalPositionCrossPipelinePass : GroundTruthCrossPipelinePass
+    {
+        const string k_ShaderName = "Perception/LocalPosition";
+        static readonly int k_Center = Shader.PropertyToID("_Center");
+        static readonly int k_Size = Shader.PropertyToID("_Size");
+
+        static int s_LastFrameExecuted = -1;
+
+        IdLabelConfig m_LabelConfig;
+
+        // NOTICE: Serialize the shader so that the shader asset is included in player builds when the SemanticSegmentationPass is used.
+        // Currently commented out and shaders moved to Resources folder due to serialization crashes when it is enabled.
+        // See https://fogbugz.unity3d.com/f/cases/1187378/
+        // [SerializeField]
+        Shader m_LocalPositionShader;
+        Material m_OverrideMaterial;
+
+        public LocalPositionCrossPipelinePass(
+            Camera targetCamera, IdLabelConfig labelConfig) : base(targetCamera)
+        {
+            m_LabelConfig = labelConfig;
+        }
+
+        public override void Setup()
+        {
+            base.Setup();
+            m_LocalPositionShader = Shader.Find(k_ShaderName);
+
+            var shaderVariantCollection = new ShaderVariantCollection();
+
+            if (shaderVariantCollection != null)
+            {
+                shaderVariantCollection.Add(
+                    new ShaderVariantCollection.ShaderVariant(m_LocalPositionShader, PassType.ScriptableRenderPipeline));
+            }
+
+            m_OverrideMaterial = new Material(m_LocalPositionShader);
+
+            shaderVariantCollection.WarmUp();
+        }
+
+        protected override void ExecutePass(
+            ScriptableRenderContext renderContext, CommandBuffer cmd, Camera camera, CullingResults cullingResult)
+        {
+            if (s_LastFrameExecuted == Time.frameCount)
+                return;
+
+            s_LastFrameExecuted = Time.frameCount;
+            var renderList = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, -1);
+            cmd.ClearRenderTarget(true, true, Color.black);
+            DrawRendererList(renderContext, cmd, RendererList.Create(renderList));
+        }
+
+        public override void SetupMaterialProperties(
+            MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            var bounds = renderer.GetComponentInChildren<MeshFilter>().sharedMesh.bounds;
+            mpb.SetVector(k_Center, bounds.center);
+            mpb.SetVector(k_Size, bounds.size);
+        }
+
+        public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
+        {
+            // mpb.SetVector(k_Center, Color.black);
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LocalPositionCrossPipelinePass.cs.meta

+fileFormatVersion: 2
+guid: 03d5f59306ff7384b9c0a18e8d2d4402
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs

+#if URP_PRESENT
+using UnityEngine.Rendering;
+using UnityEngine.Rendering.Universal;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    class LocalPositionUrpPass : ScriptableRenderPass
+    {
+        public LocalPositionCrossPipelinePass localPositionCrossPipelinePass;
+
+        public LocalPositionUrpPass(Camera camera, RenderTexture targetTexture, IdLabelConfig labelConfig)
+        {
+            localPositionCrossPipelinePass = new LocalPositionCrossPipelinePass(camera, labelConfig);
+            ConfigureTarget(targetTexture, targetTexture.depthBuffer);
+            localPositionCrossPipelinePass.Setup();
+        }
+
+        public override void Execute(ScriptableRenderContext context, ref RenderingData renderingData)
+        {
+            var commandBuffer = CommandBufferPool.Get(nameof(SemanticSegmentationUrpPass));
+            localPositionCrossPipelinePass.Execute(context, commandBuffer, renderingData.cameraData.camera, renderingData.cullResults);
+            CommandBufferPool.Release(commandBuffer);
+        }
+
+        public void Cleanup()
+        {
+            localPositionCrossPipelinePass.Cleanup();
+        }
+    }
+}
+#endif

com.unity.perception/Runtime/GroundTruth/RenderPasses/UrpPasses/LocalPositionUrpPass.cs.meta

+fileFormatVersion: 2
+guid: 560eab869b74a79438b543fec20f847e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader

+Shader "Perception/LocalPosition"
+{
+    Properties
+    {
+        _Center ("Center", Vector) = (0, 0, 0, 0)
+        _Size ("Size", Vector) = (1, 1, 1, 0)
+    }
+
+    HLSLINCLUDE
+
+    #pragma target 4.5
+    #pragma only_renderers d3d11 ps4 xboxone vulkan metal switch
+
+    //enable GPU instancing support
+    #pragma multi_compile_instancing
+
+    ENDHLSL
+
+    SubShader
+    {
+        Tags { "RenderType"="Opaque" }
+        LOD 100
+
+        Pass
+        {
+            Tags { "LightMode" = "SRP" }
+
+            Blend Off
+            ZWrite On
+            ZTest LEqual
+
+            Cull Back
+
+            CGPROGRAM
+            #pragma vertex vert
+            #pragma fragment frag
+
+            #include "UnityCG.cginc"
+
+            float4 _Center;
+            float4 _Size;
+            
+            struct app_to_vertex
+            {
+                float4 vertex : POSITION;
+            };
+
+            struct vertex_to_fragment
+            {
+                float4 vertex : SV_POSITION;
+                float4 vertex_object_space : COLOR;
+            };
+
+            vertex_to_fragment vert (app_to_vertex input)
+            {
+                vertex_to_fragment o;
+                o.vertex = UnityObjectToClipPos(input.vertex);
+                o.vertex_object_space = (input.vertex - _Center) / _Size + float4(0.5, 0.5, 0.5, 1);
+                o.vertex_object_space.w = 1;
+                return o;
+            }
+
+            float4 frag (vertex_to_fragment input) : SV_Target
+            {
+                return input.vertex_object_space;
+            }
+            ENDCG
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Resources/LocalPosition.shader.meta

+fileFormatVersion: 2
+guid: ef272c80b160cd247988b3e0723ed71e
+ShaderImporter:
+  externalObjects: {}
+  defaultTextures: []
+  nonModifiableTextures: []
+  preprocessorOverride: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: