Partially implemented refactor to address labeling bug.

com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs

 using System;
 using Unity.Simulation;
 using UnityEngine.EventSystems;
+using UnityEngine.Rendering;
 using UnityEngine.UI;
 using UnityEngine.UIElements;
 
         /// Called just before the camera renders each frame the the labeler is enabled and <see cref="SensorHandle.ShouldCaptureThisFrame"/> is true.
         /// </summary>
         protected virtual void OnBeginRendering() {}
+
-        protected virtual void OnEndRendering() {}
+        /// <param name="scriptableRenderContext"></param>
+        protected virtual void OnEndRendering(ScriptableRenderContext scriptableRenderContext) {}
         /// <summary>
         /// Labeling pass to display labeler's visualization components, if applicable. Important note, all labeler's visualizations need
         /// to use Unity's Immediate Mode GUI (IMGUI) <see cref="https://docs.unity3d.com/Manual/GUIScriptingGuide.html"/> system.
         }
         internal void InternalOnUpdate() => OnUpdate();
         internal void InternalOnBeginRendering() => OnBeginRendering();
-        internal void InternalOnEndRendering() => OnEndRendering();
+        internal void InternalOnEndRendering(ScriptableRenderContext context) => OnEndRendering(context);
         internal void InternalCleanup() => Cleanup();
         internal void InternalVisualize() => OnVisualize();
 

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

 using System.Collections.Generic;
 using System.Linq;
 using Unity.Collections;
+using UnityEngine.Rendering;
 
 namespace UnityEngine.Perception.GroundTruth
 {
             asyncAnnotation.annotation.ReportValues(m_ToReport);
         }
 
+        /// <param name="scriptableRenderContext"></param>
-        protected override void OnEndRendering()
+        protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
         {
             m_CurrentFrame = Time.frameCount;
 

com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs

                 "PNG",
                 id: Guid.Parse(annotationId));
 
-            m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture, myCamera,
-                (frameCount, data, tex) => OnSemanticSegmentationImageRead(frameCount, data));
-
+            m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture);
             visualizationEnabled = supportsVisualization;
         }
 
             asyncRequest.Execute();
         }
 
-        /// <inheritdoc/>
-        protected override void OnBeginRendering()
+        protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
+            m_SemanticSegmentationTextureReader.Capture(scriptableRenderContext,
+                (frameCount, data, renderTexture) => OnSemanticSegmentationImageRead(frameCount, data));
+
         }
 
         /// <inheritdoc/>

com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs

 using System.Linq;
 using JetBrains.Annotations;
 using Newtonsoft.Json;
+using Unity.Collections;
 
 namespace UnityEngine.Perception.GroundTruth {
     /// <summary>
                 label_id = l.id,
                 label_name = l.label,
             }).ToArray();
+        }
+
+        public static IdLabelMap GetIdLabelCache(Allocator allocator = Allocator.Temp)
+        {
+            return new IdLabelMap();
+        }
+    }
+
+    public struct IdLabelMap : IDisposable
+    {
+        public void Dispose()
+        {
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

             }
         }
 
-        void OnBeginCameraRendering(ScriptableRenderContext _, Camera cam)
+        void OnBeginCameraRendering(ScriptableRenderContext scriptableRenderContext, Camera cam)
         {
             if (!ShouldCallLabelers(cam, m_LastFrameCaptured))
                 return;
             CallOnLabelers(l => l.InternalOnBeginRendering());
         }
 
-        void OnEndCameraRendering(ScriptableRenderContext _, Camera cam)
+        void OnEndCameraRendering(ScriptableRenderContext scriptableRenderContext, Camera cam)
-            CallOnLabelers(l => l.InternalOnEndRendering());
+            CaptureInstanceSegmentation(scriptableRenderContext);
+            CallOnLabelers(l => l.InternalOnEndRendering(scriptableRenderContext));
         }
 
         void CallOnLabelers(Action<CameraLabeler> action)

com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs

 using System;
 using Unity.Collections;
+using Unity.Simulation;
+using UnityEngine.Rendering;
 #if HDRP_PRESENT
 using UnityEngine.Rendering.HighDefinition;
 #elif URP_PRESENT
         /// Invoked when instance segmentation images are read back from the graphics system. The first parameter is the
         /// Time.frameCount at which the objects were rendered. May be invoked many frames after the objects were rendered.
         /// </summary>
-        public event Action<int, NativeArray<Color32>, RenderTexture> InstanceSegmentationImageReadback;
+        public event Action<int, NativeArray<Color32>, RenderTexture/*, List<IdLabelEntry>*/> InstanceSegmentationImageReadback;
-        public event Action<int, NativeArray<RenderedObjectInfo>> RenderedObjectInfosCalculated;
+        public event Action<int, NativeArray<RenderedObjectInfo>/*, List<IdLabelEntry>*/> RenderedObjectInfosCalculated;
 
         RenderedObjectInfoGenerator m_RenderedObjectInfoGenerator;
         RenderTexture m_InstanceSegmentationTexture;
                 m_LensDistortionIntensityOverride;
 #endif
 
-            m_InstanceSegmentationReader = new RenderTextureReader<Color32>(m_InstanceSegmentationTexture, myCamera, (frameCount, data, tex) =>
+            m_InstanceSegmentationReader = new RenderTextureReader<Color32>(m_InstanceSegmentationTexture);
+
+        }
+
+        void CaptureInstanceSegmentation(ScriptableRenderContext scriptableRenderContext)
+        {
+            //var myCache = new List<IdLabelEntry>();
+            var cache = IdLabelConfig.GetIdLabelCache();
+            var width = m_InstanceSegmentationTexture.width;
+
+            m_InstanceSegmentationReader.Capture(scriptableRenderContext,  (frameCount, data, renderTexture) =>
-                InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
-                if (RenderedObjectInfosCalculated != null)
+                InstanceSegmentationImageReadback?.Invoke(frameCount, data, m_InstanceSegmentationTexture/*, myCache*/);
+
+                if(RenderedObjectInfosCalculated != null)
-                    m_RenderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var renderedObjectInfos, Allocator.Temp);
-                    RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
+                    m_RenderedObjectInfoGenerator.Compute(data, width,
+                        BoundingBoxOrigin.TopLeft, cache, out var renderedObjectInfos, Allocator.Temp);
+                    RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos/*, myCache*/);
+
+                cache.Dispose();
+            m_InstanceSegmentationReader?.WaitForAllImages();
+            m_InstanceSegmentationReader?.Dispose();
+            m_InstanceSegmentationReader = null;
+
-
-            m_InstanceSegmentationReader?.WaitForAllImages();
-            m_InstanceSegmentationReader?.Dispose();
-            m_InstanceSegmentationReader = null;
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/RenderTextureReader.cs

     /// RenderTextureReader reads a RenderTexture from the GPU each frame and passes the data back through a provided callback.
     /// </summary>
     /// <typeparam name="T">The type of the raw texture data to be provided.</typeparam>
-    public class RenderTextureReader<T> : IDisposable where T : struct
+    class RenderTextureReader<T> : IDisposable where T : struct
-        Action<int, NativeArray<T>, RenderTexture> m_ImageReadCallback;
-
-        int m_NextFrameToCapture;
-
-        Camera m_CameraRenderingToSource;
-        /// <param name="cameraRenderingToSource">The <see cref="Camera"/> which renders to the given renderTexture. This is used to determine when to read from the texture.</param>
-        /// <param name="imageReadCallback">The callback to call after reading the texture</param>
-        public RenderTextureReader(RenderTexture source, Camera cameraRenderingToSource, Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
+        public RenderTextureReader(RenderTexture source)
-            m_ImageReadCallback = imageReadCallback;
-            m_CameraRenderingToSource = cameraRenderingToSource;
-            m_NextFrameToCapture = Time.frameCount;
-
-            RenderPipelineManager.endFrameRendering += OnEndFrameRendering;
-        void OnEndFrameRendering(ScriptableRenderContext context, Camera[] cameras)
+        public void Capture(ScriptableRenderContext context, Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
-#if UNITY_EDITOR
-            if (UnityEditor.EditorApplication.isPaused)
-                return;
-#endif
-            if (!cameras.Contains(m_CameraRenderingToSource))
-                return;
-
-            if (m_NextFrameToCapture > Time.frameCount)
-                return;
-
-            m_NextFrameToCapture = Time.frameCount + 1;
+// #if UNITY_EDITOR
+//             if (UnityEditor.EditorApplication.isPaused)
+//                 return;
+// #endif
+//             if (!cameras.Contains(m_CameraRenderingToSource))
+//                 return;
+//
+//             if (m_NextFrameToCapture > Time.frameCount)
+//                 return;
+//
+//             m_NextFrameToCapture = Time.frameCount + 1;
 
             if (!GraphicsUtilities.SupportsAsyncReadback())
             {
                     0, 0);
                 RenderTexture.active = null;
                 var data = m_CpuTexture.GetRawTextureData<T>();
-                m_ImageReadCallback(Time.frameCount, data, m_Source);
+                imageReadCallback(Time.frameCount, data, m_Source);
-                commandBuffer.RequestAsyncReadback(m_Source, r => OnGpuReadback(r, frameCount));
+                commandBuffer.RequestAsyncReadback(m_Source, r => OnGpuReadback(r, frameCount, imageReadCallback));
                 context.ExecuteCommandBuffer(commandBuffer);
                 context.Submit();
                 CommandBufferPool.Release(commandBuffer);
-        void OnGpuReadback(AsyncGPUReadbackRequest request, int frameCount)
+        void OnGpuReadback(AsyncGPUReadbackRequest request, int frameCount,
+            Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
-            else if (request.done && m_ImageReadCallback != null)
+            else if (request.done && imageReadCallback != null)
-                m_ImageReadCallback(frameCount, request.GetData<T>(), m_Source);
+                imageReadCallback(frameCount, request.GetData<T>(), m_Source);
             }
         }
 
         public void Dispose()
         {
             WaitForAllImages();
-
-            RenderPipelineManager.endFrameRendering -= OnEndFrameRendering;
             if (m_CpuTexture != null)
             {
                 Object.Destroy(m_CpuTexture);

com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs

         /// <param name="boundingBoxOrigin">Whether bounding boxes should be top-left or bottom-right-based.</param>
         /// <param name="renderedObjectInfos">When this method returns, filled with RenderedObjectInfo entries for each object visible in the frame.</param>
         /// <param name="allocator">The allocator to use for allocating renderedObjectInfos and perLabelEntryObjectCount.</param>
-        public void Compute(NativeArray<Color32> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
+        public void Compute(NativeArray<Color32> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, IdLabelMap idLabelMap, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
         {
             const int jobCount = 24;
             var height = instanceSegmentationRawData.Length / stride;

com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs

 
             var dataNativeArray = new NativeArray<Color32>(producesCorrectObjectInfoData.data, Allocator.Persistent);
 
-            renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, out var boundingBoxes, Allocator.Temp);
+            var idLabelMap = IdLabelConfig.GetIdLabelCache();
+            renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, idLabelMap, out var boundingBoxes, Allocator.Temp);
+            idLabelMap.Dispose();
         }
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

 
         void Awake()
         {
-            m_Reader = new RenderTextureReader<Color32>(source, cameraSource, ImageReadCallback);
+            m_Reader = new RenderTextureReader<Color32>(source);
+            RenderPipelineManager.endCameraRendering += (context, camera) =>
+                m_Reader.Capture(context,
+                    (frameCount, data, renderTexture) => ImageReadCallback(frameCount, data, renderTexture));
         }
 
         void ImageReadCallback(int frameCount, NativeArray<Color32> data, RenderTexture renderTexture)
                 if (frames < 10)
                     return;
 
+                var idLabelMap = IdLabelConfig.GetIdLabelCache();
+
                 // Calculate the bounding box
                 if (fLensDistortionEnabled == false)
                 {
-                    renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp);
+                    renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, idLabelMap, out var boundingBoxes, Allocator.Temp);
 
                     boundingBoxWithoutLensDistortion = boundingBoxes[0].boundingBox;
 
                 {
                     var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
 
-                    renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp);
+                    renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, idLabelMap, out var boundingBoxes, Allocator.Temp);
 
                     boundingBoxWithLensDistortion = boundingBoxes[0].boundingBox;
 
                     fDone = true;
                 }
+                idLabelMap.Dispose();
             }
 
             cameraObject = SetupCamera(out perceptionCamera, false);