Moving instance segmentation to InstanceSegmentationLabeler. Updating tests.

TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity

   - component: {fileID: 963194229}
   - component: {fileID: 963194227}
   - component: {fileID: 963194231}
+  - component: {fileID: 963194232}
+  - component: {fileID: 963194233}
   m_Layer: 0
   m_Name: Main Camera
   m_TagString: MainCamera
   m_EditorClassIdentifier: 
   light: {fileID: 705507993}
   target: {fileID: 1640252278}
+--- !u!114 &963194232
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 963194225}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d0127208de324c00a881af84cdd63da3, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  annotationId: F9F22E05-443F-4602-A422-EBE4EA9B55CB
+  labelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
+    type: 2}
+--- !u!114 &963194233
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 963194225}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 43085b9d882c415b90fb9ebe4d765f26, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
 --- !u!1 &1640252278
 GameObject:
   m_ObjectHideFlags: 0

com.unity.perception/Runtime/GroundTruth/GroundTruthRendererFeature.cs

 #if URP_PRESENT
 using System;
+using System.Collections.Generic;
 using UnityEditor;
 using UnityEngine;
 using UnityEngine.Rendering;
             if (!EditorApplication.isPlaying)
                 return;
 #endif
-
-            renderer.EnqueuePass(perceptionCamera.instanceSegmentationUrpPass);
-            renderer.EnqueuePass(perceptionCamera.semanticSegmentationUrpPass);
+            foreach (var pass in perceptionCamera.passes)
+                renderer.EnqueuePass(pass);
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs

 using Unity.Collections;
 using Unity.Profiling;
 
-namespace UnityEngine.Perception.GroundTruth {
+namespace UnityEngine.Perception.GroundTruth
+{
+    [RequireComponent(typeof(InstanceSegmentationLabeler))]
     public class BoundingBoxLabeler : MonoBehaviour
     {
         public string annotationId = "F9F22E05-443F-4602-A422-EBE4EA9B55CB";

com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs

 using System;
+using Unity.Collections;
+using UnityEngine.Experimental.Rendering;
+    [RequireComponent(typeof(Camera))]
+    [RequireComponent(typeof(PerceptionCamera))]
-        public bool saveImages = false;
-        public string annotationId = "E657461D-B950-42E1-8141-BEC9B4810241";
+        //Uncomment when we support saving instance segmentation labels
+        //public bool saveImages = false;
+        //public string annotationId = "E657461D-B950-42E1-8141-BEC9B4810241";
+
+        RenderTexture m_SegmentationTexture;
+        RenderTextureReader<uint> m_SegmentationReader;
+
+#if URP_PRESENT
+        [NonSerialized]
+        InstanceSegmentationUrpPass m_InstanceSegmentationUrpPass;
+#endif
+
+        public RenderTexture InstanceSegmentationRenderTexture => m_SegmentationTexture;
+
+        public event Action<int, NativeArray<uint>, RenderTexture> InstanceSegmentationImageReadback;
+
+        public void Start()
+        {
+            var myCamera = GetComponent<Camera>();
+            var perceptionCamera = GetComponent<PerceptionCamera>();
+            var width = myCamera.pixelWidth;
+            var height = myCamera.pixelHeight;
+            m_SegmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
+            m_SegmentationTexture.name = "Segmentation";
+
+#if HDRP_PRESENT
+            var customPassVolume = this.GetComponent<CustomPassVolume>() ?? gameObject.AddComponent<CustomPassVolume>();
+            customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
+            customPassVolume.isGlobal = true;
+            m_SegmentationPass = new InstanceSegmentationPass()
+            {
+                name = "Segmentation Pass",
+                targetCamera = myCamera,
+                targetTexture = segmentationTexture
+            };
+            m_SegmentationPass.EnsureInit();
+            customPassVolume.customPasses.Add(m_SegmentationPass);
+#endif
+#if URP_PRESENT
+            perceptionCamera.AddScriptableRenderPass(new InstanceSegmentationUrpPass(myCamera, m_SegmentationTexture));
+#endif
+
+            m_SegmentationReader = new RenderTextureReader<uint>(m_SegmentationTexture, myCamera, (frameCount, data, tex) =>
+            {
+                InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
+            });
+        }
+
+        public void OnDisable()
+        {
+            if (m_SegmentationTexture != null)
+                m_SegmentationTexture.Release();
+
+            m_SegmentationTexture = null;
+
+            m_SegmentationReader?.WaitForAllImages();
+            m_SegmentationReader?.Dispose();
+            m_SegmentationReader = null;
+        }
     }
 }

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

 using UnityEngine.Experimental.Rendering;
 using UnityEngine.Profiling;
 using UnityEngine.Rendering;
+using UnityEngine.Rendering.Universal;
 using UnityEngine.Serialization;
 #if HDRP_PRESENT
 using UnityEngine.Rendering.HighDefinition;
 
         [NonSerialized]
         internal RenderTexture labelingTexture;
-        [NonSerialized]
-        internal RenderTexture segmentationTexture;
-        RenderTextureReader<uint> m_SegmentationReader;
-        [NonSerialized]
-        internal InstanceSegmentationUrpPass instanceSegmentationUrpPass;
-        [NonSerialized]
-        internal SemanticSegmentationUrpPass semanticSegmentationUrpPass;
+        internal List<ScriptableRenderPass> passes = new List<ScriptableRenderPass>();
+        public void AddScriptableRenderPass(ScriptableRenderPass pass)
+        {
+            passes.Add(pass);
+        }
 #endif
 
         bool m_CapturedLastFrame;
         RenderedObjectInfoValue[] m_VisiblePixelsValues;
 
 #if HDRP_PRESENT
-        InstanceSegmentationPass m_SegmentationPass;
         SemanticSegmentationPass m_SemanticSegmentationPass;
 #endif
         MetricDefinition m_ObjectCountMetricDefinition;
                 produceBoundingBoxAnnotations = false;
             }
 
-            segmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
-            segmentationTexture.name = "Segmentation";
             labelingTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
             labelingTexture.name = "Labeling";
 
             customPassVolume.isGlobal = true;
-            m_SegmentationPass = new InstanceSegmentationPass()
-            {
-                name = "Segmentation Pass",
-                targetCamera = myCamera,
-                targetTexture = segmentationTexture
-            };
-            m_SegmentationPass.EnsureInit();
             m_SemanticSegmentationPass = new SemanticSegmentationPass(myCamera, labelingTexture, LabelingConfiguration)
             {
                 name = "Labeling Pass"
 #endif
 #if URP_PRESENT
-            instanceSegmentationUrpPass = new InstanceSegmentationUrpPass(myCamera, segmentationTexture);
-            semanticSegmentationUrpPass = new SemanticSegmentationUrpPass(myCamera, labelingTexture, LabelingConfiguration);
+            AddScriptableRenderPass(new SemanticSegmentationUrpPass(myCamera, labelingTexture, LabelingConfiguration));
 #endif
 
             if (produceSegmentationImages)
                 m_RenderedObjectInfoGenerator = new RenderedObjectInfoGenerator(LabelingConfiguration);
                 World.DefaultGameObjectInjectionWorld.GetExistingSystem<GroundTruthLabelSetupSystem>().Activate(m_RenderedObjectInfoGenerator);
 
-                m_SegmentationReader = new RenderTextureReader<uint>(segmentationTexture, myCamera, (frameCount, data, tex) =>
+                var instanceSegmentationLabeler = GetComponent<InstanceSegmentationLabeler>();
+                if (instanceSegmentationLabeler != null)
-                    if (segmentationImageReceived != null)
-                        segmentationImageReceived(frameCount, data);
+                    instanceSegmentationLabeler.InstanceSegmentationImageReadback += (frameCount, data, tex) =>
+                    {
+                        segmentationImageReceived?.Invoke(frameCount, data);
-                    m_RenderedObjectInfoGenerator.Compute(data, tex.width, boundingBoxOrigin, out var renderedObjectInfos, out var classCounts, Allocator.Temp);
+                        m_RenderedObjectInfoGenerator.Compute(data, tex.width, boundingBoxOrigin, out var renderedObjectInfos, out var classCounts, Allocator.Temp);
-                    using (s_RenderedObjectInfosCalculatedEvent.Auto())
-                        renderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
+                        using (s_RenderedObjectInfosCalculatedEvent.Auto())
+                            renderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
-                    if (produceObjectCountAnnotations)
-                        OnObjectCountsReceived(classCounts, LabelingConfiguration.LabelEntries, frameCount);
+                        if (produceObjectCountAnnotations)
+                            OnObjectCountsReceived(classCounts, LabelingConfiguration.LabelEntries, frameCount);
-                    if (produceRenderedObjectInfoMetric)
-                        ProduceRenderedObjectInfoMetric(renderedObjectInfos, frameCount);
-                });
+                        if (produceRenderedObjectInfoMetric)
+                            ProduceRenderedObjectInfoMetric(renderedObjectInfos, frameCount);
+                    };
+                }
+                else
+                {
+                    Debug.Log("Missing InstanceSegmentationLabeler. Will not generate metrics.");
+                }
             }
 
             RenderPipelineManager.beginCameraRendering += OnBeginCameraRendering;
 #if HDRP_PRESENT
         void SetupPasses(CustomPassVolume customPassVolume)
         {
-            customPassVolume.customPasses.Remove(m_SegmentationPass);
-
-            if (produceSegmentationImages || produceObjectCountAnnotations)
-                customPassVolume.customPasses.Add(m_SegmentationPass);
 
             if (produceSegmentationImages)
                 customPassVolume.customPasses.Add(m_SemanticSegmentationPass);
-
-        void ProduceBoundingBoxesAnnotation(NativeArray<RenderedObjectInfo> renderedObjectInfos, List<LabelEntry> labelingConfigurations, int frameCount)
-        {
-        }
 
         /// <summary>
         /// Returns the class ID for the given instance ID resolved by <see cref="LabelingConfiguration"/>. Only valid when bounding boxes are being computed.
             m_ClassLabelingTextureReader?.Dispose();
             m_ClassLabelingTextureReader = null;
 
-            m_SegmentationReader?.WaitForAllImages();
-            m_SegmentationReader?.Dispose();
-            m_SegmentationReader = null;
-
             RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering;
         }
 
 
             m_ClassLabelingTextureReader?.Dispose();
             m_ClassLabelingTextureReader = null;
-            if (segmentationTexture != null)
-                segmentationTexture.Release();
-            segmentationTexture = null;
             if (labelingTexture != null)
                 labelingTexture.Release();
 

com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountTests.cs

             camera.orthographic = true;
             camera.orthographicSize = 1;
 
-#if HDRP_PRESENT
-            cameraObject.AddComponent<HDAdditionalCameraData>();
-            var customPassVolume = cameraObject.AddComponent<CustomPassVolume>();
-            customPassVolume.isGlobal = true;
-            var rt = new RenderTexture(128, 128, 1, GraphicsFormat.R8G8B8A8_UNorm);
-            rt.Create();
-            var instanceSegmentationPass = new InstanceSegmentationPass()
-            {
-                targetCamera = camera,
-                targetTexture = rt
-            };
-            instanceSegmentationPass.name = nameof(instanceSegmentationPass);
-            instanceSegmentationPass.EnsureInit();
-            customPassVolume.customPasses.Add(instanceSegmentationPass);
-            var objectCountPass = new ObjectCountPass(camera);
-            objectCountPass.SegmentationTexture = rt;
-            objectCountPass.LabelingConfiguration = labelingConfiguration;
-            objectCountPass.name = nameof(objectCountPass);
-            customPassVolume.customPasses.Add(objectCountPass);
-
-            objectCountPass.ClassCountsReceived += onClassCountsReceived;
-#endif
-#if URP_PRESENT
             var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
             perceptionCamera.LabelingConfiguration = labelingConfiguration;
             perceptionCamera.captureRgbImages = false;
-#endif
+            cameraObject.AddComponent<InstanceSegmentationLabeler>();
+
             return cameraObject;
         }
     }

com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

         {
             int timesSegmentationImageReceived = 0;
             int? frameStart = null;
-            Action<int, NativeArray<uint>> onSegmentationImageReceived = (frameCount, data) =>
+            Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived = (frameCount, data, tex) =>
             {
                 if (frameStart == null || frameStart > frameCount)
                     return;
                 var skinnedMeshRenderer = planeObject.AddComponent<SkinnedMeshRenderer>();
                 skinnedMeshRenderer.sharedMesh = meshFilter.sharedMesh;
                 skinnedMeshRenderer.material = meshRenderer.material;
-                
+
                 Object.DestroyImmediate(oldObject);
             }
             planeObject.transform.SetPositionAndRotation(new Vector3(0, 0, 10), Quaternion.Euler(90, 0, 0));
 
             //TestHelper.LoadAndStartRenderDocCapture(out var gameView);
 
-            Action<int, NativeArray<uint>> onSegmentationImageReceived = (frameCount, data) =>
+            Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived = (frameCount, data, tex) =>
             {
                 if (expectedLabelAtFrame == null || !expectedLabelAtFrame.ContainsKey(frameCount))
                     return;
             Assert.AreEqual(3, timesSegmentationImageReceived);
         }
 
-        GameObject SetupCamera(Action<int, NativeArray<uint>> onSegmentationImageReceived)
+        GameObject SetupCamera(Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
         {
             var cameraObject = new GameObject();
             cameraObject.SetActive(false);
-
-#if HDRP_PRESENT
-            cameraObject.AddComponent<HDAdditionalCameraData>();
-            var customPassVolume = cameraObject.AddComponent<CustomPassVolume>();
-            customPassVolume.isGlobal = true;
-            var rt = new RenderTexture(128, 128, 1, GraphicsFormat.R8G8B8A8_UNorm);
-            rt.Create();
-            var instanceSegmentationPass = new InstanceSegmentationPass();
-            instanceSegmentationPass.targetCamera = camera;
-            instanceSegmentationPass.targetTexture = rt;
-            customPassVolume.customPasses.Add(instanceSegmentationPass);
-            instanceSegmentationPass.name = nameof(instanceSegmentationPass);
-            instanceSegmentationPass.EnsureInit();
-
-            var reader = cameraObject.AddComponent<ImageReaderBehaviour>();
-            reader.source = rt;
-            reader.cameraSource = camera;
-
-            reader.SegmentationImageReceived += onSegmentationImageReceived;
-#endif
-#if URP_PRESENT
             var labelingConfiguration = ScriptableObject.CreateInstance<LabelingConfiguration>();
             var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
             perceptionCamera.LabelingConfiguration = labelingConfiguration;
-            perceptionCamera.segmentationImageReceived += onSegmentationImageReceived;
-#endif
+
+            var instanceSegmentationLabeler = cameraObject.AddComponent<InstanceSegmentationLabeler>();
+            instanceSegmentationLabeler.InstanceSegmentationImageReadback += onSegmentationImageReceived;
+
             AddTestObjectForCleanup(cameraObject);
             cameraObject.SetActive(true);
             return cameraObject;