Labelers are working. Moving SemanticSegmentationLabeler to colors instead of integer values.

TestProjects/PerceptionHDRP/Packages/manifest.json

     "com.unity.collab-proxy": "1.2.16",
     "com.unity.ext.nunit": "1.0.0",
     "com.unity.ide.rider": "1.1.4",
-    "com.unity.ide.vscode": "1.2.0",
+    "com.unity.ide.vscode": "1.2.1",
     "com.unity.package-validation-suite": "0.9.1-preview",
     "com.unity.perception": "file:../../../com.unity.perception",
     "com.unity.render-pipelines.core": "7.3.1",
-    "com.unity.test-framework": "1.1.13",
+    "com.unity.test-framework": "1.1.14",
     "com.unity.testtools.codecoverage": "0.2.2-preview",
     "com.unity.textmeshpro": "2.0.1",
     "com.unity.ugui": "1.0.0",

com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs

         {
             PerceptionCamera = perceptionCamera;
             SensorHandle = perceptionCamera.SensorHandle;
+            Setup();
         }
     }
 }

com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs

 
         public override void Setup()
         {
+            if (labelingConfiguration == null)
+                throw new InvalidOperationException("LabelingConfiguration must be supplied");
+
             PerceptionCamera.RenderedObjectInfosCalculated += (frameCount, objectInfo) =>
             {
                 NativeArray<uint> objectCounts = ComputeObjectCounts(objectInfo);

com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs

         /// </summary>
         public SemanticSegmentationLabelConfig labelConfig;
 
-        public event Action<int,NativeArray<uint>,RenderTexture> SemanticSegmentationImageReadback;
+        public event Action<int,NativeArray<Color32>,RenderTexture> SemanticSegmentationImageReadback;
-        RenderTextureReader<uint> m_SemanticSegmentationTextureReader;
+        RenderTextureReader<Color32> m_SemanticSegmentationTextureReader;
 
 #if HDRP_PRESENT
         SemanticSegmentationPass m_SemanticSegmentationPass;
             [UsedImplicitly]
             public string label_name;
             [UsedImplicitly]
-            public int pixel_value;
+            public Color pixel_value;
-            public NativeArray<uint> data;
+            public NativeArray<Color32> data;
             public int width;
             public int height;
             public string path;
             var specs = labelConfig.LabelEntries.Select((l) => new SemanticSegmentationSpec()
             {
                 label_name = l.label,
-                pixel_value = l.pixelValue
+                pixel_value = l.color
-            m_SemanticSegmentationTextureReader = new RenderTextureReader<uint>(semanticSegmentationTexture, myCamera,
+            m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(semanticSegmentationTexture, myCamera,
-        void OnSemanticSegmentationImageRead(int frameCount, NativeArray<uint> data)
+        void OnSemanticSegmentationImageRead(int frameCount, NativeArray<Color32> data)
         {
             var dxLocalPath = Path.Combine(k_SemanticSegmentationDirectory, k_SegmentationFilePrefix) + frameCount + ".png";
             var path = Path.Combine(Manager.Instance.GetDirectoryFor(k_SemanticSegmentationDirectory), k_SegmentationFilePrefix) + frameCount + ".png";
             SemanticSegmentationImageReadback?.Invoke(frameCount, data, semanticSegmentationTexture);
             asyncRequest.data = new AsyncSemanticSegmentationWrite()
             {
-                data = new NativeArray<uint>(data, Allocator.TempJob),
+                data = new NativeArray<Color32>(data, Allocator.TempJob),
                 width = semanticSegmentationTexture.width,
                 height = semanticSegmentationTexture.height,
                 path = path

com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs

         public LabelEntryMatchCache(LabelingConfiguration labelingConfiguration)
         {
             m_LabelingConfiguration = labelingConfiguration;
-            World.DefaultGameObjectInjectionWorld.GetOrCreateSystem<GroundTruthLabelSetupSystem>().Activate(this);
+            World.DefaultGameObjectInjectionWorld.GetOrCreateSystem<GroundTruthLabelSetupSystem>().Activate(this);
         }
 
         public bool TryGetLabelEntryFromInstanceId(uint instanceId, out LabelEntry labelEntry, out int index)

com.unity.perception/Runtime/GroundTruth/Labeling/LabelingConfiguration2.cs

     {
         string ILabelEntry.label => this.label;
         public string label;
-        public int pixelValue;
+        public Color color;
     }
     /// <summary>
     /// A definition for how a <see cref="Labeling"/> should be resolved to a single label and id for ground truth generation.

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

 
             SetupInstanceSegmentation();
 
+            foreach (var labeler in labelers)
+            {
+                labeler.Init(this);
+            }
+
             RenderPipelineManager.beginCameraRendering += OnBeginCameraRendering;
             SimulationManager.SimulationEnding += OnSimulationEnding;
         }
 
             var cam = GetComponent<Camera>();
             cam.enabled = SensorHandle.ShouldCaptureThisFrame;
+
+            foreach (var labeler in labelers)
+                labeler.OnUpdate();
         }
 
         void CaptureRgbData(Camera cam)
 
         void OnSimulationEnding()
         {
-            RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering;
+            foreach (var labeler in labelers)
+                labeler.OnSimulationEnding();
         }
 
         void OnBeginCameraRendering(ScriptableRenderContext _, Camera cam)
                 return;
 #endif
             CaptureRgbData(cam);
+
+            foreach (var labeler in labelers)
+                labeler.OnBeginRendering();
+            RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering;
 
             OnSimulationEnding();
 

com.unity.perception/Runtime/GroundTruth/Resources/SemanticSegmentation.shader

 {
     Properties
     {
-        [PerObjectData] LabelingId("Labeling Id", int) = 0
+        [PerObjectData] LabelingId("Labeling Id", Color) = (1,1,1,1)
     }
 
     HLSLINCLUDE
             #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Common.hlsl"
             #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Packing.hlsl"
 
-            uint LabelingId;
+            float4 LabelingId;
 
             struct appdata
             {
             {
                 float4 vertex : SV_POSITION;
             };
-
-            uint _SegmentationId;
 
             v2f vert (appdata v)
             {
 
             fixed4 frag (v2f i) : SV_Target
             {
-                return float4(UnpackUIntToFloat((uint)LabelingId, 0, 8), UnpackUIntToFloat((uint)LabelingId, 8, 8), 0, 1.0);
+                //return float4(UnpackUIntToFloat((uint)LabelingId, 0, 8), UnpackUIntToFloat((uint)LabelingId, 8, 8), 0, 1.0);
+                return LabelingId;
             }
 
             ENDCG

com.unity.perception/Runtime/GroundTruth/SemanticSegmentationCrossPipelinePass.cs

             }
 
             //Set the labeling ID so that it can be accessed in ClassSemanticSegmentationPass.shader
-            mpb.SetInt(k_LabelingId, entry.pixelValue);
+            mpb.SetColor(k_LabelingId, entry.color);
         }
     }
 }

com.unity.perception/Tests/Runtime/GroundTruthTests/PerceptionCameraIntegrationTests.cs

                 new SemanticSegmentationLabelEntry()
                 {
                     label = label,
-                    pixelValue = 500
+                    color = Color.blue
                 }
             };
             return labelingConfiguration;

com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

     [UnityPlatform(exclude = new[] {RuntimePlatform.LinuxEditor, RuntimePlatform.LinuxPlayer})]
     public class SegmentationPassTests : GroundTruthTestBase
     {
-        const int k_SemanticPixelValue = 4;
+        static readonly Color32 k_SemanticPixelValue = Color.blue;
 
         public enum SegmentationKind
         {
             [Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind)
         {
             int timesSegmentationImageReceived = 0;
-            int expectedPixelValue = segmentationKind == SegmentationKind.Instance ? 1 : k_SemanticPixelValue;
-            Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived = (frameCount, data, tex) =>
+            GameObject cameraObject = null;
+
+            object expectedPixelValue = null;
+            void OnSegmentationImageReceived<T>(int frameCount, NativeArray<T> data, RenderTexture tex) where T : struct
-                if (frameStart == null || frameStart > frameCount)
-                    return;
+                if (frameStart == null || frameStart > frameCount) return;
-            };
+            }
+
+            switch (segmentationKind)
+            {
+                case SegmentationKind.Instance:
+                    expectedPixelValue = 1;
+                    cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived);
+                    break;
+                case SegmentationKind.Semantic:
+                    expectedPixelValue = k_SemanticPixelValue;
+                    cameraObject = SetupCameraSemanticSegmentation(OnSegmentationImageReceived);
+                    break;
+            }
-            var cameraObject = SetupCamera(segmentationKind, onSegmentationImageReceived);
             //
             // // Arbitrary wait for 5 frames for shaders to load. Workaround for issue with Shader.WarmupAllShaders()
             // for (int i=0 ; i<5 ; ++i)
             }
             planeObject.transform.SetPositionAndRotation(new Vector3(0, 0, 10), Quaternion.Euler(90, 0, 0));
             planeObject.transform.localScale = new Vector3(10, -1, 10);
-            planeObject.AddComponent<Labeling>();
+            var labeling = planeObject.AddComponent<Labeling>();
+            labeling.labels.Add("label");
+
             AddTestObjectForCleanup(planeObject);
 
             yield return null;
             [Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind)
         {
             int timesSegmentationImageReceived = 0;
-            Dictionary<int, int> expectedLabelAtFrame = null;
+            Dictionary<int, object> expectedLabelAtFrame = null;
-            Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived = (frameCount, data, tex) =>
+            void OnSegmentationImageReceived<T>(int frameCount, NativeArray<T> data, RenderTexture tex) where T : struct
-                if (expectedLabelAtFrame == null || !expectedLabelAtFrame.ContainsKey(frameCount))
-                    return;
+                if (expectedLabelAtFrame == null || !expectedLabelAtFrame.ContainsKey(frameCount)) return;
 
                 timesSegmentationImageReceived++;
 
                 {
                     CollectionAssert.AreEqual(Enumerable.Repeat(expectedLabelAtFrame[frameCount], data.Length), data);
                 }
+
                 // ReSharper disable once RedundantCatchClause
                 catch (Exception)
                 {
                 }
-            };
+            }
-            var cameraObject = SetupCamera(segmentationKind, onSegmentationImageReceived);
+            var cameraObject = segmentationKind == SegmentationKind.Instance ?
+                SetupCameraInstanceSegmentation(OnSegmentationImageReceived<uint>) :
+                SetupCameraSemanticSegmentation(OnSegmentationImageReceived<Color32>);
-            int expectedPixelValue = segmentationKind == SegmentationKind.Instance ? 1 : k_SemanticPixelValue;
-            expectedLabelAtFrame = new Dictionary<int, int>
+            object expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) 1 : k_SemanticPixelValue;
+            expectedLabelAtFrame = new Dictionary<int, object>
             {
                 {Time.frameCount    , expectedPixelValue},
                 {Time.frameCount + 1, expectedPixelValue},
             Assert.AreEqual(3, timesSegmentationImageReceived);
         }
 
-        GameObject SetupCamera(SegmentationKind segmentationKind, Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
+        GameObject SetupCameraInstanceSegmentation(Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
+        {
+            var cameraObject = SetupCamera(out var perceptionCamera);
+            perceptionCamera.InstanceSegmentationImageReadback += onSegmentationImageReceived;
+            cameraObject.SetActive(true);
+            return cameraObject;
+        }
+
+        GameObject SetupCameraSemanticSegmentation(Action<int, NativeArray<Color32>, RenderTexture> onSegmentationImageReceived)
+        {
+            var cameraObject = SetupCamera(out var perceptionCamera);
+            var labelConfig = ScriptableObject.CreateInstance<SemanticSegmentationLabelConfig>();
+            labelConfig.LabelEntries = new List<SemanticSegmentationLabelEntry>()
+            {
+                new SemanticSegmentationLabelEntry()
+                {
+                    label = "label",
+                    color = k_SemanticPixelValue
+                }
+            };
+            var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
+            semanticSegmentationLabeler.SemanticSegmentationImageReadback += onSegmentationImageReceived;
+            perceptionCamera.labelers.Add(semanticSegmentationLabeler);
+            cameraObject.SetActive(true);
+            return cameraObject;
+        }
+
+        GameObject SetupCamera(out PerceptionCamera perceptionCamera)
         {
             var cameraObject = new GameObject();
             cameraObject.SetActive(false);
-            var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
+            perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
-            if (segmentationKind == SegmentationKind.Instance)
-                perceptionCamera.InstanceSegmentationImageReadback += onSegmentationImageReceived;
-            else
-            {
-                var labelConfig = new SemanticSegmentationLabelConfig();
-                labelConfig.LabelEntries = new List<SemanticSegmentationLabelEntry>()
-                {
-                    new SemanticSegmentationLabelEntry()
-                    {
-                        label = "label",
-                        pixelValue = k_SemanticPixelValue
-                    }
-                };
-                var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
-                semanticSegmentationLabeler.SemanticSegmentationImageReadback += onSegmentationImageReceived;
-                perceptionCamera.labelers.Add(semanticSegmentationLabeler);
-            }
-            cameraObject.SetActive(true);
             return cameraObject;
         }
     }