Adding sky color control to semantic segmentation (#274)

* Adding background color for semantic segmentation and bad UI. Need to come back to this later.

* Updating UI

* Changing 'background' to 'sky' and improving tests.

* some UI polish

* Updating label config and PerceptionURP

Co-authored-by: Mohsen Kamalzadeh <>

TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset

     color: {r: 0, g: 1, b: 0.16973758, a: 1}
   - label: Terrain
     color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
+  skyColor: {r: 0, g: 0, b: 0, a: 1}

TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset

     color: {r: 0, g: 1, b: 0.16973758, a: 1}
   - label: Terrain
     color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
+  skyColor: {r: 0, g: 0, b: 0, a: 1}

TestProjects/PerceptionURP/Packages/packages-lock.json

         "com.unity.collections": "0.9.0-preview.6",
         "com.unity.nuget.newtonsoft-json": "1.1.2",
         "com.unity.render-pipelines.core": "7.1.6",
-        "com.unity.simulation.capture": "0.0.10-preview.19",
+        "com.unity.simulation.capture": "0.0.10-preview.20",
         "com.unity.simulation.client": "0.0.10-preview.10",
         "com.unity.simulation.core": "0.0.10-preview.22"
       }
       "url": "https://packages.unity.com"
     },
     "com.unity.simulation.capture": {
-      "version": "0.0.10-preview.19",
+      "version": "0.0.10-preview.20",
       "depth": 1,
       "source": "registry",
       "dependencies": {

com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs

 
             m_StartingIdEnumField.SetEnabled(AutoAssign);
 
+            m_SkyColorUi.style.display = DisplayStyle.None;
+
             AutoAssignIdsIfNeeded();
             m_MoveDownButton.clicked += MoveSelectedItemDown;
             m_MoveUpButton.clicked += MoveSelectedItemUp;

com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs

         protected Button m_MoveDownButton;
         protected VisualElement m_MoveButtons;
         protected VisualElement m_IdSpecificUi;
+        protected VisualElement m_SkyColorUi;
+
+        protected ColorField m_SkyColorField;
+        protected Label m_SkyHexLabel;
 
 
         public void OnEnable()
             m_StartingIdEnumField = m_Root.Q<EnumField>("starting-id-dropdown");
             m_AutoIdToggle = m_Root.Q<Toggle>("auto-id-toggle");
             m_IdSpecificUi = m_Root.Q<VisualElement>("id-specific-ui");
+            m_SkyColorUi = m_Root.Q<VisualElement>("sky-color-ui");
+            m_SkyColorField = m_Root.Q<ColorField>("sky-color-value");
+            m_SkyHexLabel = m_Root.Q<Label>("sky-color-hex");
 
             m_SaveButton.SetEnabled(false);
 

com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs

         {
             m_MoveButtons.style.display = DisplayStyle.None;
             m_IdSpecificUi.style.display = DisplayStyle.None;
+
+            var skyColorProperty = serializedObject.FindProperty(nameof(SemanticSegmentationLabelConfig.skyColor));
+            m_SkyColorField.BindProperty(skyColorProperty);
+            m_SkyColorField.RegisterValueChangedCallback(e => UpdateSkyHexLabel(e.newValue));
+            UpdateSkyHexLabel(skyColorProperty.colorValue);
+        }
+
+        private void UpdateSkyHexLabel(Color colorValue)
+        {
+            m_SkyHexLabel.text = "#" + ColorUtility.ToHtmlStringRGBA(colorValue);
         }
 
         public override void PostRemoveOperations()

com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml

 <UXML xmlns="UnityEngine.UIElements" xmlns:editor="UnityEditor.UIElements">
     <VisualElement class="added-label" style="padding-top: 3px;">
         <Button name="remove-button" class="labeling__remove-item-button"/>
+        <TextField name="label-value" class="labeling__added-label-value"/>
-        <TextField name="label-value" class="labeling__added-label-value"/>
         <VisualElement style="min-width:20px; flex-direction: row; display:none">
             <Button name="move-up-button" class="move-label-in-config-button move-up" style="margin-right:-2px"/>
             <Button name="move-down-button" class="move-label-in-config-button move-down"/>

com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml

     <VisualElement class="outer-container" name="outer-container">
         <Style src="../Uss/Styles.uss"/>
         <VisualElement class="inner-container" name="id-specific-ui">
+            <Label text="Options" name="options-title" class="title-label"/>
+            </VisualElement>
+        </VisualElement>
+        <VisualElement class="inner-container" name="sky-color-ui">
+            <Label text="Options" name="options-title" class="title-label"/>
+            <VisualElement style="flex-direction: row; flex-grow: 1;">
+                <Label text="Sky Color" name="added-labels-title"
+                       style="flex-grow: 1; font-size: 11; min-width : 60px; align-self:center;"/>
+                <VisualElement class="generic-hover"
+                               style="min-width : 137px; flex-direction: row; padding: 3px 5px 3px 5px; margin-left: 3px; margin-right: 3px; border-width: 1px; border-color: #666666; border-radius: 4px;">
+                    <editor:ColorField name="sky-color-value" style="min-width : 60px; max-width: 60px; align-self:center;"/>
+                    <Label name="sky-color-hex"
+                           style="font-size: 11; min-width : 60px; max-width: 60px; align-self:center; margin: 2px"/>
+                </VisualElement>
             </VisualElement>
         </VisualElement>
         <VisualElement name="added-labels" class="inner-container" style="margin-top:5px">

com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs

             Color.yellow,
             Color.gray
         };
+
+
+        /// <summary>
+        /// The color to use for the sky in semantic segmentation images
+        /// </summary>
+        public Color skyColor = Color.black;
     }
 
     /// <summary>

com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs

 
             s_LastFrameExecuted = Time.frameCount;
             var renderList = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, -1);
-            cmd.ClearRenderTarget(true, true, Color.black);
+            cmd.ClearRenderTarget(true, true, m_LabelConfig.skyColor);
             DrawRendererList(renderContext, cmd, RendererList.Create(renderList));
         }
 

com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

     {
         static readonly Color32 k_SemanticPixelValue = new Color32(10, 20, 30, Byte.MaxValue);
         private static readonly Color32 k_InstanceSegmentationPixelValue = new Color32(255,0,0, 255);
+        private static readonly Color32 k_SkyValue = new Color32(10, 20, 30, 40);
 
         public enum SegmentationKind
         {
             switch (segmentationKind)
             {
                 case SegmentationKind.Instance:
-                    //expectedPixelValue = new Color32(0, 74, 255, 255);
                     expectedPixelValue = k_InstanceSegmentationPixelValue;
                     cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived);
                     break;
                 CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
             }
 
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
 
             AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(label: "non-matching"));
             yield return null;
                 CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
             }
 
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
 
             var gameObject = TestHelper.CreateLabeledPlane();
             gameObject.GetComponent<Labeling>().enabled = false;
         }
 
         [UnityTest]
-        public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesBlack([Values(false, true)] bool showVisualizations)
+        public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesSky([Values(false, true)] bool showVisualizations)
-            var expectedPixelValue = new Color32(0, 0, 0, 255);
+            var expectedPixelValue = k_SkyValue;
             void OnSegmentationImageReceived(NativeArray<Color32> data)
             {
                 timesSegmentationImageReceived++;
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations, expectedPixelValue);
 
             //TestHelper.LoadAndStartRenderDocCapture(out var gameView);
             yield return null;
         }
 
         [UnityTest]
+        public IEnumerator SemanticSegmentationPass_WithNoObjects_ProducesSky()
+        {
+            int timesSegmentationImageReceived = 0;
+            var expectedPixelValue = k_SkyValue;
+            void OnSegmentationImageReceived(NativeArray<Color32> data)
+            {
+                timesSegmentationImageReceived++;
+                CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
+            }
+
+            var cameraObject = SetupCameraSemanticSegmentation(
+                a => OnSegmentationImageReceived(a.data), false, expectedPixelValue);
+
+            yield return null;
+            //destroy the object to force all pending segmented image readbacks to finish and events to be fired.
+            DestroyTestObject(cameraObject);
+            Assert.AreEqual(1, timesSegmentationImageReceived);
+        }
+
+        [UnityTest]
         public IEnumerator SemanticSegmentationPass_WithTextureOverride_RendersToOverride([Values(true, false)] bool showVisualizations)
         {
             var expectedPixelValue = new Color32(0, 0, 255, 255);
             return cameraObject;
         }
 
-        GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations)
+        GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations, Color? backgroundColor = null)
         {
             var cameraObject = SetupCamera(out var perceptionCamera, showVisualizations);
             var labelConfig = ScriptableObject.CreateInstance<SemanticSegmentationLabelConfig>();
                     color = k_SemanticPixelValue
                 }
             });
+            if (backgroundColor != null)
+            {
+                labelConfig.skyColor = backgroundColor.Value;
+            }
             var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
             semanticSegmentationLabeler.imageReadback += onSegmentationImageReceived;
             perceptionCamera.AddLabeler(semanticSegmentationLabeler);