Moving visualization to screen space - camera and unifying into a single canvas to ensure good behavior with multiple cameras.

TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity

     m_TrainingDataDestination: TrainingData
     m_LightProbeSampleCountMultiplier: 4
   m_LightingDataAsset: {fileID: 0}
-  m_UseShadowmask: 0
+  m_UseShadowmask: 1
 --- !u!196 &4
 NavMeshSettings:
   serializedVersion: 2
   m_BounceIntensity: 1
   m_ColorTemperature: 6570
   m_UseColorTemperature: 0
-  m_BoundingSphereOverride: {x: 0, y: 1e-45, z: 1.8643e-41, w: 5.9e-44}
+  m_BoundingSphereOverride: {x: 4.7e-43, y: 0, z: 0, w: 0}
   m_UseBoundingSphereOverride: 0
   m_ShadowRadius: 0
   m_ShadowAngle: 0
       type: {class: SemanticSegmentationLabeler, ns: UnityEngine.Perception.GroundTruth,
         asm: Unity.Perception.Runtime}
       data:
-        enabled: 1
+        enabled: 0
         annotationId: 12F94D8D-5425-4DEB-9B21-5E53AD957D66
         labelConfig: {fileID: 11400000, guid: c140c5aa05dd09e4fadaa26de31b1f39, type: 2}
         m_TargetTextureOverride: {fileID: 0}
   m_Script: {fileID: 11500000, guid: 53f4c974fdf704444959724a41de0cfe, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  cam: {fileID: 963194229}
-  renderTexture: {fileID: 0}
-  secondCamera: {fileID: 0}
 --- !u!1 &1640252278
 GameObject:
   m_ObjectHideFlags: 0

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs

                     };
                 }
 
-                if (!CaptureOptions.useAsyncReadbackIfSupported && frameCount != Time.frameCount) 
+                if (!CaptureOptions.useAsyncReadbackIfSupported && frameCount != Time.frameCount)
                     Debug.LogWarning("Not on current frame: " + frameCount + "(" + Time.frameCount + ")");
 
                 if (perceptionCamera.visualizationEnabled && visualizationEnabled)
-                
+
                 asyncAnnotation.ReportValues(m_BoundingBoxValues);
             }
         }
             objectPool = new List<GameObject>();
 
             visualizationHolder = new GameObject("BoundsHolder" + Time.frameCount);
-            canvas.AddComponent(visualizationHolder);
+            visualizationCanvas.AddComponent(visualizationHolder);
         }
 
         void ClearObjectPool(int count)
 
                 if (i >= objectPool.Count)
                 {
-                    objectPool.Add(GameObject.Instantiate(Resources.Load<GameObject>("BoundingBoxPrefab")));
-                    (objectPool[i].transform as RectTransform).parent = visualizationHolder.transform;
+                    var boundingBoxObject = GameObject.Instantiate(Resources.Load<GameObject>("BoundingBoxPrefab"));
+                    objectPool.Add(boundingBoxObject);
+                    var rectTransform = (RectTransform)boundingBoxObject.transform;
+                    rectTransform.localScale = Vector3.one;
+                    rectTransform.parent = visualizationHolder.transform;
                 }
 
                 if (!objectPool[i].activeSelf) objectPool[i].SetActive(true);
 
                 var rectTrans = objectPool[i].transform as RectTransform;
-                
+
+                rectTrans.localScale = Vector3.one;
+                rectTrans.localPosition = Vector3.zero;
                 rectTrans.anchoredPosition = new Vector2(boxVal.x, -boxVal.y);
                 rectTrans.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, boxVal.width);
                 rectTrans.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, boxVal.height);
         /// <inheritdoc/>
-        override protected void OnVisualizerActiveStateChanged(bool enabled)
+        override protected void OnVisualizerEnabledChanged(bool enabled)
-            if (visualizationHolder != null) 
+            if (visualizationHolder != null)
                 visualizationHolder.SetActive(enabled);
         }
     }

com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs

 
         /// <summary>
         /// Labelers should set this in their setup to define if they support realtime
-        /// visualization of their data. 
+        /// visualization of their data.
         /// </summary>
         protected abstract bool supportsVisualization
         {
         /// <summary>
         /// Static counter shared by all lableler classes. If any visualizers are
-        /// active then perception camera cannot operate in asynchronous mode. 
+        /// active then perception camera cannot operate in asynchronous mode.
-        private static VisualizationCanvas m_Canvas = null;
-        
-        /// Retrieve a handle to the visualization canvas <see cref="VisualizationCanvas". This is the specific canvas that all visualization
+        /// Retrieve a handle to the visualization canvas <see cref="VisualizationCanvas"/>. This is the specific canvas that all visualization
-        public VisualizationCanvas canvas
-        {
-            get
-            {
-                if (m_Canvas == null)
-                {
-                    m_Canvas = GameObject.Instantiate(Resources.Load<GameObject>("VisualizationUI")).GetComponent<VisualizationCanvas>();
-                };
-                return m_Canvas;
-            }
-        }
+        public VisualizationCanvas visualizationCanvas { get; private set; }
-        public ControlPanel controlPanel
-        {
-            get
-            {
-                return canvas.controlPanel;
-            }
-        }
+        public ControlPanel controlPanel => visualizationCanvas != null ? visualizationCanvas.controlPanel : null;
-        public HUDPanel hudPanel
-        {
-            get
-            {
-                return canvas.hudPanel;
-            }
-        }
+        public HUDPanel hudPanel => visualizationCanvas != null ? visualizationCanvas.hudPanel : null;
 
         /// <summary>
         /// The <see cref="PerceptionCamera"/> that contains this labeler.
         /// <summary>
         /// Called when the labeler's visualization capability is turned on or off.
         /// </summary>
-        protected virtual void OnVisualizerActiveStateChanged(bool enabled) {}
+        protected virtual void OnVisualizerEnabledChanged(bool enabled) {}
         /// <summary>
         /// Called during the Update each frame the the labeler is enabled and <see cref="SensorHandle.ShouldCaptureThisFrame"/> is true.
         /// </summary>
 
         internal void InternalSetup() => Setup();
         internal void InternalPopulateVisualizationPanel(GameObject panel) => PopulateVisualizationPanel(controlPanel);
-        internal void InternalVisualizerActiveStateChanged(bool enabled) => OnVisualizerActiveStateChanged(enabled);
+        internal void InternalVisualizerActiveStateChanged(bool enabled) => OnVisualizerEnabledChanged(enabled);
-        private bool m_VisualizationEnabled = true;
-        
+        private bool m_VisualizationEnabled = false;
+
-        /// Turns on/off the labeler's realtime visualization capability. If a labeler
-        /// does not support realtime visualization (<see cref="supportsVisualization"/>)
+        /// Turns on/off the labeler's realtime visualization capability. If a labeler does not support realtime
+        /// visualization (<see cref="supportsVisualization"/>) or visualization is not enabled on the PerceptionCamera
         /// this will not function.
         /// </summary>
         protected bool visualizationEnabled
             set
             {
                 if (!supportsVisualization) return;
+                if (visualizationCanvas == null) return;
-                    
+
                     if (m_VisualizationEnabled)
                         activeVisualizers++;
                     else
                     else
                         CaptureOptions.useAsyncReadbackIfSupported = true;
 
-                    OnVisualizerActiveStateChanged(m_VisualizationEnabled);
+                    OnVisualizerEnabledChanged(m_VisualizationEnabled);
-        
-        internal void Init(PerceptionCamera newPerceptionCamera)
+
+        internal void Init(PerceptionCamera newPerceptionCamera, VisualizationCanvas visualizationCanvas)
+                this.visualizationCanvas = visualizationCanvas;
-                if (supportsVisualization)
+                if (supportsVisualization && visualizationCanvas != null)
+                    m_VisualizationEnabled = true;
                     InitVisualizationUI();
                 }
             }

com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs

         }
 
         /// <inheritdoc/>
-        override protected void OnVisualizerActiveStateChanged(bool enabled)
+        override protected void OnVisualizerEnabledChanged(bool enabled)
         {
             if (!enabled)
             {

com.unity.perception/Runtime/GroundTruth/Labelers/RenderedObjectInfoLabeler.cs

         }
 
         /// <inheritdoc/>
-        override protected void OnVisualizerActiveStateChanged(bool enabled)
+        override protected void OnVisualizerEnabledChanged(bool enabled)
         {
             if (!enabled)
             {

com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs

         /// the camera resolution.
         /// </summary>
         public RenderTexture targetTexture => m_TargetTextureOverride;
-        
+
         [Tooltip("(Optional) The RenderTexture on which semantic segmentation images will be drawn. Will be reformatted on startup.")]
         [SerializeField]
         RenderTexture m_TargetTextureOverride;
 
             m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture, myCamera,
                 (frameCount, data, tex) => OnSemanticSegmentationImageRead(frameCount, data));
-        
+
             visualizationEnabled = supportsVisualization;
         }
 
             annotation.ReportFile(datasetRelativePath);
 
             var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncSemanticSegmentationWrite>>();
-            
+
             if (visualizationEnabled && perceptionCamera.visualizationEnabled)
                 VisualizeSegmentationTexture(data, targetTexture);
 
             rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, camWidth);
             rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, camHeight);
 
-            canvas.AddComponent(segVisual, setAsLowestElement: true);
+            visualizationCanvas.AddComponent(segVisual, setAsLowestElement: true);
         }
 
         void VisualizeSegmentationTexture(NativeArray<Color32> data, RenderTexture texture)
             cpuTexture.Apply();
-            
+
-        override protected void OnVisualizerActiveStateChanged(bool enabled)
+        override protected void OnVisualizerEnabledChanged(bool enabled)
-            if (segVisual != null) 
+            if (segVisual != null)
                 segVisual.SetActive(enabled);
         }
     }

com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/HUDPanel.cs

         /// </summary>
         public void UpdateEntry(string key, string value)
         {
-            if (!entries.ContainsKey(key)) 
+            if (!entries.ContainsKey(key))
-                entries[key].transform.SetParent(contentPanel.transform, true);
+                entries[key].transform.SetParent(contentPanel.transform, false);
             }
             entries[key].SetValue(value);
         }

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

         /// be set to false if there is more than one camera in the scene, and this is
         /// not considered the active camera.
         /// </summary>
-        bool m_VisualizationBlocked = false;
+        bool m_VisualizationAllowed = false;
 
         [SerializeField]
         bool m_VisualizationEnabled = true;
             {
-                return !m_VisualizationBlocked && m_VisualizationEnabled;
+                return m_VisualizationAllowed && m_VisualizationEnabled;
-                if (m_VisualizationBlocked)
+                if (m_VisualizationAllowed)
                     return;
 
                 m_VisualizationEnabled = value;
         }
 #endif
 
+        VisualizationCanvas visualizationCanvas => m_VisualizationAllowed ? s_VisualizationCanvas.GetComponent<VisualizationCanvas>() : null;
+
         /// <summary>
         /// Add a data object which will be added to the dataset with each capture. Overrides existing sensor data associated with the given key.
         /// </summary>
             if (s_VisualizedPerceptionCamera != null)
             {
                 Debug.LogWarning($"Currently only one PerceptionCamera may be visualized at a time. Disabling visualization on {gameObject.name}.");
-                m_VisualizationBlocked = true;
+            m_VisualizationAllowed = true;
             s_VisualizedPerceptionCamera = this;
 
             // set up to render to a render texture instead of the screen
             s_VisualizationCamera = new GameObject(cam.name + "_VisualizationCamera");
             var visualizationCameraComponent = s_VisualizationCamera.AddComponent<Camera>();
             int layerMask = 1 << LayerMask.NameToLayer("UI");
+            visualizationCameraComponent.orthographic = true;
-            s_VisualizationCanvas = new GameObject(cam.name + "_VisualizationCanvas");
-            var canvas = s_VisualizationCanvas.AddComponent<Canvas>();
-            s_VisualizationCanvas.AddComponent<CanvasScaler>();
-            s_VisualizationCanvas.AddComponent<GraphicRaycaster>();
+            s_VisualizationCanvas = GameObject.Instantiate(Resources.Load<GameObject>("VisualizationUI"));
+            s_VisualizationCanvas.name = cam.name + "_VisualizationCanvas";
+            var canvas = s_VisualizationCanvas.GetComponent<Canvas>();
             canvas.renderMode = RenderMode.ScreenSpaceCamera;
             canvas.worldCamera = visualizationCameraComponent;
 
 
             var rect = imgObj.transform as RectTransform;
             rect.SetParent(s_VisualizationCanvas.transform, false);
+            //ensure the rgb image is rendered in the back
+            rect.SetAsFirstSibling();
             rect.anchorMin = new Vector2(0, 0);
             rect.anchorMax = new Vector2(1, 1);
             rect.pivot = new Vector2(0.5f, 0.5f);
                     continue;
 
                 if (!labeler.isInitialized)
-                    labeler.Init(this);
+                {
+                    labeler.Init(this, visualizationCanvas);
+                }
 
                 labeler.InternalOnUpdate();
             }
                     continue;
 
                 if (!labeler.isInitialized)
-                    labeler.Init(this);
+                    labeler.Init(this, visualizationCanvas);
 
                 labeler.InternalOnBeginRendering();
             }