using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using NUnit.Framework; using Unity.Collections; using Unity.Simulation; using UnityEngine; using UnityEngine.Perception.GroundTruth; using UnityEngine.Rendering; #if HDRP_PRESENT using UnityEngine.Rendering.HighDefinition; #elif URP_PRESENT using UnityEngine.Rendering.Universal; #endif using UnityEngine.TestTools; using Object = UnityEngine.Object; namespace GroundTruthTests { public class ImageReaderBehaviour : MonoBehaviour { public RenderTexture source; public Camera cameraSource; RenderTextureReader m_Reader; public event Action> SegmentationImageReceived; void Awake() { m_Reader = new RenderTextureReader(source); RenderPipelineManager.endCameraRendering += (context, camera) => m_Reader.Capture(context, (frameCount, data, renderTexture) => ImageReadCallback(frameCount, data, renderTexture)); } void ImageReadCallback(int frameCount, NativeArray data, RenderTexture renderTexture) { if (SegmentationImageReceived != null) SegmentationImageReceived(frameCount, data); } void OnDestroy() { m_Reader.Dispose(); m_Reader = null; } } public enum RendererType { MeshRenderer, SkinnedMeshRenderer, Terrain } //Graphics issues with OpenGL Linux Editor. https://jira.unity3d.com/browse/AISV-422 [UnityPlatform(exclude = new[] {RuntimePlatform.LinuxEditor, RuntimePlatform.LinuxPlayer})] public class SegmentationPassTests : GroundTruthTestBase { static readonly Color32 k_SemanticPixelValue = new Color32(10, 20, 30, Byte.MaxValue); private static readonly Color32 k_InstanceSegmentationPixelValue = new Color32(255,0,0, 255); private static readonly Color32 k_SkyValue = new Color32(10, 20, 30, 40); public enum SegmentationKind { Instance, Semantic } // A UnityTest behaves like a coroutine in Play Mode. In Edit Mode you can use // `yield return null;` to skip a frame. [UnityTest] public IEnumerator SegmentationPassTestsWithEnumeratorPasses( [Values(RendererType.MeshRenderer, RendererType.SkinnedMeshRenderer, RendererType.Terrain)] RendererType rendererType, [Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind) { int timesSegmentationImageReceived = 0; int? frameStart = null; GameObject cameraObject = null; object expectedPixelValue; void OnSegmentationImageReceived(int frameCount, NativeArray data, RenderTexture tex) where T : struct { if (frameStart == null || frameStart > frameCount) return; timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray()); } switch (segmentationKind) { case SegmentationKind.Instance: expectedPixelValue = k_InstanceSegmentationPixelValue; cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived); break; case SegmentationKind.Semantic: expectedPixelValue = k_SemanticPixelValue; cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.frameCount, a.data, a.sourceTexture), false); break; } //Put a plane in front of the camera GameObject planeObject; if (rendererType == RendererType.Terrain) { var terrainData = new TerrainData(); AddTestObjectForCleanup(terrainData); //look down because terrains cannot be rotated cameraObject.transform.rotation = Quaternion.LookRotation(Vector3.down, Vector3.forward); planeObject = Terrain.CreateTerrainGameObject(terrainData); planeObject.transform.SetPositionAndRotation(new Vector3(-10, -10, -10), Quaternion.identity); } else { planeObject = GameObject.CreatePrimitive(PrimitiveType.Plane); if (rendererType == RendererType.SkinnedMeshRenderer) { var oldObject = planeObject; planeObject = new GameObject(); var meshFilter = oldObject.GetComponent(); var meshRenderer = oldObject.GetComponent(); var skinnedMeshRenderer = planeObject.AddComponent(); skinnedMeshRenderer.sharedMesh = meshFilter.sharedMesh; skinnedMeshRenderer.material = meshRenderer.material; Object.DestroyImmediate(oldObject); } planeObject.transform.SetPositionAndRotation(new Vector3(0, 0, 10), Quaternion.Euler(90, 0, 0)); planeObject.transform.localScale = new Vector3(10, -1, 10); } var labeling = planeObject.AddComponent(); labeling.labels.Add("label"); frameStart = Time.frameCount; AddTestObjectForCleanup(planeObject); yield return null; yield return null; yield return null; yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); DestroyTestObject(planeObject); Assert.AreEqual(4, timesSegmentationImageReceived); } // Lens Distortion is only applicable in URP or HDRP pipelines // As such, this test will always fail if URP or HDRP are not present (and also not really compile either) #if HDRP_PRESENT || URP_PRESENT [UnityTest] public IEnumerator SemanticSegmentationPass_WithLensDistortion() { GameObject cameraObject = null; PerceptionCamera perceptionCamera; var fLensDistortionEnabled = false; var fDone = false; var frames = 0; #if false var dataBBox = new Color32[] { Color.blue, Color.blue, Color.blue, Color.blue }; #endif var boundingBoxWithoutLensDistortion = new Rect(); var boundingBoxWithLensDistortion = new Rect(); void OnSegmentationImageReceived(int frameCount, NativeArray data, RenderTexture tex) { frames++; if (frames < 10) return; // Calculate the bounding box if (fLensDistortionEnabled == false) { fLensDistortionEnabled = true; var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator(); renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp); boundingBoxWithoutLensDistortion = boundingBoxes[0].boundingBox; // Add lens distortion perceptionCamera.OverrideLensDistortionIntensity(0.715f); frames = 0; } else { var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator(); renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp); boundingBoxWithLensDistortion = boundingBoxes[0].boundingBox; Assert.AreNotEqual(boundingBoxWithoutLensDistortion, boundingBoxWithLensDistortion); Assert.Greater(boundingBoxWithLensDistortion.width, boundingBoxWithoutLensDistortion.width); fDone = true; } } cameraObject = SetupCamera(out perceptionCamera, false); perceptionCamera.InstanceSegmentationImageReadback += OnSegmentationImageReceived; cameraObject.SetActive(true); // Put a plane in front of the camera var planeObject = GameObject.CreatePrimitive(PrimitiveType.Plane); planeObject.transform.SetPositionAndRotation(new Vector3(0, 0, 10), Quaternion.Euler(90, 0, 0)); planeObject.transform.localScale = new Vector3(0.1f, -1, 0.1f); var labeling = planeObject.AddComponent(); labeling.labels.Add("label"); AddTestObjectForCleanup(planeObject); perceptionCamera.OverrideLensDistortionIntensity(0.5f); while (fDone != true) { yield return null; } // Destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); DestroyTestObject(planeObject); } #endif // ! HDRP_PRESENT || URP_PRESENT [UnityTest] public IEnumerator SemanticSegmentationPass_WithLabeledButNotMatchingObject_ProducesBlack() { int timesSegmentationImageReceived = 0; var expectedPixelValue = new Color32(0, 0, 0, 255); void OnSegmentationImageReceived(NativeArray data) { timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray()); } var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue); AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(label: "non-matching")); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithMatchingButDisabledLabel_ProducesBlack() { int timesSegmentationImageReceived = 0; var expectedPixelValue = new Color32(0, 0, 0, 255); void OnSegmentationImageReceived(NativeArray data) { timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray()); } var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue); var gameObject = TestHelper.CreateLabeledPlane(); gameObject.GetComponent().enabled = false; AddTestObjectForCleanup(gameObject); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator InstanceSegmentationPass_WithMatchingButDisabledLabel_ProducesBlack() { int timesSegmentationImageReceived = 0; var expectedPixelValue = new Color32(0, 0, 0, 255); void OnSegmentationImageReceived(NativeArray data) { CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data); timesSegmentationImageReceived++; } var cameraObject = SetupCameraInstanceSegmentation((frame, data, renderTexture) => OnSegmentationImageReceived(data)); var gameObject = TestHelper.CreateLabeledPlane(); gameObject.GetComponent().enabled = false; AddTestObjectForCleanup(gameObject); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesSky([Values(false, true)] bool showVisualizations) { int timesSegmentationImageReceived = 0; var expectedPixelValue = k_SkyValue; void OnSegmentationImageReceived(NativeArray data) { timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray()); } var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations, expectedPixelValue); //TestHelper.LoadAndStartRenderDocCapture(out var gameView); yield return null; var segLabeler = (SemanticSegmentationLabeler)cameraObject.GetComponent().labelers[0]; var request = AsyncGPUReadback.Request(segLabeler.targetTexture, callback: r => { CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, segLabeler.targetTexture.width * segLabeler.targetTexture.height), r.GetData()); }); AsyncGPUReadback.WaitAllRequests(); //RenderDoc.EndCaptureRenderDoc(gameView); //request.WaitForCompletion(); Assert.IsTrue(request.done); Assert.IsFalse(request.hasError); //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithNoObjects_ProducesSky() { int timesSegmentationImageReceived = 0; var expectedPixelValue = k_SkyValue; void OnSegmentationImageReceived(NativeArray data) { timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray()); } var cameraObject = SetupCameraSemanticSegmentation( a => OnSegmentationImageReceived(a.data), false, expectedPixelValue); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithTextureOverride_RendersToOverride([Values(true, false)] bool showVisualizations) { var expectedPixelValue = new Color32(0, 0, 255, 255); var targetTextureOverride = new RenderTexture(2, 2, 1, RenderTextureFormat.R8); var cameraObject = SetupCamera(out var perceptionCamera, showVisualizations); var labelConfig = ScriptableObject.CreateInstance(); labelConfig.Init(new List() { new SemanticSegmentationLabelEntry() { label = "label", color = expectedPixelValue } }); var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig, targetTextureOverride); perceptionCamera.AddLabeler(semanticSegmentationLabeler); cameraObject.SetActive(true); AddTestObjectForCleanup(cameraObject); AddTestObjectForCleanup(TestHelper.CreateLabeledPlane()); yield return null; TestHelper.ReadRenderTextureRawData(targetTextureOverride, data => { CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, targetTextureOverride.width * targetTextureOverride.height), data); }); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithMultiMaterial_ProducesCorrectValues([Values(true, false)] bool showVisualizations) { int timesSegmentationImageReceived = 0; var expectedPixelValue = k_SemanticPixelValue; void OnSegmentationImageReceived(NativeArray data) { timesSegmentationImageReceived++; CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data); } var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false); var plane = TestHelper.CreateLabeledPlane(); var meshRenderer = plane.GetComponent(); var baseMaterial = meshRenderer.material; meshRenderer.materials = new[] { baseMaterial, baseMaterial }; MaterialPropertyBlock mpb = new MaterialPropertyBlock(); mpb.SetFloat("float", 1f); for (int i = 0; i < 2; i++) { meshRenderer.SetPropertyBlock(mpb, i); } AddTestObjectForCleanup(plane); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SemanticSegmentationPass_WithChangingLabeling_ProducesCorrectValues([Values(true, false)] bool showVisualizations) { int timesSegmentationImageReceived = 0; var expectedPixelValue = k_SemanticPixelValue; void OnSegmentationImageReceived(NativeArray data) { if (timesSegmentationImageReceived == 1) { CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data); } timesSegmentationImageReceived++; } var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false); var plane = TestHelper.CreateLabeledPlane(label: "non-matching"); AddTestObjectForCleanup(plane); yield return null; var labeling = plane.GetComponent(); labeling.labels = new List { "label" }; labeling.RefreshLabeling(); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(2, timesSegmentationImageReceived); } [UnityTest] public IEnumerator InstanceSegmentationPass_WithSeparateDisabledPerceptionCamera_ProducesCorrectValues() { int timesSegmentationImageReceived = 0; void OnSegmentationImageReceived(NativeArray data) { CollectionAssert.AreEqual(Enumerable.Repeat(k_InstanceSegmentationPixelValue, data.Length), data); timesSegmentationImageReceived++; } var cameraObject = SetupCameraInstanceSegmentation((frame, data, renderTexture) => OnSegmentationImageReceived(data)); var cameraObject2 = SetupCameraInstanceSegmentation(null); cameraObject2.SetActive(false); var plane = TestHelper.CreateLabeledPlane(); AddTestObjectForCleanup(plane); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); DestroyTestObject(cameraObject2); Assert.AreEqual(1, timesSegmentationImageReceived); } [UnityTest] public IEnumerator SegmentationPassProducesCorrectValuesEachFrame( [Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind) { int timesSegmentationImageReceived = 0; Dictionary expectedLabelAtFrame = null; //TestHelper.LoadAndStartRenderDocCapture(out var gameView); void OnSegmentationImageReceived(int frameCount, NativeArray data, RenderTexture tex) where T : struct { if (expectedLabelAtFrame == null || !expectedLabelAtFrame.ContainsKey(frameCount)) return; timesSegmentationImageReceived++; Debug.Log($"Segmentation image received. FrameCount: {frameCount}"); try { CollectionAssert.AreEqual(Enumerable.Repeat(expectedLabelAtFrame[frameCount], data.Length), data.ToArray()); } // ReSharper disable once RedundantCatchClause catch (Exception) { //uncomment to get RenderDoc captures while this check is failing //UnityEditorInternal.RenderDoc.EndCaptureRenderDoc(gameView); throw; } } var cameraObject = segmentationKind == SegmentationKind.Instance ? SetupCameraInstanceSegmentation(OnSegmentationImageReceived) : SetupCameraSemanticSegmentation((a) => OnSegmentationImageReceived(a.frameCount, a.data, a.sourceTexture), false); //object expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) new Color32(0, 74, 255, 255) : k_SemanticPixelValue; var expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) k_InstanceSegmentationPixelValue : k_SemanticPixelValue; expectedLabelAtFrame = new Dictionary { {Time.frameCount , expectedPixelValue}, {Time.frameCount + 1, expectedPixelValue}, {Time.frameCount + 2, expectedPixelValue} }; GameObject planeObject; //Put a plane in front of the camera planeObject = TestHelper.CreateLabeledPlane(); yield return null; //UnityEditorInternal.RenderDoc.EndCaptureRenderDoc(gameView); Object.DestroyImmediate(planeObject); planeObject = TestHelper.CreateLabeledPlane(); //TestHelper.LoadAndStartRenderDocCapture(out gameView); yield return null; //UnityEditorInternal.RenderDoc.EndCaptureRenderDoc(gameView); Object.DestroyImmediate(planeObject); planeObject = TestHelper.CreateLabeledPlane(); yield return null; Object.DestroyImmediate(planeObject); yield return null; //destroy the object to force all pending segmented image readbacks to finish and events to be fired. DestroyTestObject(cameraObject); Assert.AreEqual(3, timesSegmentationImageReceived); } GameObject SetupCameraInstanceSegmentation(Action, RenderTexture> onSegmentationImageReceived) { var cameraObject = SetupCamera(out var perceptionCamera, false); perceptionCamera.InstanceSegmentationImageReadback += onSegmentationImageReceived; cameraObject.SetActive(true); return cameraObject; } GameObject SetupCameraSemanticSegmentation(Action onSegmentationImageReceived, bool showVisualizations, Color? backgroundColor = null) { var cameraObject = SetupCamera(out var perceptionCamera, showVisualizations); var labelConfig = ScriptableObject.CreateInstance(); labelConfig.Init(new List() { new SemanticSegmentationLabelEntry() { label = "label", color = k_SemanticPixelValue } }); if (backgroundColor != null) { labelConfig.skyColor = backgroundColor.Value; } var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig); semanticSegmentationLabeler.imageReadback += onSegmentationImageReceived; perceptionCamera.AddLabeler(semanticSegmentationLabeler); cameraObject.SetActive(true); return cameraObject; } GameObject SetupCamera(out PerceptionCamera perceptionCamera, bool showVisualizations) { var cameraObject = new GameObject(); cameraObject.SetActive(false); var camera = cameraObject.AddComponent(); camera.orthographic = true; camera.orthographicSize = 1; perceptionCamera = cameraObject.AddComponent(); perceptionCamera.captureRgbImages = false; perceptionCamera.showVisualizations = showVisualizations; AddTestObjectForCleanup(cameraObject); return cameraObject; } } }