using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using JetBrains.Annotations; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using Unity.Entities; using Unity.Profiling; using Unity.Simulation; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Profiling; using UnityEngine.Rendering; using UnityEngine.Serialization; #if HDRP_PRESENT using UnityEngine.Rendering.HighDefinition; #endif #if URP_PRESENT using UnityEngine.Rendering.Universal; #endif namespace UnityEngine.Perception.GroundTruth { /// /// Captures ground truth from the associated Camera. /// [RequireComponent(typeof(Camera))] public class PerceptionCamera : MonoBehaviour { //TODO: Remove the Guid path when we have proper dataset merging in USim/Thea internal static string RgbDirectory { get; } = $"RGB{Guid.NewGuid()}"; static string s_RgbFilePrefix = "rgb_"; /// /// A human-readable description of the camera. /// public string description; /// /// The period in seconds that the Camera should render /// public float period = .0166f; /// /// The start time in seconds of the first frame in the simulation. /// public float startTime; /// /// Whether camera output should be captured to disk /// public bool captureRgbImages = true; /// /// Event invoked after the camera finishes rendering during a frame. /// [SerializeReference] public List labelers = new List(); Dictionary m_PersistentSensorData = new Dictionary(); #if URP_PRESENT internal List passes = new List(); public void AddScriptableRenderPass(ScriptableRenderPass pass) { passes.Add(pass); } #endif bool m_CapturedLastFrame; Ego m_EgoMarker; /// /// The associated with this camera. Use this to report additional annotations and metrics at runtime. /// public SensorHandle SensorHandle { get; private set; } struct AsyncCaptureInfo { public int FrameCount; public AsyncAnnotation SegmentationAsyncAnnotation; public AsyncMetric ClassCountAsyncMetric; public AsyncMetric RenderedObjectInfoAsyncMetric; public AsyncAnnotation BoundingBoxAsyncMetric; } List m_AsyncCaptureInfos = new List(); static ProfilerMarker s_WriteFrame = new ProfilerMarker("Write Frame (PerceptionCamera)"); static ProfilerMarker s_FlipY = new ProfilerMarker("Flip Y (PerceptionCamera)"); static ProfilerMarker s_EncodeAndSave = new ProfilerMarker("Encode and save (PerceptionCamera)"); /// /// Add a data object which will be added to the dataset with each capture. Overrides existing sensor data associated with the given key. /// /// The key to associate with the data. /// An object containing the data. Will be serialized into json. public void SetPersistentSensorData(string key, object data) { m_PersistentSensorData[key] = data; } /// /// Removes a persistent sensor data object. /// /// The key of the object to remove. /// True if a data object was removed. False if it was not set. public bool RemovePersistentSensorData(string key) { return m_PersistentSensorData.Remove(key); } // Start is called before the first frame update void Awake() { m_EgoMarker = this.GetComponentInParent(); var ego = m_EgoMarker == null ? SimulationManager.RegisterEgo("") : m_EgoMarker.EgoHandle; SensorHandle = SimulationManager.RegisterSensor(ego, "camera", description, period, startTime); var myCamera = GetComponent(); var width = myCamera.pixelWidth; var height = myCamera.pixelHeight; RenderPipelineManager.beginCameraRendering += OnBeginCameraRendering; SimulationManager.SimulationEnding += OnSimulationEnding; } // Update is called once per frame void Update() { if (!SensorHandle.IsValid) return; var cam = GetComponent(); cam.enabled = SensorHandle.ShouldCaptureThisFrame; } void CaptureRgbData(Camera cam) { Profiler.BeginSample("CaptureDataFromLastFrame"); if (!captureRgbImages) return; var captureFilename = Path.Combine(Manager.Instance.GetDirectoryFor(RgbDirectory), $"{s_RgbFilePrefix}{Time.frameCount}.png"); var dxRootPath = Path.Combine(RgbDirectory, $"{s_RgbFilePrefix}{Time.frameCount}.png"); SensorHandle.ReportCapture(dxRootPath, SensorSpatialData.FromGameObjects(m_EgoMarker == null ? null : m_EgoMarker.gameObject, gameObject), m_PersistentSensorData.Select(kvp => (kvp.Key, kvp.Value)).ToArray()); Func, AsyncRequest.Result> colorFunctor = null; var width = cam.pixelWidth; var height = cam.pixelHeight; var flipY = ShouldFlipY(cam); colorFunctor = r => { using (s_WriteFrame.Auto()) { var dataColorBuffer = (byte[])r.data.colorBuffer; if (flipY) FlipImageY(dataColorBuffer, height); byte[] encodedData; using (s_EncodeAndSave.Auto()) { encodedData = ImageConversion.EncodeArrayToPNG(dataColorBuffer, GraphicsFormat.R8G8B8A8_UNorm, (uint)width, (uint)height); } return !FileProducer.Write(captureFilename, encodedData) ? AsyncRequest.Result.Error : AsyncRequest.Result.Completed; } }; CaptureCamera.Capture(cam, colorFunctor); Profiler.EndSample(); } // ReSharper disable once ParameterHidesMember bool ShouldFlipY(Camera camera) { #if HDRP_PRESENT var hdAdditionalCameraData = GetComponent(); //Based on logic in HDRenderPipeline.PrepareFinalBlitParameters return camera.targetTexture != null || hdAdditionalCameraData.flipYMode == HDAdditionalCameraData.FlipYMode.ForceFlipY || camera.cameraType == CameraType.Game; #elif URP_PRESENT return (SystemInfo.graphicsDeviceType == GraphicsDeviceType.Direct3D11 || SystemInfo.graphicsDeviceType == GraphicsDeviceType.Metal) && (camera.targetTexture != null || camera.cameraType == CameraType.Game); #else return false; #endif } static unsafe void FlipImageY(byte[] dataColorBuffer, int height) { using (s_FlipY.Auto()) { var stride = dataColorBuffer.Length / height; var buffer = new NativeArray(stride, Allocator.TempJob, NativeArrayOptions.UninitializedMemory); fixed(byte* colorBufferPtr = &dataColorBuffer[0]) { var unsafePtr = (byte*)buffer.GetUnsafePtr(); for (var row = 0; row < height / 2; row++) { var nearRowStartPtr = colorBufferPtr + stride * row; var oppositeRowStartPtr = colorBufferPtr + stride * (height - row - 1); UnsafeUtility.MemCpy(unsafePtr, oppositeRowStartPtr, stride); UnsafeUtility.MemCpy(oppositeRowStartPtr, nearRowStartPtr, stride); UnsafeUtility.MemCpy(nearRowStartPtr, unsafePtr, stride); } } buffer.Dispose(); } } void OnSimulationEnding() { RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering; } void OnBeginCameraRendering(ScriptableRenderContext _, Camera cam) { if (cam != GetComponent()) return; #if UNITY_EDITOR if (UnityEditor.EditorApplication.isPaused) return; #endif BeginRendering?.Invoke(); CaptureRgbData(cam); } void OnDisable() { SimulationManager.SimulationEnding -= OnSimulationEnding; OnSimulationEnding(); if (SensorHandle.IsValid) SensorHandle.Dispose(); SensorHandle = default; } } [Serializable] public abstract class CameraLabeler { public bool enabled; public bool foldout; } }