using UnityEngine; using UnityEngine.Rendering; namespace Unity.MLAgents.Sensors { /// /// A sensor that wraps a Camera object to generate visual observations for an agent. /// public class CameraSensor : ISensor, IBuiltInSensor { Camera m_Camera; int m_Width; int m_Height; bool m_Grayscale; string m_Name; private ObservationSpec m_ObservationSpec; SensorCompressionType m_CompressionType; /// /// The Camera used for rendering the sensor observations. /// public Camera Camera { get { return m_Camera; } set { m_Camera = value; } } /// /// The compression type used by the sensor. /// public SensorCompressionType CompressionType { get { return m_CompressionType; } set { m_CompressionType = value; } } /// /// Creates and returns the camera sensor. /// /// Camera object to capture images from. /// The width of the generated visual observation. /// The height of the generated visual observation. /// Whether to convert the generated image to grayscale or keep color. /// The name of the camera sensor. /// The compression to apply to the generated image. public CameraSensor( Camera camera, int width, int height, bool grayscale, string name, SensorCompressionType compression) { m_Camera = camera; m_Width = width; m_Height = height; m_Grayscale = grayscale; m_Name = name; var channels = grayscale ? 1 : 3; m_ObservationSpec = ObservationSpec.Visual(height, width, channels); m_CompressionType = compression; } /// /// Accessor for the name of the sensor. /// /// Sensor name. public string GetName() { return m_Name; } /// public ObservationSpec GetObservationSpec() { return m_ObservationSpec; } /// /// Generates a compressed image. This can be valuable in speeding-up training. /// /// Compressed image. public byte[] GetCompressedObservation() { using (TimerStack.Instance.Scoped("CameraSensor.GetCompressedObservation")) { var texture = ObservationToTexture(m_Camera, m_Width, m_Height); // TODO support more types here, e.g. JPG var compressed = texture.EncodeToPNG(); DestroyTexture(texture); return compressed; } } /// /// Writes out the generated, uncompressed image to the provided . /// /// Where the observation is written to. /// public int Write(ObservationWriter writer) { using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor")) { var texture = ObservationToTexture(m_Camera, m_Width, m_Height); var numWritten = writer.WriteTexture(texture, m_Grayscale); DestroyTexture(texture); return numWritten; } } /// public void Update() { } /// public void Reset() { } /// public SensorCompressionType GetCompressionType() { return m_CompressionType; } /// /// Renders a Camera instance to a 2D texture at the corresponding resolution. /// /// The 2D texture. /// Camera. /// Width of resulting 2D texture. /// Height of resulting 2D texture. /// Texture2D to render to. public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height) { if (SystemInfo.graphicsDeviceType == GraphicsDeviceType.Null) { Debug.LogError("GraphicsDeviceType is Null. This will likely crash when trying to render."); } var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false); var oldRec = obsCamera.rect; obsCamera.rect = new Rect(0f, 0f, 1f, 1f); var depth = 24; var format = RenderTextureFormat.Default; var readWrite = RenderTextureReadWrite.Default; var tempRt = RenderTexture.GetTemporary(width, height, depth, format, readWrite); var prevActiveRt = RenderTexture.active; var prevCameraRt = obsCamera.targetTexture; // render to offscreen texture (readonly from CPU side) RenderTexture.active = tempRt; obsCamera.targetTexture = tempRt; obsCamera.Render(); texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0); obsCamera.targetTexture = prevCameraRt; obsCamera.rect = oldRec; RenderTexture.active = prevActiveRt; RenderTexture.ReleaseTemporary(tempRt); return texture2D; } /// /// Computes the observation shape for a camera sensor based on the height, width /// and grayscale flag. /// /// Width of the image captures from the camera. /// Height of the image captures from the camera. /// Whether or not to convert the image to grayscale. /// The observation shape. internal static int[] GenerateShape(int width, int height, bool grayscale) { return new[] { height, width, grayscale ? 1 : 3 }; } static void DestroyTexture(Texture2D texture) { if (Application.isEditor) { // Edit Mode tests complain if we use Destroy() // TODO move to extension methods for UnityEngine.Object? Object.DestroyImmediate(texture); } else { Object.Destroy(texture); } } /// public BuiltInSensorType GetBuiltInSensorType() { return BuiltInSensorType.CameraSensor; } } }