using System; using UnityEngine; using UnityEngine.Rendering; namespace Unity.MLAgents.Sensors { /// /// A sensor that wraps a Camera object to generate visual observations for an agent. /// public class CameraSensor : ISensor, IBuiltInSensor, IDisposable { Camera m_Camera; int m_Width; int m_Height; bool m_Grayscale; string m_Name; private ObservationSpec m_ObservationSpec; SensorCompressionType m_CompressionType; Texture2D m_Texture; /// /// The Camera used for rendering the sensor observations. /// public Camera Camera { get { return m_Camera; } set { m_Camera = value; } } /// /// The compression type used by the sensor. /// public SensorCompressionType CompressionType { get { return m_CompressionType; } set { m_CompressionType = value; } } /// /// Creates and returns the camera sensor. /// /// Camera object to capture images from. /// The width of the generated visual observation. /// The height of the generated visual observation. /// Whether to convert the generated image to grayscale or keep color. /// The name of the camera sensor. /// The compression to apply to the generated image. /// The type of observation. public CameraSensor( Camera camera, int width, int height, bool grayscale, string name, SensorCompressionType compression, ObservationType observationType = ObservationType.Default) { m_Camera = camera; m_Width = width; m_Height = height; m_Grayscale = grayscale; m_Name = name; var channels = grayscale ? 1 : 3; m_ObservationSpec = ObservationSpec.Visual(height, width, channels, observationType); m_CompressionType = compression; m_Texture = new Texture2D(width, height, TextureFormat.RGB24, false); } /// /// Accessor for the name of the sensor. /// /// Sensor name. public string GetName() { return m_Name; } /// /// Returns a description of the observations that will be generated by the sensor. /// The shape will be h x w x 1 for grayscale and h x w x 3 for color. /// The dimensions have translational equivariance along width and height, /// and no property along the channels dimension. /// /// public ObservationSpec GetObservationSpec() { return m_ObservationSpec; } /// /// Generates a compressed image. This can be valuable in speeding-up training. /// /// Compressed image. public byte[] GetCompressedObservation() { using (TimerStack.Instance.Scoped("CameraSensor.GetCompressedObservation")) { ObservationToTexture(m_Camera, m_Texture, m_Width, m_Height); // TODO support more types here, e.g. JPG var compressed = m_Texture.EncodeToPNG(); return compressed; } } /// /// Writes out the generated, uncompressed image to the provided . /// /// Where the observation is written to. /// public int Write(ObservationWriter writer) { using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor")) { ObservationToTexture(m_Camera, m_Texture, m_Width, m_Height); var numWritten = writer.WriteTexture(m_Texture, m_Grayscale); return numWritten; } } /// public void Update() { } /// public void Reset() { } /// public CompressionSpec GetCompressionSpec() { return new CompressionSpec(m_CompressionType); } /// /// Renders a Camera instance to a 2D texture at the corresponding resolution. /// /// Camera. /// Texture2D to render to. /// Width of resulting 2D texture. /// Height of resulting 2D texture. public static void ObservationToTexture(Camera obsCamera, Texture2D texture2D, int width, int height) { if (SystemInfo.graphicsDeviceType == GraphicsDeviceType.Null) { Debug.LogError("GraphicsDeviceType is Null. This will likely crash when trying to render."); } var oldRec = obsCamera.rect; obsCamera.rect = new Rect(0f, 0f, 1f, 1f); var depth = 24; var format = RenderTextureFormat.Default; var readWrite = RenderTextureReadWrite.Default; var tempRt = RenderTexture.GetTemporary(width, height, depth, format, readWrite); var prevActiveRt = RenderTexture.active; var prevCameraRt = obsCamera.targetTexture; // render to offscreen texture (readonly from CPU side) RenderTexture.active = tempRt; obsCamera.targetTexture = tempRt; obsCamera.Render(); texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0); obsCamera.targetTexture = prevCameraRt; obsCamera.rect = oldRec; RenderTexture.active = prevActiveRt; RenderTexture.ReleaseTemporary(tempRt); } /// public BuiltInSensorType GetBuiltInSensorType() { return BuiltInSensorType.CameraSensor; } /// /// Clean up the owned Texture2D. /// public void Dispose() { if (!ReferenceEquals(null, m_Texture)) { Utilities.DestroyTexture(m_Texture); m_Texture = null; } } } }