using UnityEngine; namespace Unity.MLAgents.Sensors { /// /// Sensor class that wraps a [RenderTexture](https://docs.unity3d.com/ScriptReference/RenderTexture.html) instance. /// public class RenderTextureSensor : ISensor, IBuiltInSensor { RenderTexture m_RenderTexture; bool m_Grayscale; string m_Name; private ObservationSpec m_ObservationSpec; SensorCompressionType m_CompressionType; /// /// The compression type used by the sensor. /// public SensorCompressionType CompressionType { get { return m_CompressionType; } set { m_CompressionType = value; } } /// /// Initializes the sensor. /// /// The [RenderTexture](https://docs.unity3d.com/ScriptReference/RenderTexture.html) /// instance to wrap. /// Whether to convert it to grayscale or not. /// Name of the sensor. /// Compression method for the render texture. /// [GameObject]: https://docs.unity3d.com/Manual/GameObjects.html public RenderTextureSensor( RenderTexture renderTexture, bool grayscale, string name, SensorCompressionType compressionType) { m_RenderTexture = renderTexture; var width = renderTexture != null ? renderTexture.width : 0; var height = renderTexture != null ? renderTexture.height : 0; m_Grayscale = grayscale; m_Name = name; m_ObservationSpec = ObservationSpec.Visual(height, width, grayscale ? 1 : 3); m_CompressionType = compressionType; } /// public string GetName() { return m_Name; } /// public ObservationSpec GetObservationSpec() { return m_ObservationSpec; } /// public byte[] GetCompressedObservation() { using (TimerStack.Instance.Scoped("RenderTextureSensor.GetCompressedObservation")) { var texture = ObservationToTexture(m_RenderTexture); // TODO support more types here, e.g. JPG var compressed = texture.EncodeToPNG(); DestroyTexture(texture); return compressed; } } /// public int Write(ObservationWriter writer) { using (TimerStack.Instance.Scoped("RenderTextureSensor.Write")) { var texture = ObservationToTexture(m_RenderTexture); var numWritten = writer.WriteTexture(texture, m_Grayscale); DestroyTexture(texture); return numWritten; } } /// public void Update() { } /// public void Reset() { } /// public CompressionSpec GetCompressionSpec() { return new CompressionSpec(m_CompressionType); } /// public BuiltInSensorType GetBuiltInSensorType() { return BuiltInSensorType.RenderTextureSensor; } /// /// Converts a RenderTexture to a 2D texture. /// /// The 2D texture. /// RenderTexture. /// Texture2D to render to. public static Texture2D ObservationToTexture(RenderTexture obsTexture) { var height = obsTexture.height; var width = obsTexture.width; var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false); var prevActiveRt = RenderTexture.active; RenderTexture.active = obsTexture; texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0); texture2D.Apply(); RenderTexture.active = prevActiveRt; return texture2D; } static void DestroyTexture(Texture2D texture) { if (Application.isEditor) { // Edit Mode tests complain if we use Destroy() // TODO move to extension methods for UnityEngine.Object? Object.DestroyImmediate(texture); } else { Object.Destroy(texture); } } } }