using System;
using MLAgents.InferenceBrain;
using UnityEngine;
namespace MLAgents.Sensor
{
public class CameraSensor : ISensor
{
private Camera m_Camera;
private int m_Width;
private int m_Height;
private bool m_Grayscale;
private string m_Name;
private int[] m_Shape;
public CameraSensor(Camera camera, int width, int height, bool grayscale, string name)
{
m_Camera = camera;
m_Width = width;
m_Height = height;
m_Grayscale = grayscale;
m_Name = name;
m_Shape = new[] { width, height, grayscale ? 1 : 3 };
}
public string GetName()
{
return m_Name;
}
public int[] GetFloatObservationShape()
{
return m_Shape;
}
public byte[] GetCompressedObservation()
{
using (TimerStack.Instance.Scoped("CameraSensor.GetCompressedObservation"))
{
var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
// TODO support more types here, e.g. JPG
var compressed = texture.EncodeToPNG();
UnityEngine.Object.Destroy(texture);
return compressed;
}
}
public void WriteToTensor(TensorProxy tensorProxy, int agentIndex)
{
using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor"))
{
var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
Utilities.TextureToTensorProxy(texture, tensorProxy, m_Grayscale, agentIndex);
UnityEngine.Object.Destroy(texture);
}
}
public CompressionType GetCompressionType()
{
return CompressionType.PNG;
}
///
/// Converts a m_Camera and corresponding resolution to a 2D texture.
///
/// The 2D texture.
/// Camera.
/// Width of resulting 2D texture.
/// Height of resulting 2D texture.
/// Texture2D to render to.
public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height)
{
var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
var oldRec = obsCamera.rect;
obsCamera.rect = new Rect(0f, 0f, 1f, 1f);
var depth = 24;
var format = RenderTextureFormat.Default;
var readWrite = RenderTextureReadWrite.Default;
var tempRt =
RenderTexture.GetTemporary(width, height, depth, format, readWrite);
var prevActiveRt = RenderTexture.active;
var prevCameraRt = obsCamera.targetTexture;
// render to offscreen texture (readonly from CPU side)
RenderTexture.active = tempRt;
obsCamera.targetTexture = tempRt;
obsCamera.Render();
texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
obsCamera.targetTexture = prevCameraRt;
obsCamera.rect = oldRec;
RenderTexture.active = prevActiveRt;
RenderTexture.ReleaseTemporary(tempRt);
return texture2D;
}
}
}