External Contribution: Use RenderTexture instead of Camera for Visual Observation (#1824)

* Added RenderTexture support for visual observations * Cleaned up new ObservationToTexture function * Added check for to width/height of RenderTexture * Added check to hide HelpBox unless both cameras and RenderTextures are used * Added documentation for Visual Observations using RenderTextures * Added GridWorldRenderTexture Example scene * Adjusted image size of doc images * Added GridWorld example reference * Fixed missing reference in the GridWorldRenderTexture scene and resaved the agent prefab * Fix prefab instantiation and render timing in GridWorldRenderTexture * Added screenshot and reworded documentation * Unchecked control box * Rename renderTexture * Make RenderTexture scene default for GridWorld Co-authored-by: Mads Johansen <pyjamads@gmail.com>
6 年前 · 6f8fc130
--- a/UnitySDK/Assets/ML-Agents/Editor/AgentEditor.cs
+++ b/UnitySDK/Assets/ML-Agents/Editor/AgentEditor.cs
                "agentParameters.onDemandDecision");
            SerializedProperty cameras = serializedAgent.FindProperty(
                "agentParameters.agentCameras");
+            SerializedProperty renderTextures = serializedAgent.FindProperty(
+                "agentParameters.agentRenderTextures");
+            if (cameras.arraySize > 0 && renderTextures.arraySize > 0)
+            {
+                EditorGUILayout.HelpBox("Brain visual observations created by first getting all cameras then all render textures.", MessageType.Info);    
+            }
+            
            EditorGUILayout.LabelField("Agent Cameras");
            for (int i = 0; i < cameras.arraySize; i++)
            {
            }

            EditorGUILayout.EndHorizontal();
+            
+            EditorGUILayout.LabelField("Agent RenderTextures");
+            for (int i = 0; i < renderTextures.arraySize; i++)
+            {
+                EditorGUILayout.PropertyField(
+                    renderTextures.GetArrayElementAtIndex(i),
+                    new GUIContent("RenderTexture " + (i + 1).ToString() + ": "));
+            }
+
+            EditorGUILayout.BeginHorizontal();
+            if (GUILayout.Button("Add RenderTextures", EditorStyles.miniButton))
+            {
+                renderTextures.arraySize++;
+            }
+
+            if (GUILayout.Button("Remove RenderTextures", EditorStyles.miniButton))
+            {
+                renderTextures.arraySize--;
+            }
+
+            EditorGUILayout.EndHorizontal();
+

            EditorGUILayout.PropertyField(
                maxSteps,
--- a/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity
+++ b/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity
  m_ReflectionIntensity: 1
  m_CustomReflection: {fileID: 0}
  m_Sun: {fileID: 0}
-  m_IndirectSpecularColor: {r: 0.43667555, g: 0.4842717, b: 0.56452394, a: 1}
+  m_IndirectSpecularColor: {r: 0.43668893, g: 0.4842832, b: 0.56452656, a: 1}
 --- !u!157 &3
 LightmapSettings:
  m_ObjectHideFlags: 0
      value: 1
  actorObjs: []
  players: 
-  trueAgent: {fileID: 2008405822}
+  trueAgent: {fileID: 858783320}
-  camObject: {fileID: 231883441}
+  camObject: {fileID: 99095112}
  agentPref: {fileID: 1657514749044530, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
  goalPref: {fileID: 1508142483324970, guid: 1ec4e4e96e7514d45b7ebc3ba5a9a481, type: 2}
  pitPref: {fileID: 1811317785436014, guid: d13ee2db77b3a4dcc8664d2fe2a0f219, type: 2}
  m_Father: {fileID: 0}
  m_RootOrder: 1
  m_LocalEulerAnglesHint: {x: 90, y: 0, z: 0}
--- !u!1 &231883441
+--- !u!1 &99095112
 GameObject:
  m_ObjectHideFlags: 0
  m_PrefabParentObject: {fileID: 0}
-  - component: {fileID: 231883447}
-  - component: {fileID: 231883446}
-  - component: {fileID: 231883445}
-  - component: {fileID: 231883444}
+  - component: {fileID: 99095116}
+  - component: {fileID: 99095115}
+  - component: {fileID: 99095114}
+  - component: {fileID: 99095113}
  m_Layer: 0
  m_Name: Main Camera
  m_TagString: MainCamera
  m_IsActive: 1
--- !u!124 &231883444
+--- !u!124 &99095113
-  m_GameObject: {fileID: 231883441}
+  m_GameObject: {fileID: 99095112}
--- !u!92 &231883445
+--- !u!92 &99095114
-  m_GameObject: {fileID: 231883441}
+  m_GameObject: {fileID: 99095112}
--- !u!20 &231883446
+--- !u!20 &99095115
-  m_GameObject: {fileID: 231883441}
+  m_GameObject: {fileID: 99095112}
  m_Enabled: 1
  serializedVersion: 2
  m_ClearFlags: 2
  m_OcclusionCulling: 1
  m_StereoConvergence: 10
  m_StereoSeparation: 0.022
--- !u!4 &231883447
+--- !u!4 &99095116
-  m_GameObject: {fileID: 231883441}
+  m_GameObject: {fileID: 99095112}
  m_LocalRotation: {x: 0.35355338, y: 0.35355338, z: -0.1464466, w: 0.8535535}
  m_LocalPosition: {x: -5, y: 8.2, z: -5}
  m_LocalScale: {x: 1, y: 1, z: 1}
  m_LocalScale: {x: 0, y: 0, z: 0}
  m_Children:
  - {fileID: 1553342943}
+  - {fileID: 918893359}
+  - {fileID: 1305247360}
  m_Father: {fileID: 0}
  m_RootOrder: 4
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
    serializedVersion: 2
    m_Bits: 4294967295
  m_RenderingPath: -1
-  m_TargetTexture: {fileID: 0}
+  m_TargetTexture: {fileID: 8400000, guid: 114608d5384404f89bff4b6f88432958, type: 2}
  m_TargetDisplay: 0
  m_TargetEye: 3
  m_HDR: 0
  m_Father: {fileID: 0}
  m_RootOrder: 3
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &858783320 stripped
+GameObject:
+  m_PrefabParentObject: {fileID: 1657514749044530, guid: 628960e910f094ad1909ecc88cc8016d,
+    type: 2}
+  m_PrefabInternal: {fileID: 1477921759}
+--- !u!1 &918893358
+GameObject:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 5
+  m_Component:
+  - component: {fileID: 918893359}
+  - component: {fileID: 918893361}
+  - component: {fileID: 918893360}
+  m_Layer: 5
+  m_Name: Text
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &918893359
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 918893358}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 363761400}
+  m_RootOrder: 1
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0.5, y: 0.5}
+  m_AnchorMax: {x: 0.5, y: 0.5}
+  m_AnchoredPosition: {x: -369.5, y: -62.2}
+  m_SizeDelta: {x: 160, y: 55.6}
+  m_Pivot: {x: 0.5, y: 0.5}
+--- !u!114 &918893360
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 918893358}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 708705254, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1}
+  m_RaycastTarget: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+    m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI,
+      Version=1.0.0.0, Culture=neutral, PublicKeyToken=null
+  m_FontData:
+    m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0}
+    m_FontSize: 20
+    m_FontStyle: 0
+    m_BestFit: 0
+    m_MinSize: 2
+    m_MaxSize: 40
+    m_Alignment: 7
+    m_AlignByGeometry: 0
+    m_RichText: 1
+    m_HorizontalOverflow: 0
+    m_VerticalOverflow: 0
+    m_LineSpacing: 1
+  m_Text: 'Agent View
+
+    (RenderTexture)'
+--- !u!222 &918893361
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 918893358}
 --- !u!1 &959566328
 GameObject:
  m_ObjectHideFlags: 0
  m_Father: {fileID: 486401524}
  m_RootOrder: 0
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &1305247359
+GameObject:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 5
+  m_Component:
+  - component: {fileID: 1305247360}
+  - component: {fileID: 1305247362}
+  - component: {fileID: 1305247361}
+  m_Layer: 5
+  m_Name: RawImage
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &1305247360
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1305247359}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 363761400}
+  m_RootOrder: 2
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0.5, y: 0.5}
+  m_AnchorMax: {x: 0.5, y: 0.5}
+  m_AnchoredPosition: {x: -369.5, y: -197}
+  m_SizeDelta: {x: 200, y: 200}
+  m_Pivot: {x: 0.5, y: 0.5}
+--- !u!114 &1305247361
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1305247359}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: -98529514, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 1, g: 1, b: 1, a: 1}
+  m_RaycastTarget: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+    m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI,
+      Version=1.0.0.0, Culture=neutral, PublicKeyToken=null
+  m_Texture: {fileID: 8400000, guid: 114608d5384404f89bff4b6f88432958, type: 2}
+  m_UVRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+--- !u!222 &1305247362
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1305247359}
+--- !u!1001 &1477921759
+Prefab:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  m_Modification:
+    m_TransformParent: {fileID: 0}
+    m_Modifications:
+    - target: {fileID: 114143683117020968, guid: 628960e910f094ad1909ecc88cc8016d,
+        type: 2}
+      propertyPath: agentParameters.agentCameras.Array.size
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 114143683117020968, guid: 628960e910f094ad1909ecc88cc8016d,
+        type: 2}
+      propertyPath: agentParameters.agentRenderTextures.Array.size
+      value: 1
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalPosition.x
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalPosition.y
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalPosition.z
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalRotation.x
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalRotation.y
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalRotation.z
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_LocalRotation.w
+      value: 1
+      objectReference: {fileID: 0}
+    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_RootOrder
+      value: 7
+      objectReference: {fileID: 0}
+    - target: {fileID: 1657514749044530, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+      propertyPath: m_Name
+      value: trueAgent
+      objectReference: {fileID: 0}
+    - target: {fileID: 114143683117020968, guid: 628960e910f094ad1909ecc88cc8016d,
+        type: 2}
+      propertyPath: agentParameters.agentRenderTextures.Array.data[0]
+      value: 
+      objectReference: {fileID: 8400000, guid: 114608d5384404f89bff4b6f88432958, type: 2}
+    - target: {fileID: 114143683117020968, guid: 628960e910f094ad1909ecc88cc8016d,
+        type: 2}
+      propertyPath: renderCamera
+      value: 
+      objectReference: {fileID: 489340228}
+    m_RemovedComponents: []
+  m_ParentPrefab: {fileID: 100100000, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
+  m_IsPrefabParent: 0
 --- !u!1 &1553342942
 GameObject:
  m_ObjectHideFlags: 0
  m_Father: {fileID: 486401524}
  m_RootOrder: 3
  m_LocalEulerAnglesHint: {x: 0, y: 90, z: 0}
--- !u!1001 &2008405821
-Prefab:
-  m_ObjectHideFlags: 0
-  serializedVersion: 2
-  m_Modification:
-    m_TransformParent: {fileID: 0}
-    m_Modifications:
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalPosition.x
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalPosition.y
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalPosition.z
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalRotation.x
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalRotation.y
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalRotation.z
-      value: 0
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_LocalRotation.w
-      value: 1
-      objectReference: {fileID: 0}
-    - target: {fileID: 4587181662868872, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_RootOrder
-      value: 7
-      objectReference: {fileID: 0}
-    - target: {fileID: 1657514749044530, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-      propertyPath: m_Name
-      value: trueAgent
-      objectReference: {fileID: 0}
-    - target: {fileID: 114143683117020968, guid: 628960e910f094ad1909ecc88cc8016d,
-        type: 2}
-      propertyPath: agentParameters.agentCameras.Array.data[0]
-      value: 
-      objectReference: {fileID: 489340228}
-    m_RemovedComponents: []
-  m_ParentPrefab: {fileID: 100100000, guid: 628960e910f094ad1909ecc88cc8016d, type: 2}
-  m_IsPrefabParent: 0
--- !u!1 &2008405822 stripped
-GameObject:
-  m_PrefabParentObject: {fileID: 1657514749044530, guid: 628960e910f094ad1909ecc88cc8016d,
-    type: 2}
-  m_PrefabInternal: {fileID: 2008405821}
--- a/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity.meta
+++ b/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity.meta
 fileFormatVersion: 2
-guid: 7c777442467e245108558a5155153927
-timeCreated: 1502222754
-licenseType: Free
+guid: c6fdbb551d80c435abe6fc911b1ea11f
+  externalObjects: {}
  userData: 
  assetBundleName: 
  assetBundleVariant: 
--- a/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs
+++ b/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs
    public float timeBetweenDecisionsAtInference;
    private float timeSinceDecision;

+    [Tooltip("Because we want an observation right before making a decision, we can force " + 
+             "a camera to render before making a decision. Place the agentCam here if using " +
+             "RenderTexture as observations.")]
+    public Camera renderCamera;
+
    [Tooltip("Selecting will turn on action masking. Note that a model trained with action " +
             "masking turned on may not behave optimally when action masking is turned off.")]
    public bool maskActions = true;

    private void WaitTimeInference()
    {
+        if(renderCamera != null)
+        {
+            renderCamera.Render();
+        }
+
        if (!academy.GetIsInference())
        {
            RequestDecision();
--- a/UnitySDK/Assets/ML-Agents/Scripts/Agent.cs
+++ b/UnitySDK/Assets/ML-Agents/Scripts/Agent.cs
        /// observations.
        /// </summary>
        public List<Camera> agentCameras = new List<Camera>();
+        
+        /// <summary>
+        /// The list of the RenderTextures the agent uses for visual
+        /// observations.
+        /// </summary>
+        public List<RenderTexture> agentRenderTextures = new List<RenderTexture>();
+

        /// <summary>
        /// The maximum number of steps the agent takes before being done. 
        /// becomes enabled or active.
        void OnEnable()
        {
-            textureArray = new Texture2D[agentParameters.agentCameras.Count];
-            for (int i = 0; i < agentParameters.agentCameras.Count; i++)
+            var textureCount = agentParameters.agentCameras.Count+agentParameters.agentRenderTextures.Count;
+            textureArray = new Texture2D[textureCount];
+            for (int i = 0; i < textureCount; i++)
            {
                textureArray[i] = new Texture2D(1, 1, TextureFormat.RGB24, false);
            }
            info.stackedVectorObservation.AddRange(info.vectorObservation);

            info.visualObservations.Clear();
-            if (param.cameraResolutions.Length > agentParameters.agentCameras.Count)
+            var visualObservationCount = agentParameters.agentCameras.Count+agentParameters.agentRenderTextures.Count;
+            if (param.cameraResolutions.Length > visualObservationCount)
-                    "Not enough cameras for agent {0} : Bain {1} expecting at " +
-                    "least {2} cameras but only {3} were present.",
+                    "Not enough cameras/renderTextures for agent {0} : Brain {1} expecting at " +
+                    "least {2} cameras/renderTextures but only {3} were present.",
-                    agentParameters.agentCameras.Count));
+                    visualObservationCount));
-            for (int i = 0; i < brain.brainParameters.cameraResolutions.Length; i++)
+            //First add all cameras
+            for (int i = 0; i < agentParameters.agentCameras.Count; i++)
            {
                ObservationToTexture(
                    agentParameters.agentCameras[i],
                info.visualObservations.Add(textureArray[i]);
            }
+            
+            //Then add all renderTextures
+            var camCount = agentParameters.agentCameras.Count;
+            for (int i = 0; i < agentParameters.agentRenderTextures.Count; i++)
+            {
+                ObservationToTexture(
+                    agentParameters.agentRenderTextures[i],
+                    param.cameraResolutions[camCount+i].width,
+                    param.cameraResolutions[camCount+i].height,
+                    ref textureArray[i]);
+                info.visualObservations.Add(textureArray[i]);
+            }

            info.reward = reward;
            info.done = done;
            obsCamera.rect = oldRec;
            RenderTexture.active = prevActiveRT;
            RenderTexture.ReleaseTemporary(tempRT);
+        }
+        
+        /// <summary>
+        /// Converts a RenderTexture and correspinding resolution to a 2D texture.
+        /// </summary>
+        /// <returns>The 2D texture.</returns>
+        /// <param name="obsTexture">RenderTexture.</param>
+        /// <param name="width">Width of resulting 2D texture.</param>
+        /// <param name="height">Height of resulting 2D texture.</param>
+        /// <param name="texture2D">Texture2D to render to.</param>
+        public static void ObservationToTexture(RenderTexture obsTexture, int width, int height, ref Texture2D texture2D)
+        {
+            if (width != texture2D.width || height != texture2D.height)
+            {
+                texture2D.Resize(width, height);
+            }
+            
+            if(width != obsTexture.width || height != obsTexture.height)
+            {
+                throw new UnityAgentsException(string.Format(
+                    "RenderTexture {0} : width/height is {1}/{2} brain is expecting {3}/{4}.",
+                    obsTexture.name, obsTexture.width, obsTexture.height, width, height));
+            }
+
+            var prevActiveRT = RenderTexture.active;
+            RenderTexture.active = obsTexture;
+
+            texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
+            texture2D.Apply();
+            RenderTexture.active = prevActiveRT;
        }

        /// <summary>
--- a/docs/Learning-Environment-Design-Agents.md
+++ b/docs/Learning-Environment-Design-Agents.md

 * **Vector Observation** — a feature vector consisting of an array of floating
  point numbers.
-* **Visual Observations** — one or more camera images.
+* **Visual Observations** — one or more camera images and/or render textures.
-**Visual Observations**, you only need to identify which Unity Camera objects
-will provide images and the base Agent class handles the rest. You do not need
-to implement the `CollectObservations()` method when your Agent uses visual
-observations (unless it also uses vector observations).
+**Visual Observations**, you only need to identify which Unity Camera objects 
+or RenderTextures will provide images and the base Agent class handles the rest. 
+You do not need to implement the `CollectObservations()` method when your Agent 
+uses visual observations (unless it also uses vector observations).

 ### Vector Observation Space: Feature Vectors


 ### Multiple Visual Observations

-Camera observations use rendered textures from one or more cameras in a scene.
-The Brain vectorizes the textures into a 3D Tensor which can be fed into a
-convolutional neural network (CNN). For more information on CNNs, see [this
-guide](http://cs231n.github.io/convolutional-networks/). You can use camera
-observations along side vector observations.
+Visual observations use rendered textures directly or from one or more 
+cameras in a scene. The Brain vectorizes the textures into a 3D Tensor which 
+can be fed into a convolutional neural network (CNN). For more information on 
+CNNs, see [this guide](http://cs231n.github.io/convolutional-networks/). You 
+can use visual observations along side vector observations.
-Agents using camera images can capture state of arbitrary complexity and are
-useful when the state is difficult to describe numerically. However, they are
-also typically less efficient and slower to train, and sometimes don't succeed
-at all.  
+Agents using visual observations can capture state of arbitrary complexity and 
+are useful when the state is difficult to describe numerically. However, they 
+are also typically less efficient and slower to train, and sometimes don't 
+succeed at all.
-To add a visual observation to an Agent, click on the `Add Camera` button in the
-Agent inspector. Then drag the camera you want to add to the `Camera` field. You
-can have more than one camera attached to an Agent.
+Visual observations can be derived from Cameras or RenderTextures within your scene. 
+To add a visual observation to an Agent, either click on the `Add Camera` or 
+`Add RenderTexture` button in the Agent inspector. Then drag the camera or 
+render texture you want to add to the `Camera` or `RenderTexture` field. 
+You can have more than one camera or render texture and even use a combination 
+of both attached to an Agent.
+
+or
+
+![Agent RenderTexture](images/visual-observation-rendertexture.png)
-specify the number of Cameras the Agent is using for its visual observations.
+specify the number of Resolutions the Agent is using for its visual observations.
-is checked).
+is checked). 
+
+For instance, if you are using two cameras and one render texture on your Agent,
+three **Visual Observations** have to be added to the **Brain Parameters**. 
+During runtime, if a combination of `Cameras` and `RenderTextures` is used, all 
+cameras are captured first, then all render textures will be added, in the
+order they appear in the editor. 
+
+![Agent Camera and RenderTexture combination](images/visual-observation-combination.png)
+
+RenderTexture observations will throw an `Exception` if the width/height doesn't 
+match the resolution specified under **Brain Parameters** > **Visual Observations**.
+
+When using `RenderTexture` visual observations, a handy feature for debugging is 
+adding a `Canvas`, then adding a `Raw Image` with it's texture set to the Agent's 
+`RenderTexture`. This will render the agent observation on the game screen. 
+
+![RenderTexture with Raw Image](images/visual-observation-rawimage.png)
+
+The [GridWorld environment](Learning-Environment-Examples.md#gridworld) 
+is an example on how to use a RenderTexure for both debugging and observation. Note 
+that in this example, a Camera is rendered to a RenderTexture, which is then used for 
+observations and debugging. To update the RenderTexture, the Camera must be asked to 
+render every time a decision is requested within the game code. When using Cameras 
+as observations directly, this is done automatically by the Agent.
+
+![Agent RenderTexture Debug](images/visual-observation-debug.png)

 ## Vector Actions


 * `Brain` - The Brain to register this Agent to. Can be dragged into the
  inspector using the Editor.
-* `Visual Observations` - A list of `Cameras` which will be used to generate
-  observations.
+* `Visual Observations` - A list of `Cameras` or `RenderTextures` which will 
+  be used to generate observations.
 * `Max Step` - The per-agent maximum number of steps. Once this number is
  reached, the Agent will be reset if `Reset On Done` is checked.
 * `Reset On Done` - Whether the Agent's `AgentReset()` function should be called
--- a/docs/images/agent.png
+++ b/docs/images/agent.png
--- a/docs/images/visual-observation.png
+++ b/docs/images/visual-observation.png
--- a/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Resources/agentRenderTexture.renderTexture
+++ b/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Resources/agentRenderTexture.renderTexture
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!84 &8400000
+RenderTexture:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_Name: camRenderTexture
+  m_ImageContentsHash:
+    serializedVersion: 2
+    Hash: 00000000000000000000000000000000
+  m_ForcedFallbackFormat: 4
+  m_DownscaleFallback: 0
+  m_Width: 84
+  m_Height: 84
+  m_AntiAliasing: 1
+  m_DepthFormat: 1
+  m_ColorFormat: 0
+  m_MipMap: 0
+  m_GenerateMips: 0
+  m_SRGB: 0
+  m_UseDynamicScale: 0
+  m_BindMS: 0
+  m_TextureSettings:
+    serializedVersion: 2
+    m_FilterMode: 0
+    m_Aniso: 0
+    m_MipBias: 0
+    m_WrapU: 1
+    m_WrapV: 1
+    m_WrapW: 1
+  m_Dimension: 2
+  m_VolumeDepth: 1
--- a/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Resources/agentRenderTexture.renderTexture.meta
+++ b/UnitySDK/Assets/ML-Agents/Examples/GridWorld/Resources/agentRenderTexture.renderTexture.meta
+fileFormatVersion: 2
+guid: 114608d5384404f89bff4b6f88432958
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/docs/images/visual-observation-combination.png
+++ b/docs/images/visual-observation-combination.png
--- a/docs/images/visual-observation-debug.png
+++ b/docs/images/visual-observation-debug.png
--- a/docs/images/visual-observation-rawimage.png
+++ b/docs/images/visual-observation-rawimage.png
--- a/docs/images/visual-observation-rendertexture.png
+++ b/docs/images/visual-observation-rendertexture.png