[WIP] ISensor interface and use for visual observations (#2731)

* ISensor and SensorBase

* camera and rendertex first pass

* use isensors for visual obs

* Update gridworld with CameraSensors

* compressed obs for reals

* Remove AgentInfo.visualObservations

* better separation of train and inference sensor calls

* compressed obs proto - need CI to generate code

* int32

* get proto name right

* run protoc locally for new fiels

* apply generated proto patch (pyi files were weird)

* don't repeat bytes

* hook up compressedobs

* dont send BrainParameters until there's an AgentInfo

* python BrainParameters now needs an AgentInfo to create

* remove last (I hope) dependency on camerares

* remove CameraResolutions and AgentInfo.visual_observations

* update mypy-protobuf version

* cleanup todos

* python cleanup

* more unit test fixes

* more unit test fix

* camera sensors for VisualFood collector, record demo

* SensorCompon...

UnitySDK/Assets/ML-Agents/Editor/AgentEditor.cs

                 "agentParameters.resetOnDone");
             var isOdd = serializedAgent.FindProperty(
                 "agentParameters.onDemandDecision");
-            var cameras = serializedAgent.FindProperty(
-                "agentParameters.agentCameras");
-            var renderTextures = serializedAgent.FindProperty(
-                "agentParameters.agentRenderTextures");
-
-            if (cameras.arraySize > 0 && renderTextures.arraySize > 0)
-            {
-                EditorGUILayout.HelpBox("Brain visual observations created by first getting all cameras then all render textures.", MessageType.Info);
-            }
-
-            EditorGUILayout.LabelField("Agent Cameras");
-            for (var i = 0; i < cameras.arraySize; i++)
-            {
-                EditorGUILayout.PropertyField(
-                    cameras.GetArrayElementAtIndex(i),
-                    new GUIContent("Camera " + (i + 1) + ": "));
-            }
-
-            EditorGUILayout.BeginHorizontal();
-            if (GUILayout.Button("Add Camera", EditorStyles.miniButton))
-            {
-                cameras.arraySize++;
-            }
-
-            if (GUILayout.Button("Remove Camera", EditorStyles.miniButton))
-            {
-                cameras.arraySize--;
-            }
-
-            EditorGUILayout.EndHorizontal();
-
-            EditorGUILayout.LabelField("Agent RenderTextures");
-            for (var i = 0; i < renderTextures.arraySize; i++)
-            {
-                EditorGUILayout.PropertyField(
-                    renderTextures.GetArrayElementAtIndex(i),
-                    new GUIContent("RenderTexture " + (i + 1) + ": "));
-            }
-
-            EditorGUILayout.BeginHorizontal();
-            if (GUILayout.Button("Add RenderTextures", EditorStyles.miniButton))
-            {
-                renderTextures.arraySize++;
-            }
-
-            if (GUILayout.Button("Remove RenderTextures", EditorStyles.miniButton))
-            {
-                renderTextures.arraySize--;
-            }
-
-            EditorGUILayout.EndHorizontal();
 
 
             EditorGUILayout.PropertyField(

UnitySDK/Assets/ML-Agents/Editor/BrainParametersDrawer.cs

         // The height of a line in the Unity Inspectors
         private const float k_LineHeight = 17f;
         private const int k_VecObsNumLine = 3;
-        private const string k_CamResPropName = "cameraResolutions";
-        private const string k_CamWidthPropName = "width";
-        private const string k_CamHeightPropName = "height";
-        private const string k_CamGrayPropName = "blackAndWhite";
-        private const int k_DefaultCameraWidth = 84;
-        private const int k_DefaultCameraHeight = 84;
-        private const bool k_DefaultCameraGray = false;
 
         /// <inheritdoc />
         public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
                 return k_LineHeight +
                     GetHeightDrawVectorObservation() +
-                    GetHeightDrawVisualObservation(property) +
                     GetHeightDrawVectorAction(property) +
                     GetHeightDrawVectorActionDescriptions(property);
             }
                 DrawVectorObservation(position, property);
                 position.y += GetHeightDrawVectorObservation();
 
-                //Visual Observations
-                DrawVisualObservations(position, property);
-                position.y += GetHeightDrawVisualObservation(property);
-
                 // Vector Action
                 DrawVectorAction(position, property);
                 position.y += GetHeightDrawVectorAction(property);
         private static float GetHeightDrawVectorObservation()
         {
             return k_VecObsNumLine * k_LineHeight;
-        }
-
-        /// <summary>
-        /// Draws the Visual Observations parameters for the Brain Parameters
-        /// </summary>
-        /// <param name="position">Rectangle on the screen to use for the property GUI.</param>
-        /// <param name="property">The SerializedProperty of the BrainParameters
-        /// to make the custom GUI for.</param>
-        private static void DrawVisualObservations(Rect position, SerializedProperty property)
-        {
-            EditorGUI.LabelField(position, "Visual Observations");
-            position.y += k_LineHeight;
-            var quarter = position.width / 4;
-            var resolutions = property.FindPropertyRelative(k_CamResPropName);
-            DrawVisualObsButtons(position, resolutions);
-            position.y += k_LineHeight;
-
-            // Display the labels for the columns : Index, Width, Height and Gray
-            var indexRect = new Rect(position.x, position.y, quarter, position.height);
-            var widthRect = new Rect(position.x + quarter, position.y, quarter, position.height);
-            var heightRect = new Rect(position.x + 2 * quarter, position.y, quarter, position.height);
-            var bwRect = new Rect(position.x + 3 * quarter, position.y, quarter, position.height);
-            EditorGUI.indentLevel++;
-            if (resolutions.arraySize > 0)
-            {
-                EditorGUI.LabelField(indexRect, "Index");
-                indexRect.y += k_LineHeight;
-                EditorGUI.LabelField(widthRect, "Width");
-                widthRect.y += k_LineHeight;
-                EditorGUI.LabelField(heightRect, "Height");
-                heightRect.y += k_LineHeight;
-                EditorGUI.LabelField(bwRect, "Gray");
-                bwRect.y += k_LineHeight;
-            }
-
-            // Iterate over the resolutions
-            for (var i = 0; i < resolutions.arraySize; i++)
-            {
-                EditorGUI.LabelField(indexRect, "Obs " + i);
-                indexRect.y += k_LineHeight;
-                var res = resolutions.GetArrayElementAtIndex(i);
-                var w = res.FindPropertyRelative("width");
-                w.intValue = EditorGUI.IntField(widthRect, w.intValue);
-                widthRect.y += k_LineHeight;
-                var h = res.FindPropertyRelative("height");
-                h.intValue = EditorGUI.IntField(heightRect, h.intValue);
-                heightRect.y += k_LineHeight;
-                var bw = res.FindPropertyRelative("blackAndWhite");
-                bw.boolValue = EditorGUI.Toggle(bwRect, bw.boolValue);
-                bwRect.y += k_LineHeight;
-            }
-            EditorGUI.indentLevel--;
-        }
-
-        /// <summary>
-        /// Draws the buttons to add and remove the visual observations parameters
-        /// </summary>
-        /// <param name="position">Rectangle on the screen to use for the property GUI.</param>
-        /// <param name="resolutions">The SerializedProperty of the resolution array
-        /// to make the custom GUI for.</param>
-        private static void DrawVisualObsButtons(Rect position, SerializedProperty resolutions)
-        {
-            var widthEighth = position.width / 8;
-            var addButtonRect = new Rect(position.x + widthEighth, position.y,
-                3 * widthEighth, position.height);
-            var removeButtonRect = new Rect(position.x + 4 * widthEighth, position.y,
-                3 * widthEighth, position.height);
-            if (resolutions.arraySize == 0)
-            {
-                addButtonRect.width *= 2;
-            }
-            // Display the buttons
-            if (GUI.Button(addButtonRect, "Add New", EditorStyles.miniButton))
-            {
-                resolutions.arraySize += 1;
-                var newRes = resolutions.GetArrayElementAtIndex(resolutions.arraySize - 1);
-                newRes.FindPropertyRelative(k_CamWidthPropName).intValue = k_DefaultCameraWidth;
-                newRes.FindPropertyRelative(k_CamHeightPropName).intValue = k_DefaultCameraHeight;
-                newRes.FindPropertyRelative(k_CamGrayPropName).boolValue = k_DefaultCameraGray;
-            }
-            if (resolutions.arraySize > 0)
-            {
-                if (GUI.Button(removeButtonRect, "Remove Last", EditorStyles.miniButton))
-                {
-                    resolutions.arraySize -= 1;
-                }
-            }
-        }
-
-        /// <summary>
-        /// The Height required to draw the Visual Observations parameters
-        /// </summary>
-        /// <returns>The height of the drawer of the Visual Observations </returns>
-        private static float GetHeightDrawVisualObservation(SerializedProperty property)
-        {
-            var visObsSize = property.FindPropertyRelative(k_CamResPropName).arraySize + 2;
-            if (property.FindPropertyRelative(k_CamResPropName).arraySize > 0)
-            {
-                visObsSize += 1;
-            }
-            return k_LineHeight * visObsSize;
         }
 
         /// <summary>

UnitySDK/Assets/ML-Agents/Editor/DemonstrationDrawer.cs

         return actionLabel.ToString();
     }
 
-    /// <summary>
-    /// Constructs complex label for each CameraResolution object.
-    /// An example of this could be `[ 84 X 84 ]`
-    /// for a single camera with 84 pixels height and width.
-    /// </summary>
-    private static string BuildCameraResolutionLabel(SerializedProperty cameraArray)
-    {
-        var numCameras = cameraArray.arraySize;
-        var cameraLabel = new StringBuilder("[ ");
-        for (var i = 0; i < numCameras; i++)
-        {
-            var camHeightPropName =
-                cameraArray.GetArrayElementAtIndex(i).FindPropertyRelative("height");
-            cameraLabel.Append(camHeightPropName.intValue);
-            cameraLabel.Append(" X ");
-            var camWidthPropName =
-                cameraArray.GetArrayElementAtIndex(i).FindPropertyRelative("width");
-            cameraLabel.Append(camWidthPropName.intValue);
-            if (i < numCameras - 1)
-            {
-                cameraLabel.Append(", ");
-            }
-        }
-
-        cameraLabel.Append(" ]");
-        return cameraLabel.ToString();
-    }
 
     /// <summary>
     /// Renders Inspector UI for Brain Parameters of Demonstration.
         var vecObsSizeProp = property.FindPropertyRelative("vectorObservationSize");
         var numStackedProp = property.FindPropertyRelative("numStackedVectorObservations");
         var actSizeProperty = property.FindPropertyRelative("vectorActionSize");
-        var camResProp = property.FindPropertyRelative("cameraResolutions");
         var actSpaceTypeProp = property.FindPropertyRelative("vectorActionSpaceType");
 
         var vecObsSizeLabel = vecObsSizeProp.displayName + ": " + vecObsSizeProp.intValue;
-        var camResLabel = camResProp.displayName + ": " + BuildCameraResolutionLabel(camResProp);
         var actSpaceTypeLabel = actSpaceTypeProp.displayName + ": " +
             (SpaceType)actSpaceTypeProp.enumValueIndex;
 
-        EditorGUILayout.LabelField(camResLabel);
         EditorGUILayout.LabelField(actSpaceTypeLabel);
     }
 

UnitySDK/Assets/ML-Agents/Editor/Tests/DemonstrationTests.cs

             {
                 vectorObservationSize = 3,
                 numStackedVectorObservations = 2,
-                cameraResolutions = new[] {new Resolution()},
                 vectorActionDescriptions = new[] {"TestActionA", "TestActionB"},
                 vectorActionSize = new[] {2, 2},
                 vectorActionSpaceType = SpaceType.Discrete
             var agentInfo = new AgentInfo
             {
                 reward = 1f,
-                visualObservations = new List<Texture2D>(),
                 actionMasks = new[] {false, true},
                 done = true,
                 id = 5,

UnitySDK/Assets/ML-Agents/Editor/Tests/MLAgentsEditModeTest.cs

 using UnityEngine;
 using NUnit.Framework;
 using System.Reflection;
+using MLAgents.Sensor;
+using MLAgents.InferenceBrain;
 
 namespace MLAgents.Tests
 {
         public override void InitializeAgent()
         {
             initializeAgentCalls += 1;
+
+            // Add in some custom sensors so we can confirm they get sorted as expected.
+            var sensor1 = new TestSensor("testsensor1");
+            var sensor2 = new TestSensor("testsensor2");
+
+            m_Sensors.Add(sensor2);
+            m_Sensors.Add(sensor1);
         }
 
         public override void CollectObservations()
         }
     }
 
+    public class TestSensor : ISensor
+    {
+        public string sensorName;
+
+        public TestSensor(string n)
+        {
+            sensorName = n;
+        }
+
+        public int[] GetFloatObservationShape() {
+            return new[] {1};
+        }
+
+        public void WriteToTensor(TensorProxy tensorProxy, int agentIndex) { }
+
+        public byte[] GetCompressedObservation()
+        {
+            return null;
+        }
+
+        public CompressionType GetCompressionType()
+        {
+            return CompressionType.None;
+        }
+
+        public string GetName()
+        {
+            return sensorName;
+        }
+    }
+
     // This is an empty class for testing the behavior of agents and academy
     // It is left empty because we are not testing any brain behavior
     public class TestBrain : Brain
             Assert.AreEqual(1, agent2.initializeAgentCalls);
             Assert.AreEqual(0, agent1.agentActionCalls);
             Assert.AreEqual(0, agent2.agentActionCalls);
+
+            // Make sure the sensors were sorted
+            Assert.AreEqual(agent1.m_Sensors[0].GetName(), "testsensor1");
+            Assert.AreEqual(agent1.m_Sensors[1].GetName(), "testsensor2");
         }
     }
 
             agent2.agentParameters.onDemandDecision = true;
             // agent2 will request decisions only when RequestDecision is called
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
             agent2.agentParameters.onDemandDecision = true;
             // agent2 will request decisions only when RequestDecision is called
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
             agent1.agentParameters.resetOnDone = false;
             agent2.agentParameters.resetOnDone = false;
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
             // agent2 will request decisions only when RequestDecision is called
             agent1.agentParameters.maxStep = 20;
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 

UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DDecision.cs

 using System.Collections.Generic;
 using UnityEngine;
 using MLAgents;
+using MLAgents.Sensor;
 
 public class Ball3DDecision : Decision
 {
         List<float> vectorObs,
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)
 
     public override List<float> MakeMemory(
         List<float> vectorObs,
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)

UnitySDK/Assets/ML-Agents/Examples/Basic/Scripts/BasicDecision.cs

-using System.Collections.Generic;
+using System.Collections.Generic;
+using MLAgents.Sensor;
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)
 
     public override List<float> MakeMemory(
         List<float> vectorObs,
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)

UnitySDK/Assets/ML-Agents/Examples/FoodCollector/Prefabs/VisualFoodCollectorArea.prefab

   - component: {fileID: 65932430430952094}
   - component: {fileID: 54338553804475180}
   - component: {fileID: 114080992426104682}
+  - component: {fileID: 114741414993459624}
   m_Layer: 0
   m_Name: Agent (3)
   m_TagString: agent
   - component: {fileID: 65178766753981896}
   - component: {fileID: 54088750604404324}
   - component: {fileID: 114257846618384398}
+  - component: {fileID: 114708558732651708}
   m_Layer: 0
   m_Name: Agent
   m_TagString: agent
   - component: {fileID: 65441205843344002}
   - component: {fileID: 54753735395303746}
   - component: {fileID: 114625025344622832}
+  - component: {fileID: 114401766178763868}
   m_Layer: 0
   m_Name: Agent (1)
   m_TagString: agent
   - component: {fileID: 65540878552782854}
   - component: {fileID: 54626777054134360}
   - component: {fileID: 114855044884614820}
+  - component: {fileID: 114778458791730518}
   m_Layer: 0
   m_Name: Agent (2)
   m_TagString: agent
   myLaser: {fileID: 1580603984718180}
   contribute: 0
   useVectorObs: 0
+--- !u!114 &114401766178763868
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1661627863960872}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 41cb6bf4b09974bf583f5b2fef0c08a7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!114 &114552799170750468
 MonoBehaviour:
   m_ObjectHideFlags: 1
   myLaser: {fileID: 1335947341879056}
   contribute: 0
   useVectorObs: 0
+--- !u!114 &114708558732651708
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1443673828307814}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 41cb6bf4b09974bf583f5b2fef0c08a7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  width: 84
+  height: 84
+  grayscale: 0
+--- !u!114 &114741414993459624
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1169383371125934}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 41cb6bf4b09974bf583f5b2fef0c08a7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  width: 84
+  height: 84
+  grayscale: 0
+--- !u!114 &114778458791730518
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1964761937961574}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 41cb6bf4b09974bf583f5b2fef0c08a7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!114 &114855044884614820
 MonoBehaviour:
   m_ObjectHideFlags: 1

UnitySDK/Assets/ML-Agents/Examples/FoodCollector/Scenes/VisualFoodCollector.unity

       propertyPath: m_Materials.Array.data[0]
       value: 
       objectReference: {fileID: 2100000, guid: 580f2003972f64189826f085e2498080, type: 3}
+    - target: {fileID: 114708558732651708, guid: c85b585836e104587b4efdc4d8b9d62b,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1081822021}
+    - target: {fileID: 114401766178763868, guid: c85b585836e104587b4efdc4d8b9d62b,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1081822020}
+    - target: {fileID: 114778458791730518, guid: c85b585836e104587b4efdc4d8b9d62b,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1081822019}
+    - target: {fileID: 114741414993459624, guid: c85b585836e104587b4efdc4d8b9d62b,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1081822018}
+--- !u!20 &1081822018 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20825339371991526, guid: c85b585836e104587b4efdc4d8b9d62b,
+    type: 2}
+  m_PrefabInternal: {fileID: 1081822017}
+--- !u!20 &1081822019 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20416796192002010, guid: c85b585836e104587b4efdc4d8b9d62b,
+    type: 2}
+  m_PrefabInternal: {fileID: 1081822017}
+--- !u!20 &1081822020 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20184829288622090, guid: c85b585836e104587b4efdc4d8b9d62b,
+    type: 2}
+  m_PrefabInternal: {fileID: 1081822017}
+--- !u!20 &1081822021 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20890573870796418, guid: c85b585836e104587b4efdc4d8b9d62b,
+    type: 2}
+  m_PrefabInternal: {fileID: 1081822017}
 --- !u!1 &1086444495
 GameObject:
   m_ObjectHideFlags: 0
   m_Script: {fileID: 11500000, guid: 4fe57113e76a5426297487dd6faadc5b, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  broadcastHub:
-    brainsToControl:
-    - {fileID: 11400000, guid: 24e823594179d48189b2c78003c50ce0, type: 2}
   m_TrainingConfiguration:
     width: 500
     height: 500

UnitySDK/Assets/ML-Agents/Examples/GridWorld/Prefabs/Area.prefab

   - component: {fileID: 33823710649932060}
   - component: {fileID: 65073501172061214}
   - component: {fileID: 114650561397225712}
+  - component: {fileID: 114399612900818788}
   m_Layer: 8
   m_Name: Agent
   m_TagString: agent
   serializedVersion: 2
   m_Size: {x: 1, y: 1, z: 1}
   m_Center: {x: 0, y: 0, z: 0}
+--- !u!114 &114399612900818788
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1488387672112076}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 282f342c2ab144bf38be65d4d0c4e07d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 20743940359151984}
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!114 &114650561397225712
 MonoBehaviour:
   m_ObjectHideFlags: 1

UnitySDK/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity

   m_ReflectionIntensity: 1
   m_CustomReflection: {fileID: 0}
   m_Sun: {fileID: 0}
-  m_IndirectSpecularColor: {r: 0.44971162, g: 0.49977726, b: 0.5756362, a: 1}
+  m_IndirectSpecularColor: {r: 0.4497121, g: 0.49977785, b: 0.57563704, a: 1}
 --- !u!157 &3
 LightmapSettings:
   m_ObjectHideFlags: 0
   - component: {fileID: 125487789}
   - component: {fileID: 125487788}
   - component: {fileID: 125487787}
+  - component: {fileID: 125487790}
   m_Layer: 8
   m_Name: Agent
   m_TagString: agent
   m_EditorClassIdentifier: 
   brain: {fileID: 11400000, guid: 2c1d51b7167874f31beda0b0cf0af468, type: 2}
   agentParameters:
-    agentCameras: []
-    agentRenderTextures:
-    - {fileID: 8400000, guid: 114608d5384404f89bff4b6f88432958, type: 2}
-  m_Area: {fileID: 1795599557}
+  area: {fileID: 1795599557}
   timeBetweenDecisionsAtInference: 0.15
   renderCamera: {fileID: 797520692}
   maskActions: 1
   m_PrefabInternal: {fileID: 0}
   m_GameObject: {fileID: 125487785}
   m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
+--- !u!114 &125487790
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 125487785}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 132e1194facb64429b007ea1edf562d0, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  renderTexture: {fileID: 8400000, guid: 114608d5384404f89bff4b6f88432958, type: 2}
+  sensorName: RenderTextureSensor
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!1 &260425459
 GameObject:
   m_ObjectHideFlags: 0

UnitySDK/Assets/ML-Agents/Examples/Hallway/Prefabs/VisualSymbolFinderArea.prefab

   - component: {fileID: 65678389736547598}
   - component: {fileID: 54606255118850520}
   - component: {fileID: 114451776683649118}
+  - component: {fileID: 114516857402348526}
   m_Layer: 0
   m_Name: Agent
   m_TagString: agent
   m_EditorClassIdentifier: 
   brain: {fileID: 11400000, guid: a36aad05c06144991a0a5e87de40d003, type: 2}
   agentParameters:
-    agentCameras:
-    - {fileID: 20961984019151212}
-    agentRenderTextures: []
     maxStep: 3000
     resetOnDone: 1
     onDemandDecision: 0
   symbolO: {fileID: 1945451279398622}
   symbolX: {fileID: 1162012575508818}
   useVectorObs: 0
+--- !u!114 &114516857402348526
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1234267001558658}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 282f342c2ab144bf38be65d4d0c4e07d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  sensorName: CameraSensor
+  width: 84
+  height: 84
+  grayscale: 0

UnitySDK/Assets/ML-Agents/Examples/Hallway/Scenes/VisualHallway.unity

       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 32612447}
+--- !u!20 &32612447 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 32612446}
 --- !u!1 &255077123
 GameObject:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 341018563}
+--- !u!20 &341018563 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 341018562}
 --- !u!1 &365376270
 GameObject:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 721234460}
+--- !u!20 &721234460 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 721234459}
 --- !u!1001 &977960505
 Prefab:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 977960506}
+--- !u!20 &977960506 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 977960505}
 --- !u!1 &1257687048
 GameObject:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1353330548}
+--- !u!20 &1353330548 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 1353330547}
 --- !u!1001 &1388008248
 Prefab:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1388008249}
+--- !u!20 &1388008249 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 1388008248}
 --- !u!1001 &1436760868
 Prefab:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1436760869}
+--- !u!20 &1436760869 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 1436760868}
 --- !u!1 &1574236047
 GameObject:
   m_ObjectHideFlags: 0
   m_Script: {fileID: 11500000, guid: 40db664a3061b46a0a0628f90b2264f7, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  broadcastHub:
-    brainsToControl:
-    - {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2, type: 2}
   m_TrainingConfiguration:
     width: 128
     height: 128
       value: 
       objectReference: {fileID: 11400000, guid: fe56dd72ed38a4c2fb5419aba1e2d5f2,
         type: 2}
+    - target: {fileID: 114516857402348526, guid: f2281a3adc3e640b490f89407c2e12d1,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1746153507}
+--- !u!20 &1746153507 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20309822448307506, guid: f2281a3adc3e640b490f89407c2e12d1,
+    type: 2}
+  m_PrefabInternal: {fileID: 1746153506}
 --- !u!1001 &2025898844
 Prefab:
   m_ObjectHideFlags: 0

UnitySDK/Assets/ML-Agents/Examples/PushBlock/Prefabs/PushBlockVisualArea.prefab

   - component: {fileID: 114812843792483960}
   - component: {fileID: 114650520402303970}
   - component: {fileID: 65891831092422300}
+  - component: {fileID: 114024228081418500}
   m_Layer: 0
   m_Name: Agent
   m_TagString: agent
   serializedVersion: 2
   m_Size: {x: 1, y: 1, z: 1}
   m_Center: {x: 0, y: 0, z: 0}
+--- !u!114 &114024228081418500
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1626010291821672}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 282f342c2ab144bf38be65d4d0c4e07d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  sensorName: CameraSensor
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!114 &114650520402303970
 MonoBehaviour:
   m_ObjectHideFlags: 1
   m_EditorClassIdentifier: 
   brain: {fileID: 11400000, guid: cc62140bff6494e0399caaed0b56020d, type: 2}
   agentParameters:
-    agentCameras:
-    - {fileID: 20961401228419460}
-    agentRenderTextures: []
     maxStep: 5000
     resetOnDone: 1
     onDemandDecision: 0

UnitySDK/Assets/ML-Agents/Examples/PushBlock/Scenes/VisualPushBlock.unity

       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 116640260}
+--- !u!20 &116640260 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 116640259}
 --- !u!1 &255077123
 GameObject:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 731659952}
+--- !u!20 &731659952 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 731659951}
 --- !u!1 &762086410
 GameObject:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 912811241}
+--- !u!20 &912811241 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 912811240}
 --- !u!1 &1009000883
 GameObject:
   m_ObjectHideFlags: 0
   m_Script: {fileID: 11500000, guid: a2ca406dad5ec4ede8184998f4f9067d, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  broadcastHub:
-    brainsToControl:
-    - {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f, type: 2}
   m_TrainingConfiguration:
     width: 1280
     height: 720
       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1878756100}
+--- !u!20 &1878756100 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 1878756099}
 --- !u!1001 &1942601654
 Prefab:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1942601655}
+--- !u!20 &1942601655 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 1942601654}
 --- !u!1001 &1954420364
 Prefab:
   m_ObjectHideFlags: 0
       value: 
       objectReference: {fileID: 11400000, guid: d359d2290a825421e930c94284994e3f,
         type: 2}
+    - target: {fileID: 114024228081418500, guid: 9d9b85a2a80e74e5294bdfb248825335,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1954420365}
+--- !u!20 &1954420365 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20961401228419460, guid: 9d9b85a2a80e74e5294bdfb248825335,
+    type: 2}
+  m_PrefabInternal: {fileID: 1954420364}
 --- !u!1001 &1969300026
 Prefab:
   m_ObjectHideFlags: 0

UnitySDK/Assets/ML-Agents/Examples/Pyramids/Prefabs/VisualAreaPyramids.prefab

   - component: {fileID: 114741503533626942}
   - component: {fileID: 114027965503222182}
   - component: {fileID: 65882754134362954}
+  - component: {fileID: 114538851081060382}
   m_Layer: 0
   m_Name: Agent
   m_TagString: agent
   onMaterial: {fileID: 2100000, guid: c67450f290f3e4897bc40276a619e78d, type: 2}
   offMaterial: {fileID: 2100000, guid: 5c59bd36def9a4684be49a21cf201a2d, type: 2}
   myButton: {fileID: 1756074776765382}
+--- !u!114 &114538851081060382
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1736680821577442}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 282f342c2ab144bf38be65d4d0c4e07d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  camera: {fileID: 0}
+  sensorName: CameraSensor
+  width: 84
+  height: 84
+  grayscale: 0
 --- !u!114 &114659265491970150
 MonoBehaviour:
   m_ObjectHideFlags: 1
   m_EditorClassIdentifier: 
   brain: {fileID: 11400000, guid: 60f0ffcd08c3b43a6bdc746cfc0c4059, type: 2}
   agentParameters:
-    agentCameras:
-    - {fileID: 20712684238256298}
-    agentRenderTextures: []
     maxStep: 5000
     resetOnDone: 1
     onDemandDecision: 0

UnitySDK/Assets/ML-Agents/Examples/Pyramids/Scenes/VisualPyramids.unity

       propertyPath: m_RootOrder
       value: 5
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 133343957}
+--- !u!20 &133343957 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 133343956}
+--- !u!20 &177604012 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 309299717}
 --- !u!1001 &281839921
 Prefab:
   m_ObjectHideFlags: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 973199703}
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: 0567215293abe487b932aec366b57c8e, type: 2}
   m_IsPrefabParent: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 177604012}
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: 0567215293abe487b932aec366b57c8e, type: 2}
   m_IsPrefabParent: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1529303581}
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: 0567215293abe487b932aec366b57c8e, type: 2}
   m_IsPrefabParent: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 816767823}
+--- !u!20 &816767823 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 816767822}
+--- !u!20 &828837071 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 1728325040}
+--- !u!20 &973199703 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 281839921}
 --- !u!1 &1009000883
 GameObject:
   m_ObjectHideFlags: 0
   m_OcclusionCulling: 1
   m_StereoConvergence: 10
   m_StereoSeparation: 0.022
+--- !u!20 &1074152210 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 1818326666}
 --- !u!1001 &1155497957
 Prefab:
   m_ObjectHideFlags: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1194295937}
+--- !u!20 &1194295937 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 1155497957}
+--- !u!20 &1529303581 stripped
+Camera:
+  m_PrefabParentObject: {fileID: 20712684238256298, guid: 0567215293abe487b932aec366b57c8e,
+    type: 2}
+  m_PrefabInternal: {fileID: 714012435}
 --- !u!1 &1574236047
 GameObject:
   m_ObjectHideFlags: 0
   m_Script: {fileID: 11500000, guid: dba8df9c8b16946dc88d331a301d0ab3, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  broadcastHub:
-    brainsToControl:
-    - {fileID: 11400000, guid: 60f0ffcd08c3b43a6bdc746cfc0c4059, type: 2}
   m_TrainingConfiguration:
     width: 80
     height: 80
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 828837071}
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: 0567215293abe487b932aec366b57c8e, type: 2}
   m_IsPrefabParent: 0
       propertyPath: m_IsActive
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 114538851081060382, guid: 0567215293abe487b932aec366b57c8e,
+        type: 2}
+      propertyPath: camera
+      value: 
+      objectReference: {fileID: 1074152210}
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: 0567215293abe487b932aec366b57c8e, type: 2}
   m_IsPrefabParent: 0

UnitySDK/Assets/ML-Agents/Examples/Reacher/Scripts/ReacherDecision.cs

-using System.Collections.Generic;
+using System.Collections.Generic;
+using MLAgents.Sensor;
-    public override float[] Decide(List<float> state, List<Texture2D> observation, float reward, bool done, List<float> memory)
+    public override float[] Decide(List<float> state, List<ISensor> sensors, float reward, bool done, List<float> memory)
     {
         var action = new float[4];
         for (var i = 0; i < 4; i++)
         return action;
     }
 
-    public override List<float> MakeMemory(List<float> state, List<Texture2D> observation, float reward, bool done, List<float> memory)
+    public override List<float> MakeMemory(List<float> state, List<ISensor> sensors, float reward, bool done, List<float> memory)
     {
         return new List<float>();
     }

UnitySDK/Assets/ML-Agents/Examples/SharedAssets/Scripts/RandomDecision.cs

-using System.Collections.Generic;
+using System.Collections.Generic;
+using MLAgents.Sensor;
 using UnityEngine;
 
 namespace MLAgents
         public override float[] Decide(
             List<float> vectorObs,
-            List<Texture2D> visualObs,
+            List<ISensor> sensors,
             float reward,
             bool done,
             List<float> memory)
 
         public override List<float> MakeMemory(
             List<float> vectorObs,
-            List<Texture2D> visualObs,
+            List<ISensor> sensors,
             float reward,
             bool done,
             List<float> memory)

UnitySDK/Assets/ML-Agents/Examples/Template/Scripts/TemplateDecision.cs

-using System.Collections.Generic;
+using System.Collections.Generic;
+using MLAgents.Sensor;
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)
 
     public override List<float> MakeMemory(
         List<float> vectorObs,
-        List<Texture2D> visualObs,
+        List<ISensor> sensors,
         float reward,
         bool done,
         List<float> memory)

UnitySDK/Assets/ML-Agents/Scripts/Agent.cs

 using System.Collections.Generic;
 using UnityEngine;
+using MLAgents.Sensor;
 
 
 namespace MLAgents
         public List<float> stackedVectorObservation;
 
         /// <summary>
-        /// Most recent agent camera (i.e. texture) observation.
+        /// Most recent compressed observations.
-        public List<Texture2D> visualObservations;
+        public List<CompressedObservation> compressedObservations;
 
         /// <summary>
         /// Most recent text observation.
         /// TODO(cgoy): All references to protobuf objects should be removed.
         /// </summary>
         public CommunicatorObjects.CustomObservationProto customObservation;
-
-        /// <summary>
-        /// Remove the visual observations from memory. Call at each timestep
-        /// to avoid memory leaks.
-        /// </summary>
-        public void ClearVisualObs()
-        {
-            foreach (var obs in visualObservations)
-            {
-                Object.Destroy(obs);
-            }
-            visualObservations.Clear();
-        }
     }
 
     /// <summary>
     public class AgentParameters
     {
         /// <summary>
-        /// The list of the Camera GameObjects the agent uses for visual
-        /// observations.
-        /// </summary>
-        public List<Camera> agentCameras = new List<Camera>();
-
-        /// <summary>
-        /// The list of the RenderTextures the agent uses for visual
-        /// observations.
-        /// </summary>
-        public List<RenderTexture> agentRenderTextures = new List<RenderTexture>();
-
-
-        /// <summary>
         /// The maximum number of steps the agent takes before being done.
         /// </summary>
         /// <remarks>
         /// </summary>
         private DemonstrationRecorder m_Recorder;
 
+        public List<ISensor> m_Sensors;
+
         /// Monobehavior function that is called when the attached GameObject
         /// becomes enabled or active.
         void OnEnable()
         {
             m_Info = new AgentInfo();
             m_Action = new AgentAction();
+            m_Sensors = new List<ISensor>();
 
             if (academy == null)
             {
             }
 
             InitializeAgent();
+            InitializeSensors();
         }
 
         /// Monobehavior function that is called when the attached GameObject
                 new float[param.vectorObservationSize
                           * param.numStackedVectorObservations]);
 
-            m_Info.visualObservations = new List<Texture2D>();
+            m_Info.compressedObservations = new List<CompressedObservation>();
             m_Info.customObservation = null;
         }
 
         }
 
         /// <summary>
+        /// Set up the list of ISensors on the Agent. By default, this will select any
+        /// SensorBase's attached to the Agent.
+        /// </summary>
+        public void InitializeSensors()
+        {
+            var attachedSensorComponents = GetComponents<SensorComponent>();
+            m_Sensors.Capacity += attachedSensorComponents.Length;
+            foreach (var component in attachedSensorComponents)
+            {
+                m_Sensors.Add(component.CreateSensor());
+            }
+
+            // Sort the sensors by name to ensure determinism
+            m_Sensors.Sort((x, y) => x.GetName().CompareTo(y.GetName()));
+
+#if DEBUG
+            // Make sure the names are actually unique
+            for (var i = 0; i < m_Sensors.Count - 1; i++)
+            {
+                Debug.Assert(!m_Sensors[i].GetName().Equals(m_Sensors[i+1].GetName()), "Sensor names must be unique.");
+            }
+#endif
+        }
+
+        /// <summary>
         /// Sends the Agent info to the linked Brain.
         /// </summary>
         void SendInfoToBrain()
             m_Info.storedVectorActions = m_Action.vectorActions;
             m_Info.storedTextActions = m_Action.textActions;
             m_Info.vectorObservation.Clear();
+            m_Info.compressedObservations.Clear();
             m_ActionMasker.ResetMask();
             using (TimerStack.Instance.Scoped("CollectObservations"))
             {
             Utilities.ReplaceRange(m_Info.stackedVectorObservation, m_Info.vectorObservation,
                 m_Info.stackedVectorObservation.Count - m_Info.vectorObservation.Count);
 
-            m_Info.visualObservations.Clear();
-            var visualObservationCount = agentParameters.agentCameras.Count + agentParameters.agentRenderTextures.Count;
-            if (param.cameraResolutions.Length > visualObservationCount)
-            {
-                throw new UnityAgentsException(string.Format(
-                    "Not enough cameras/renderTextures for agent {0} : Brain {1} expecting at " +
-                    "least {2} cameras/renderTextures but only {3} were present.",
-                    gameObject.name, brain.name,
-                    brain.brainParameters.cameraResolutions.Length,
-                    visualObservationCount));
-            }
-
-            //First add all cameras
-            for (var i = 0; i < agentParameters.agentCameras.Count; i++)
-            {
-                var obsTexture = ObservationToTexture(
-                    agentParameters.agentCameras[i],
-                    param.cameraResolutions[i].width,
-                    param.cameraResolutions[i].height);
-                m_Info.visualObservations.Add(obsTexture);
-            }
-
-            //Then add all renderTextures
-            var camCount = agentParameters.agentCameras.Count;
-            for (var i = 0; i < agentParameters.agentRenderTextures.Count; i++)
-            {
-                var obsTexture = ObservationToTexture(
-                    agentParameters.agentRenderTextures[i],
-                    param.cameraResolutions[camCount + i].width,
-                    param.cameraResolutions[camCount + i].height);
-                m_Info.visualObservations.Add(obsTexture);
-            }
-
             m_Info.reward = m_Reward;
             m_Info.done = m_Done;
             m_Info.maxStepReached = m_MaxStepReached;
 
             if (m_Recorder != null && m_Recorder.record && Application.isEditor)
             {
+                // This is a bit of a hack - if we're in inference mode, compressed observations won't be generated
+                // But we need these to be generated for the recorder. So generate them here.
+                if (m_Info.compressedObservations.Count == 0)
+                {
+                    GenerateSensorData();
+                }
+
                 m_Recorder.WriteExperience(m_Info);
             }
 
-        public void ClearVisualObservations()
+        /// <summary>
+        /// Generate data for each sensor and store it on the Agent's AgentInfo.
+        /// NOTE: At the moment, this is only called during training or when using a DemonstrationRecorder;
+        /// during inference the sensors are used to write directly to the Tensor data. This will likely change in the
+        /// future to be controlled by the type of brain being used.
+        /// </summary>
+        public void GenerateSensorData()
-            m_Info.ClearVisualObs();
+            // Generate data for all sensors
+            // TODO add bool argument indicating when to compress? For now, we always will compress.
+            for (var i = 0; i<m_Sensors.Count; i++)
+            {
+                var sensor = m_Sensors[i];
+                var compressedObs = new CompressedObservation
+                {
+                    Data = sensor.GetCompressedObservation(),
+                    Shape = sensor.GetFloatObservationShape(),
+                    CompressionType = sensor.GetCompressionType()
+                };
+                m_Info.compressedObservations.Add(compressedObs);
+            }
         }
 
         /// <summary>
                     RequestDecision();
                 }
             }
-        }
-
-        /// <summary>
-        /// Converts a camera and corresponding resolution to a 2D texture.
-        /// </summary>
-        /// <returns>The 2D texture.</returns>
-        /// <param name="obsCamera">Camera.</param>
-        /// <param name="width">Width of resulting 2D texture.</param>
-        /// <param name="height">Height of resulting 2D texture.</param>
-        /// <returns name="texture2D">Texture2D to render to.</returns>
-        public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height)
-        {
-            var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
-            var oldRec = obsCamera.rect;
-            obsCamera.rect = new Rect(0f, 0f, 1f, 1f);
-            var depth = 24;
-            var format = RenderTextureFormat.Default;
-            var readWrite = RenderTextureReadWrite.Default;
-
-            var tempRt =
-                RenderTexture.GetTemporary(width, height, depth, format, readWrite);
-
-            var prevActiveRt = RenderTexture.active;
-            var prevCameraRt = obsCamera.targetTexture;
-
-            // render to offscreen texture (readonly from CPU side)
-            RenderTexture.active = tempRt;
-            obsCamera.targetTexture = tempRt;
-
-            obsCamera.Render();
-
-            texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
-
-            obsCamera.targetTexture = prevCameraRt;
-            obsCamera.rect = oldRec;
-            RenderTexture.active = prevActiveRt;
-            RenderTexture.ReleaseTemporary(tempRt);
-            return texture2D;
-        }
-
-        /// <summary>
-        /// Converts a RenderTexture and correspinding resolution to a 2D texture.
-        /// </summary>
-        /// <returns>The 2D texture.</returns>
-        /// <param name="obsTexture">RenderTexture.</param>
-        /// <param name="width">Width of resulting 2D texture.</param>
-        /// <param name="height">Height of resulting 2D texture.</param>
-        /// <returns name="texture2D">Texture2D to render to.</returns>
-        public static Texture2D ObservationToTexture(RenderTexture obsTexture, int width, int height)
-        {
-            var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
-
-            if (width != texture2D.width || height != texture2D.height)
-            {
-                texture2D.Resize(width, height);
-            }
-
-            if (width != obsTexture.width || height != obsTexture.height)
-            {
-                throw new UnityAgentsException(string.Format(
-                    "RenderTexture {0} : width/height is {1}/{2} brain is expecting {3}/{4}.",
-                    obsTexture.name, obsTexture.width, obsTexture.height, width, height));
-            }
-
-            var prevActiveRt = RenderTexture.active;
-            RenderTexture.active = obsTexture;
-
-            texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
-            texture2D.Apply();
-            RenderTexture.active = prevActiveRt;
-            return texture2D;
         }
 
         /// <summary>

UnitySDK/Assets/ML-Agents/Scripts/Brain.cs

 using System;
 using System.Collections.Generic;
+using Barracuda;
 using UnityEngine;
 
 namespace MLAgents
         /// List of agents subscribed for decisions.
         /// </summary>
         protected List<Agent> m_Agents = new List<Agent>(1024);
+
+        /// <summary>
+        /// Sensor shapes for the associated Agents. All Agents must have the same shapes for their sensors.
+        /// </summary>
+        List<int[]> m_SensorShapes;
 
         [NonSerialized]
         private bool m_IsInitialized;
         public void SubscribeAgentForDecision(Agent agent)
         {
             LazyInitialize();
+#if DEBUG
+            ValidateAgentSensorShapes(agent);
+#endif
             m_Agents.Add(agent);
         }
 
                     academy.BrainDecideAction += BrainDecideAction;
                     academy.DestroyAction += Shutdown;
                     Initialize();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Check that the Agent sensors are the same shape as the the other Agents using the same Brain.
+        /// If this is the first Agent being checked, its Sensor sizes will be saved.
+        /// </summary>
+        /// <param name="agent">The Agent to check</param>
+        private void ValidateAgentSensorShapes(Agent agent)
+        {
+            if (m_SensorShapes == null)
+            {
+                m_SensorShapes = new List<int[]>(agent.m_Sensors.Count);
+                // First agent, save the sensor sizes
+                foreach(var sensor in agent.m_Sensors)
+                {
+                    m_SensorShapes.Add(sensor.GetFloatObservationShape());
+                }
+            }
+            else
+            {
+                // Check for compatibility with the other Agents' sensors
+                // TODO make sure this only checks once per agent
+                Debug.Assert(m_SensorShapes.Count == agent.m_Sensors.Count, $"Number of sensors must match. {m_SensorShapes.Count} != {agent.m_Sensors.Count}");
+                for (var i = 0; i<m_SensorShapes.Count; i++)
+                {
+                    var cachedShape = m_SensorShapes[i];
+                    var sensorShape = agent.m_Sensors[i].GetFloatObservationShape();
+                    Debug.Assert(cachedShape.Length == sensorShape.Length, "Sensor dimensions must match.");
+                    for (var j = 0; j < cachedShape.Length; j++)
+                    {
+                        Debug.Assert(cachedShape[j] == sensorShape[j], "Sensor sizes much match.");
+                    }
                 }
             }
         }

UnitySDK/Assets/ML-Agents/Scripts/BrainParameters.cs

     }
 
     /// <summary>
-    /// The resolution of a camera used by an agent.
-    /// The width defines the number of pixels on the horizontal axis.
-    /// The height defines the number of pixels on the verical axis.
-    /// blackAndWhite defines whether or not the image is grayscale.
-    /// </summary>
-    [Serializable]
-    public struct Resolution
-    {
-        /// <summary>The width of the observation in pixels </summary>
-        public int width;
-
-        /// <summary>The height of the observation in pixels</summary>
-        public int height;
-
-        /// <summary>
-        /// If true, the image will be in black and white.
-        /// If false, it will be in colors RGB
-        /// </summary>
-        public bool blackAndWhite;
-    }
-
-    /// <summary>
     /// Holds information about the Brain. It defines what are the inputs and outputs of the
     /// decision process.
     /// </summary>
         /// </summary>
         public int[] vectorActionSize = new[] {1};
 
-        /// <summary> The list of observation resolutions for the brain</summary>
-        public Resolution[] cameraResolutions;
-
         /// <summary></summary>The list of strings describing what the actions correpond to */
         public string[] vectorActionDescriptions;
 
                 vectorObservationSize = vectorObservationSize,
                 numStackedVectorObservations = numStackedVectorObservations,
                 vectorActionSize = (int[])vectorActionSize.Clone(),
-                cameraResolutions = (Resolution[])cameraResolutions.Clone(),
                 vectorActionDescriptions = (string[])vectorActionDescriptions.Clone(),
                 vectorActionSpaceType = vectorActionSpaceType
             };

UnitySDK/Assets/ML-Agents/Scripts/Decision.cs

 using System.Collections.Generic;
+using MLAgents.Sensor;
 using UnityEngine;
 
 namespace MLAgents
         /// </summary>
         /// <returns>Vector action vector.</returns>
         /// <param name="vectorObs">The vector observations of the agent.</param>
-        /// <param name="visualObs">The cameras the agent uses for visual observations.</param>
+        /// <param name="sensors">The sensors the agent uses for observations.</param>
         /// <param name="reward">The reward the agent received at the previous step.</param>
         /// <param name="done">Whether or not the agent is done.</param>
         /// <param name="memory">
         public abstract float[] Decide(
-            List<float>
-            vectorObs,
-            List<Texture2D> visualObs,
+            List<float> vectorObs,
+            List<ISensor> sensors,
             float reward,
             bool done,
             List<float> memory);
         /// </summary>
         /// <returns>The vector of memories the agent will use at the next step.</returns>
         /// <param name="vectorObs">The vector observations of the agent.</param>
-        /// <param name="visualObs">The cameras the agent uses for visual observations.</param>
+        /// <param name="sensors">The sensors the agent uses for observations.</param>
         /// <param name="reward">The reward the agent received at the previous step.</param>
         /// <param name="done">Whether or not the agent is done.</param>
         /// <param name="memory">
             List<float> vectorObs,
-            List<Texture2D> visualObs,
+            List<ISensor> sensors,
             float reward,
             bool done,
             List<float> memory);

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/AgentInfo.cs

       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
             "CjNtbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL2FnZW50X2lu",
-            "Zm8ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGjttbGFnZW50cy9lbnZz",
-            "L2NvbW11bmljYXRvcl9vYmplY3RzL2N1c3RvbV9vYnNlcnZhdGlvbi5wcm90",
-            "byLcAgoOQWdlbnRJbmZvUHJvdG8SIgoac3RhY2tlZF92ZWN0b3Jfb2JzZXJ2",
-            "YXRpb24YASADKAISGwoTdmlzdWFsX29ic2VydmF0aW9ucxgCIAMoDBIYChB0",
-            "ZXh0X29ic2VydmF0aW9uGAMgASgJEh0KFXN0b3JlZF92ZWN0b3JfYWN0aW9u",
-            "cxgEIAMoAhIbChNzdG9yZWRfdGV4dF9hY3Rpb25zGAUgASgJEhAKCG1lbW9y",
-            "aWVzGAYgAygCEg4KBnJld2FyZBgHIAEoAhIMCgRkb25lGAggASgIEhgKEG1h",
-            "eF9zdGVwX3JlYWNoZWQYCSABKAgSCgoCaWQYCiABKAUSEwoLYWN0aW9uX21h",
-            "c2sYCyADKAgSSAoSY3VzdG9tX29ic2VydmF0aW9uGAwgASgLMiwuY29tbXVu",
-            "aWNhdG9yX29iamVjdHMuQ3VzdG9tT2JzZXJ2YXRpb25Qcm90b0IfqgIcTUxB",
-            "Z2VudHMuQ29tbXVuaWNhdG9yT2JqZWN0c2IGcHJvdG8z"));
+            "Zm8ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGj9tbGFnZW50cy9lbnZz",
+            "L2NvbW11bmljYXRvcl9vYmplY3RzL2NvbXByZXNzZWRfb2JzZXJ2YXRpb24u",
+            "cHJvdG8aO21sYWdlbnRzL2VudnMvY29tbXVuaWNhdG9yX29iamVjdHMvY3Vz",
+            "dG9tX29ic2VydmF0aW9uLnByb3RvIpgDCg5BZ2VudEluZm9Qcm90bxIiChpz",
+            "dGFja2VkX3ZlY3Rvcl9vYnNlcnZhdGlvbhgBIAMoAhIYChB0ZXh0X29ic2Vy",
+            "dmF0aW9uGAMgASgJEh0KFXN0b3JlZF92ZWN0b3JfYWN0aW9ucxgEIAMoAhIb",
+            "ChNzdG9yZWRfdGV4dF9hY3Rpb25zGAUgASgJEhAKCG1lbW9yaWVzGAYgAygC",
+            "Eg4KBnJld2FyZBgHIAEoAhIMCgRkb25lGAggASgIEhgKEG1heF9zdGVwX3Jl",
+            "YWNoZWQYCSABKAgSCgoCaWQYCiABKAUSEwoLYWN0aW9uX21hc2sYCyADKAgS",
+            "SAoSY3VzdG9tX29ic2VydmF0aW9uGAwgASgLMiwuY29tbXVuaWNhdG9yX29i",
+            "amVjdHMuQ3VzdG9tT2JzZXJ2YXRpb25Qcm90bxJRChdjb21wcmVzc2VkX29i",
+            "c2VydmF0aW9ucxgNIAMoCzIwLmNvbW11bmljYXRvcl9vYmplY3RzLkNvbXBy",
+            "ZXNzZWRPYnNlcnZhdGlvblByb3RvSgQIAhADQh+qAhxNTEFnZW50cy5Db21t",
+            "dW5pY2F0b3JPYmplY3RzYgZwcm90bzM="));
-          new pbr::FileDescriptor[] { global::MLAgents.CommunicatorObjects.CustomObservationReflection.Descriptor, },
+          new pbr::FileDescriptor[] { global::MLAgents.CommunicatorObjects.CompressedObservationReflection.Descriptor, global::MLAgents.CommunicatorObjects.CustomObservationReflection.Descriptor, },
-            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.AgentInfoProto), global::MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "StackedVectorObservation", "VisualObservations", "TextObservation", "StoredVectorActions", "StoredTextActions", "Memories", "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "CustomObservation" }, null, null, null)
+            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.AgentInfoProto), global::MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "StackedVectorObservation", "TextObservation", "StoredVectorActions", "StoredTextActions", "Memories", "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "CustomObservation", "CompressedObservations" }, null, null, null)
           }));
     }
     #endregion
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public AgentInfoProto(AgentInfoProto other) : this() {
       stackedVectorObservation_ = other.stackedVectorObservation_.Clone();
-      visualObservations_ = other.visualObservations_.Clone();
       textObservation_ = other.textObservation_;
       storedVectorActions_ = other.storedVectorActions_.Clone();
       storedTextActions_ = other.storedTextActions_;
       id_ = other.id_;
       actionMask_ = other.actionMask_.Clone();
       CustomObservation = other.customObservation_ != null ? other.CustomObservation.Clone() : null;
+      compressedObservations_ = other.compressedObservations_.Clone();
       _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
     }
 
       get { return stackedVectorObservation_; }
     }
 
-    /// <summary>Field number for the "visual_observations" field.</summary>
-    public const int VisualObservationsFieldNumber = 2;
-    private static readonly pb::FieldCodec<pb::ByteString> _repeated_visualObservations_codec
-        = pb::FieldCodec.ForBytes(18);
-    private readonly pbc::RepeatedField<pb::ByteString> visualObservations_ = new pbc::RepeatedField<pb::ByteString>();
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public pbc::RepeatedField<pb::ByteString> VisualObservations {
-      get { return visualObservations_; }
-    }
-
     /// <summary>Field number for the "text_observation" field.</summary>
     public const int TextObservationFieldNumber = 3;
     private string textObservation_ = "";
       }
     }
 
+    /// <summary>Field number for the "compressed_observations" field.</summary>
+    public const int CompressedObservationsFieldNumber = 13;
+    private static readonly pb::FieldCodec<global::MLAgents.CommunicatorObjects.CompressedObservationProto> _repeated_compressedObservations_codec
+        = pb::FieldCodec.ForMessage(106, global::MLAgents.CommunicatorObjects.CompressedObservationProto.Parser);
+    private readonly pbc::RepeatedField<global::MLAgents.CommunicatorObjects.CompressedObservationProto> compressedObservations_ = new pbc::RepeatedField<global::MLAgents.CommunicatorObjects.CompressedObservationProto>();
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public pbc::RepeatedField<global::MLAgents.CommunicatorObjects.CompressedObservationProto> CompressedObservations {
+      get { return compressedObservations_; }
+    }
+
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public override bool Equals(object other) {
       return Equals(other as AgentInfoProto);
         return true;
       }
       if(!stackedVectorObservation_.Equals(other.stackedVectorObservation_)) return false;
-      if(!visualObservations_.Equals(other.visualObservations_)) return false;
       if (TextObservation != other.TextObservation) return false;
       if(!storedVectorActions_.Equals(other.storedVectorActions_)) return false;
       if (StoredTextActions != other.StoredTextActions) return false;
       if (Id != other.Id) return false;
       if(!actionMask_.Equals(other.actionMask_)) return false;
       if (!object.Equals(CustomObservation, other.CustomObservation)) return false;
+      if(!compressedObservations_.Equals(other.compressedObservations_)) return false;
       return Equals(_unknownFields, other._unknownFields);
     }
 
       hash ^= stackedVectorObservation_.GetHashCode();
-      hash ^= visualObservations_.GetHashCode();
       if (TextObservation.Length != 0) hash ^= TextObservation.GetHashCode();
       hash ^= storedVectorActions_.GetHashCode();
       if (StoredTextActions.Length != 0) hash ^= StoredTextActions.GetHashCode();
       if (Id != 0) hash ^= Id.GetHashCode();
       hash ^= actionMask_.GetHashCode();
       if (customObservation_ != null) hash ^= CustomObservation.GetHashCode();
+      hash ^= compressedObservations_.GetHashCode();
       if (_unknownFields != null) {
         hash ^= _unknownFields.GetHashCode();
       }
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public void WriteTo(pb::CodedOutputStream output) {
       stackedVectorObservation_.WriteTo(output, _repeated_stackedVectorObservation_codec);
-      visualObservations_.WriteTo(output, _repeated_visualObservations_codec);
       if (TextObservation.Length != 0) {
         output.WriteRawTag(26);
         output.WriteString(TextObservation);
         output.WriteRawTag(98);
         output.WriteMessage(CustomObservation);
       }
+      compressedObservations_.WriteTo(output, _repeated_compressedObservations_codec);
       if (_unknownFields != null) {
         _unknownFields.WriteTo(output);
       }
     public int CalculateSize() {
       int size = 0;
       size += stackedVectorObservation_.CalculateSize(_repeated_stackedVectorObservation_codec);
-      size += visualObservations_.CalculateSize(_repeated_visualObservations_codec);
       if (TextObservation.Length != 0) {
         size += 1 + pb::CodedOutputStream.ComputeStringSize(TextObservation);
       }
       if (customObservation_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(CustomObservation);
       }
+      size += compressedObservations_.CalculateSize(_repeated_compressedObservations_codec);
       if (_unknownFields != null) {
         size += _unknownFields.CalculateSize();
       }
         return;
       }
       stackedVectorObservation_.Add(other.stackedVectorObservation_);
-      visualObservations_.Add(other.visualObservations_);
       if (other.TextObservation.Length != 0) {
         TextObservation = other.TextObservation;
       }
         }
         CustomObservation.MergeFrom(other.CustomObservation);
       }
+      compressedObservations_.Add(other.compressedObservations_);
       _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
     }
 
             stackedVectorObservation_.AddEntriesFrom(input, _repeated_stackedVectorObservation_codec);
             break;
           }
-          case 18: {
-            visualObservations_.AddEntriesFrom(input, _repeated_visualObservations_codec);
-            break;
-          }
           case 26: {
             TextObservation = input.ReadString();
             break;
               customObservation_ = new global::MLAgents.CommunicatorObjects.CustomObservationProto();
             }
             input.ReadMessage(customObservation_);
+            break;
+          }
+          case 106: {
+            compressedObservations_.AddEntriesFrom(input, _repeated_compressedObservations_codec);
             break;
           }
         }

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/BrainParameters.cs

           string.Concat(
             "CjltbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL2JyYWluX3Bh",
             "cmFtZXRlcnMucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGjNtbGFnZW50",
-            "cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL3Jlc29sdXRpb24ucHJvdG8a",
-            "M21sYWdlbnRzL2VudnMvY29tbXVuaWNhdG9yX29iamVjdHMvc3BhY2VfdHlw",
-            "ZS5wcm90byLUAgoUQnJhaW5QYXJhbWV0ZXJzUHJvdG8SHwoXdmVjdG9yX29i",
-            "c2VydmF0aW9uX3NpemUYASABKAUSJwofbnVtX3N0YWNrZWRfdmVjdG9yX29i",
-            "c2VydmF0aW9ucxgCIAEoBRIaChJ2ZWN0b3JfYWN0aW9uX3NpemUYAyADKAUS",
-            "QQoSY2FtZXJhX3Jlc29sdXRpb25zGAQgAygLMiUuY29tbXVuaWNhdG9yX29i",
-            "amVjdHMuUmVzb2x1dGlvblByb3RvEiIKGnZlY3Rvcl9hY3Rpb25fZGVzY3Jp",
-            "cHRpb25zGAUgAygJEkYKGHZlY3Rvcl9hY3Rpb25fc3BhY2VfdHlwZRgGIAEo",
-            "DjIkLmNvbW11bmljYXRvcl9vYmplY3RzLlNwYWNlVHlwZVByb3RvEhIKCmJy",
-            "YWluX25hbWUYByABKAkSEwoLaXNfdHJhaW5pbmcYCCABKAhCH6oCHE1MQWdl",
-            "bnRzLkNvbW11bmljYXRvck9iamVjdHNiBnByb3RvMw=="));
+            "cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL3NwYWNlX3R5cGUucHJvdG8i",
+            "lwIKFEJyYWluUGFyYW1ldGVyc1Byb3RvEh8KF3ZlY3Rvcl9vYnNlcnZhdGlv",
+            "bl9zaXplGAEgASgFEicKH251bV9zdGFja2VkX3ZlY3Rvcl9vYnNlcnZhdGlv",
+            "bnMYAiABKAUSGgoSdmVjdG9yX2FjdGlvbl9zaXplGAMgAygFEiIKGnZlY3Rv",
+            "cl9hY3Rpb25fZGVzY3JpcHRpb25zGAUgAygJEkYKGHZlY3Rvcl9hY3Rpb25f",
+            "c3BhY2VfdHlwZRgGIAEoDjIkLmNvbW11bmljYXRvcl9vYmplY3RzLlNwYWNl",
+            "VHlwZVByb3RvEhIKCmJyYWluX25hbWUYByABKAkSEwoLaXNfdHJhaW5pbmcY",
+            "CCABKAhKBAgEEAVCH6oCHE1MQWdlbnRzLkNvbW11bmljYXRvck9iamVjdHNi",
+            "BnByb3RvMw=="));
-          new pbr::FileDescriptor[] { global::MLAgents.CommunicatorObjects.ResolutionReflection.Descriptor, global::MLAgents.CommunicatorObjects.SpaceTypeReflection.Descriptor, },
+          new pbr::FileDescriptor[] { global::MLAgents.CommunicatorObjects.SpaceTypeReflection.Descriptor, },
-            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.BrainParametersProto), global::MLAgents.CommunicatorObjects.BrainParametersProto.Parser, new[]{ "VectorObservationSize", "NumStackedVectorObservations", "VectorActionSize", "CameraResolutions", "VectorActionDescriptions", "VectorActionSpaceType", "BrainName", "IsTraining" }, null, null, null)
+            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.BrainParametersProto), global::MLAgents.CommunicatorObjects.BrainParametersProto.Parser, new[]{ "VectorObservationSize", "NumStackedVectorObservations", "VectorActionSize", "VectorActionDescriptions", "VectorActionSpaceType", "BrainName", "IsTraining" }, null, null, null)
           }));
     }
     #endregion
       vectorObservationSize_ = other.vectorObservationSize_;
       numStackedVectorObservations_ = other.numStackedVectorObservations_;
       vectorActionSize_ = other.vectorActionSize_.Clone();
-      cameraResolutions_ = other.cameraResolutions_.Clone();
       vectorActionDescriptions_ = other.vectorActionDescriptions_.Clone();
       vectorActionSpaceType_ = other.vectorActionSpaceType_;
       brainName_ = other.brainName_;
       get { return vectorActionSize_; }
     }
 
-    /// <summary>Field number for the "camera_resolutions" field.</summary>
-    public const int CameraResolutionsFieldNumber = 4;
-    private static readonly pb::FieldCodec<global::MLAgents.CommunicatorObjects.ResolutionProto> _repeated_cameraResolutions_codec
-        = pb::FieldCodec.ForMessage(34, global::MLAgents.CommunicatorObjects.ResolutionProto.Parser);
-    private readonly pbc::RepeatedField<global::MLAgents.CommunicatorObjects.ResolutionProto> cameraResolutions_ = new pbc::RepeatedField<global::MLAgents.CommunicatorObjects.ResolutionProto>();
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public pbc::RepeatedField<global::MLAgents.CommunicatorObjects.ResolutionProto> CameraResolutions {
-      get { return cameraResolutions_; }
-    }
-
     /// <summary>Field number for the "vector_action_descriptions" field.</summary>
     public const int VectorActionDescriptionsFieldNumber = 5;
     private static readonly pb::FieldCodec<string> _repeated_vectorActionDescriptions_codec
       if (VectorObservationSize != other.VectorObservationSize) return false;
       if (NumStackedVectorObservations != other.NumStackedVectorObservations) return false;
       if(!vectorActionSize_.Equals(other.vectorActionSize_)) return false;
-      if(!cameraResolutions_.Equals(other.cameraResolutions_)) return false;
       if(!vectorActionDescriptions_.Equals(other.vectorActionDescriptions_)) return false;
       if (VectorActionSpaceType != other.VectorActionSpaceType) return false;
       if (BrainName != other.BrainName) return false;
       if (VectorObservationSize != 0) hash ^= VectorObservationSize.GetHashCode();
       if (NumStackedVectorObservations != 0) hash ^= NumStackedVectorObservations.GetHashCode();
       hash ^= vectorActionSize_.GetHashCode();
-      hash ^= cameraResolutions_.GetHashCode();
       hash ^= vectorActionDescriptions_.GetHashCode();
       if (VectorActionSpaceType != 0) hash ^= VectorActionSpaceType.GetHashCode();
       if (BrainName.Length != 0) hash ^= BrainName.GetHashCode();
         output.WriteInt32(NumStackedVectorObservations);
       }
       vectorActionSize_.WriteTo(output, _repeated_vectorActionSize_codec);
-      cameraResolutions_.WriteTo(output, _repeated_cameraResolutions_codec);
       vectorActionDescriptions_.WriteTo(output, _repeated_vectorActionDescriptions_codec);
       if (VectorActionSpaceType != 0) {
         output.WriteRawTag(48);
         size += 1 + pb::CodedOutputStream.ComputeInt32Size(NumStackedVectorObservations);
       }
       size += vectorActionSize_.CalculateSize(_repeated_vectorActionSize_codec);
-      size += cameraResolutions_.CalculateSize(_repeated_cameraResolutions_codec);
       size += vectorActionDescriptions_.CalculateSize(_repeated_vectorActionDescriptions_codec);
       if (VectorActionSpaceType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) VectorActionSpaceType);
         NumStackedVectorObservations = other.NumStackedVectorObservations;
       }
       vectorActionSize_.Add(other.vectorActionSize_);
-      cameraResolutions_.Add(other.cameraResolutions_);
       vectorActionDescriptions_.Add(other.vectorActionDescriptions_);
       if (other.VectorActionSpaceType != 0) {
         VectorActionSpaceType = other.VectorActionSpaceType;
           case 26:
           case 24: {
             vectorActionSize_.AddEntriesFrom(input, _repeated_vectorActionSize_codec);
-            break;
-          }
-          case 34: {
-            cameraResolutions_.AddEntriesFrom(input, _repeated_cameraResolutions_codec);
             break;
           }
           case 42: {

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/SpaceType.cs

       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
             "CjNtbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL3NwYWNlX3R5",
-            "cGUucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGjNtbGFnZW50cy9lbnZz",
-            "L2NvbW11bmljYXRvcl9vYmplY3RzL3Jlc29sdXRpb24ucHJvdG8qLgoOU3Bh",
-            "Y2VUeXBlUHJvdG8SDAoIZGlzY3JldGUQABIOCgpjb250aW51b3VzEAFCH6oC",
-            "HE1MQWdlbnRzLkNvbW11bmljYXRvck9iamVjdHNiBnByb3RvMw=="));
+            "cGUucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzKi4KDlNwYWNlVHlwZVBy",
+            "b3RvEgwKCGRpc2NyZXRlEAASDgoKY29udGludW91cxABQh+qAhxNTEFnZW50",
+            "cy5Db21tdW5pY2F0b3JPYmplY3RzYgZwcm90bzM="));
-          new pbr::FileDescriptor[] { global::MLAgents.CommunicatorObjects.ResolutionReflection.Descriptor, },
+          new pbr::FileDescriptor[] { },
           new pbr::GeneratedClrTypeInfo(new[] {typeof(global::MLAgents.CommunicatorObjects.SpaceTypeProto), }, null));
     }
     #endregion

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/CompressedObservation.cs.meta

 fileFormatVersion: 2
-guid: f19fefeaaa2594e70b23e74adfaf0d5a
+guid: 55ac40ee8d5b74b9e80d3def9d4ef6e0
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

UnitySDK/Assets/ML-Agents/Scripts/Grpc/GrpcExtensions.cs

 using Google.Protobuf;
 using Google.Protobuf.Collections;
 using MLAgents.CommunicatorObjects;
+using MLAgents.Sensor;
 using UnityEngine;
 
 namespace MLAgents
                 agentInfoProto.ActionMask.AddRange(ai.actionMasks);
             }
 
-            foreach (var obs in ai.visualObservations)
+            if (ai.compressedObservations != null)
-                using (TimerStack.Instance.Scoped("encodeVisualObs"))
+                foreach (var obs in ai.compressedObservations)
-                    agentInfoProto.VisualObservations.Add(
-                        ByteString.CopyFrom(obs.EncodeToPNG())
-                    );
+                    agentInfoProto.CompressedObservations.Add(obs.ToProto());
+
-        /// Converts a Brain into to a Protobuff BrainInfoProto so it can be sent
+        /// Converts a Brain into to a Protobuf BrainInfoProto so it can be sent
         /// </summary>
         /// <returns>The BrainInfoProto generated.</returns>
         /// <param name="bp">The instance of BrainParameter to extend.</param>
                 IsTraining = isTraining
             };
             brainParametersProto.VectorActionDescriptions.AddRange(bp.vectorActionDescriptions);
-            foreach (var res in bp.cameraResolutions)
-            {
-                brainParametersProto.CameraResolutions.Add(
-                    new ResolutionProto
-                    {
-                        Width = res.width,
-                        Height = res.height,
-                        GrayScale = res.blackAndWhite
-                    });
-            }
-
             return brainParametersProto;
         }
 
         }
 
         /// <summary>
-        /// Converts Resolution protobuf array to C# Resolution array.
-        /// </summary>
-        private static Resolution[] ResolutionProtoToNative(IReadOnlyList<ResolutionProto> resolutionProtos)
-        {
-            var localCameraResolutions = new Resolution[resolutionProtos.Count];
-            for (var i = 0; i < resolutionProtos.Count; i++)
-            {
-                localCameraResolutions[i] = new Resolution
-                {
-                    height = resolutionProtos[i].Height,
-                    width = resolutionProtos[i].Width,
-                    blackAndWhite = resolutionProtos[i].GrayScale
-                };
-            }
-
-            return localCameraResolutions;
-        }
-
-        /// <summary>
         /// Convert a BrainParametersProto to a BrainParameters struct.
         /// </summary>
         /// <param name="bpp">An instance of a brain parameters protobuf object.</param>
             var bp = new BrainParameters
             {
                 vectorObservationSize = bpp.VectorObservationSize,
-                cameraResolutions = ResolutionProtoToNative(
-                    bpp.CameraResolutions
-                    ),
                 numStackedVectorObservations = bpp.NumStackedVectorObservations,
                 vectorActionSize = bpp.VectorActionSize.ToArray(),
                 vectorActionDescriptions = bpp.VectorActionDescriptions.ToArray(),
                 agentActions.Add(ap.ToAgentAction());
             }
             return agentActions;
+        }
+
+        public static CompressedObservationProto ToProto(this CompressedObservation obs)
+        {
+            var obsProto = new CompressedObservationProto
+            {
+                Data = ByteString.CopyFrom(obs.Data),
+                CompressionType = (CompressionTypeProto) obs.CompressionType,
+            };
+            obsProto.Shape.AddRange(obs.Shape);
+            return obsProto;
         }
     }
 }

UnitySDK/Assets/ML-Agents/Scripts/Grpc/RpcCommunicator.cs

         Dictionary<string, Dictionary<Agent, AgentAction>> m_LastActionsReceived =
             new Dictionary<string, Dictionary<Agent, AgentAction>>();
 
-        private UnityRLInitializationOutputProto m_CurrentUnityRlInitializationOutput;
+        // Brains that we have sent over the communicator with agents.
+        HashSet<string> m_sentBrainKeys = new HashSet<string>();
+        Dictionary<string, BrainParameters> m_unsentBrainKeys = new Dictionary<string, BrainParameters>();
 
 
 # if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
             m_CurrentAgents[brainKey] = new List<Agent>(k_NumAgents);
             m_CurrentUnityRlOutput.AgentInfos.Add(
                 brainKey,
-                new CommunicatorObjects.UnityRLOutputProto.Types.ListAgentInfoProto());
-            if (m_CurrentUnityRlInitializationOutput == null)
-            {
-                m_CurrentUnityRlInitializationOutput = new UnityRLInitializationOutputProto();
-            }
-            m_CurrentUnityRlInitializationOutput.BrainParameters.Add(brainParameters.ToProto(brainKey, true));
+                new UnityRLOutputProto.Types.ListAgentInfoProto()
+            );
+
+            CacheBrainParameters(brainKey, brainParameters);
         }
 
         void UpdateEnvironmentWithInput(UnityRLInputProto rlInput)
             // the message and update hasSentState
             if (m_CurrentAgents[brainKey].Count > 0)
             {
-                foreach (var agent in m_CurrentAgents[brainKey])
+                using (TimerStack.Instance.Scoped("AgentInfo.ToProto"))
-                    var agentInfoProto = agent.Info.ToProto();
-                    m_CurrentUnityRlOutput.AgentInfos[brainKey].Value.Add(agentInfoProto);
-                    // Avoid visual obs memory leak. This should be called AFTER we are done with the visual obs.
-                    // e.g. after recording them to demo and using them for inference.
-                    agent.ClearVisualObservations();
+                    foreach (var agent in m_CurrentAgents[brainKey])
+                    {
+                        // Update the sensor data on the AgentInfo
+                        agent.GenerateSensorData();
+                        var agentInfoProto = agent.Info.ToProto();
+                        m_CurrentUnityRlOutput.AgentInfos[brainKey].Value.Add(agentInfoProto);
+                    }
                 }
 
                 m_HasData[brainKey] = true;
             {
                 RlOutput = m_CurrentUnityRlOutput,
             };
-            if (m_CurrentUnityRlInitializationOutput != null)
+            var tempUnityRlInitializationOutput = GetTempUnityRlInitializationOutput();
+            if (tempUnityRlInitializationOutput != null)
-                message.RlInitializationOutput = m_CurrentUnityRlInitializationOutput;
+                message.RlInitializationOutput = tempUnityRlInitializationOutput;
-            m_CurrentUnityRlInitializationOutput = null;
+            UpdateSentBrainParameters(tempUnityRlInitializationOutput);
 
             foreach (var k in m_CurrentUnityRlOutput.AgentInfos.Keys)
             {
             if (state == PlayModeStateChange.ExitingPlayMode)
             {
                 Dispose();
+            }
+        }
+
+        private void CacheBrainParameters(string brainKey, BrainParameters brainParameters)
+        {
+            if (m_sentBrainKeys.Contains(brainKey))
+            {
+                return;
+            }
+
+            // TODO We should check that if m_unsentBrainKeys has brainKey, it equals brainParameters
+            m_unsentBrainKeys[brainKey] = brainParameters;
+        }
+
+        private UnityRLInitializationOutputProto GetTempUnityRlInitializationOutput()
+        {
+            UnityRLInitializationOutputProto output = null;
+            foreach(var brainKey in m_unsentBrainKeys.Keys)
+            {
+                if (m_CurrentUnityRlOutput.AgentInfos.ContainsKey(brainKey))
+                {
+                    if (output == null)
+                    {
+                        output = new UnityRLInitializationOutputProto();
+                    }
+
+                    var brainParameters = m_unsentBrainKeys[brainKey];
+                    output.BrainParameters.Add(brainParameters.ToProto(brainKey, true));
+                }
+            }
+
+            return output;
+        }
+
+        private void UpdateSentBrainParameters(UnityRLInitializationOutputProto output)
+        {
+            if (output == null)
+            {
+                return;
+            }
+
+            foreach (var brainProto in output.BrainParameters)
+            {
+                m_sentBrainKeys.Add(brainProto.BrainName);
+                m_unsentBrainKeys.Remove(brainProto.BrainName);
             }
         }
 

UnitySDK/Assets/ML-Agents/Scripts/HeuristicBrain.cs

                 var info = agent.Info;
                 agent.UpdateVectorAction(decision.Decide(
                     info.stackedVectorObservation,
-                    info.visualObservations,
+                    agent.m_Sensors, // TODO set on Info? property?
                     info.reward,
                     info.done,
                     info.memories));
                 var info = agent.Info;
                 agent.UpdateMemoriesAction(decision.MakeMemory(
                     info.stackedVectorObservation,
-                    info.visualObservations,
+                    agent.m_Sensors,
                     info.reward,
                     info.done,
                     info.memories));

UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/BarracudaModelParamLoader.cs

                     "The model does not contain a Vector Observation  Placeholder Input. " +
                     "You must set the Vector Observation Space Size to 0.");
             }
-            // If there are not enough Visual Observation Input compared to what the
-            // Brain Parameters expect.
-            for (var visObsIndex = 0;
-                 visObsIndex < brainParameters.cameraResolutions.Length;
-                 visObsIndex++)
-            {
-                if (!tensorsNames.Contains(
-                    TensorNames.VisualObservationPlaceholderPrefix + visObsIndex))
-                {
-                    failedModelChecks.Add(
-                        "The model does not contain a Visual Observation Placeholder Input " +
-                        "for visual observation " + visObsIndex + ".");
-                }
-            }
+
+            // TODO reenable checks there are enough Visual Observation Placeholder in the model.
 
             // If the model has a non-negative memory size but requires a recurrent input
             if (memory > 0)
                 tensorTester[mem.input] = ((bp, tensor) => null);
             }
 
-            for (var obsIndex = 0; obsIndex < brainParameters.cameraResolutions.Length; obsIndex++)
-            {
-                var index = obsIndex;
-                tensorTester[TensorNames.VisualObservationPlaceholderPrefix + obsIndex] =
-                    (bp, tensor) => CheckVisualObsShape(bp, tensor, index);
-            }
+            // TODO reenable checks on visual observation shapes.
+
             // If the model expects an input but it is not in this list
             foreach (var tensor in GetInputTensors(model))
             {
             {
                 return "Previous Action Size of the model does not match. " +
                     $"Received {numberActionsBp} but was expecting {numberActionsT}.";
-            }
-            return null;
-        }
-
-        /// <summary>
-        /// Checks that the shape of the visual observation input placeholder is the same in the
-        /// model and in the Brain Parameters.
-        /// </summary>
-        /// <param name="brainParameters">
-        /// The BrainParameters that are used verify the compatibility with the InferenceEngine
-        /// </param>
-        /// <param name="tensorProxy">The tensor that is expected by the model</param>
-        /// <param name="visObsIndex">The index of the visual observation.</param>
-        /// <returns>
-        /// If the Check failed, returns a string containing information about why the
-        /// check failed. If the check passed, returns null.
-        /// </returns>
-        private static string CheckVisualObsShape(
-            BrainParameters brainParameters, TensorProxy tensorProxy, int visObsIndex)
-        {
-            var resolutionBp = brainParameters.cameraResolutions[visObsIndex];
-            var widthBp = resolutionBp.width;
-            var heightBp = resolutionBp.height;
-            var pixelBp = resolutionBp.blackAndWhite ? 1 : 3;
-            var heightT = tensorProxy.shape[1];
-            var widthT = tensorProxy.shape[2];
-            var pixelT = tensorProxy.shape[3];
-            if ((widthBp != widthT) || (heightBp != heightT) || (pixelBp != pixelT))
-            {
-                return $"The visual Observation {visObsIndex} of the model does not match. " +
-                    $"Received TensorProxy of shape [?x{widthBp}x{heightBp}x{pixelBp}] but " +
-                    $"was expecting [?x{widthT}x{heightT}x{pixelT}].";
             }
             return null;
         }

UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/GeneratorImpl.cs

         private readonly ITensorAllocator m_Allocator;
 
         public VisualObservationInputGenerator(
-            int index, bool grayScale, ITensorAllocator allocator)
+            int index, ITensorAllocator allocator)
-            m_GrayScale = grayScale;
             m_Allocator = allocator;
         }
 
-            var textures = agents.Select(
-                agent => agent.Info.visualObservations[m_Index]).ToList();
-
-            Utilities.TextureToTensorProxy(textures, tensorProxy, m_GrayScale);
+            var agentIndex = 0;
+            foreach (var agent in agents)
+            {
+                // TODO direct access to sensors list here - should we do it differently?
+                // TODO m_Index here is the visual observation index. Will work for now but not if we add more sensor types.
+                agent.m_Sensors[m_Index].WriteToTensor(tensorProxy, agentIndex);
+                agentIndex++;
+            }
         }
     }
 }

UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/ModelRunner.cs

 using System.Collections.Generic;
+using System.Linq;
 using UnityEngine;
 using Barracuda;
 using UnityEngine.Profiling;
         private string[] m_OutputNames;
         private IReadOnlyList<TensorProxy> m_InferenceInputs;
         private IReadOnlyList<TensorProxy> m_InferenceOutputs;
+
+        private bool m_visualObservationsInitialized = false;
 
         /// <summary>
         /// Initializes the Brain with the Model that it will use when selecting actions for
             if (currentBatchSize == 0)
             {
                 return;
+            }
+
+            if (!m_visualObservationsInitialized)
+            {
+                // Just grab the first agent in the collection (any will suffice, really).
+                // We check for an empty Collection above, so this will always return successfully.
+                var firstAgent = agents.First();
+                m_TensorGenerator.InitializeVisualObservations(firstAgent, m_TensorAllocator);
+                m_visualObservationsInitialized = true;
             }
 
             Profiler.BeginSample("LearningBrain.DecideAction");

UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/TensorGenerator.cs

                 new ActionMaskInputGenerator(allocator);
             m_Dict[TensorNames.RandomNormalEpsilonPlaceholder] =
                 new RandomNormalInputGenerator(seed, allocator);
-            if (bp.cameraResolutions != null)
-            {
-                for (var visIndex = 0;
-                     visIndex < bp.cameraResolutions.Length;
-                     visIndex++)
-                {
-                    var index = visIndex;
-                    var bw = bp.cameraResolutions[visIndex].blackAndWhite;
-                    m_Dict[TensorNames.VisualObservationPlaceholderPrefix + visIndex] =
-                        new VisualObservationInputGenerator(index, bw, allocator);
-                }
-            }
+
+        }
+
+        public void InitializeVisualObservations(Agent agent, ITensorAllocator allocator)
+        {
+            for (var visIndex = 0; visIndex < agent.m_Sensors.Count; visIndex++)
+            {
+                // TODO handle non-visual sensors too - need to index better
+                m_Dict[TensorNames.VisualObservationPlaceholderPrefix + visIndex] =
+                    new VisualObservationInputGenerator(visIndex, allocator);
+            }
         }
 
         /// <summary>

UnitySDK/Assets/ML-Agents/Scripts/Utilities.cs

             var numTextures = textures.Count;
             var width = textures[0].width;
             var height = textures[0].height;
+
+            for (var t = 0; t < numTextures; t++)
+            {
+                var texture = textures[t];
+                Debug.Assert(width == texture.width, "All Textures must have the same dimension");
+                Debug.Assert(height == texture.height, "All Textures must have the same dimension");
+                TextureToTensorProxy(texture, tensorProxy, grayScale, t);
+            }
+        }
+
+        /// <summary>
+        /// Puts a Texture2D into a TensorProxy.
+        /// </summary>
+        /// <param name="texture">
+        /// The texture to be put into the tensor.
+        /// </param>
+        /// <param name="tensorProxy">
+        /// TensorProxy to fill with Texture data.
+        /// </param>
+        /// <param name="grayScale">
+        /// If set to <c>true</c> the textures will be converted to grayscale before
+        /// being stored in the tensor.
+        /// </param>
+        /// <param name="textureOffset">
+        /// Index of the texture being written.
+        /// </param>
+        public static void TextureToTensorProxy(
+            Texture2D texture,
+            TensorProxy tensorProxy,
+            bool grayScale,
+            int textureOffset = 0)
+        {
+            var width = texture.width;
+            var height = texture.height;
-            for (var t = 0; t < numTextures; t++)
+            var t = textureOffset;
+            var texturePixels = texture.GetPixels32();
+            for (var h = height - 1; h >= 0; h--)
-                var texturePixels = textures[t].GetPixels32();
-                for (var h = height - 1; h >= 0; h--)
+                for (var w = 0; w < width; w++)
-                    for (var w = 0; w < width; w++)
+                    var currentPixel = texturePixels[(height - h - 1) * width + w];
+                    if (grayScale)
-                        var currentPixel = texturePixels[(height - h - 1) * width + w];
-                        if (grayScale)
-                        {
-                            data[t, h, w, 0] =
-                                (currentPixel.r + currentPixel.g + currentPixel.b) / 3f / 255.0f;
-                        }
-                        else
-                        {
-                            // For Color32, the r, g and b values are between 0 and 255.
-                            data[t, h, w, 0] = currentPixel.r / 255.0f;
-                            data[t, h, w, 1] = currentPixel.g / 255.0f;
-                            data[t, h, w, 2] = currentPixel.b / 255.0f;
-                        }
+                        data[t, h, w, 0] =
+                            (currentPixel.r + currentPixel.g + currentPixel.b) / 3f / 255.0f;
+                    }
+                    else
+                    {
+                        // For Color32, the r, g and b values are between 0 and 255.
+                        data[t, h, w, 0] = currentPixel.r / 255.0f;
+                        data[t, h, w, 1] = currentPixel.g / 255.0f;
+                        data[t, h, w, 2] = currentPixel.b / 255.0f;
+
         }
 
         /// <summary>

ml-agents-envs/mlagents/envs/brain.py

 class CameraResolution(NamedTuple):
     height: int
     width: int
-    gray_scale: bool
+    num_channels: int
-    def num_channels(self) -> int:
-        return 1 if self.gray_scale else 3
-
-    @staticmethod
-    def from_proto(p):
-        return CameraResolution(height=p.height, width=p.width, gray_scale=p.gray_scale)
+    def gray_scale(self) -> bool:
+        return self.num_channels == 1
 
 
 class BrainParameters:
         )
 
     @staticmethod
-    def from_proto(brain_param_proto: BrainParametersProto) -> "BrainParameters":
+    def from_proto(
+        brain_param_proto: BrainParametersProto, agent_info: AgentInfoProto
+    ) -> "BrainParameters":
-        resolution = [
-            CameraResolution.from_proto(x) for x in brain_param_proto.camera_resolutions
+        resolutions = [
+            CameraResolution(x.shape[0], x.shape[1], x.shape[2])
+            for x in agent_info.compressed_observations
+
-            resolution,
+            resolutions,
             list(brain_param_proto.vector_action_size),
             list(brain_param_proto.vector_action_descriptions),
             brain_param_proto.vector_action_space_type,
         for i in range(brain_params.number_visual_observations):
             obs = [
                 BrainInfo.process_pixels(
-                    x.visual_observations[i],
+                    x.compressed_observations[i].data,
                     brain_params.camera_resolutions[i].gray_scale,
                 )
                 for x in agent_info_list

ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.py

 _sym_db = _symbol_database.Default()
 
 
+from mlagents.envs.communicator_objects import compressed_observation_pb2 as mlagents_dot_envs_dot_communicator__objects_dot_compressed__observation__pb2
 from mlagents.envs.communicator_objects import custom_observation_pb2 as mlagents_dot_envs_dot_communicator__objects_dot_custom__observation__pb2
 
 
   syntax='proto3',
-  serialized_pb=_b('\n3mlagents/envs/communicator_objects/agent_info.proto\x12\x14\x63ommunicator_objects\x1a;mlagents/envs/communicator_objects/custom_observation.proto\"\xdc\x02\n\x0e\x41gentInfoProto\x12\"\n\x1astacked_vector_observation\x18\x01 \x03(\x02\x12\x1b\n\x13visual_observations\x18\x02 \x03(\x0c\x12\x18\n\x10text_observation\x18\x03 \x01(\t\x12\x1d\n\x15stored_vector_actions\x18\x04 \x03(\x02\x12\x1b\n\x13stored_text_actions\x18\x05 \x01(\t\x12\x10\n\x08memories\x18\x06 \x03(\x02\x12\x0e\n\x06reward\x18\x07 \x01(\x02\x12\x0c\n\x04\x64one\x18\x08 \x01(\x08\x12\x18\n\x10max_step_reached\x18\t \x01(\x08\x12\n\n\x02id\x18\n \x01(\x05\x12\x13\n\x0b\x61\x63tion_mask\x18\x0b \x03(\x08\x12H\n\x12\x63ustom_observation\x18\x0c \x01(\x0b\x32,.communicator_objects.CustomObservationProtoB\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
+  serialized_pb=_b('\n3mlagents/envs/communicator_objects/agent_info.proto\x12\x14\x63ommunicator_objects\x1a?mlagents/envs/communicator_objects/compressed_observation.proto\x1a;mlagents/envs/communicator_objects/custom_observation.proto\"\x98\x03\n\x0e\x41gentInfoProto\x12\"\n\x1astacked_vector_observation\x18\x01 \x03(\x02\x12\x18\n\x10text_observation\x18\x03 \x01(\t\x12\x1d\n\x15stored_vector_actions\x18\x04 \x03(\x02\x12\x1b\n\x13stored_text_actions\x18\x05 \x01(\t\x12\x10\n\x08memories\x18\x06 \x03(\x02\x12\x0e\n\x06reward\x18\x07 \x01(\x02\x12\x0c\n\x04\x64one\x18\x08 \x01(\x08\x12\x18\n\x10max_step_reached\x18\t \x01(\x08\x12\n\n\x02id\x18\n \x01(\x05\x12\x13\n\x0b\x61\x63tion_mask\x18\x0b \x03(\x08\x12H\n\x12\x63ustom_observation\x18\x0c \x01(\x0b\x32,.communicator_objects.CustomObservationProto\x12Q\n\x17\x63ompressed_observations\x18\r \x03(\x0b\x32\x30.communicator_objects.CompressedObservationProtoJ\x04\x08\x02\x10\x03\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
-  dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_custom__observation__pb2.DESCRIPTOR,])
+  dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_compressed__observation__pb2.DESCRIPTOR,mlagents_dot_envs_dot_communicator__objects_dot_custom__observation__pb2.DESCRIPTOR,])
 
 
 
       is_extension=False, extension_scope=None,
       options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='visual_observations', full_name='communicator_objects.AgentInfoProto.visual_observations', index=1,
-      number=2, type=12, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None, file=DESCRIPTOR),
-    _descriptor.FieldDescriptor(
-      name='text_observation', full_name='communicator_objects.AgentInfoProto.text_observation', index=2,
+      name='text_observation', full_name='communicator_objects.AgentInfoProto.text_observation', index=1,
       number=3, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
-      name='stored_vector_actions', full_name='communicator_objects.AgentInfoProto.stored_vector_actions', index=3,
+      name='stored_vector_actions', full_name='communicator_objects.AgentInfoProto.stored_vector_actions', index=2,
       number=4, type=2, cpp_type=6, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
-      name='stored_text_actions', full_name='communicator_objects.AgentInfoProto.stored_text_actions', index=4,
+      name='stored_text_actions', full_name='communicator_objects.AgentInfoProto.stored_text_actions', index=3,
       number=5, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
-      name='memories', full_name='communicator_objects.AgentInfoProto.memories', index=5,
+      name='memories', full_name='communicator_objects.AgentInfoProto.memories', index=4,
       number=6, type=2, cpp_type=6, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
-      name='reward', full_name='communicator_objects.AgentInfoProto.reward', index=6,
+      name='reward', full_name='communicator_objects.AgentInfoProto.reward', index=5,
       number=7, type=2, cpp_type=6, label=1,
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
-      name='done', full_name='communicator_objects.AgentInfoProto.done', index=7,
+      name='done', full_name='communicator_objects.AgentInfoProto.done', index=6,
       number=8, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
-      name='max_step_reached', full_name='communicator_objects.AgentInfoProto.max_step_reached', index=8,
+      name='max_step_reached', full_name='communicator_objects.AgentInfoProto.max_step_reached', index=7,
       number=9, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
-      name='id', full_name='communicator_objects.AgentInfoProto.id', index=9,
+      name='id', full_name='communicator_objects.AgentInfoProto.id', index=8,
       number=10, type=5, cpp_type=1, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
-      name='action_mask', full_name='communicator_objects.AgentInfoProto.action_mask', index=10,
+      name='action_mask', full_name='communicator_objects.AgentInfoProto.action_mask', index=9,
       number=11, type=8, cpp_type=7, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
-      name='custom_observation', full_name='communicator_objects.AgentInfoProto.custom_observation', index=11,
+      name='custom_observation', full_name='communicator_objects.AgentInfoProto.custom_observation', index=10,
+    _descriptor.FieldDescriptor(
+      name='compressed_observations', full_name='communicator_objects.AgentInfoProto.compressed_observations', index=11,
+      number=13, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=139,
-  serialized_end=487,
+  serialized_start=204,
+  serialized_end=612,
+_AGENTINFOPROTO.fields_by_name['compressed_observations'].message_type = mlagents_dot_envs_dot_communicator__objects_dot_compressed__observation__pb2._COMPRESSEDOBSERVATIONPROTO
 DESCRIPTOR.message_types_by_name['AgentInfoProto'] = _AGENTINFOPROTO
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 

ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.pyi

 )
 
 from google.protobuf.internal.containers import (
+    RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer,
+)
+
+from mlagents.envs.communicator_objects.compressed_observation_pb2 import (
+    CompressedObservationProto as mlagents___envs___communicator_objects___compressed_observation_pb2___CompressedObservationProto,
 )
 
 from mlagents.envs.communicator_objects.custom_observation_pb2 import (
 class AgentInfoProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
     stacked_vector_observation = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float]
-    visual_observations = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bytes]
     text_observation = ... # type: typing___Text
     stored_vector_actions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float]
     stored_text_actions = ... # type: typing___Text
     @property
     def custom_observation(self) -> mlagents___envs___communicator_objects___custom_observation_pb2___CustomObservationProto: ...
 
+    @property
+    def compressed_observations(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents___envs___communicator_objects___compressed_observation_pb2___CompressedObservationProto]: ...
+
-        visual_observations : typing___Optional[typing___Iterable[builtin___bytes]] = None,
         text_observation : typing___Optional[typing___Text] = None,
         stored_vector_actions : typing___Optional[typing___Iterable[builtin___float]] = None,
         stored_text_actions : typing___Optional[typing___Text] = None,
         id : typing___Optional[builtin___int] = None,
         action_mask : typing___Optional[typing___Iterable[builtin___bool]] = None,
         custom_observation : typing___Optional[mlagents___envs___communicator_objects___custom_observation_pb2___CustomObservationProto] = None,
+        compressed_observations : typing___Optional[typing___Iterable[mlagents___envs___communicator_objects___compressed_observation_pb2___CompressedObservationProto]] = None,
         ) -> None: ...
     @classmethod
     def FromString(cls, s: builtin___bytes) -> AgentInfoProto: ...
         def HasField(self, field_name: typing_extensions___Literal[u"custom_observation"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"custom_observation",u"done",u"id",u"max_step_reached",u"memories",u"reward",u"stacked_vector_observation",u"stored_text_actions",u"stored_vector_actions",u"text_observation",u"visual_observations"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"compressed_observations",u"custom_observation",u"done",u"id",u"max_step_reached",u"memories",u"reward",u"stacked_vector_observation",u"stored_text_actions",u"stored_vector_actions",u"text_observation"]) -> None: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"custom_observation",b"custom_observation",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"memories",b"memories",u"reward",b"reward",u"stacked_vector_observation",b"stacked_vector_observation",u"stored_text_actions",b"stored_text_actions",u"stored_vector_actions",b"stored_vector_actions",u"text_observation",b"text_observation",u"visual_observations",b"visual_observations"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"compressed_observations",b"compressed_observations",u"custom_observation",b"custom_observation",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"memories",b"memories",u"reward",b"reward",u"stacked_vector_observation",b"stacked_vector_observation",u"stored_text_actions",b"stored_text_actions",u"stored_vector_actions",b"stored_vector_actions",u"text_observation",b"text_observation"]) -> None: ...

ml-agents-envs/mlagents/envs/communicator_objects/brain_parameters_pb2.py

 _sym_db = _symbol_database.Default()
 
 
-from mlagents.envs.communicator_objects import resolution_pb2 as mlagents_dot_envs_dot_communicator__objects_dot_resolution__pb2
 from mlagents.envs.communicator_objects import space_type_pb2 as mlagents_dot_envs_dot_communicator__objects_dot_space__type__pb2
 
 
   syntax='proto3',
-  serialized_pb=_b('\n9mlagents/envs/communicator_objects/brain_parameters.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents/envs/communicator_objects/resolution.proto\x1a\x33mlagents/envs/communicator_objects/space_type.proto\"\xd4\x02\n\x14\x42rainParametersProto\x12\x1f\n\x17vector_observation_size\x18\x01 \x01(\x05\x12\'\n\x1fnum_stacked_vector_observations\x18\x02 \x01(\x05\x12\x1a\n\x12vector_action_size\x18\x03 \x03(\x05\x12\x41\n\x12\x63\x61mera_resolutions\x18\x04 \x03(\x0b\x32%.communicator_objects.ResolutionProto\x12\"\n\x1avector_action_descriptions\x18\x05 \x03(\t\x12\x46\n\x18vector_action_space_type\x18\x06 \x01(\x0e\x32$.communicator_objects.SpaceTypeProto\x12\x12\n\nbrain_name\x18\x07 \x01(\t\x12\x13\n\x0bis_training\x18\x08 \x01(\x08\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
+  serialized_pb=_b('\n9mlagents/envs/communicator_objects/brain_parameters.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents/envs/communicator_objects/space_type.proto\"\x97\x02\n\x14\x42rainParametersProto\x12\x1f\n\x17vector_observation_size\x18\x01 \x01(\x05\x12\'\n\x1fnum_stacked_vector_observations\x18\x02 \x01(\x05\x12\x1a\n\x12vector_action_size\x18\x03 \x03(\x05\x12\"\n\x1avector_action_descriptions\x18\x05 \x03(\t\x12\x46\n\x18vector_action_space_type\x18\x06 \x01(\x0e\x32$.communicator_objects.SpaceTypeProto\x12\x12\n\nbrain_name\x18\x07 \x01(\t\x12\x13\n\x0bis_training\x18\x08 \x01(\x08J\x04\x08\x04\x10\x05\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
-  dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_resolution__pb2.DESCRIPTOR,mlagents_dot_envs_dot_communicator__objects_dot_space__type__pb2.DESCRIPTOR,])
+  dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_space__type__pb2.DESCRIPTOR,])
 
 
 
       is_extension=False, extension_scope=None,
       options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='camera_resolutions', full_name='communicator_objects.BrainParametersProto.camera_resolutions', index=3,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None, file=DESCRIPTOR),
-    _descriptor.FieldDescriptor(
-      name='vector_action_descriptions', full_name='communicator_objects.BrainParametersProto.vector_action_descriptions', index=4,
+      name='vector_action_descriptions', full_name='communicator_objects.BrainParametersProto.vector_action_descriptions', index=3,
       number=5, type=9, cpp_type=9, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
-      name='vector_action_space_type', full_name='communicator_objects.BrainParametersProto.vector_action_space_type', index=5,
+      name='vector_action_space_type', full_name='communicator_objects.BrainParametersProto.vector_action_space_type', index=4,
       number=6, type=14, cpp_type=8, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
-      name='brain_name', full_name='communicator_objects.BrainParametersProto.brain_name', index=6,
+      name='brain_name', full_name='communicator_objects.BrainParametersProto.brain_name', index=5,
       number=7, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
-      name='is_training', full_name='communicator_objects.BrainParametersProto.is_training', index=7,
+      name='is_training', full_name='communicator_objects.BrainParametersProto.is_training', index=6,
       number=8, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=190,
-  serialized_end=530,
+  serialized_start=137,
+  serialized_end=416,
-_BRAINPARAMETERSPROTO.fields_by_name['camera_resolutions'].message_type = mlagents_dot_envs_dot_communicator__objects_dot_resolution__pb2._RESOLUTIONPROTO
 _BRAINPARAMETERSPROTO.fields_by_name['vector_action_space_type'].enum_type = mlagents_dot_envs_dot_communicator__objects_dot_space__type__pb2._SPACETYPEPROTO
 DESCRIPTOR.message_types_by_name['BrainParametersProto'] = _BRAINPARAMETERSPROTO
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)

ml-agents-envs/mlagents/envs/communicator_objects/brain_parameters_pb2.pyi

 )
 
 from google.protobuf.internal.containers import (
-    RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer,
-)
-
-from mlagents.envs.communicator_objects.resolution_pb2 import (
-    ResolutionProto as mlagents___envs___communicator_objects___resolution_pb2___ResolutionProto,
 )
 
 from mlagents.envs.communicator_objects.space_type_pb2 import (
     brain_name = ... # type: typing___Text
     is_training = ... # type: builtin___bool
 
-    @property
-    def camera_resolutions(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents___envs___communicator_objects___resolution_pb2___ResolutionProto]: ...
-
-        camera_resolutions : typing___Optional[typing___Iterable[mlagents___envs___communicator_objects___resolution_pb2___ResolutionProto]] = None,
         vector_action_descriptions : typing___Optional[typing___Iterable[typing___Text]] = None,
         vector_action_space_type : typing___Optional[mlagents___envs___communicator_objects___space_type_pb2___SpaceTypeProto] = None,
         brain_name : typing___Optional[typing___Text] = None,
     def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
     def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
     if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",u"camera_resolutions",u"is_training",u"num_stacked_vector_observations",u"vector_action_descriptions",u"vector_action_size",u"vector_action_space_type",u"vector_observation_size"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",u"is_training",u"num_stacked_vector_observations",u"vector_action_descriptions",u"vector_action_size",u"vector_action_space_type",u"vector_observation_size"]) -> None: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",b"brain_name",u"camera_resolutions",b"camera_resolutions",u"is_training",b"is_training",u"num_stacked_vector_observations",b"num_stacked_vector_observations",u"vector_action_descriptions",b"vector_action_descriptions",u"vector_action_size",b"vector_action_size",u"vector_action_space_type",b"vector_action_space_type",u"vector_observation_size",b"vector_observation_size"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",b"brain_name",u"is_training",b"is_training",u"num_stacked_vector_observations",b"num_stacked_vector_observations",u"vector_action_descriptions",b"vector_action_descriptions",u"vector_action_size",b"vector_action_size",u"vector_action_space_type",b"vector_action_space_type",u"vector_observation_size",b"vector_observation_size"]) -> None: ...

ml-agents-envs/mlagents/envs/communicator_objects/space_type_pb2.py

 _sym_db = _symbol_database.Default()
 
 
-from mlagents.envs.communicator_objects import resolution_pb2 as mlagents_dot_envs_dot_communicator__objects_dot_resolution__pb2
 
 
 DESCRIPTOR = _descriptor.FileDescriptor(
-  serialized_pb=_b('\n3mlagents/envs/communicator_objects/space_type.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents/envs/communicator_objects/resolution.proto*.\n\x0eSpaceTypeProto\x12\x0c\n\x08\x64iscrete\x10\x00\x12\x0e\n\ncontinuous\x10\x01\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
-  ,
-  dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_resolution__pb2.DESCRIPTOR,])
+  serialized_pb=_b('\n3mlagents/envs/communicator_objects/space_type.proto\x12\x14\x63ommunicator_objects*.\n\x0eSpaceTypeProto\x12\x0c\n\x08\x64iscrete\x10\x00\x12\x0e\n\ncontinuous\x10\x01\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
+)
 
 _SPACETYPEPROTO = _descriptor.EnumDescriptor(
   name='SpaceTypeProto',
   ],
   containing_type=None,
   options=None,
-  serialized_start=130,
-  serialized_end=176,
+  serialized_start=77,
+  serialized_end=123,
 )
 _sym_db.RegisterEnumDescriptor(_SPACETYPEPROTO)
 

ml-agents-envs/mlagents/envs/environment.py

 from mlagents.envs.communicator_objects.environment_parameters_pb2 import (
     EnvironmentParametersProto,
 )
+from mlagents.envs.communicator_objects.unity_output_pb2 import UnityOutputProto
-from mlagents.envs.communicator_objects.unity_rl_initialization_output_pb2 import (
-    UnityRLInitializationOutputProto,
-)
+
 from mlagents.envs.communicator_objects.unity_input_pb2 import UnityInputProto
 from mlagents.envs.communicator_objects.custom_action_pb2 import CustomActionProto
 
 
         rl_init_parameters_in = UnityRLInitializationInputProto(seed=seed)
         try:
-            aca_params = self.send_academy_parameters(rl_init_parameters_in)
+            aca_output = self.send_academy_parameters(rl_init_parameters_in)
+            aca_params = aca_output.rl_initialization_output
         except UnityTimeOutException:
             self._close()
             raise
         self._brains: Dict[str, BrainParameters] = {}
         self._external_brain_names: List[str] = []
         self._num_external_brains = 0
-        self._update_brain_parameters(aca_params)
+        self._update_brain_parameters(aca_output)
         self._resetParameters = dict(aca_params.environment_parameters.float_parameters)
         logger.info(
             "\n'{0}' started successfully!\n{1}".format(self._academy_name, str(self))
             )
             if outputs is None:
                 raise UnityCommunicationException("Communicator has stopped.")
-            self._update_brain_parameters(outputs.rl_initialization_output)
+            self._update_brain_parameters(outputs)
             rl_output = outputs.rl_output
             s = self._get_state(rl_output)
             for _b in self._external_brain_names:
                 outputs = self.communicator.exchange(step_input)
             if outputs is None:
                 raise UnityCommunicationException("Communicator has stopped.")
-            self._update_brain_parameters(outputs.rl_initialization_output)
+            self._update_brain_parameters(outputs)
             rl_output = outputs.rl_output
             state = self._get_state(rl_output)
             for _b in self._external_brain_names:
             )
         return _data
 
-    def _update_brain_parameters(
-        self, init_output: Optional[UnityRLInitializationOutputProto]
-    ) -> None:
-        if init_output is not None:
-            for brain_param in init_output.brain_parameters:
+    def _update_brain_parameters(self, output: UnityOutputProto) -> None:
+        init_output = output.rl_initialization_output
+
+        for brain_param in init_output.brain_parameters:
+            # Each BrainParameter in the rl_initialization_output should have at least one AgentInfo
+            # Get that agent, because we need some of its observations.
+            agent_infos = output.rl_output.agentInfos[brain_param.brain_name]
+            if agent_infos.value:
+                agent = agent_infos.value[0]
-                    brain_param
+                    brain_param, agent
-            self._external_brain_names = list(self._brains.keys())
-            self._num_external_brains = len(self._external_brain_names)
+        self._external_brain_names = list(self._brains.keys())
+        self._num_external_brains = len(self._external_brain_names)
 
     @timed
     def _generate_step_input(
 
     def send_academy_parameters(
         self, init_parameters: UnityRLInitializationInputProto
-    ) -> UnityRLInitializationOutputProto:
+    ) -> UnityOutputProto:
-        return self.communicator.initialize(inputs).rl_initialization_output
+        return self.communicator.initialize(inputs)
 
     @staticmethod
     def wrap_unity_input(rl_input: UnityRLInputProto) -> UnityInputProto:

ml-agents-envs/mlagents/envs/mock_communicator.py

 )
 from mlagents.envs.communicator_objects.unity_input_pb2 import UnityInputProto
 from mlagents.envs.communicator_objects.unity_output_pb2 import UnityOutputProto
-from mlagents.envs.communicator_objects.resolution_pb2 import ResolutionProto
+from mlagents.envs.communicator_objects.compressed_observation_pb2 import (
+    CompressedObservationProto,
+    CompressionTypeProto,
+)
 
 
 class MockCommunicator(Communicator):
             self.num_stacks = 1
 
     def initialize(self, inputs: UnityInputProto) -> UnityOutputProto:
-        resolutions = [
-            ResolutionProto(width=30, height=40, gray_scale=False)
-            for i in range(self.visual_inputs)
-        ]
-            camera_resolutions=resolutions,
             vector_action_descriptions=["", ""],
             vector_action_space_type=int(not self.is_discrete),
             brain_name=self.brain_name,
             name="RealFakeAcademy", version="API-10", log_path="", brain_parameters=[bp]
         )
-        return UnityOutputProto(rl_initialization_output=rl_init)
+        output = UnityRLOutputProto(agentInfos=self._get_agent_infos())
+        return UnityOutputProto(rl_initialization_output=rl_init, rl_output=output)
-    def exchange(self, inputs: UnityInputProto) -> UnityOutputProto:
+    def _get_agent_infos(self):
         dict_agent_info = {}
         if self.is_discrete:
             vector_action = [1]
         else:
             observation = [1, 2, 3, 1, 2, 3]
 
+        compressed_obs = [
+            CompressedObservationProto(
+                data=None, shape=[30, 40, 3], compression_type=CompressionTypeProto.PNG
+            )
+            for _ in range(self.visual_inputs)
+        ]
+
         for i in range(self.num_agents):
             list_agent_info.append(
                 AgentInfoProto(
                     done=(i == 2),
                     max_step_reached=False,
                     id=i,
+                    compressed_observations=compressed_obs,
-        result = UnityRLOutputProto(agentInfos=dict_agent_info)
+        return dict_agent_info
+
+    def exchange(self, inputs: UnityInputProto) -> UnityOutputProto:
+        result = UnityRLOutputProto(agentInfos=self._get_agent_infos())
         return UnityOutputProto(rl_output=result)
 
     def close(self):

ml-agents/mlagents/trainers/demo_loader.py

         )
 
     brain_params = None
+    brain_param_proto = None
     brain_infos = []
     total_expected = 0
     for _file_path in file_paths:
             if obs_decoded == 1:
                 brain_param_proto = BrainParametersProto()
                 brain_param_proto.ParseFromString(data[pos : pos + next_pos])
-                brain_params = BrainParameters.from_proto(brain_param_proto)
+
+                if brain_params is None:
+                    brain_params = BrainParameters.from_proto(
+                        brain_param_proto, agent_info
+                    )
                 brain_info = BrainInfo.from_agent_proto(0, [agent_info], brain_params)
                 brain_infos.append(brain_info)
                 if len(brain_infos) == total_expected:

ml-agents/mlagents/trainers/tests/mock_brain.py

 import unittest.mock as mock
 import numpy as np
 
-from mlagents.envs.brain import CameraResolution
+from mlagents.envs.brain import CameraResolution, BrainParameters
 from mlagents.trainers.buffer import Buffer
 
 
     mock_brain.return_value.vector_observation_space_size = (
         vector_observation_space_size
     )
-    camrez = CameraResolution(height=84, width=84, gray_scale=False)
+    camrez = CameraResolution(height=84, width=84, num_channels=3)
     mock_brain.return_value.camera_resolutions = [camrez] * number_visual_observations
     mock_brain.return_value.vector_action_space_size = vector_action_space_size
     mock_brain.return_value.brain_name = "MockBrain"
         vector_observation_space_size=0,
     )
     return mock_brain
+
+
+def make_brain_parameters(
+    discrete_action: bool = False,
+    visual_inputs: int = 0,
+    stack: bool = True,
+    brain_name: str = "RealFakeBrain",
+    vec_obs_size: int = 3,
+) -> BrainParameters:
+    resolutions = [
+        CameraResolution(width=30, height=40, num_channels=3)
+        for _ in range(visual_inputs)
+    ]
+
+    return BrainParameters(
+        vector_observation_space_size=vec_obs_size,
+        num_stacked_vector_observations=2 if stack else 1,
+        camera_resolutions=resolutions,
+        vector_action_space_size=[2],
+        vector_action_descriptions=["", ""],
+        vector_action_space_type=int(not discrete_action),
+        brain_name=brain_name,
+    )

ml-agents/mlagents/trainers/tests/test_bc.py

 from mlagents.trainers.bc.offline_trainer import BCTrainer
 from mlagents.envs.environment import UnityEnvironment
 from mlagents.envs.mock_communicator import MockCommunicator
+from mlagents.trainers.tests.mock_brain import make_brain_parameters
 
 
 @pytest.fixture
     env.close()
 
 
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_cc_bc_model(mock_communicator, mock_launcher):
+def test_cc_bc_model():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=0
+            model = BehavioralCloningModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=0)
-            env = UnityEnvironment(" ")
-            model = BehavioralCloningModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
+            # env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_dc_bc_model(mock_communicator, mock_launcher):
+def test_dc_bc_model():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=0
+            model = BehavioralCloningModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=0)
-            env = UnityEnvironment(" ")
-            model = BehavioralCloningModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_visual_dc_bc_model(mock_communicator, mock_launcher):
+def test_visual_dc_bc_model():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=2
+            model = BehavioralCloningModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=2)
-            env = UnityEnvironment(" ")
-            model = BehavioralCloningModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_visual_cc_bc_model(mock_communicator, mock_launcher):
+def test_visual_cc_bc_model():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=2
+            model = BehavioralCloningModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=2)
-            env = UnityEnvironment(" ")
-            model = BehavioralCloningModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.visual_in[1]: np.ones([2, 40, 30, 3]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
 
 
 if __name__ == "__main__":

ml-agents/mlagents/trainers/tests/test_ppo.py

 from mlagents.envs.environment import UnityEnvironment
 from mlagents.envs.mock_communicator import MockCommunicator
 from mlagents.trainers.tests import mock_brain as mb
+from mlagents.trainers.tests.mock_brain import make_brain_parameters
 
 
 @pytest.fixture
     env.close()
 
 
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_cc_vector(mock_communicator, mock_launcher):
+def test_ppo_model_cc_vector():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=0
+            model = PPOModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=0)
-            env = UnityEnvironment(" ")
-
-            model = PPOModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.epsilon: np.array([[0, 1], [2, 3]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_cc_visual(mock_communicator, mock_launcher):
+def test_ppo_model_cc_visual():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=2
+            model = PPOModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=2)
-            env = UnityEnvironment(" ")
-
-            model = PPOModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.epsilon: np.array([[0, 1], [2, 3]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_dc_visual(mock_communicator, mock_launcher):
+def test_ppo_model_dc_visual():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=2
+            model = PPOModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=2)
-            env = UnityEnvironment(" ")
-            model = PPOModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_dc_vector(mock_communicator, mock_launcher):
+def test_ppo_model_dc_vector():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=0
+            model = PPOModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=0)
-            env = UnityEnvironment(" ")
-            model = PPOModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_dc_vector_rnn(mock_communicator, mock_launcher):
+def test_ppo_model_dc_vector_rnn():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=0
-            )
-            env = UnityEnvironment(" ")
-                env.brains["RealFakeBrain"], use_recurrent=True, m_size=memory_size
+                make_brain_parameters(discrete_action=True, visual_inputs=0),
+                use_recurrent=True,
+                m_size=memory_size,
             )
             init = tf.global_variables_initializer()
             sess.run(init)
                 model.action_masks: np.ones([1, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_ppo_model_cc_vector_rnn(mock_communicator, mock_launcher):
+def test_ppo_model_cc_vector_rnn():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=0
-            )
-            env = UnityEnvironment(" ")
-                env.brains["RealFakeBrain"], use_recurrent=True, m_size=memory_size
+                make_brain_parameters(discrete_action=False, visual_inputs=0),
+                use_recurrent=True,
+                m_size=memory_size,
             )
             init = tf.global_variables_initializer()
             sess.run(init)
                 model.epsilon: np.array([[0, 1]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
 
 
 def test_rl_functions():

ml-agents/mlagents/trainers/tests/test_sac.py

 from mlagents.trainers.sac.models import SACModel
 from mlagents.trainers.sac.policy import SACPolicy
 from mlagents.trainers.sac.trainer import SACTrainer
-from mlagents.envs.environment import UnityEnvironment
-from mlagents.envs.mock_communicator import MockCommunicator
+from mlagents.trainers.tests.mock_brain import make_brain_parameters
 
 
 @pytest.fixture
     env.close()
 
 
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_cc_vector(mock_communicator, mock_launcher):
+def test_sac_model_cc_vector():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=0
+            model = SACModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=0)
-            env = UnityEnvironment(" ")
-
-            model = SACModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_cc_visual(mock_communicator, mock_launcher):
+def test_sac_model_cc_visual():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=2
+            model = SACModel(
+                make_brain_parameters(discrete_action=False, visual_inputs=2)
-            env = UnityEnvironment(" ")
-
-            model = SACModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.visual_in[1]: np.ones([2, 40, 30, 3]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_dc_visual(mock_communicator, mock_launcher):
+def test_sac_model_dc_visual():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=2
+            model = SACModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=2)
-            env = UnityEnvironment(" ")
-            model = SACModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_dc_vector(mock_communicator, mock_launcher):
+def test_sac_model_dc_vector():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=0
+            model = SACModel(
+                make_brain_parameters(discrete_action=True, visual_inputs=0)
-            env = UnityEnvironment(" ")
-            model = SACModel(env.brains["RealFakeBrain"])
             init = tf.global_variables_initializer()
             sess.run(init)
 
                 model.action_masks: np.ones([2, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_dc_vector_rnn(mock_communicator, mock_launcher):
+def test_sac_model_dc_vector_rnn():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=True, visual_inputs=0
-            )
-            env = UnityEnvironment(" ")
-                env.brains["RealFakeBrain"], use_recurrent=True, m_size=memory_size
+                make_brain_parameters(discrete_action=True, visual_inputs=0),
+                use_recurrent=True,
+                m_size=memory_size,
             )
             init = tf.global_variables_initializer()
             sess.run(init)
                 model.action_masks: np.ones([1, 2]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
-@mock.patch("mlagents.envs.environment.UnityEnvironment.executable_launcher")
-@mock.patch("mlagents.envs.environment.UnityEnvironment.get_communicator")
-def test_sac_model_cc_vector_rnn(mock_communicator, mock_launcher):
+def test_sac_model_cc_vector_rnn():
-            mock_communicator.return_value = MockCommunicator(
-                discrete_action=False, visual_inputs=0
-            )
-            env = UnityEnvironment(" ")
-                env.brains["RealFakeBrain"], use_recurrent=True, m_size=memory_size
+                make_brain_parameters(discrete_action=False, visual_inputs=0),
+                use_recurrent=True,
+                m_size=memory_size,
             )
             init = tf.global_variables_initializer()
             sess.run(init)
                 model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]),
             }
             sess.run(run_list, feed_dict=feed_dict)
-            env.close()
 
 
 def test_sac_save_load_buffer(tmpdir):

protobuf-definitions/proto/mlagents/envs/communicator_objects/agent_info.proto

 syntax = "proto3";
 
+import "mlagents/envs/communicator_objects/compressed_observation.proto";
 import "mlagents/envs/communicator_objects/custom_observation.proto";
 
 option csharp_namespace = "MLAgents.CommunicatorObjects";
     repeated float stacked_vector_observation = 1;
-    repeated bytes visual_observations = 2;
+    reserved 2; // deprecated repeated bytes visual_observations = 2;
     string text_observation = 3;
     repeated float stored_vector_actions = 4;
     string stored_text_actions = 5;
     int32 id = 10;
     repeated bool action_mask = 11;
     CustomObservationProto custom_observation = 12;
+    repeated CompressedObservationProto compressed_observations = 13;
 }

protobuf-definitions/proto/mlagents/envs/communicator_objects/brain_parameters.proto

 syntax = "proto3";
 
-import "mlagents/envs/communicator_objects/resolution.proto";
 import "mlagents/envs/communicator_objects/space_type.proto";
 
 option csharp_namespace = "MLAgents.CommunicatorObjects";
     int32 vector_observation_size = 1;
     int32 num_stacked_vector_observations = 2;
     repeated int32 vector_action_size = 3;
-    repeated ResolutionProto camera_resolutions = 4;
+    reserved 4; // deprecated repeated ResolutionProto camera_resolutions
     repeated string vector_action_descriptions = 5;
     SpaceTypeProto vector_action_space_type = 6;
     string brain_name = 7;

protobuf-definitions/proto/mlagents/envs/communicator_objects/space_type.proto

 syntax = "proto3";
 
-import "mlagents/envs/communicator_objects/resolution.proto";
-
 option csharp_namespace = "MLAgents.CommunicatorObjects";
 package communicator_objects;
 

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/CompressedObservation.cs

+// <auto-generated>
+//     Generated by the protocol buffer compiler.  DO NOT EDIT!
+//     source: mlagents/envs/communicator_objects/compressed_observation.proto
+// </auto-generated>
+#pragma warning disable 1591, 0612, 3021
+#region Designer generated code
+
+using pb = global::Google.Protobuf;
+using pbc = global::Google.Protobuf.Collections;
+using pbr = global::Google.Protobuf.Reflection;
+using scg = global::System.Collections.Generic;
+namespace MLAgents.CommunicatorObjects {
+
+  /// <summary>Holder for reflection information generated from mlagents/envs/communicator_objects/compressed_observation.proto</summary>
+  public static partial class CompressedObservationReflection {
+
+    #region Descriptor
+    /// <summary>File descriptor for mlagents/envs/communicator_objects/compressed_observation.proto</summary>
+    public static pbr::FileDescriptor Descriptor {
+      get { return descriptor; }
+    }
+    private static pbr::FileDescriptor descriptor;
+
+    static CompressedObservationReflection() {
+      byte[] descriptorData = global::System.Convert.FromBase64String(
+          string.Concat(
+            "Cj9tbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL2NvbXByZXNz",
+            "ZWRfb2JzZXJ2YXRpb24ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzIn8K",
+            "GkNvbXByZXNzZWRPYnNlcnZhdGlvblByb3RvEg0KBXNoYXBlGAEgAygFEkQK",
+            "EGNvbXByZXNzaW9uX3R5cGUYAiABKA4yKi5jb21tdW5pY2F0b3Jfb2JqZWN0",
+            "cy5Db21wcmVzc2lvblR5cGVQcm90bxIMCgRkYXRhGAMgASgMKikKFENvbXBy",
+            "ZXNzaW9uVHlwZVByb3RvEggKBE5PTkUQABIHCgNQTkcQAUIfqgIcTUxBZ2Vu",
+            "dHMuQ29tbXVuaWNhdG9yT2JqZWN0c2IGcHJvdG8z"));
+      descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
+          new pbr::FileDescriptor[] { },
+          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::MLAgents.CommunicatorObjects.CompressionTypeProto), }, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.CompressedObservationProto), global::MLAgents.CommunicatorObjects.CompressedObservationProto.Parser, new[]{ "Shape", "CompressionType", "Data" }, null, null, null)
+          }));
+    }
+    #endregion
+
+  }
+  #region Enums
+  public enum CompressionTypeProto {
+    [pbr::OriginalName("NONE")] None = 0,
+    [pbr::OriginalName("PNG")] Png = 1,
+  }
+
+  #endregion
+
+  #region Messages
+  public sealed partial class CompressedObservationProto : pb::IMessage<CompressedObservationProto> {
+    private static readonly pb::MessageParser<CompressedObservationProto> _parser = new pb::MessageParser<CompressedObservationProto>(() => new CompressedObservationProto());
+    private pb::UnknownFieldSet _unknownFields;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pb::MessageParser<CompressedObservationProto> Parser { get { return _parser; } }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pbr::MessageDescriptor Descriptor {
+      get { return global::MLAgents.CommunicatorObjects.CompressedObservationReflection.Descriptor.MessageTypes[0]; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    pbr::MessageDescriptor pb::IMessage.Descriptor {
+      get { return Descriptor; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public CompressedObservationProto() {
+      OnConstruction();
+    }
+
+    partial void OnConstruction();
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public CompressedObservationProto(CompressedObservationProto other) : this() {
+      shape_ = other.shape_.Clone();
+      compressionType_ = other.compressionType_;
+      data_ = other.data_;
+      _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public CompressedObservationProto Clone() {
+      return new CompressedObservationProto(this);
+    }
+
+    /// <summary>Field number for the "shape" field.</summary>
+    public const int ShapeFieldNumber = 1;
+    private static readonly pb::FieldCodec<int> _repeated_shape_codec
+        = pb::FieldCodec.ForInt32(10);
+    private readonly pbc::RepeatedField<int> shape_ = new pbc::RepeatedField<int>();
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public pbc::RepeatedField<int> Shape {
+      get { return shape_; }
+    }
+
+    /// <summary>Field number for the "compression_type" field.</summary>
+    public const int CompressionTypeFieldNumber = 2;
+    private global::MLAgents.CommunicatorObjects.CompressionTypeProto compressionType_ = 0;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public global::MLAgents.CommunicatorObjects.CompressionTypeProto CompressionType {
+      get { return compressionType_; }
+      set {
+        compressionType_ = value;
+      }
+    }
+
+    /// <summary>Field number for the "data" field.</summary>
+    public const int DataFieldNumber = 3;
+    private pb::ByteString data_ = pb::ByteString.Empty;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public pb::ByteString Data {
+      get { return data_; }
+      set {
+        data_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override bool Equals(object other) {
+      return Equals(other as CompressedObservationProto);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public bool Equals(CompressedObservationProto other) {
+      if (ReferenceEquals(other, null)) {
+        return false;
+      }
+      if (ReferenceEquals(other, this)) {
+        return true;
+      }
+      if(!shape_.Equals(other.shape_)) return false;
+      if (CompressionType != other.CompressionType) return false;
+      if (Data != other.Data) return false;
+      return Equals(_unknownFields, other._unknownFields);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override int GetHashCode() {
+      int hash = 1;
+      hash ^= shape_.GetHashCode();
+      if (CompressionType != 0) hash ^= CompressionType.GetHashCode();
+      if (Data.Length != 0) hash ^= Data.GetHashCode();
+      if (_unknownFields != null) {
+        hash ^= _unknownFields.GetHashCode();
+      }
+      return hash;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override string ToString() {
+      return pb::JsonFormatter.ToDiagnosticString(this);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void WriteTo(pb::CodedOutputStream output) {
+      shape_.WriteTo(output, _repeated_shape_codec);
+      if (CompressionType != 0) {
+        output.WriteRawTag(16);
+        output.WriteEnum((int) CompressionType);
+      }
+      if (Data.Length != 0) {
+        output.WriteRawTag(26);
+        output.WriteBytes(Data);
+      }
+      if (_unknownFields != null) {
+        _unknownFields.WriteTo(output);
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public int CalculateSize() {
+      int size = 0;
+      size += shape_.CalculateSize(_repeated_shape_codec);
+      if (CompressionType != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) CompressionType);
+      }
+      if (Data.Length != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeBytesSize(Data);
+      }
+      if (_unknownFields != null) {
+        size += _unknownFields.CalculateSize();
+      }
+      return size;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(CompressedObservationProto other) {
+      if (other == null) {
+        return;
+      }
+      shape_.Add(other.shape_);
+      if (other.CompressionType != 0) {
+        CompressionType = other.CompressionType;
+      }
+      if (other.Data.Length != 0) {
+        Data = other.Data;
+      }
+      _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(pb::CodedInputStream input) {
+      uint tag;
+      while ((tag = input.ReadTag()) != 0) {
+        switch(tag) {
+          default:
+            _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input);
+            break;
+          case 10:
+          case 8: {
+            shape_.AddEntriesFrom(input, _repeated_shape_codec);
+            break;
+          }
+          case 16: {
+            compressionType_ = (global::MLAgents.CommunicatorObjects.CompressionTypeProto) input.ReadEnum();
+            break;
+          }
+          case 26: {
+            Data = input.ReadBytes();
+            break;
+          }
+        }
+      }
+    }
+
+  }
+
+  #endregion
+
+}
+
+#endregion Designer generated code

UnitySDK/Assets/ML-Agents/Scripts/Sensor.meta

+fileFormatVersion: 2
+guid: 96db1714def024608a0ff58ed9eae82d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

ml-agents-envs/mlagents/envs/communicator_objects/compressed_observation_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: mlagents/envs/communicator_objects/compressed_observation.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='mlagents/envs/communicator_objects/compressed_observation.proto',
+  package='communicator_objects',
+  syntax='proto3',
+  serialized_pb=_b('\n?mlagents/envs/communicator_objects/compressed_observation.proto\x12\x14\x63ommunicator_objects\"\x7f\n\x1a\x43ompressedObservationProto\x12\r\n\x05shape\x18\x01 \x03(\x05\x12\x44\n\x10\x63ompression_type\x18\x02 \x01(\x0e\x32*.communicator_objects.CompressionTypeProto\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c*)\n\x14\x43ompressionTypeProto\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03PNG\x10\x01\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
+)
+
+_COMPRESSIONTYPEPROTO = _descriptor.EnumDescriptor(
+  name='CompressionTypeProto',
+  full_name='communicator_objects.CompressionTypeProto',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='NONE', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='PNG', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=218,
+  serialized_end=259,
+)
+_sym_db.RegisterEnumDescriptor(_COMPRESSIONTYPEPROTO)
+
+CompressionTypeProto = enum_type_wrapper.EnumTypeWrapper(_COMPRESSIONTYPEPROTO)
+NONE = 0
+PNG = 1
+
+
+
+_COMPRESSEDOBSERVATIONPROTO = _descriptor.Descriptor(
+  name='CompressedObservationProto',
+  full_name='communicator_objects.CompressedObservationProto',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='shape', full_name='communicator_objects.CompressedObservationProto.shape', index=0,
+      number=1, type=5, cpp_type=1, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='compression_type', full_name='communicator_objects.CompressedObservationProto.compression_type', index=1,
+      number=2, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='data', full_name='communicator_objects.CompressedObservationProto.data', index=2,
+      number=3, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=89,
+  serialized_end=216,
+)
+
+_COMPRESSEDOBSERVATIONPROTO.fields_by_name['compression_type'].enum_type = _COMPRESSIONTYPEPROTO
+DESCRIPTOR.message_types_by_name['CompressedObservationProto'] = _COMPRESSEDOBSERVATIONPROTO
+DESCRIPTOR.enum_types_by_name['CompressionTypeProto'] = _COMPRESSIONTYPEPROTO
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+CompressedObservationProto = _reflection.GeneratedProtocolMessageType('CompressedObservationProto', (_message.Message,), dict(
+  DESCRIPTOR = _COMPRESSEDOBSERVATIONPROTO,
+  __module__ = 'mlagents.envs.communicator_objects.compressed_observation_pb2'
+  # @@protoc_insertion_point(class_scope:communicator_objects.CompressedObservationProto)
+  ))
+_sym_db.RegisterMessage(CompressedObservationProto)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\034MLAgents.CommunicatorObjects'))
+# @@protoc_insertion_point(module_scope)

ml-agents-envs/mlagents/envs/communicator_objects/compressed_observation_pb2.pyi

+# @generated by generate_proto_mypy_stubs.py.  Do not edit!
+import sys
+from google.protobuf.descriptor import (
+    Descriptor as google___protobuf___descriptor___Descriptor,
+    EnumDescriptor as google___protobuf___descriptor___EnumDescriptor,
+)
+
+from google.protobuf.internal.containers import (
+    RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer,
+)
+
+from google.protobuf.message import (
+    Message as google___protobuf___message___Message,
+)
+
+from typing import (
+    Iterable as typing___Iterable,
+    List as typing___List,
+    Optional as typing___Optional,
+    Tuple as typing___Tuple,
+    cast as typing___cast,
+)
+
+from typing_extensions import (
+    Literal as typing_extensions___Literal,
+)
+
+
+builtin___bool = bool
+builtin___bytes = bytes
+builtin___float = float
+builtin___int = int
+builtin___str = str
+
+
+class CompressionTypeProto(builtin___int):
+    DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ...
+    @classmethod
+    def Name(cls, number: builtin___int) -> builtin___str: ...
+    @classmethod
+    def Value(cls, name: builtin___str) -> 'CompressionTypeProto': ...
+    @classmethod
+    def keys(cls) -> typing___List[builtin___str]: ...
+    @classmethod
+    def values(cls) -> typing___List['CompressionTypeProto']: ...
+    @classmethod
+    def items(cls) -> typing___List[typing___Tuple[builtin___str, 'CompressionTypeProto']]: ...
+    NONE = typing___cast('CompressionTypeProto', 0)
+    PNG = typing___cast('CompressionTypeProto', 1)
+NONE = typing___cast('CompressionTypeProto', 0)
+PNG = typing___cast('CompressionTypeProto', 1)
+
+class CompressedObservationProto(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    shape = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int]
+    compression_type = ... # type: CompressionTypeProto
+    data = ... # type: builtin___bytes
+
+    def __init__(self,
+        *,
+        shape : typing___Optional[typing___Iterable[builtin___int]] = None,
+        compression_type : typing___Optional[CompressionTypeProto] = None,
+        data : typing___Optional[builtin___bytes] = None,
+        ) -> None: ...
+    @classmethod
+    def FromString(cls, s: builtin___bytes) -> CompressedObservationProto: ...
+    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
+    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
+    if sys.version_info >= (3,):
+        def ClearField(self, field_name: typing_extensions___Literal[u"compression_type",u"data",u"shape"]) -> None: ...
+    else:
+        def ClearField(self, field_name: typing_extensions___Literal[u"compression_type",b"compression_type",u"data",b"data",u"shape",b"shape"]) -> None: ...

protobuf-definitions/proto/mlagents/envs/communicator_objects/compressed_observation.proto

+syntax = "proto3";
+
+option csharp_namespace = "MLAgents.CommunicatorObjects";
+package communicator_objects;
+
+enum CompressionTypeProto {
+    NONE = 0;
+    PNG = 1;
+}
+
+message CompressedObservationProto {
+    repeated int32 shape = 1;
+    CompressionTypeProto compression_type = 2;
+    bytes data = 3;
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CameraSensor.cs

+using System;
+using MLAgents.InferenceBrain;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    public class CameraSensor : ISensor
+    {
+        private Camera m_Camera;
+        private int m_Width;
+        private int m_Height;
+        private bool m_Grayscale;
+        private string m_Name;
+        private int[] m_Shape;
+
+        public CameraSensor(Camera camera, int width, int height, bool grayscale, string name)
+        {
+            m_Camera = camera;
+            m_Width = width;
+            m_Height = height;
+            m_Grayscale = grayscale;
+            m_Name = name;
+            m_Shape = new[] { width, height, grayscale ? 1 : 3 };
+        }
+
+        public string GetName()
+        {
+            return m_Name;
+        }
+
+        public int[] GetFloatObservationShape()
+        {
+            return m_Shape;
+        }
+
+        public byte[] GetCompressedObservation()
+        {
+            using (TimerStack.Instance.Scoped("CameraSensor.GetCompressedObservation"))
+            {
+                var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
+                // TODO support more types here, e.g. JPG
+                var compressed = texture.EncodeToPNG();
+                UnityEngine.Object.Destroy(texture);
+                return compressed;
+            }
+        }
+
+        public void WriteToTensor(TensorProxy tensorProxy, int agentIndex)
+        {
+            using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor"))
+            {
+                var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
+                Utilities.TextureToTensorProxy(texture, tensorProxy, m_Grayscale, agentIndex);
+                UnityEngine.Object.Destroy(texture);
+            }
+        }
+
+        public CompressionType GetCompressionType()
+        {
+            return CompressionType.PNG;
+        }
+
+        /// <summary>
+        /// Converts a m_Camera and corresponding resolution to a 2D texture.
+        /// </summary>
+        /// <returns>The 2D texture.</returns>
+        /// <param name="obsCamera">Camera.</param>
+        /// <param name="width">Width of resulting 2D texture.</param>
+        /// <param name="height">Height of resulting 2D texture.</param>
+        /// <returns name="texture2D">Texture2D to render to.</returns>
+        public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height)
+        {
+            var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
+            var oldRec = obsCamera.rect;
+            obsCamera.rect = new Rect(0f, 0f, 1f, 1f);
+            var depth = 24;
+            var format = RenderTextureFormat.Default;
+            var readWrite = RenderTextureReadWrite.Default;
+
+            var tempRt =
+                RenderTexture.GetTemporary(width, height, depth, format, readWrite);
+
+            var prevActiveRt = RenderTexture.active;
+            var prevCameraRt = obsCamera.targetTexture;
+
+            // render to offscreen texture (readonly from CPU side)
+            RenderTexture.active = tempRt;
+            obsCamera.targetTexture = tempRt;
+
+            obsCamera.Render();
+
+            texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
+
+            obsCamera.targetTexture = prevCameraRt;
+            obsCamera.rect = oldRec;
+            RenderTexture.active = prevActiveRt;
+            RenderTexture.ReleaseTemporary(tempRt);
+            return texture2D;
+        }
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CameraSensor.cs.meta

+fileFormatVersion: 2
+guid: 41cb6bf4b09974bf583f5b2fef0c08a7
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CameraSensorComponent.cs

+using System;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    public class CameraSensorComponent : SensorComponent
+    {
+        public new Camera camera;
+        public string sensorName = "CameraSensor";
+        public int width = 84;
+        public int height = 84;
+        public bool grayscale = false;
+
+        public override ISensor CreateSensor()
+        {
+            return new CameraSensor(camera, width, height, grayscale, sensorName);
+        }
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CameraSensorComponent.cs.meta

+fileFormatVersion: 2
+guid: 282f342c2ab144bf38be65d4d0c4e07d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CompressedObservation.cs

+using System;
+using MLAgents.InferenceBrain;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    public struct CompressedObservation
+    {
+        /// <summary>
+        /// The compressed data.
+        /// </summary>
+        public byte[] Data;
+
+        /// <summary>
+        /// The format of the compressed data
+        /// </summary>
+        public CompressionType CompressionType;
+
+        /// <summary>
+        /// The uncompressed dimensions of the data.
+        /// E.g. for RGB visual observations, this will be {Width, Height, 3}
+        /// </summary>
+        public int[] Shape;
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/CompressedObservation.cs.meta

+fileFormatVersion: 2
+guid: e9292f3e3b1864e0fbe1e9bb9a29d4ed
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/ISensor.cs

+using MLAgents.InferenceBrain;
+
+namespace MLAgents.Sensor
+{
+    public enum CompressionType
+    {
+        None,
+        PNG,
+    }
+
+    /// <summary>
+    /// Sensor interface for generating observations.
+    /// For custom implementations, it is recommended to SensorBase instead.
+    /// </summary>
+    public interface ISensor {
+        /// <summary>
+        /// Returns the size of the observations that will be generated.
+        /// For example, a sensor that observes the velocity of a rigid body (in 3D) would return new {3}.
+        /// A sensor that returns an RGB image would return new [] {Width, Height, 3}
+        /// </summary>
+        /// <returns></returns>
+        int[] GetFloatObservationShape();
+
+        /// <summary>
+        /// Write the observation data directly to the TensorProxy.
+        /// This is considered an advanced interface; for a simpler approach, use SensorBase and override WriteFloats instead.
+        /// </summary>
+        /// <param name="tensorProxy"></param>
+        /// <param name="agentIndex"></param>
+        void WriteToTensor(TensorProxy tensorProxy, int agentIndex);
+
+        /// <summary>
+        /// Return a compressed representation of the observation. For small observations, this should generally not be
+        /// implemented. However, compressing large observations (such as visual results) can significantly improve
+        /// model training time.
+        /// </summary>
+        /// <returns></returns>
+        byte[] GetCompressedObservation();
+
+        /// <summary>
+        /// Return the compression type being used. If no compression is used, return CompressionType.None
+        /// </summary>
+        /// <returns></returns>
+        CompressionType GetCompressionType();
+
+        /// <summary>
+        /// Get the name of the sensor. This is used to ensure deterministic sorting of the sensors on an Agent,
+        /// so the naming must be consistent across all sensors and agents.
+        /// </summary>
+        /// <returns>The name of the sensor</returns>
+        string GetName();
+    }
+
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/ISensor.cs.meta

+fileFormatVersion: 2
+guid: 4bb5e09a94c6d4cd9a46c60b084e4952
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/RenderTextureSensor.cs

+using System;
+using System.Threading;
+using MLAgents.InferenceBrain;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    class RenderTextureSensor : ISensor
+    {
+        private RenderTexture m_RenderTexture;
+        private int m_Width;
+        private int m_Height;
+        private bool m_Grayscale;
+        private string m_Name;
+        private int[] m_Shape;
+
+        public RenderTextureSensor(RenderTexture renderTexture, int width, int height, bool grayscale, string name)
+        {
+            m_RenderTexture = renderTexture;
+            m_Width = width;
+            m_Height = height;
+            m_Grayscale = grayscale;
+            m_Name = name;
+            m_Shape = new[] { width, height, grayscale ? 1 : 3 };
+        }
+
+        public string GetName()
+        {
+            return m_Name;
+        }
+
+        public int[] GetFloatObservationShape()
+        {
+            return m_Shape;
+        }
+
+        public byte[] GetCompressedObservation()
+        {
+            using(TimerStack.Instance.Scoped("RenderTexSensor.GetCompressedObservation"))
+            {
+                var texture = ObservationToTexture(m_RenderTexture, m_Width, m_Height);
+                // TODO support more types here, e.g. JPG
+                var compressed = texture.EncodeToPNG();
+                UnityEngine.Object.Destroy(texture);
+                return compressed;
+            }
+        }
+
+        public void WriteToTensor(TensorProxy tensorProxy, int index)
+        {
+            using (TimerStack.Instance.Scoped("RenderTexSensor.GetCompressedObservation"))
+            {
+                var texture = ObservationToTexture(m_RenderTexture, m_Width, m_Height);
+                Utilities.TextureToTensorProxy(texture, tensorProxy, m_Grayscale, index);
+                UnityEngine.Object.Destroy(texture);
+            }
+        }
+
+        public CompressionType GetCompressionType()
+        {
+            return CompressionType.PNG;
+        }
+
+        /// <summary>
+        /// Converts a RenderTexture and correspinding resolution to a 2D texture.
+        /// </summary>
+        /// <returns>The 2D texture.</returns>
+        /// <param name="obsTexture">RenderTexture.</param>
+        /// <param name="width">Width of resulting 2D texture.</param>
+        /// <param name="height">Height of resulting 2D texture.</param>
+        /// <returns name="texture2D">Texture2D to render to.</returns>
+        public static Texture2D ObservationToTexture(RenderTexture obsTexture, int width, int height)
+        {
+            var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
+
+            if (width != texture2D.width || height != texture2D.height)
+            {
+                texture2D.Resize(width, height);
+            }
+
+            if (width != obsTexture.width || height != obsTexture.height)
+            {
+                throw new UnityAgentsException(string.Format(
+                    "RenderTexture {0} : width/height is {1}/{2} brain is expecting {3}/{4}.",
+                    obsTexture.name, obsTexture.width, obsTexture.height, width, height));
+            }
+
+            var prevActiveRt = RenderTexture.active;
+            RenderTexture.active = obsTexture;
+
+            texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
+            texture2D.Apply();
+            RenderTexture.active = prevActiveRt;
+            return texture2D;
+        }
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/RenderTextureSensor.cs.meta

+fileFormatVersion: 2
+guid: 958f1f6bb9058405cae3c03266ad9899
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/RenderTextureSensorComponent.cs

+using System;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    public class RenderTextureSensorComponent : SensorComponent
+    {
+        public RenderTexture renderTexture;
+        public string sensorName = "RenderTextureSensor";
+        public int width = 84;
+        public int height = 84;
+        public bool grayscale = false;
+
+        public override ISensor CreateSensor()
+        {
+            return new RenderTextureSensor(renderTexture, width, height, grayscale, sensorName);
+        }
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/RenderTextureSensorComponent.cs.meta

+fileFormatVersion: 2
+guid: 132e1194facb64429b007ea1edf562d0
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/SensorBase.cs

+using MLAgents.InferenceBrain;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    public abstract class SensorBase : ISensor
+    {
+        /// <summary>
+        /// Write the observations to the output buffer. This size of the buffer will be product of the sizes returned
+        /// by GetFloatObservationShape().
+        /// </summary>
+        /// <param name="output"></param>
+        public abstract void WriteObservation(float[] output);
+
+        public abstract int[] GetFloatObservationShape();
+
+        public abstract string GetName();
+
+        /// <summary>
+        /// Default implementation of WriteToTensor interface. This creates a temporary array, calls WriteObservation,
+        /// and then writes the results to the TensorProxy.
+        /// </summary>
+        /// <param name="tensorProxy"></param>
+        /// <param name="agentIndex"></param>
+        public virtual void WriteToTensor(TensorProxy tensorProxy, int agentIndex)
+        {
+            // TODO reuse buffer for similar agents, don't call GetFloatObservationShape()
+            int[] shape = GetFloatObservationShape();
+            int numFloats = 1;
+            foreach (var dim in shape)
+            {
+                numFloats *= dim;
+            }
+
+            float[] buffer = new float[numFloats];
+            WriteObservation(buffer);
+
+            for (var i = 0; i < numFloats; i++)
+            {
+                tensorProxy.data[agentIndex, i] = buffer[i];
+            }
+        }
+
+        public virtual byte[] GetCompressedObservation()
+        {
+            return null;
+        }
+
+        public virtual CompressionType GetCompressionType()
+        {
+            return CompressionType.None;
+        }
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/SensorBase.cs.meta

+fileFormatVersion: 2
+guid: 553b05a1b59a94260b3e545f13190389
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Sensor/SensorComponent.cs

+using System;
+using UnityEngine;
+
+namespace MLAgents.Sensor
+{
+    /// <summary>
+    /// Editor components for creating Sensors. Generally an ISensor implementation should have a corresponding
+    /// SensorComponent to create it.
+    /// </summary>
+    public abstract class SensorComponent : MonoBehaviour
+    {
+        /// <summary>
+        /// Create the ISensor. This is called by the Agent when it is initialized.
+        /// </summary>
+        /// <returns></returns>
+        public abstract ISensor CreateSensor();
+    }
+}

UnitySDK/Assets/ML-Agents/Scripts/Sensor/SensorComponent.cs.meta

+fileFormatVersion: 2
+guid: 4f1dad589959a4b598d09e54f61fbe02
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/Resolution.cs

-// <auto-generated>
-//     Generated by the protocol buffer compiler.  DO NOT EDIT!
-//     source: mlagents/envs/communicator_objects/resolution.proto
-// </auto-generated>
-#pragma warning disable 1591, 0612, 3021
-#region Designer generated code
-
-using pb = global::Google.Protobuf;
-using pbc = global::Google.Protobuf.Collections;
-using pbr = global::Google.Protobuf.Reflection;
-using scg = global::System.Collections.Generic;
-namespace MLAgents.CommunicatorObjects {
-
-  /// <summary>Holder for reflection information generated from mlagents/envs/communicator_objects/resolution.proto</summary>
-  public static partial class ResolutionReflection {
-
-    #region Descriptor
-    /// <summary>File descriptor for mlagents/envs/communicator_objects/resolution.proto</summary>
-    public static pbr::FileDescriptor Descriptor {
-      get { return descriptor; }
-    }
-    private static pbr::FileDescriptor descriptor;
-
-    static ResolutionReflection() {
-      byte[] descriptorData = global::System.Convert.FromBase64String(
-          string.Concat(
-            "CjNtbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL3Jlc29sdXRp",
-            "b24ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzIkQKD1Jlc29sdXRpb25Q",
-            "cm90bxINCgV3aWR0aBgBIAEoBRIOCgZoZWlnaHQYAiABKAUSEgoKZ3JheV9z",
-            "Y2FsZRgDIAEoCEIfqgIcTUxBZ2VudHMuQ29tbXVuaWNhdG9yT2JqZWN0c2IG",
-            "cHJvdG8z"));
-      descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
-          new pbr::FileDescriptor[] { },
-          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
-            new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.ResolutionProto), global::MLAgents.CommunicatorObjects.ResolutionProto.Parser, new[]{ "Width", "Height", "GrayScale" }, null, null, null)
-          }));
-    }
-    #endregion
-
-  }
-  #region Messages
-  public sealed partial class ResolutionProto : pb::IMessage<ResolutionProto> {
-    private static readonly pb::MessageParser<ResolutionProto> _parser = new pb::MessageParser<ResolutionProto>(() => new ResolutionProto());
-    private pb::UnknownFieldSet _unknownFields;
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public static pb::MessageParser<ResolutionProto> Parser { get { return _parser; } }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public static pbr::MessageDescriptor Descriptor {
-      get { return global::MLAgents.CommunicatorObjects.ResolutionReflection.Descriptor.MessageTypes[0]; }
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    pbr::MessageDescriptor pb::IMessage.Descriptor {
-      get { return Descriptor; }
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public ResolutionProto() {
-      OnConstruction();
-    }
-
-    partial void OnConstruction();
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public ResolutionProto(ResolutionProto other) : this() {
-      width_ = other.width_;
-      height_ = other.height_;
-      grayScale_ = other.grayScale_;
-      _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public ResolutionProto Clone() {
-      return new ResolutionProto(this);
-    }
-
-    /// <summary>Field number for the "width" field.</summary>
-    public const int WidthFieldNumber = 1;
-    private int width_;
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public int Width {
-      get { return width_; }
-      set {
-        width_ = value;
-      }
-    }
-
-    /// <summary>Field number for the "height" field.</summary>
-    public const int HeightFieldNumber = 2;
-    private int height_;
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public int Height {
-      get { return height_; }
-      set {
-        height_ = value;
-      }
-    }
-
-    /// <summary>Field number for the "gray_scale" field.</summary>
-    public const int GrayScaleFieldNumber = 3;
-    private bool grayScale_;
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public bool GrayScale {
-      get { return grayScale_; }
-      set {
-        grayScale_ = value;
-      }
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public override bool Equals(object other) {
-      return Equals(other as ResolutionProto);
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public bool Equals(ResolutionProto other) {
-      if (ReferenceEquals(other, null)) {
-        return false;
-      }
-      if (ReferenceEquals(other, this)) {
-        return true;
-      }
-      if (Width != other.Width) return false;
-      if (Height != other.Height) return false;
-      if (GrayScale != other.GrayScale) return false;
-      return Equals(_unknownFields, other._unknownFields);
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public override int GetHashCode() {
-      int hash = 1;
-      if (Width != 0) hash ^= Width.GetHashCode();
-      if (Height != 0) hash ^= Height.GetHashCode();
-      if (GrayScale != false) hash ^= GrayScale.GetHashCode();
-      if (_unknownFields != null) {
-        hash ^= _unknownFields.GetHashCode();
-      }
-      return hash;
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public override string ToString() {
-      return pb::JsonFormatter.ToDiagnosticString(this);
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public void WriteTo(pb::CodedOutputStream output) {
-      if (Width != 0) {
-        output.WriteRawTag(8);
-        output.WriteInt32(Width);
-      }
-      if (Height != 0) {
-        output.WriteRawTag(16);
-        output.WriteInt32(Height);
-      }
-      if (GrayScale != false) {
-        output.WriteRawTag(24);
-        output.WriteBool(GrayScale);
-      }
-      if (_unknownFields != null) {
-        _unknownFields.WriteTo(output);
-      }
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public int CalculateSize() {
-      int size = 0;
-      if (Width != 0) {
-        size += 1 + pb::CodedOutputStream.ComputeInt32Size(Width);
-      }
-      if (Height != 0) {
-        size += 1 + pb::CodedOutputStream.ComputeInt32Size(Height);
-      }
-      if (GrayScale != false) {
-        size += 1 + 1;
-      }
-      if (_unknownFields != null) {
-        size += _unknownFields.CalculateSize();
-      }
-      return size;
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public void MergeFrom(ResolutionProto other) {
-      if (other == null) {
-        return;
-      }
-      if (other.Width != 0) {
-        Width = other.Width;
-      }
-      if (other.Height != 0) {
-        Height = other.Height;
-      }
-      if (other.GrayScale != false) {
-        GrayScale = other.GrayScale;
-      }
-      _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
-    }
-
-    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-    public void MergeFrom(pb::CodedInputStream input) {
-      uint tag;
-      while ((tag = input.ReadTag()) != 0) {
-        switch(tag) {
-          default:
-            _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input);
-            break;
-          case 8: {
-            Width = input.ReadInt32();
-            break;
-          }
-          case 16: {
-            Height = input.ReadInt32();
-            break;
-          }
-          case 24: {
-            GrayScale = input.ReadBool();
-            break;
-          }
-        }
-      }
-    }
-
-  }
-
-  #endregion
-
-}
-
-#endregion Designer generated code

ml-agents-envs/mlagents/envs/communicator_objects/resolution_pb2.py

-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: mlagents/envs/communicator_objects/resolution.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='mlagents/envs/communicator_objects/resolution.proto',
-  package='communicator_objects',
-  syntax='proto3',
-  serialized_pb=_b('\n3mlagents/envs/communicator_objects/resolution.proto\x12\x14\x63ommunicator_objects\"D\n\x0fResolutionProto\x12\r\n\x05width\x18\x01 \x01(\x05\x12\x0e\n\x06height\x18\x02 \x01(\x05\x12\x12\n\ngray_scale\x18\x03 \x01(\x08\x42\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
-)
-
-
-
-
-_RESOLUTIONPROTO = _descriptor.Descriptor(
-  name='ResolutionProto',
-  full_name='communicator_objects.ResolutionProto',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='width', full_name='communicator_objects.ResolutionProto.width', index=0,
-      number=1, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None, file=DESCRIPTOR),
-    _descriptor.FieldDescriptor(
-      name='height', full_name='communicator_objects.ResolutionProto.height', index=1,
-      number=2, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None, file=DESCRIPTOR),
-    _descriptor.FieldDescriptor(
-      name='gray_scale', full_name='communicator_objects.ResolutionProto.gray_scale', index=2,
-      number=3, type=8, cpp_type=7, label=1,
-      has_default_value=False, default_value=False,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None, file=DESCRIPTOR),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=77,
-  serialized_end=145,
-)
-
-DESCRIPTOR.message_types_by_name['ResolutionProto'] = _RESOLUTIONPROTO
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-ResolutionProto = _reflection.GeneratedProtocolMessageType('ResolutionProto', (_message.Message,), dict(
-  DESCRIPTOR = _RESOLUTIONPROTO,
-  __module__ = 'mlagents.envs.communicator_objects.resolution_pb2'
-  # @@protoc_insertion_point(class_scope:communicator_objects.ResolutionProto)
-  ))
-_sym_db.RegisterMessage(ResolutionProto)
-
-
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\034MLAgents.CommunicatorObjects'))
-# @@protoc_insertion_point(module_scope)

ml-agents-envs/mlagents/envs/communicator_objects/resolution_pb2.pyi

-# @generated by generate_proto_mypy_stubs.py.  Do not edit!
-import sys
-from google.protobuf.descriptor import (
-    Descriptor as google___protobuf___descriptor___Descriptor,
-)
-
-from google.protobuf.message import (
-    Message as google___protobuf___message___Message,
-)
-
-from typing import (
-    Optional as typing___Optional,
-)
-
-from typing_extensions import (
-    Literal as typing_extensions___Literal,
-)
-
-
-builtin___bool = bool
-builtin___bytes = bytes
-builtin___float = float
-builtin___int = int
-
-
-class ResolutionProto(google___protobuf___message___Message):
-    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
-    width = ... # type: builtin___int
-    height = ... # type: builtin___int
-    gray_scale = ... # type: builtin___bool
-
-    def __init__(self,
-        *,
-        width : typing___Optional[builtin___int] = None,
-        height : typing___Optional[builtin___int] = None,
-        gray_scale : typing___Optional[builtin___bool] = None,
-        ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> ResolutionProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"gray_scale",u"height",u"width"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"gray_scale",b"gray_scale",u"height",b"height",u"width",b"width"]) -> None: ...

protobuf-definitions/proto/mlagents/envs/communicator_objects/resolution.proto

-syntax = "proto3";
-
-option csharp_namespace = "MLAgents.CommunicatorObjects";
-package communicator_objects;
-
-message ResolutionProto {
-    int32 width = 1;
-    int32 height = 2;
-    bool gray_scale = 3;
-}
-