Send sensor types to Python

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs

         {
             return SensorCompressionType.None;
         }
+
+        /// <inheritdoc/>
+        public virtual SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
     }
 }

com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs

                     CompressionType = (CompressionTypeProto)obs.CompressionType,
                 };
             }
-
             obsProto.Shape.AddRange(obs.Shape);
             return obsProto;
         }
                     CompressionType = (CompressionTypeProto)sensor.GetCompressionType(),
                 };
             }
+            observationProto.SensorType = (SensorTypeProto)sensor.GetSensorType();
             observationProto.Shape.AddRange(shape);
             return observationProto;
         }

com.unity.ml-agents/Runtime/Grpc/CommunicatorObjects/Observation.cs

       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
             "CjRtbGFnZW50c19lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL29ic2VydmF0",
-            "aW9uLnByb3RvEhRjb21tdW5pY2F0b3Jfb2JqZWN0cyL5AQoQT2JzZXJ2YXRp",
+            "aW9uLnByb3RvEhRjb21tdW5pY2F0b3Jfb2JqZWN0cyK1AgoQT2JzZXJ2YXRp",
-            "RmxvYXREYXRhSAAaGQoJRmxvYXREYXRhEgwKBGRhdGEYASADKAJCEgoQb2Jz",
-            "ZXJ2YXRpb25fZGF0YSopChRDb21wcmVzc2lvblR5cGVQcm90bxIICgROT05F",
-            "EAASBwoDUE5HEAFCJaoCIlVuaXR5Lk1MQWdlbnRzLkNvbW11bmljYXRvck9i",
-            "amVjdHNiBnByb3RvMw=="));
+            "RmxvYXREYXRhSAASOgoLc2Vuc29yX3R5cGUYBSABKA4yJS5jb21tdW5pY2F0",
+            "b3Jfb2JqZWN0cy5TZW5zb3JUeXBlUHJvdG8aGQoJRmxvYXREYXRhEgwKBGRh",
+            "dGEYASADKAJCEgoQb2JzZXJ2YXRpb25fZGF0YSopChRDb21wcmVzc2lvblR5",
+            "cGVQcm90bxIICgROT05FEAASBwoDUE5HEAEqRAoPU2Vuc29yVHlwZVByb3Rv",
+            "Eg8KC09CU0VSVkFUSU9OEAASFAoQUEFSQU1FVEVSSVpBVElPThABEgoKBlJF",
+            "V0FSRBACQiWqAiJVbml0eS5NTEFnZW50cy5Db21tdW5pY2F0b3JPYmplY3Rz",
+            "YgZwcm90bzM="));
-          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Unity.MLAgents.CommunicatorObjects.CompressionTypeProto), }, new pbr::GeneratedClrTypeInfo[] {
-            new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.ObservationProto), global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Parser, new[]{ "Shape", "CompressionType", "CompressedData", "FloatData" }, new[]{ "ObservationData" }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Types.FloatData), global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Types.FloatData.Parser, new[]{ "Data" }, null, null, null)})
+          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Unity.MLAgents.CommunicatorObjects.CompressionTypeProto), typeof(global::Unity.MLAgents.CommunicatorObjects.SensorTypeProto), }, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.ObservationProto), global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Parser, new[]{ "Shape", "CompressionType", "CompressedData", "FloatData", "SensorType" }, new[]{ "ObservationData" }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Types.FloatData), global::Unity.MLAgents.CommunicatorObjects.ObservationProto.Types.FloatData.Parser, new[]{ "Data" }, null, null, null)})
           }));
     }
     #endregion
   internal enum CompressionTypeProto {
     [pbr::OriginalName("NONE")] None = 0,
     [pbr::OriginalName("PNG")] Png = 1,
+  }
+
+  internal enum SensorTypeProto {
+    [pbr::OriginalName("OBSERVATION")] Observation = 0,
+    [pbr::OriginalName("PARAMETERIZATION")] Parameterization = 1,
+    [pbr::OriginalName("REWARD")] Reward = 2,
   }
 
   #endregion
     public ObservationProto(ObservationProto other) : this() {
       shape_ = other.shape_.Clone();
       compressionType_ = other.compressionType_;
+      sensorType_ = other.sensorType_;
       switch (other.ObservationDataCase) {
         case ObservationDataOneofCase.CompressedData:
           CompressedData = other.CompressedData;
       }
     }
 
+    /// <summary>Field number for the "sensor_type" field.</summary>
+    public const int SensorTypeFieldNumber = 5;
+    private global::Unity.MLAgents.CommunicatorObjects.SensorTypeProto sensorType_ = 0;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public global::Unity.MLAgents.CommunicatorObjects.SensorTypeProto SensorType {
+      get { return sensorType_; }
+      set {
+        sensorType_ = value;
+      }
+    }
+
     private object observationData_;
     /// <summary>Enum of possible cases for the "observation_data" oneof.</summary>
     public enum ObservationDataOneofCase {
       if (CompressionType != other.CompressionType) return false;
       if (CompressedData != other.CompressedData) return false;
       if (!object.Equals(FloatData, other.FloatData)) return false;
+      if (SensorType != other.SensorType) return false;
       if (ObservationDataCase != other.ObservationDataCase) return false;
       return Equals(_unknownFields, other._unknownFields);
     }
       if (CompressionType != 0) hash ^= CompressionType.GetHashCode();
       if (observationDataCase_ == ObservationDataOneofCase.CompressedData) hash ^= CompressedData.GetHashCode();
       if (observationDataCase_ == ObservationDataOneofCase.FloatData) hash ^= FloatData.GetHashCode();
+      if (SensorType != 0) hash ^= SensorType.GetHashCode();
       hash ^= (int) observationDataCase_;
       if (_unknownFields != null) {
         hash ^= _unknownFields.GetHashCode();
         output.WriteRawTag(34);
         output.WriteMessage(FloatData);
       }
+      if (SensorType != 0) {
+        output.WriteRawTag(40);
+        output.WriteEnum((int) SensorType);
+      }
       if (_unknownFields != null) {
         _unknownFields.WriteTo(output);
       }
       if (observationDataCase_ == ObservationDataOneofCase.FloatData) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(FloatData);
       }
+      if (SensorType != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) SensorType);
+      }
       if (_unknownFields != null) {
         size += _unknownFields.CalculateSize();
       }
       shape_.Add(other.shape_);
       if (other.CompressionType != 0) {
         CompressionType = other.CompressionType;
+      }
+      if (other.SensorType != 0) {
+        SensorType = other.SensorType;
       }
       switch (other.ObservationDataCase) {
         case ObservationDataOneofCase.CompressedData:
             }
             input.ReadMessage(subBuilder);
             FloatData = subBuilder;
+            break;
+          }
+          case 40: {
+            sensorType_ = (global::Unity.MLAgents.CommunicatorObjects.SensorTypeProto) input.ReadEnum();
             break;
           }
         }

com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs

         }
 
         /// <summary>
+        /// Camera sensors are always Observations.
+        /// </summary>
+        /// <returns>Sensor type of observation.</returns>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+
+        /// <summary>
         /// Generates a compressed image. This can be valuable in speeding-up training.
         /// </summary>
         /// <returns>Compressed image.</returns>

com.unity.ml-agents/Runtime/Sensors/ISensor.cs

     }
 
     /// <summary>
+    /// The compression setting for visual/camera observations.
+    /// </summary>
+    public enum SensorType
+    {
+        /// <summary>
+        /// Sensor represents an agent's observation.
+        /// </summary>
+        Observation,
+
+        /// <summary>
+        /// Sensor represents an agent's task/goal parameterization.
+        /// </summary>
+        Parameterization,
+
+        /// <summary>
+        /// Sensor represents one or more reward signals.
+        /// </summary>
+        Reward
+    }
+
+    /// <summary>
     /// Sensor interface for generating observations.
     /// </summary>
     public interface ISensor
         /// </summary>
         /// <returns>The name of the sensor.</returns>
         string GetName();
+
+
+        /// <summary>
+        /// Get the semantic meaning of the sensor, i.e. whether it is an observation or other type
+        /// of data to be sent to the Agent.
+        /// </summary>
+        /// <returns>The type of the sensor.</returns>
+        SensorType GetSensorType();
     }
 
 

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs

             return SensorCompressionType.None;
         }
 
+        /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         /// <summary>
         /// Evaluates the raycasts to be used as part of an observation of an agent.
         /// </summary>

com.unity.ml-agents/Runtime/Sensors/Reflection/ReflectionSensorBase.cs

         }
 
         /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <inheritdoc/>
         public string GetName()
         {
             return m_SensorName;

com.unity.ml-agents/Runtime/Sensors/RenderTextureSensor.cs

         }
 
         /// <summary>
+        /// RenderTexture sensors are always Observations.
+        /// </summary>
+        /// <returns>Sensor type of observation.</returns>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <summary>
         /// Converts a RenderTexture to a 2D texture.
         /// </summary>
         /// <returns>The 2D texture.</returns>

com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs

             return SensorCompressionType.None;
         }
 
+        /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         // TODO support stacked compressed observations (byte stream)
     }
 }

com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs

             return SensorCompressionType.None;
         }
 
+        /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         void Clear()
         {
             m_Observations.Clear();

com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs

         {
             numResetCalls++;
         }
+
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
     }
 
     [TestFixture]

com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs

             return SensorCompressionType.None;
         }
 
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         public string GetName()
         {
             return m_Name;

com.unity.ml-agents/Tests/Editor/Sensor/FloatVisualSensorTests.cs

         {
             return SensorCompressionType.None;
         }
+
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
     }
 
     public class FloatVisualSensorTests

com.unity.ml-agents/Tests/Editor/Sensor/SensorShapeValidatorTests.cs

         {
             return SensorCompressionType.None;
         }
+
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
     }
 
     public class SensorShapeValidatorTests

ml-agents-envs/mlagents_envs/base_env.py

     CONTINUOUS = 1
 
 
+class SensorType(Enum):
+    OBSERVATION = 0
+    PARAMETERIZATION = 1
+    REWARD = 2
+
+
 class BehaviorSpec(NamedTuple):
     """
     A NamedTuple to containing information about the observations and actions
     """
 
     observation_shapes: List[Tuple]
+    sensor_types: List[SensorType]
     action_type: ActionType
     action_shape: Union[int, Tuple[int, ...]]
 

ml-agents-envs/mlagents_envs/communicator_objects/agent_action_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    vector_actions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float]
-    value = ... # type: builtin___float
+    vector_actions: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] = ...
+    value: builtin___float = ...
 
     def __init__(self,
         *,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> AgentActionProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"value",u"vector_actions"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value",u"vector_actions",b"vector_actions"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value",u"vector_actions",b"vector_actions"]) -> None: ...
+type___AgentActionProto = AgentActionProto

ml-agents-envs/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class AgentInfoActionPairProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
 
         agent_info : typing___Optional[mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto] = None,
         action_info : typing___Optional[mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> AgentInfoActionPairProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"action_info",u"agent_info"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_info",u"agent_info"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> None: ...
+type___AgentInfoActionPairProto = AgentInfoActionPairProto

ml-agents-envs/mlagents_envs/communicator_objects/agent_info_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    reward = ... # type: builtin___float
-    done = ... # type: builtin___bool
-    max_step_reached = ... # type: builtin___bool
-    id = ... # type: builtin___int
-    action_mask = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bool]
+    reward: builtin___float = ...
+    done: builtin___bool = ...
+    max_step_reached: builtin___bool = ...
+    id: builtin___int = ...
+    action_mask: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bool] = ...
 
     @property
     def observations(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___observation_pb2___ObservationProto]: ...
         action_mask : typing___Optional[typing___Iterable[builtin___bool]] = None,
         observations : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___observation_pb2___ObservationProto]] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> AgentInfoProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"done",u"id",u"max_step_reached",u"observations",u"reward"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"observations",b"observations",u"reward",b"reward"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"observations",b"observations",u"reward",b"reward"]) -> None: ...
+type___AgentInfoProto = AgentInfoProto

ml-agents-envs/mlagents_envs/communicator_objects/brain_parameters_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 )
 
 from mlagents_envs.communicator_objects.space_type_pb2 import (
-    SpaceTypeProto as mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto,
+    SpaceTypeProtoValue as mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProtoValue,
 )
 
 from typing import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    vector_action_size = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int]
-    vector_action_descriptions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text]
-    vector_action_space_type = ... # type: mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto
-    brain_name = ... # type: typing___Text
-    is_training = ... # type: builtin___bool
+    vector_action_size: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] = ...
+    vector_action_descriptions: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] = ...
+    vector_action_space_type: mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProtoValue = ...
+    brain_name: typing___Text = ...
+    is_training: builtin___bool = ...
-        vector_action_space_type : typing___Optional[mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto] = None,
+        vector_action_space_type : typing___Optional[mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProtoValue] = None,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> BrainParametersProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",u"is_training",u"vector_action_descriptions",u"vector_action_size",u"vector_action_space_type"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",b"brain_name",u"is_training",b"is_training",u"vector_action_descriptions",b"vector_action_descriptions",u"vector_action_size",b"vector_action_size",u"vector_action_space_type",b"vector_action_space_type"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"brain_name",b"brain_name",u"is_training",b"is_training",u"vector_action_descriptions",b"vector_action_descriptions",u"vector_action_size",b"vector_action_size",u"vector_action_space_type",b"vector_action_space_type"]) -> None: ...
+type___BrainParametersProto = BrainParametersProto

ml-agents-envs/mlagents_envs/communicator_objects/capabilities_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    baseRLCapabilities = ... # type: builtin___bool
+    baseRLCapabilities: builtin___bool = ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityRLCapabilitiesProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"baseRLCapabilities"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"baseRLCapabilities",b"baseRLCapabilities"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"baseRLCapabilities",b"baseRLCapabilities"]) -> None: ...
+type___UnityRLCapabilitiesProto = UnityRLCapabilitiesProto

ml-agents-envs/mlagents_envs/communicator_objects/command_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     EnumDescriptor as google___protobuf___descriptor___EnumDescriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
+
+from google.protobuf.internal.enum_type_wrapper import (
+    _EnumTypeWrapper as google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper,
 )
 
 from google.protobuf.message import (
 from typing import (
-    List as typing___List,
-    Tuple as typing___Tuple,
+    NewType as typing___NewType,
-builtin___str = str
-class CommandProto(builtin___int):
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+CommandProtoValue = typing___NewType('CommandProtoValue', builtin___int)
+type___CommandProtoValue = CommandProtoValue
+CommandProto: _CommandProto
+class _CommandProto(google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper[CommandProtoValue]):
-    @classmethod
-    def Name(cls, number: builtin___int) -> builtin___str: ...
-    @classmethod
-    def Value(cls, name: builtin___str) -> 'CommandProto': ...
-    @classmethod
-    def keys(cls) -> typing___List[builtin___str]: ...
-    @classmethod
-    def values(cls) -> typing___List['CommandProto']: ...
-    @classmethod
-    def items(cls) -> typing___List[typing___Tuple[builtin___str, 'CommandProto']]: ...
-    STEP = typing___cast('CommandProto', 0)
-    RESET = typing___cast('CommandProto', 1)
-    QUIT = typing___cast('CommandProto', 2)
-STEP = typing___cast('CommandProto', 0)
-RESET = typing___cast('CommandProto', 1)
-QUIT = typing___cast('CommandProto', 2)
+    STEP = typing___cast(CommandProtoValue, 0)
+    RESET = typing___cast(CommandProtoValue, 1)
+    QUIT = typing___cast(CommandProtoValue, 2)
+STEP = typing___cast(CommandProtoValue, 0)
+RESET = typing___cast(CommandProtoValue, 1)
+QUIT = typing___cast(CommandProtoValue, 2)
+type___CommandProto = CommandProto

ml-agents-envs/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 
-builtin___bytes = bytes
-
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
 
 class CustomResetParametersProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> CustomResetParametersProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
+type___CustomResetParametersProto = CustomResetParametersProto

ml-agents-envs/mlagents_envs/communicator_objects/demonstration_meta_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    api_version = ... # type: builtin___int
-    demonstration_name = ... # type: typing___Text
-    number_steps = ... # type: builtin___int
-    number_episodes = ... # type: builtin___int
-    mean_reward = ... # type: builtin___float
+    api_version: builtin___int = ...
+    demonstration_name: typing___Text = ...
+    number_steps: builtin___int = ...
+    number_episodes: builtin___int = ...
+    mean_reward: builtin___float = ...
 
     def __init__(self,
         *,
         number_episodes : typing___Optional[builtin___int] = None,
         mean_reward : typing___Optional[builtin___float] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> DemonstrationMetaProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"api_version",u"demonstration_name",u"mean_reward",u"number_episodes",u"number_steps"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"api_version",b"api_version",u"demonstration_name",b"demonstration_name",u"mean_reward",b"mean_reward",u"number_episodes",b"number_episodes",u"number_steps",b"number_steps"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"api_version",b"api_version",u"demonstration_name",b"demonstration_name",u"mean_reward",b"mean_reward",u"number_episodes",b"number_episodes",u"number_steps",b"number_steps"]) -> None: ...
+type___DemonstrationMetaProto = DemonstrationMetaProto

ml-agents-envs/mlagents_envs/communicator_objects/engine_configuration_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    width = ... # type: builtin___int
-    height = ... # type: builtin___int
-    quality_level = ... # type: builtin___int
-    time_scale = ... # type: builtin___float
-    target_frame_rate = ... # type: builtin___int
-    show_monitor = ... # type: builtin___bool
+    width: builtin___int = ...
+    height: builtin___int = ...
+    quality_level: builtin___int = ...
+    time_scale: builtin___float = ...
+    target_frame_rate: builtin___int = ...
+    show_monitor: builtin___bool = ...
 
     def __init__(self,
         *,
         target_frame_rate : typing___Optional[builtin___int] = None,
         show_monitor : typing___Optional[builtin___bool] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> EngineConfigurationProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"height",u"quality_level",u"show_monitor",u"target_frame_rate",u"time_scale",u"width"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"height",b"height",u"quality_level",b"quality_level",u"show_monitor",b"show_monitor",u"target_frame_rate",b"target_frame_rate",u"time_scale",b"time_scale",u"width",b"width"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"height",b"height",u"quality_level",b"quality_level",u"show_monitor",b"show_monitor",u"target_frame_rate",b"target_frame_rate",u"time_scale",b"time_scale",u"width",b"width"]) -> None: ...
+type___EngineConfigurationProto = EngineConfigurationProto

ml-agents-envs/mlagents_envs/communicator_objects/header_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    status = ... # type: builtin___int
-    message = ... # type: typing___Text
+    status: builtin___int = ...
+    message: typing___Text = ...
 
     def __init__(self,
         *,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> HeaderProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"message",u"status"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"message",b"message",u"status",b"status"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"message",b"message",u"status",b"status"]) -> None: ...
+type___HeaderProto = HeaderProto

ml-agents-envs/mlagents_envs/communicator_objects/observation_pb2.py

   name='mlagents_envs/communicator_objects/observation.proto',
   package='communicator_objects',
   syntax='proto3',
-  serialized_pb=_b('\n4mlagents_envs/communicator_objects/observation.proto\x12\x14\x63ommunicator_objects\"\xf9\x01\n\x10ObservationProto\x12\r\n\x05shape\x18\x01 \x03(\x05\x12\x44\n\x10\x63ompression_type\x18\x02 \x01(\x0e\x32*.communicator_objects.CompressionTypeProto\x12\x19\n\x0f\x63ompressed_data\x18\x03 \x01(\x0cH\x00\x12\x46\n\nfloat_data\x18\x04 \x01(\x0b\x32\x30.communicator_objects.ObservationProto.FloatDataH\x00\x1a\x19\n\tFloatData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x03(\x02\x42\x12\n\x10observation_data*)\n\x14\x43ompressionTypeProto\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03PNG\x10\x01\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3')
+  serialized_pb=_b('\n4mlagents_envs/communicator_objects/observation.proto\x12\x14\x63ommunicator_objects\"\xb5\x02\n\x10ObservationProto\x12\r\n\x05shape\x18\x01 \x03(\x05\x12\x44\n\x10\x63ompression_type\x18\x02 \x01(\x0e\x32*.communicator_objects.CompressionTypeProto\x12\x19\n\x0f\x63ompressed_data\x18\x03 \x01(\x0cH\x00\x12\x46\n\nfloat_data\x18\x04 \x01(\x0b\x32\x30.communicator_objects.ObservationProto.FloatDataH\x00\x12:\n\x0bsensor_type\x18\x05 \x01(\x0e\x32%.communicator_objects.SensorTypeProto\x1a\x19\n\tFloatData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x03(\x02\x42\x12\n\x10observation_data*)\n\x14\x43ompressionTypeProto\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03PNG\x10\x01*D\n\x0fSensorTypeProto\x12\x0f\n\x0bOBSERVATION\x10\x00\x12\x14\n\x10PARAMETERIZATION\x10\x01\x12\n\n\x06REWARD\x10\x02\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3')
 )
 
 _COMPRESSIONTYPEPROTO = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=330,
-  serialized_end=371,
+  serialized_start=390,
+  serialized_end=431,
+_SENSORTYPEPROTO = _descriptor.EnumDescriptor(
+  name='SensorTypeProto',
+  full_name='communicator_objects.SensorTypeProto',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='OBSERVATION', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='PARAMETERIZATION', index=1, number=1,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='REWARD', index=2, number=2,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=433,
+  serialized_end=501,
+)
+_sym_db.RegisterEnumDescriptor(_SENSORTYPEPROTO)
+
+SensorTypeProto = enum_type_wrapper.EnumTypeWrapper(_SENSORTYPEPROTO)
+OBSERVATION = 0
+PARAMETERIZATION = 1
+REWARD = 2
 
 
 
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=283,
-  serialized_end=308,
+  serialized_start=343,
+  serialized_end=368,
 )
 
 _OBSERVATIONPROTO = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='sensor_type', full_name='communicator_objects.ObservationProto.sensor_type', index=4,
+      number=5, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
       index=0, containing_type=None, fields=[]),
   ],
   serialized_start=79,
-  serialized_end=328,
+  serialized_end=388,
+_OBSERVATIONPROTO.fields_by_name['sensor_type'].enum_type = _SENSORTYPEPROTO
 _OBSERVATIONPROTO.oneofs_by_name['observation_data'].fields.append(
   _OBSERVATIONPROTO.fields_by_name['compressed_data'])
 _OBSERVATIONPROTO.fields_by_name['compressed_data'].containing_oneof = _OBSERVATIONPROTO.oneofs_by_name['observation_data']
 DESCRIPTOR.message_types_by_name['ObservationProto'] = _OBSERVATIONPROTO
 DESCRIPTOR.enum_types_by_name['CompressionTypeProto'] = _COMPRESSIONTYPEPROTO
+DESCRIPTOR.enum_types_by_name['SensorTypeProto'] = _SENSORTYPEPROTO
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 ObservationProto = _reflection.GeneratedProtocolMessageType('ObservationProto', (_message.Message,), dict(

ml-agents-envs/mlagents_envs/communicator_objects/observation_pb2.pyi

 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
     EnumDescriptor as google___protobuf___descriptor___EnumDescriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
+from google.protobuf.internal.enum_type_wrapper import (
+    _EnumTypeWrapper as google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper,
+)
+
 from google.protobuf.message import (
     Message as google___protobuf___message___Message,
 )
-    List as typing___List,
+    NewType as typing___NewType,
-    Tuple as typing___Tuple,
     cast as typing___cast,
 )
 
 builtin___bytes = bytes
 builtin___float = float
 builtin___int = int
-builtin___str = str
-class CompressionTypeProto(builtin___int):
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+CompressionTypeProtoValue = typing___NewType('CompressionTypeProtoValue', builtin___int)
+type___CompressionTypeProtoValue = CompressionTypeProtoValue
+CompressionTypeProto: _CompressionTypeProto
+class _CompressionTypeProto(google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper[CompressionTypeProtoValue]):
-    @classmethod
-    def Name(cls, number: builtin___int) -> builtin___str: ...
-    @classmethod
-    def Value(cls, name: builtin___str) -> 'CompressionTypeProto': ...
-    @classmethod
-    def keys(cls) -> typing___List[builtin___str]: ...
-    @classmethod
-    def values(cls) -> typing___List['CompressionTypeProto']: ...
-    @classmethod
-    def items(cls) -> typing___List[typing___Tuple[builtin___str, 'CompressionTypeProto']]: ...
-    NONE = typing___cast('CompressionTypeProto', 0)
-    PNG = typing___cast('CompressionTypeProto', 1)
-NONE = typing___cast('CompressionTypeProto', 0)
-PNG = typing___cast('CompressionTypeProto', 1)
+    NONE = typing___cast(CompressionTypeProtoValue, 0)
+    PNG = typing___cast(CompressionTypeProtoValue, 1)
+NONE = typing___cast(CompressionTypeProtoValue, 0)
+PNG = typing___cast(CompressionTypeProtoValue, 1)
+type___CompressionTypeProto = CompressionTypeProto
+
+SensorTypeProtoValue = typing___NewType('SensorTypeProtoValue', builtin___int)
+type___SensorTypeProtoValue = SensorTypeProtoValue
+SensorTypeProto: _SensorTypeProto
+class _SensorTypeProto(google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper[SensorTypeProtoValue]):
+    DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ...
+    OBSERVATION = typing___cast(SensorTypeProtoValue, 0)
+    PARAMETERIZATION = typing___cast(SensorTypeProtoValue, 1)
+    REWARD = typing___cast(SensorTypeProtoValue, 2)
+OBSERVATION = typing___cast(SensorTypeProtoValue, 0)
+PARAMETERIZATION = typing___cast(SensorTypeProtoValue, 1)
+REWARD = typing___cast(SensorTypeProtoValue, 2)
+type___SensorTypeProto = SensorTypeProto
-        data = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float]
+        data: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] = ...
-        @classmethod
-        def FromString(cls, s: builtin___bytes) -> ObservationProto.FloatData: ...
-        def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        if sys.version_info >= (3,):
-            def ClearField(self, field_name: typing_extensions___Literal[u"data"]) -> None: ...
-        else:
-            def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> None: ...
+    type___FloatData = FloatData
-    shape = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int]
-    compression_type = ... # type: CompressionTypeProto
-    compressed_data = ... # type: builtin___bytes
+    shape: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] = ...
+    compression_type: type___CompressionTypeProtoValue = ...
+    compressed_data: builtin___bytes = ...
+    sensor_type: type___SensorTypeProtoValue = ...
-    def float_data(self) -> ObservationProto.FloatData: ...
+    def float_data(self) -> type___ObservationProto.FloatData: ...
-        compression_type : typing___Optional[CompressionTypeProto] = None,
+        compression_type : typing___Optional[type___CompressionTypeProtoValue] = None,
-        float_data : typing___Optional[ObservationProto.FloatData] = None,
+        float_data : typing___Optional[type___ObservationProto.FloatData] = None,
+        sensor_type : typing___Optional[type___SensorTypeProtoValue] = None,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> ObservationProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"compressed_data",u"float_data",u"observation_data"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"compressed_data",u"compression_type",u"float_data",u"observation_data",u"shape"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"compressed_data",b"compressed_data",u"float_data",b"float_data",u"observation_data",b"observation_data"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"compressed_data",b"compressed_data",u"compression_type",b"compression_type",u"float_data",b"float_data",u"observation_data",b"observation_data",u"shape",b"shape"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"compressed_data",b"compressed_data",u"float_data",b"float_data",u"observation_data",b"observation_data"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"compressed_data",b"compressed_data",u"compression_type",b"compression_type",u"float_data",b"float_data",u"observation_data",b"observation_data",u"sensor_type",b"sensor_type",u"shape",b"shape"]) -> None: ...
+type___ObservationProto = ObservationProto

ml-agents-envs/mlagents_envs/communicator_objects/space_type_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     EnumDescriptor as google___protobuf___descriptor___EnumDescriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
+
+from google.protobuf.internal.enum_type_wrapper import (
+    _EnumTypeWrapper as google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper,
 )
 
 from google.protobuf.message import (
 from typing import (
-    List as typing___List,
-    Tuple as typing___Tuple,
+    NewType as typing___NewType,
-builtin___str = str
-class SpaceTypeProto(builtin___int):
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+SpaceTypeProtoValue = typing___NewType('SpaceTypeProtoValue', builtin___int)
+type___SpaceTypeProtoValue = SpaceTypeProtoValue
+SpaceTypeProto: _SpaceTypeProto
+class _SpaceTypeProto(google___protobuf___internal___enum_type_wrapper____EnumTypeWrapper[SpaceTypeProtoValue]):
-    @classmethod
-    def Name(cls, number: builtin___int) -> builtin___str: ...
-    @classmethod
-    def Value(cls, name: builtin___str) -> 'SpaceTypeProto': ...
-    @classmethod
-    def keys(cls) -> typing___List[builtin___str]: ...
-    @classmethod
-    def values(cls) -> typing___List['SpaceTypeProto']: ...
-    @classmethod
-    def items(cls) -> typing___List[typing___Tuple[builtin___str, 'SpaceTypeProto']]: ...
-    discrete = typing___cast('SpaceTypeProto', 0)
-    continuous = typing___cast('SpaceTypeProto', 1)
-discrete = typing___cast('SpaceTypeProto', 0)
-continuous = typing___cast('SpaceTypeProto', 1)
+    discrete = typing___cast(SpaceTypeProtoValue, 0)
+    continuous = typing___cast(SpaceTypeProtoValue, 1)
+discrete = typing___cast(SpaceTypeProtoValue, 0)
+continuous = typing___cast(SpaceTypeProtoValue, 1)
+type___SpaceTypeProto = SpaceTypeProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_input_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class UnityInputProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
 
         rl_input : typing___Optional[mlagents_envs___communicator_objects___unity_rl_input_pb2___UnityRLInputProto] = None,
         rl_initialization_input : typing___Optional[mlagents_envs___communicator_objects___unity_rl_initialization_input_pb2___UnityRLInitializationInputProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityInputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",u"rl_input"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",u"rl_input"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> None: ...
+type___UnityInputProto = UnityInputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_message_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class UnityMessageProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
 
         unity_output : typing___Optional[mlagents_envs___communicator_objects___unity_output_pb2___UnityOutputProto] = None,
         unity_input : typing___Optional[mlagents_envs___communicator_objects___unity_input_pb2___UnityInputProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityMessageProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"header",u"unity_input",u"unity_output"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"header",u"unity_input",u"unity_output"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> None: ...
+type___UnityMessageProto = UnityMessageProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_output_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class UnityOutputProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
 
         rl_output : typing___Optional[mlagents_envs___communicator_objects___unity_rl_output_pb2___UnityRLOutputProto] = None,
         rl_initialization_output : typing___Optional[mlagents_envs___communicator_objects___unity_rl_initialization_output_pb2___UnityRLInitializationOutputProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityOutputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",u"rl_output"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",u"rl_output"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> None: ...
+type___UnityOutputProto = UnityOutputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.message import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    seed = ... # type: builtin___int
-    communication_version = ... # type: typing___Text
-    package_version = ... # type: typing___Text
+    seed: builtin___int = ...
+    communication_version: typing___Text = ...
+    package_version: typing___Text = ...
 
     @property
     def capabilities(self) -> mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto: ...
         package_version : typing___Optional[typing___Text] = None,
         capabilities : typing___Optional[mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityRLInitializationInputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"capabilities"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"capabilities",u"communication_version",u"package_version",u"seed"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"package_version",b"package_version",u"seed",b"seed"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"package_version",b"package_version",u"seed",b"seed"]) -> None: ...
+type___UnityRLInitializationInputProto = UnityRLInitializationInputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
-    name = ... # type: typing___Text
-    communication_version = ... # type: typing___Text
-    log_path = ... # type: typing___Text
-    package_version = ... # type: typing___Text
+    name: typing___Text = ...
+    communication_version: typing___Text = ...
+    log_path: typing___Text = ...
+    package_version: typing___Text = ...
 
     @property
     def brain_parameters(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___brain_parameters_pb2___BrainParametersProto]: ...
         package_version : typing___Optional[typing___Text] = None,
         capabilities : typing___Optional[mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto] = None,
         ) -> None: ...
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityRLInitializationOutputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def HasField(self, field_name: typing_extensions___Literal[u"capabilities"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_parameters",u"capabilities",u"communication_version",u"log_path",u"name",u"package_version"]) -> None: ...
-    else:
-        def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ...
-        def ClearField(self, field_name: typing_extensions___Literal[u"brain_parameters",b"brain_parameters",u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"log_path",b"log_path",u"name",b"name",u"package_version",b"package_version"]) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"brain_parameters",b"brain_parameters",u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"log_path",b"log_path",u"name",b"name",u"package_version",b"package_version"]) -> None: ...
+type___UnityRLInitializationOutputProto = UnityRLInitializationOutputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_rl_input_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 )
 
 from mlagents_envs.communicator_objects.command_pb2 import (
-    CommandProto as mlagents_envs___communicator_objects___command_pb2___CommandProto,
+    CommandProtoValue as mlagents_envs___communicator_objects___command_pb2___CommandProtoValue,
 )
 
 from typing import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class UnityRLInputProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
     class ListAgentActionProto(google___protobuf___message___Message):
             *,
             value : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto]] = None,
             ) -> None: ...
-        @classmethod
-        def FromString(cls, s: builtin___bytes) -> UnityRLInputProto.ListAgentActionProto: ...
-        def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        if sys.version_info >= (3,):
-            def ClearField(self, field_name: typing_extensions___Literal[u"value"]) -> None: ...
-        else:
-            def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ...
+    type___ListAgentActionProto = ListAgentActionProto
-        key = ... # type: typing___Text
+        key: typing___Text = ...
-        def value(self) -> UnityRLInputProto.ListAgentActionProto: ...
+        def value(self) -> type___UnityRLInputProto.ListAgentActionProto: ...
-            value : typing___Optional[UnityRLInputProto.ListAgentActionProto] = None,
+            value : typing___Optional[type___UnityRLInputProto.ListAgentActionProto] = None,
-        @classmethod
-        def FromString(cls, s: builtin___bytes) -> UnityRLInputProto.AgentActionsEntry: ...
-        def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        if sys.version_info >= (3,):
-            def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ...
-            def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ...
-        else:
-            def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ...
-            def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ...
+        def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ...
+    type___AgentActionsEntry = AgentActionsEntry
-    command = ... # type: mlagents_envs___communicator_objects___command_pb2___CommandProto
-    side_channel = ... # type: builtin___bytes
+    command: mlagents_envs___communicator_objects___command_pb2___CommandProtoValue = ...
+    side_channel: builtin___bytes = ...
-    def agent_actions(self) -> typing___MutableMapping[typing___Text, UnityRLInputProto.ListAgentActionProto]: ...
+    def agent_actions(self) -> typing___MutableMapping[typing___Text, type___UnityRLInputProto.ListAgentActionProto]: ...
-        agent_actions : typing___Optional[typing___Mapping[typing___Text, UnityRLInputProto.ListAgentActionProto]] = None,
-        command : typing___Optional[mlagents_envs___communicator_objects___command_pb2___CommandProto] = None,
+        agent_actions : typing___Optional[typing___Mapping[typing___Text, type___UnityRLInputProto.ListAgentActionProto]] = None,
+        command : typing___Optional[mlagents_envs___communicator_objects___command_pb2___CommandProtoValue] = None,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityRLInputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"agent_actions",u"command",u"side_channel"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"agent_actions",b"agent_actions",u"command",b"command",u"side_channel",b"side_channel"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"agent_actions",b"agent_actions",u"command",b"command",u"side_channel",b"side_channel"]) -> None: ...
+type___UnityRLInputProto = UnityRLInputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_rl_output_pb2.pyi

 import sys
 from google.protobuf.descriptor import (
     Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
 )
 
 from google.protobuf.internal.containers import (
 builtin___int = int
 
 
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
 class UnityRLOutputProto(google___protobuf___message___Message):
     DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
     class ListAgentInfoProto(google___protobuf___message___Message):
             *,
             value : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto]] = None,
             ) -> None: ...
-        @classmethod
-        def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto.ListAgentInfoProto: ...
-        def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        if sys.version_info >= (3,):
-            def ClearField(self, field_name: typing_extensions___Literal[u"value"]) -> None: ...
-        else:
-            def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ...
+    type___ListAgentInfoProto = ListAgentInfoProto
-        key = ... # type: typing___Text
+        key: typing___Text = ...
-        def value(self) -> UnityRLOutputProto.ListAgentInfoProto: ...
+        def value(self) -> type___UnityRLOutputProto.ListAgentInfoProto: ...
-            value : typing___Optional[UnityRLOutputProto.ListAgentInfoProto] = None,
+            value : typing___Optional[type___UnityRLOutputProto.ListAgentInfoProto] = None,
-        @classmethod
-        def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto.AgentInfosEntry: ...
-        def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-        if sys.version_info >= (3,):
-            def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ...
-            def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ...
-        else:
-            def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ...
-            def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ...
+        def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ...
+        def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ...
+    type___AgentInfosEntry = AgentInfosEntry
-    side_channel = ... # type: builtin___bytes
+    side_channel: builtin___bytes = ...
-    def agentInfos(self) -> typing___MutableMapping[typing___Text, UnityRLOutputProto.ListAgentInfoProto]: ...
+    def agentInfos(self) -> typing___MutableMapping[typing___Text, type___UnityRLOutputProto.ListAgentInfoProto]: ...
-        agentInfos : typing___Optional[typing___Mapping[typing___Text, UnityRLOutputProto.ListAgentInfoProto]] = None,
+        agentInfos : typing___Optional[typing___Mapping[typing___Text, type___UnityRLOutputProto.ListAgentInfoProto]] = None,
-    @classmethod
-    def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto: ...
-    def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
-    if sys.version_info >= (3,):
-        def ClearField(self, field_name: typing_extensions___Literal[u"agentInfos",u"side_channel"]) -> None: ...
-    else:
-        def ClearField(self, field_name: typing_extensions___Literal[u"agentInfos",b"agentInfos",u"side_channel",b"side_channel"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"agentInfos",b"agentInfos",u"side_channel",b"side_channel"]) -> None: ...
+type___UnityRLOutputProto = UnityRLOutputProto

ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2.pyi

 # @generated by generate_proto_mypy_stubs.py.  Do not edit!
 import sys
+from google.protobuf.descriptor import (
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
+
+
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...

ml-agents-envs/mlagents_envs/rpc_utils.py

 from mlagents_envs.base_env import (
     BehaviorSpec,
     ActionType,
+    SensorType,
     DecisionSteps,
     TerminalSteps,
 )
     :return: BehaviorSpec object.
     """
     observation_shape = [tuple(obs.shape) for obs in agent_info.observations]
+    sensor_type = [SensorType(obs.sensor_type) for obs in agent_info.observations]
     action_type = (
         ActionType.DISCRETE
         if brain_param_proto.vector_action_space_type == 0
         ] = brain_param_proto.vector_action_size[0]
     else:
         action_shape = tuple(brain_param_proto.vector_action_size)
-    return BehaviorSpec(observation_shape, action_type, action_shape)
+    return BehaviorSpec(observation_shape, sensor_type, action_type, action_shape)
 
 
 @timed

protobuf-definitions/proto/mlagents_envs/communicator_objects/observation.proto

     PNG = 1;
 }
 
+enum SensorTypeProto {
+    OBSERVATION = 0;
+    PARAMETERIZATION = 1;
+    REWARD = 2;
+}
+
 message ObservationProto {
     message FloatData {
         repeated float data = 1;
         bytes compressed_data = 3;
         FloatData float_data = 4;
     }
+    SensorTypeProto sensor_type = 5;
 }