Add sensor type

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs

         }
 
         /// <inheritdoc/>
+        public virtual SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <inheritdoc/>
         public virtual SensorCompressionType GetCompressionType()
         {
             return SensorCompressionType.None;

com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs

                     observationProto.CompressedChannelMapping.AddRange(compressibleSensor.GetCompressedChannelMapping());
                 }
             }
+            observationProto.SensorType = (SensorTypeProto)sensor.GetSensorType();
             observationProto.Shape.AddRange(shape);
             return observationProto;
         }

com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs

         }
 
         /// <summary>
+        /// Camera sensors are always Observations.
+        /// </summary>
+        /// <returns>Sensor type of observation.</returns>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <summary>
         /// Generates a compressed image. This can be valuable in speeding-up training.
         /// </summary>
         /// <returns>Compressed image.</returns>

com.unity.ml-agents/Runtime/Sensors/ISensor.cs

     }
 
     /// <summary>
+    /// The semantic meaning of the sensor.
+    /// </summary>
+    public enum SensorType
+    {
+        /// <summary>
+        /// Sensor represents an agent's observation.
+        /// </summary>
+        Observation,
+
+        /// <summary>
+        /// Sensor represents an agent's task/goal parameterization.
+        /// </summary>
+        Goal,
+
+        /// <summary>
+        /// Sensor represents one or more reward signals.
+        /// </summary>
+        Reward
+    }
+
+
+    /// <summary>
     /// Sensor interface for generating observations.
     /// </summary>
     public interface ISensor
         /// </summary>
         /// <returns>The name of the sensor.</returns>
         string GetName();
+
+        /// <summary>
+        /// Get the semantic meaning of the sensor, i.e. whether it is an observation or other type
+        /// of data to be sent to the Agent.
+        /// </summary>
+        /// <returns>The type of the sensor.</returns>
+        SensorType GetSensorType();
     }
 
 

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs

         }
 
         /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <inheritdoc/>
         public string GetName()
         {
             return m_Name;

com.unity.ml-agents/Runtime/Sensors/Reflection/ReflectionSensorBase.cs

         }
 
         /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <inheritdoc/>
         public void Update() { }
 
         /// <inheritdoc/>

com.unity.ml-agents/Runtime/Sensors/RenderTextureSensor.cs

         }
 
         /// <summary>
+        /// RenderTexture sensors are always Observations.
+        /// </summary>
+        /// <returns>Sensor type of observation.</returns>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <summary>
         /// Converts a RenderTexture to a 2D texture.
         /// </summary>
         /// <returns>The 2D texture.</returns>

com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs

             return m_WrappedSensor.GetCompressionType();
         }
 
+        /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         /// <summary>
         /// Create Empty PNG for initializing the buffer for stacking.
         /// </summary>

com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs

         }
 
         /// <inheritdoc/>
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
+        /// <inheritdoc/>
         public virtual byte[] GetCompressedObservation()
         {
             return null;

com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs

             return new byte[] { 0 };
         }
 
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         public SensorCompressionType GetCompressionType()
         {
             return compressionType;

com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs

             return new[] { m_Height, m_Width, m_Channels };
         }
 
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         public int Write(ObservationWriter writer)
         {
             for (int i = 0; i < m_Width * m_Height * m_Channels; i++)

com.unity.ml-agents/Tests/Editor/Sensor/FloatVisualSensorTests.cs

             m_Shape = new[] { Height, Width, 1 };
         }
 
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         public string GetName()
         {
             return m_Name;

com.unity.ml-agents/Tests/Editor/Sensor/SensorShapeValidatorTests.cs

             return m_Shape;
         }
 
+        public SensorType GetSensorType()
+        {
+            return SensorType.Observation;
+        }
+
         public byte[] GetCompressedObservation()
         {
             return null;

ml-agents-envs/mlagents_envs/base_env.py

 
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
+from enum import Enum
 from typing import (
     List,
     NamedTuple,
         )
 
 
+class SensorType(Enum):
+    OBSERVATION = 0
+    PARAMETERIZATION = 1
+    REWARD = 2
+
+
 class ActionSpec(NamedTuple):
     """
     A NamedTuple containing utility functions and information about the action spaces
     - observation_shapes is a List of Tuples of int : Each Tuple corresponds
     to an observation's dimensions. The shape tuples have the same ordering as
     the ordering of the DecisionSteps and TerminalSteps.
+    - sensor_types is a List of SensorTypes, each corresponding to the type of
+    sensor (i.e. observation, goal, etc).
+    sensor_types: List[SensorType]
     action_spec: ActionSpec
 
 

ml-agents-envs/mlagents_envs/rpc_utils.py

 from mlagents_envs.base_env import (
     BehaviorSpec,
     ActionSpec,
+    SensorType,
     DecisionSteps,
     TerminalSteps,
 )
     :return: BehaviorSpec object.
     """
     observation_shape = [tuple(obs.shape) for obs in agent_info.observations]
+    sensor_type = [SensorType(obs.sensor_type) for obs in agent_info.observations]
-    return BehaviorSpec(observation_shape, action_spec)
+    return BehaviorSpec(observation_shape, sensor_type, action_spec)
 
 
 class OffsetBytesIO:

protobuf-definitions/proto/mlagents_envs/communicator_objects/observation.proto

     PNG = 1;
 }
 
+enum SensorTypeProto {
+    OBSERVATION = 0;
+    GOAL = 1;
+    REWARD = 2;
+}
+
 message ObservationProto {
     message FloatData {
         repeated float data = 1;
         FloatData float_data = 4;
     }
     repeated int32 compressed_channel_mapping = 5;
+    SensorTypeProto sensor_type = 6;
 }