example custom sensor

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensorComponentBase.cs

         internal Color rayMissColor = Color.white;
 
         [NonSerialized]
-        RayPerceptionSensor m_RaySensor;
+        protected RayPerceptionSensor m_RaySensor;
 
         /// <summary>
         /// Get the RayPerceptionSensor that was created.

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs

         /// <returns></returns>
         public int OutputSize()
         {
-            return ((DetectableTags?.Count ?? 0) + 2) * (Angles?.Count ?? 0);
+            return OutputSizePerRay() * NumRays();
+        }
+
+        public int OutputSizePerRay()
+        {
+            return (DetectableTags?.Count ?? 0) + 2;
+        }
+
+        public int NumRays()
+        {
+            return Angles?.Count ?? 0;
         }
 
         /// <summary>
             }
         }
 
+        public int CustomObservationSizePerRay;
+
+        public virtual void GetCustomObservationData(RayOutput rayOutput, float[] buffer) { }
+
         /// <summary>
         /// RayOutput for each ray that was cast.
         /// </summary>
     public class RayPerceptionSensor : ISensor, IBuiltInSensor
     {
         float[] m_Observations;
+        float[] m_SingleRayObservation;
         ObservationSpec m_ObservationSpec;
         string m_Name;
 
             get { return m_DebugLastFrameCount; }
         }
 
+        internal int ObservationSizePerRay
+        {
+            get { return m_RayPerceptionOutput.CustomObservationSizePerRay + ((m_RayPerceptionInput.DetectableTags?.Count ?? 0) + 2); }
+        }
+
         /// <summary>
         /// Creates the RayPerceptionSensor.
         /// </summary>
         }
 
         /// <summary>
+        /// Creates the RayPerceptionSensor.
+        /// </summary>
+        /// <param name="name">The name of the sensor.</param>
+        /// <param name="rayInput">The inputs for the sensor.</param>
+        /// <param name="rayOutput">The outputs for the sensor.</param>
+        public RayPerceptionSensor(string name, RayPerceptionInput rayInput, RayPerceptionOutput rayOutput)
+        {
+            m_Name = name;
+            m_RayPerceptionInput = rayInput;
+
+            SetNumObservations((rayOutput.CustomObservationSizePerRay + rayInput.OutputSizePerRay()) * rayInput.NumRays());
+            if (rayOutput.CustomObservationSizePerRay > 0)
+            {
+                m_SingleRayObservation = new float[rayOutput.CustomObservationSizePerRay];
+            }
+
+            m_DebugLastFrameCount = Time.frameCount;
+            m_RayPerceptionOutput = rayOutput;
+        }
+
+        /// <summary>
         /// The most recent raycast results.
         /// </summary>
         public RayPerceptionOutput RayPerceptionOutput
                 for (var rayIndex = 0; rayIndex < numRays; rayIndex++)
                 {
                     m_RayPerceptionOutput.RayOutputs?[rayIndex].ToFloatArray(numDetectableTags, rayIndex, m_Observations);
+                    if (m_RayPerceptionOutput.CustomObservationSizePerRay > 0)
+                    {
+                        Array.Clear(m_SingleRayObservation, 0, m_SingleRayObservation.Length);
+                        m_RayPerceptionOutput.GetCustomObservationData(m_RayPerceptionOutput.RayOutputs[rayIndex], m_SingleRayObservation);
+                        Array.Copy(
+                            m_SingleRayObservation,
+                            0,
+                            m_Observations,
+                            ObservationSizePerRay * rayIndex + m_RayPerceptionInput.OutputSizePerRay(),
+                            m_RayPerceptionOutput.CustomObservationSizePerRay
+                        );
+                    }
                 }
 
                 // Finally, add the observations to the ObservationWriter

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensorComponentCustom3D.cs

+using UnityEngine;
+using UnityEngine.Serialization;
+
+namespace Unity.MLAgents.Sensors
+{
+    public class CustomRayPerceptionOutput : RayPerceptionOutput
+    {
+        public CustomRayPerceptionOutput()
+        {
+            CustomObservationSizePerRay = 3;
+        }
+
+        public override void GetCustomObservationData(RayOutput rayOutput, float[] buffer)
+        {
+            var rb = rayOutput.HitGameObject.GetComponent<Rigidbody>();
+            if (rb != null)
+            {
+                buffer[0] = rb.velocity.x;
+                buffer[1] = rb.velocity.z;
+                buffer[2] = rb.mass;
+            }
+        }
+    }
+
+    /// <summary>
+    /// A component for Custom 3D Ray Perception.
+    /// </summary>
+    [AddComponentMenu("ML Agents/Custom Ray Perception Sensor 3D", (int)MenuGroup.Sensors)]
+    public class RayPerceptionSensorComponentCustom3D : RayPerceptionSensorComponent3D
+    {
+        public override ISensor[] CreateSensors()
+        {
+            var rayPerceptionInput = GetRayPerceptionInput();
+            var rayPerceptionOutput = new CustomRayPerceptionOutput();
+
+            m_RaySensor = new RayPerceptionSensor(SensorName, rayPerceptionInput, rayPerceptionOutput);
+
+            if (ObservationStacks != 1)
+            {
+                var stackingSensor = new StackingSensor(m_RaySensor, ObservationStacks);
+                return new ISensor[] { stackingSensor };
+            }
+
+            return new ISensor[] { m_RaySensor };
+        }
+    }
+}

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensorComponentCustom3D.cs.meta

+fileFormatVersion: 2
+guid: a1aeaa2bcb3974a44898e86dae3193d3
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: