[MLA-1824] make SensorComponent return ISensor[] (#5181)

* Make SensorComponent return an array

* split match3 sensors, partial retrain

* docstrings, migration, changelog, cleanup

Project/Assets/ML-Agents/Examples/Basic/Scripts/BasicSensorComponent.cs

         /// Creates a BasicSensor.
         /// </summary>
         /// <returns></returns>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return new BasicSensor(basicController);
+            return new ISensor[] { new BasicSensor(basicController) };
         }
     }
 

Project/Assets/ML-Agents/TestScenes/TestCompressedTexture/TestTextureSensorComponent.cs

 
 
     /// <inheritdoc/>
-    public override ISensor CreateSensor()
+    public override ISensor[] CreateSensors()
-            return new StackingSensor(m_Sensor, ObservationStacks);
+            return new ISensor[] { new StackingSensor(m_Sensor, ObservationStacks) };
-        return m_Sensor;
+        return new ISensor[] { m_Sensor };
     }
 }
 

com.unity.ml-agents.extensions/Runtime/Match3/Match3Sensor.cs

 
 namespace Unity.MLAgents.Extensions.Match3
 {
+
+    /// <summary>
+    /// Delegate that provides integer values at a given (x,y) coordinate.
+    /// </summary>
+    /// <param name="x"></param>
+    /// <param name="y"></param>
+    public delegate int GridValueProvider(int x, int y);
+
     /// <summary>
     /// Type of observations to generate.
     ///
 
     /// <summary>
     /// Sensor for Match3 games. Can generate either vector, compressed visual,
-    /// or uncompressed visual observations. Uses AbstractBoard.GetCellType()
-    /// and AbstractBoard.GetSpecialType() to determine the observation values.
+    /// or uncompressed visual observations. Uses a GridValueProvider to determine the observation values.
-        private AbstractBoard m_Board;
-        private int[] m_SparseChannelMapping;
-        private int m_NumCellTypes;
-        private int m_NumSpecialTypes;
-
-        private int SpecialTypeSize
-        {
-            get { return m_NumSpecialTypes == 0 ? 0 : m_NumSpecialTypes + 1; }
-        }
+        private GridValueProvider m_GridValues;
+        private int m_OneHotSize;
-        /// Create a sensor for the board with the specified observation type.
+        /// Create a sensor for the GridValueProvider with the specified observation type.
-        /// <param name="board"></param>
-        /// <param name="obsType"></param>
-        /// <param name="name"></param>
-        public Match3Sensor(AbstractBoard board, Match3ObservationType obsType, string name)
+        /// <remarks>
+        /// Use Match3Sensor.CellTypeSensor() or Match3Sensor.SpecialTypeSensor() instead of calling
+        /// the constructor directly.
+        /// </remarks>
+        /// <param name="board">The abstract board. This is only used to get the size.</param>
+        /// <param name="gvp">The GridValueProvider, should be either board.GetCellType or board.GetSpecialType.</param>
+        /// <param name="oneHotSize">The number of possible values that the GridValueProvider can return.</param>
+        /// <param name="obsType">Whether to produce vector or visual observations</param>
+        /// <param name="name">Name of the sensor.</param>
+        public Match3Sensor(AbstractBoard board, GridValueProvider gvp, int oneHotSize, Match3ObservationType obsType, string name)
-            m_Board = board;
-            m_NumCellTypes = board.NumCellTypes;
-            m_NumSpecialTypes = board.NumSpecialTypes;
+            m_GridValues = gvp;
+            m_OneHotSize = oneHotSize;
-                ? ObservationSpec.Vector(m_Rows * m_Columns * (m_NumCellTypes + SpecialTypeSize))
-                : ObservationSpec.Visual(m_Rows, m_Columns, m_NumCellTypes + SpecialTypeSize);
+                ? ObservationSpec.Vector(m_Rows * m_Columns * oneHotSize)
+                : ObservationSpec.Visual(m_Rows, m_Columns, oneHotSize);
+        }
-            // See comment in GetCompressedObservation()
-            var cellTypePaddedSize = 3 * ((m_NumCellTypes + 2) / 3);
-            m_SparseChannelMapping = new int[cellTypePaddedSize + SpecialTypeSize];
-            // If we have 4 cell types and 2 special types (3 special size), we'd have
-            // [0, 1, 2, 3, -1, -1, 4, 5, 6]
-            for (var i = 0; i < m_NumCellTypes; i++)
-            {
-                m_SparseChannelMapping[i] = i;
-            }
+        /// <summary>
+        /// Create a sensor that encodes the board cells as observations.
+        /// </summary>
+        /// <param name="board">The abstract board.</param>
+        /// <param name="obsType">Whether to produce vector or visual observations</param>
+        /// <param name="name">Name of the sensor.</param>
+        /// <returns></returns>
+        public static Match3Sensor CellTypeSensor(AbstractBoard board, Match3ObservationType obsType, string name)
+        {
+            return new Match3Sensor(board, board.GetCellType, board.NumCellTypes, obsType, name);
+        }
-            for (var i = m_NumCellTypes; i < cellTypePaddedSize; i++)
-            {
-                m_SparseChannelMapping[i] = -1;
-            }
-
-            for (var i = 0; i < SpecialTypeSize; i++)
-            {
-                m_SparseChannelMapping[cellTypePaddedSize + i] = i + m_NumCellTypes;
-            }
+        /// <summary>
+        /// Create a sensor that encodes the cell special types as observations.
+        /// </summary>
+        /// <param name="board">The abstract board.</param>
+        /// <param name="obsType">Whether to produce vector or visual observations</param>
+        /// <param name="name">Name of the sensor.</param>
+        /// <returns></returns>
+        public static Match3Sensor SpecialTypeSensor(AbstractBoard board, Match3ObservationType obsType, string name)
+        {
+            var specialSize = board.NumSpecialTypes == 0 ? 0 : board.NumSpecialTypes + 1;
+            return new Match3Sensor(board, board.GetSpecialType, specialSize, obsType, name);
         }
 
         /// <inheritdoc/>
         /// <inheritdoc/>
         public int Write(ObservationWriter writer)
         {
-            if (m_Board.Rows != m_Rows || m_Board.Columns != m_Columns || m_Board.NumCellTypes != m_NumCellTypes)
-            {
-                Debug.LogWarning(
-                    $"Board shape changes since sensor initialization. This may cause unexpected results. " +
-                    $"Old shape: Rows={m_Rows} Columns={m_Columns}, NumCellTypes={m_NumCellTypes} " +
-                    $"Current shape: Rows={m_Board.Rows} Columns={m_Board.Columns}, NumCellTypes={m_Board.NumCellTypes}"
-                );
-            }
+            // if (m_Board.Rows != m_Rows || m_Board.Columns != m_Columns || m_Board.NumCellTypes != m_NumCellTypes)
+            // {
+            //     Debug.LogWarning(
+            //         $"Board shape changes since sensor initialization. This may cause unexpected results. " +
+            //         $"Old shape: Rows={m_Rows} Columns={m_Columns}, NumCellTypes={m_NumCellTypes} " +
+            //         $"Current shape: Rows={m_Board.Rows} Columns={m_Board.Columns}, NumCellTypes={m_Board.NumCellTypes}"
+            //     );
+            // }
 
             if (m_ObservationType == Match3ObservationType.Vector)
             {
                     for (var c = 0; c < m_Columns; c++)
                     {
-                        var val = m_Board.GetCellType(r, c);
-                        for (var i = 0; i < m_NumCellTypes; i++)
+                        var val = m_GridValues(r, c);
+
+                        for (var i = 0; i < m_OneHotSize; i++)
-
-                        if (m_NumSpecialTypes > 0)
-                        {
-                            var special = m_Board.GetSpecialType(r, c);
-                            for (var i = 0; i < SpecialTypeSize; i++)
-                            {
-                                writer[offset] = (i == special) ? 1.0f : 0.0f;
-                                offset++;
-                            }
-                        }
                     }
                 }
 
                 {
                     for (var c = 0; c < m_Columns; c++)
                     {
-                        var val = m_Board.GetCellType(r, c);
-                        for (var i = 0; i < m_NumCellTypes; i++)
+                        var val = m_GridValues(r, c);
+                        for (var i = 0; i < m_OneHotSize; i++)
-
-                        if (m_NumSpecialTypes > 0)
-                        {
-                            var special = m_Board.GetSpecialType(r, c);
-                            for (var i = 0; i < SpecialTypeSize; i++)
-                            {
-                                writer[offset] = (i == special) ? 1.0f : 0.0f;
-                                offset++;
-                            }
-                        }
                     }
                 }
 
             // fit in in 2 images, but we'll use 3 here (2 PNGs for the 4 cell type channels, and 1 for
             // the special types). Note that we have to also implement the sparse channel mapping.
             // Optimize this it later.
-            var numCellImages = (m_NumCellTypes + 2) / 3;
+            var numCellImages = (m_OneHotSize + 2) / 3;
-                converter.EncodeToTexture(m_Board.GetCellType, tempTexture, 3 * i);
-                bytesOut.AddRange(tempTexture.EncodeToPNG());
-            }
-
-            var numSpecialImages = (SpecialTypeSize + 2) / 3;
-            for (var i = 0; i < numSpecialImages; i++)
-            {
-                converter.EncodeToTexture(m_Board.GetSpecialType, tempTexture, 3 * i);
+                converter.EncodeToTexture(m_GridValues, tempTexture, 3 * i);
                 bytesOut.AddRange(tempTexture.EncodeToPNG());
             }
 
         /// <inheritdoc/>
         public CompressionSpec GetCompressionSpec()
         {
-            return new CompressionSpec(GetCompressionType(), m_SparseChannelMapping);
+            return new CompressionSpec(GetCompressionType());
         }
 
         /// <inheritdoc/>
         int m_Height;
         int m_Width;
         private static Color[] s_OneHotColors = { Color.red, Color.green, Color.blue };
-
-        public delegate int GridValueProvider(int x, int y);
-
 
         public OneHotToTextureUtil(int height, int width)
         {

com.unity.ml-agents.extensions/Runtime/Match3/Match3SensorComponent.cs

         public Match3ObservationType ObservationType = Match3ObservationType.Vector;
 
         /// <inheritdoc/>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return new Match3Sensor(board, ObservationType, SensorName);
+            var cellSensor = Match3Sensor.CellTypeSensor(board, ObservationType, SensorName + " (cells)");
+            if (board.NumSpecialTypes > 0)
+            {
+                var specialSensor =
+                    Match3Sensor.SpecialTypeSensor(board, ObservationType, SensorName + " (special)");
+                return new ISensor[] { cellSensor, specialSensor };
+            }
+            else
+            {
+                return new ISensor[] { cellSensor };
+            }
         }
 
     }

com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodySensorComponent.cs

         /// Creates a PhysicsBodySensor.
         /// </summary>
         /// <returns></returns>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return new PhysicsBodySensor(RootBody, Settings, sensorName);
+            return new ISensor[] {new PhysicsBodySensor(RootBody, Settings, sensorName)};
         }
 
     }

com.unity.ml-agents.extensions/Runtime/Sensors/GridSensor.cs

         private Color DebugDefaultColor = new Color(1f, 1f, 1f, 0.25f);
 
         /// <inheritdoc/>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return this;
+            return new ISensor[] { this };
         }
 
         /// <summary>

com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodySensorComponent.cs

         /// Creates a PhysicsBodySensor.
         /// </summary>
         /// <returns></returns>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return new PhysicsBodySensor(GetPoseExtractor(), Settings, _sensorName);
+            return new ISensor[] { new PhysicsBodySensor(GetPoseExtractor(), Settings, _sensorName) };
         }
 
         /// <summary>

com.unity.ml-agents.extensions/Tests/Editor/Match3/Match3SensorTests.cs

         // Whether the expected PNG data should be written to a file.
         // Only set this to true if the compressed observation format changes.
         private bool WritePNGDataToFile = false;
+        private const string k_CellObservationPng = "match3obs";
+        private const string k_SpecialObservationPng = "match3obs_special";
 
         [Test]
         public void TestVectorObservations()
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.Vector;
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
 
             var expectedShape = new InplaceArray<int>(3 * 3 * 2);
             Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.Vector;
-            var sensor = sensorComponent.CreateSensor();
+            var sensors = sensorComponent.CreateSensors();
+            var cellSensor = sensors[0];
+            var specialSensor = sensors[1];
-            var expectedShape = new InplaceArray<int>(3 * 3 * (2 + 3));
-            Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
-            var expectedObs = new float[]
+            {
+                var expectedShape = new InplaceArray<int>(3 * 3 * 2);
+                Assert.AreEqual(expectedShape, cellSensor.GetObservationSpec().Shape);
+
+                var expectedObs = new float[]
+                {
+                    1, 0, /* (0) */ 0, 1, /* (1) */ 1, 0, /* (0) */
+                    1, 0, /* (0) */ 1, 0, /* (0) */ 1, 0, /* (0) */
+                    1, 0, /* (0) */ 1, 0, /* (0) */ 1, 0, /* (0) */
+                };
+                SensorTestHelper.CompareObservation(cellSensor, expectedObs);
+            }
-                1, 0, 1, 0, 0, /* (0, 0) */ 0, 1, 1, 0, 0, /* (0, 1) */ 1, 0, 1, 0, 0, /* (0, 0) */
-                1, 0, 0, 0, 1, /* (0, 2) */ 1, 0, 1, 0, 0, /* (0, 0) */ 1, 0, 1, 0, 0, /* (0, 0) */
-                1, 0, 1, 0, 0, /* (0, 0) */ 1, 0, 0, 1, 0, /* (0, 1) */ 1, 0, 1, 0, 0, /* (0, 0) */
-            };
-            SensorTestHelper.CompareObservation(sensor, expectedObs);
+                var expectedShape = new InplaceArray<int>(3 * 3 * 3);
+                Assert.AreEqual(expectedShape, specialSensor.GetObservationSpec().Shape);
+
+                var expectedObs = new float[]
+                {
+                    1, 0, 0, /* (0) */ 1, 0, 0, /* (1) */ 1, 0, 0, /* (0) */
+                    0, 0, 1, /* (2) */ 1, 0, 0, /* (0) */ 1, 0, 0, /* (0) */
+                    1, 0, 0, /* (0) */ 0, 1, 0, /* (1) */ 1, 0, 0, /* (0) */
+                };
+                SensorTestHelper.CompareObservation(specialSensor, expectedObs);
+            }
         }
 
         [Test]
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.UncompressedVisual;
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
 
             var expectedShape = new InplaceArray<int>(3, 3, 2);
             Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.UncompressedVisual;
-            var sensor = sensorComponent.CreateSensor();
+            var sensors = sensorComponent.CreateSensors();
+            var cellSensor = sensors[0];
+            var specialSensor = sensors[1];
-            var expectedShape = new InplaceArray<int>(3, 3, 2 + 3);
-            Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
+            {
+                var expectedShape = new InplaceArray<int>(3, 3, 2);
+                Assert.AreEqual(expectedShape, cellSensor.GetObservationSpec().Shape);
-            Assert.AreEqual(SensorCompressionType.None, sensor.GetCompressionSpec().SensorCompressionType);
+                Assert.AreEqual(SensorCompressionType.None, cellSensor.GetCompressionSpec().SensorCompressionType);
+
+                var expectedObs = new float[]
+                {
+                    1, 0, /* (0) */ 0, 1, /* (1) */ 1, 0, /* (0) */
+                    1, 0, /* (0) */ 1, 0, /* (0) */ 1, 0, /* (0) */
+                    1, 0, /* (0) */ 1, 0, /* (0) */ 1, 0, /* (0) */
+                };
+                SensorTestHelper.CompareObservation(cellSensor, expectedObs);
-            var expectedObs = new float[]
+                var expectedObs3D = new float[,,]
+                {
+                    {{1, 0}, {0, 1}, {1, 0}},
+                    {{1, 0}, {1, 0}, {1, 0}},
+                    {{1, 0}, {1, 0}, {1, 0}},
+                };
+                SensorTestHelper.CompareObservation(cellSensor, expectedObs3D);
+            }
-                1, 0, 1, 0, 0, /* (0, 0) */ 0, 1, 1, 0, 0, /* (0, 1) */ 1, 0, 1, 0, 0, /* (0, 0) */
-                1, 0, 0, 0, 1, /* (0, 2) */ 1, 0, 1, 0, 0, /* (0, 0) */ 1, 0, 1, 0, 0, /* (0, 0) */
-                1, 0, 1, 0, 0, /* (0, 0) */ 1, 0, 0, 1, 0, /* (0, 1) */ 1, 0, 1, 0, 0, /* (0, 0) */
-            };
-            SensorTestHelper.CompareObservation(sensor, expectedObs);
+                var expectedShape = new InplaceArray<int>(3, 3, 3);
+                Assert.AreEqual(expectedShape, specialSensor.GetObservationSpec().Shape);
+
+                Assert.AreEqual(SensorCompressionType.None, specialSensor.GetCompressionSpec().SensorCompressionType);
-            var expectedObs3D = new float[,,]
-            {
-                {{1, 0, 1, 0, 0}, {0, 1, 1, 0, 0}, {1, 0, 1, 0, 0}},
-                {{1, 0, 0, 0, 1}, {1, 0, 1, 0, 0}, {1, 0, 1, 0, 0}},
-                {{1, 0, 1, 0, 0}, {1, 0, 0, 1, 0}, {1, 0, 1, 0, 0}},
-            };
-            SensorTestHelper.CompareObservation(sensor, expectedObs3D);
+                var expectedObs = new float[]
+                {
+                    1, 0, 0, /* (0) */ 1, 0, 0, /* (1) */ 1, 0, 0, /* (0) */
+                    0, 0, 1, /* (2) */ 1, 0, 0, /* (0) */ 1, 0, 0, /* (0) */
+                    1, 0, 0, /* (0) */ 0, 1, 0, /* (1) */ 1, 0, 0, /* (0) */
+                };
+                SensorTestHelper.CompareObservation(specialSensor, expectedObs);
+
+                var expectedObs3D = new float[,,]
+                {
+                    {{1, 0, 0}, {1, 0, 0}, {1, 0, 0}},
+                    {{0, 0, 1}, {1, 0, 0}, {1, 0, 0}},
+                    {{1, 0, 0}, {0, 1, 0}, {1, 0, 0}},
+                };
+                SensorTestHelper.CompareObservation(specialSensor, expectedObs3D);
+            }
         }
 
         [Test]
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.CompressedVisual;
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
 
             var expectedShape = new InplaceArray<int>(3, 3, 2);
             Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
             if (WritePNGDataToFile)
             {
                 // Enable this if the format of the observation changes
-                SavePNGs(pngData, "match3obs");
+                SavePNGs(pngData, k_CellObservationPng);
-            var expectedPng = LoadPNGs("match3obs", 1);
+            var expectedPng = LoadPNGs(k_CellObservationPng, 1);
             Assert.AreEqual(expectedPng, pngData);
         }
 
 
             var sensorComponent = gameObj.AddComponent<Match3SensorComponent>();
             sensorComponent.ObservationType = Match3ObservationType.CompressedVisual;
-            var sensor = sensorComponent.CreateSensor();
+            var sensors = sensorComponent.CreateSensors();
-            var expectedShape = new InplaceArray<int>(3, 3, 2 + 3);
-            Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
+            var paths = new[] { k_CellObservationPng, k_SpecialObservationPng };
+            var expectedChannels = new[] { 2, 3 };
-            Assert.AreEqual(SensorCompressionType.PNG, sensor.GetCompressionSpec().SensorCompressionType);
-
-            var concatenatedPngData = sensor.GetCompressedObservation();
-            var pathPrefix = "match3obs_special";
-            if (WritePNGDataToFile)
+            for (var i = 0; i < 2; i++)
-                // Enable this if the format of the observation changes
-                SavePNGs(concatenatedPngData, pathPrefix);
+                var sensor = sensors[i];
+                var expectedShape = new InplaceArray<int>(3, 3, expectedChannels[i]);
+                Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
+
+                Assert.AreEqual(SensorCompressionType.PNG, sensor.GetCompressionSpec().SensorCompressionType);
+
+                var pngData = sensor.GetCompressedObservation();
+                if (WritePNGDataToFile)
+                {
+                    // Enable this if the format of the observation changes
+                    SavePNGs(pngData, paths[i]);
+                }
+
+                var expectedPng = LoadPNGs(paths[i], 1);
+                Assert.AreEqual(expectedPng, pngData);
-            var expectedPng = LoadPNGs(pathPrefix, 2);
-            Assert.AreEqual(expectedPng, concatenatedPngData);
+
         }
 
         /// <summary>

com.unity.ml-agents.extensions/Tests/Editor/Match3/match3obs0.png.meta

 fileFormatVersion: 2
 guid: 3e1767bf6c63e46b1a16404dc1afe508
 TextureImporter:
-  fileIDToRecycleName: {}
+  internalIDToNameTable: []
-  serializedVersion: 9
+  serializedVersion: 11
   mipmaps:
     mipMapMode: 0
     enableMipMap: 1
   maxTextureSizeSet: 0
   compressionQualitySet: 0
   textureFormatSet: 0
+  applyGammaDecoding: 0
-  - serializedVersion: 2
+  - serializedVersion: 3
     buildTarget: DefaultTexturePlatform
     maxTextureSize: 2048
     resizeAlgorithm: 0
     allowsAlphaSplitting: 0
     overridden: 0
     androidETC2FallbackOverride: 0
+    forceMaximumCompressionQuality_BC6H_BC7: 0
   spriteSheet:
     serializedVersion: 2
     sprites: []
     spriteID: 
+    internalID: 0
+    secondaryTextures: []
   spritePackingTag: 
   pSDRemoveMatte: 0
   pSDShowRemoveMatteOption: 0

com.unity.ml-agents.extensions/Tests/Editor/Match3/match3obs_special0.png.meta

 fileFormatVersion: 2
 guid: 2e4ca31cf9cff4505acbefe44b621d6f
 TextureImporter:
-  fileIDToRecycleName: {}
+  internalIDToNameTable: []
-  serializedVersion: 9
+  serializedVersion: 11
   mipmaps:
     mipMapMode: 0
     enableMipMap: 1
   maxTextureSizeSet: 0
   compressionQualitySet: 0
   textureFormatSet: 0
+  applyGammaDecoding: 0
-  - serializedVersion: 2
+  - serializedVersion: 3
     buildTarget: DefaultTexturePlatform
     maxTextureSize: 2048
     resizeAlgorithm: 0
     allowsAlphaSplitting: 0
     overridden: 0
     androidETC2FallbackOverride: 0
+    forceMaximumCompressionQuality_BC6H_BC7: 0
   spriteSheet:
     serializedVersion: 2
     sprites: []
     spriteID: 
+    internalID: 0
+    secondaryTextures: []
   spritePackingTag: 
   pSDRemoveMatte: 0
   pSDShowRemoveMatteOption: 0

com.unity.ml-agents.extensions/Tests/Runtime/Sensors/ArticulationBodySensorTests.cs

             var gameObj = new GameObject();
 
             var sensorComponent = gameObj.AddComponent<ArticulationBodySensorComponent>();
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
             SensorTestHelper.CompareObservation(sensor, new float[0]);
         }
 
                 UseLocalSpaceRotations = true
             };
 
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
             sensor.Update();
             var expected = new[]
             {
             };
             SensorTestHelper.CompareObservation(sensor, expected);
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
         }
 
         [Test]
 #endif
             };
 
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
             sensor.Update();
             var expected = new[]
             {
 #endif
             };
             SensorTestHelper.CompareObservation(sensor, expected);
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
 
             // Update the settings to only process joint observations
             sensorComponent.Settings = new PhysicsSensorSettings
             };
 
-            sensor = sensorComponent.CreateSensor();
+            sensor = sensorComponent.CreateSensors()[0];
             sensor.Update();
 
             expected = new[]
                 0f, // joint2.force
             };
             SensorTestHelper.CompareObservation(sensor, expected);
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
         }
     }
 }

com.unity.ml-agents.extensions/Tests/Runtime/Sensors/RigidBodySensorTests.cs

             var gameObj = new GameObject();
 
             var sensorComponent = gameObj.AddComponent<RigidBodySensorComponent>();
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
             SensorTestHelper.CompareObservation(sensor, new float[0]);
         }
 
                 UseLocalSpaceRotations = true
             };
 
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
             SensorTestHelper.CompareObservation(sensor, expected);
         }
 
             };
             sensorComponent.VirtualRoot = virtualRoot;
 
-            var sensor = sensorComponent.CreateSensor();
+            var sensor = sensorComponent.CreateSensors()[0];
             sensor.Update();
 
             // Note that the VirtualRoot is ignored from the observations
                 -1f, 1f, 0f, // Attached vel
                 0f, -1f, 1f // Leaf vel
             };
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
             SensorTestHelper.CompareObservation(sensor, expected);
 
             // Update the settings to only process joint observations
                 UseJointForces = true,
             };
 
-            sensor = sensorComponent.CreateSensor();
+            sensor = sensorComponent.CreateSensors()[0];
             sensor.Update();
 
             expected = new[]
                 0f, 0f, 0f, // joint2.torque
             };
             SensorTestHelper.CompareObservation(sensor, expected);
-            Assert.AreEqual(expected.Length, sensorComponent.CreateSensor().GetObservationSpec().Shape[0]);
+            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
 
         }
     }

com.unity.ml-agents/CHANGELOG.md

 
 ## [Unreleased]
 ### Major Changes
-#### com.unity.ml-agents (C#)
+#### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
 - The minimum supported Unity version was updated to 2019.4. (#5166)
 - Several breaking interface changes were made. See the
 [Migration Guide](https://github.com/Unity-Technologies/ml-agents/blob/release_14_docs/docs/Migrating.md) for more
 - `ISensor.GetCompressionType()` was removed, and `GetCompressionSpec()` was added. The `ISparseChannelSensor`
 interface was removed. (#5164)
 - The abstract method `SensorComponent.GetObservationShape()` was no longer being called, so it has been removed. (#5172)
+- `SensorComponent.CreateSensor()` was replaced with `SensorComponent.CreateSensor()`, which returns an `ISensor[]`. (#5181)
+- `Match3Sensor` was refactored to produce cell and special type observations separately, and `Match3SensorComponent` now
+produces two `Match3Sensor`s (unless there are no special types). Previously trained models will have different observation
+sizes and will need to be retrained. (#5181)
 
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 
 ### Bug Fixes
-#### com.unity.ml-agents (C#)
+#### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 
 

com.unity.ml-agents/Runtime/Agent.cs

             sensors.Capacity += attachedSensorComponents.Length;
             foreach (var component in attachedSensorComponents)
             {
-                sensors.Add(component.CreateSensor());
+                sensors.AddRange(component.CreateSensors());
             }
 
             // Support legacy CollectObservations

com.unity.ml-agents/Runtime/Sensors/BufferSensorComponent.cs

         private BufferSensor m_Sensor;
 
         /// <inheritdoc/>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return m_Sensor;
+            return new ISensor[] { m_Sensor };
         }
 
         /// <summary>

com.unity.ml-agents/Runtime/Sensors/CameraSensorComponent.cs

         /// Creates the <see cref="CameraSensor"/>
         /// </summary>
         /// <returns>The created <see cref="CameraSensor"/> object for this component.</returns>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-                return new StackingSensor(m_Sensor, ObservationStacks);
+                return new ISensor[] { new StackingSensor(m_Sensor, ObservationStacks) };
-            return m_Sensor;
+            return new ISensor[] { m_Sensor };
         }
 
         /// <summary>

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensorComponentBase.cs

         /// Returns an initialized raycast sensor.
         /// </summary>
         /// <returns></returns>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
         {
             var rayPerceptionInput = GetRayPerceptionInput();
 
             {
                 var stackingSensor = new StackingSensor(m_RaySensor, ObservationStacks);
-                return stackingSensor;
+                return new ISensor[] { stackingSensor };
-            return m_RaySensor;
+            return new ISensor[] { m_RaySensor };
         }
 
         /// <summary>

com.unity.ml-agents/Runtime/Sensors/RenderTextureSensorComponent.cs

         }
 
         /// <inheritdoc/>
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-                return new StackingSensor(m_Sensor, ObservationStacks);
+                return new ISensor[] { new StackingSensor(m_Sensor, ObservationStacks) };
-            return m_Sensor;
+            return new ISensor[] { m_Sensor };
         }
 
         /// <summary>

com.unity.ml-agents/Runtime/Sensors/SensorComponent.cs

     public abstract class SensorComponent : MonoBehaviour
     {
         /// <summary>
-        /// Create the ISensor. This is called by the Agent when it is initialized.
+        /// Create the ISensors. This is called by the Agent when it is initialized.
-        /// <returns>Created ISensor object.</returns>
-        public abstract ISensor CreateSensor();
+        /// <returns>Created ISensor objects.</returns>
+        public abstract ISensor[] CreateSensors();
     }
 }

com.unity.ml-agents/Tests/Editor/Inference/ModelRunnerTest.cs

             var modelRunner = new ModelRunner(discreteONNXModel, actionSpec, InferenceDevice.Burst);
             var info1 = new AgentInfo();
             info1.episodeId = 1;
-            modelRunner.PutObservations(info1, new[] { sensor_21_20_3.CreateSensor() }.ToList());
+            modelRunner.PutObservations(info1, new[] { sensor_21_20_3.CreateSensors()[0] }.ToList());
-            modelRunner.PutObservations(info2, new[] { sensor_21_20_3.CreateSensor() }.ToList());
+            modelRunner.PutObservations(info2, new[] { sensor_21_20_3.CreateSensors()[0] }.ToList());
 
             modelRunner.DecideBatch();
 

com.unity.ml-agents/Tests/Editor/Inference/ParameterLoaderTest.cs

     {
         public ISensor Sensor;
 
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            return Sensor;
+            return new ISensor[] { Sensor };
         }
     }
 
 
             var errors = BarracudaModelParamLoader.CheckModel(
                 model, validBrainParameters,
-                new ISensor[] { new VectorSensor(8), sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]
+                new ISensor[]
+                {
+                    new VectorSensor(8),
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
             );
             Assert.AreEqual(0, errors.Count()); // There should not be any errors
         }
 
             var errors = BarracudaModelParamLoader.CheckModel(
                 model, validBrainParameters,
-                new ISensor[] { sensor_21_20_3.CreateSensor() }, new ActuatorComponent[0]
+                new ISensor[] { sensor_21_20_3.CreateSensors()[0] }, new ActuatorComponent[0]
             );
             Assert.AreEqual(0, errors.Count()); // There should not be any errors
         }
 
             var errors = BarracudaModelParamLoader.CheckModel(
                 model, validBrainParameters,
-                new ISensor[] { new VectorSensor(validBrainParameters.VectorObservationSize) }, new ActuatorComponent[0]
+                new ISensor[]
+                {
+                    new VectorSensor(validBrainParameters.VectorObservationSize)
+                }, new ActuatorComponent[0]
             );
             Assert.AreEqual(0, errors.Count()); // There should not be any errors
         }
             brainParameters.VectorObservationSize = 9; // Invalid observation
             var errors = BarracudaModelParamLoader.CheckModel(
                 model, brainParameters,
-                new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]
+                new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
             );
             Assert.Greater(errors.Count(), 0);
 
                 model, brainParameters,
-                new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]
+                new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
             );
             Assert.Greater(errors.Count(), 0);
         }
 
             var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
             brainParameters.VectorObservationSize = 1; // Invalid observation
-            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor() }, new ActuatorComponent[0]);
+            var errors = BarracudaModelParamLoader.CheckModel(
+                model, brainParameters, new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
             Assert.Greater(errors.Count(), 0);
         }
 
 
             var brainParameters = GetContinuous2vis8vec2actionBrainParameters();
             brainParameters.ActionSpec = ActionSpec.MakeContinuous(3); // Invalid action
-            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]);
+            var errors = BarracudaModelParamLoader.CheckModel(
+                model, brainParameters, new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
-            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]);
+            errors = BarracudaModelParamLoader.CheckModel(
+                model, brainParameters, new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
             Assert.Greater(errors.Count(), 0);
         }
 
 
             var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
             brainParameters.ActionSpec = ActionSpec.MakeDiscrete(3, 3); // Invalid action
-            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor() }, new ActuatorComponent[0]);
+            var errors = BarracudaModelParamLoader.CheckModel(
+                model, brainParameters,
+                new ISensor[] { sensor_21_20_3.CreateSensors()[0] },
+                new ActuatorComponent[0]
+            );
-            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor() }, new ActuatorComponent[0]);
+            errors = BarracudaModelParamLoader.CheckModel(
+                model,
+                brainParameters,
+                new ISensor[] { sensor_21_20_3.CreateSensors()[0] },
+                new ActuatorComponent[0]
+            );
             Assert.Greater(errors.Count(), 0);
         }
 
 
             var brainParameters = GetHybridBrainParameters();
             brainParameters.ActionSpec = new ActionSpec(3, new[] { 3 }); // Invalid discrete action size
-            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]);
+            var errors = BarracudaModelParamLoader.CheckModel(
+                model,
+                brainParameters,
+                new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
-            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]);
+            errors = BarracudaModelParamLoader.CheckModel(
+                model,
+                brainParameters,
+                new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
             Assert.Greater(errors.Count(), 0);
         }
 
             var brainParameters = GetContinuous2vis8vec2actionBrainParameters();
-            var errors = BarracudaModelParamLoader.CheckModel(null, brainParameters, new ISensor[] { sensor_21_20_3.CreateSensor(), sensor_20_22_3.CreateSensor() }, new ActuatorComponent[0]);
+            var errors = BarracudaModelParamLoader.CheckModel(
+                null,
+                brainParameters,
+                new ISensor[]
+                {
+                    sensor_21_20_3.CreateSensors()[0],
+                    sensor_20_22_3.CreateSensors()[0]
+                },
+                new ActuatorComponent[0]
+            );
             Assert.Greater(errors.Count(), 0);
         }
     }

com.unity.ml-agents/Tests/Editor/PublicAPI/PublicApiValidation.cs

             sensorComponent.RayLayerMask = 0;
             sensorComponent.ObservationStacks = 2;
 
-            sensorComponent.CreateSensor();
+            sensorComponent.CreateSensors();
         }
 #endif
     }

com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs

         public SensorComponent wrappedComponent;
         public int numStacks;
 
-        public override ISensor CreateSensor()
+        public override ISensor[] CreateSensors()
-            var wrappedSensor = wrappedComponent.CreateSensor();
-            return new StackingSensor(wrappedSensor, numStacks);
+            var wrappedSensors = wrappedComponent.CreateSensors();
+            var sensorsOut = new ISensor[wrappedSensors.Length];
+            for (var i = 0; i < wrappedSensors.Length; i++)
+            {
+                sensorsOut[i] = new StackingSensor(wrappedSensors[i], numStacks);
+            }
+
+            return sensorsOut;
         }
     }
 

com.unity.ml-agents/Tests/Runtime/Sensor/BufferSensorTest.cs

             bufferComponent.ObservableSize = 4;
             bufferComponent.SensorName = "TestName";
 
-            var sensor = bufferComponent.CreateSensor();
+            var sensor = bufferComponent.CreateSensors()[0];
             var shape = sensor.GetObservationSpec().Shape;
 
             Assert.AreEqual(shape[0], 20);

com.unity.ml-agents/Tests/Runtime/Sensor/CameraSensorComponentTest.cs

                     cameraComponent.Grayscale = grayscale;
                     cameraComponent.CompressionType = compression;
 
-                    var sensor = cameraComponent.CreateSensor();
+                    var sensor = cameraComponent.CreateSensors()[0];
                     var expectedShape = new InplaceArray<int>(height, width, grayscale ? 1 : 3);
                     Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
                     Assert.AreEqual(typeof(CameraSensor), sensor.GetType());

com.unity.ml-agents/Tests/Runtime/Sensor/RayPerceptionSensorTests.cs

             foreach (var castRadius in radii)
             {
                 perception.SphereCastRadius = castRadius;
-                var sensor = perception.CreateSensor();
+                var sensor = perception.CreateSensors()[0];
 
                 var expectedObs = (2 * perception.RaysPerDirection + 1) * (perception.DetectableTags.Count + 2);
                 Assert.AreEqual(sensor.GetObservationSpec().Shape[0], expectedObs);
             perception.DetectableTags.Add(k_CubeTag);
             perception.DetectableTags.Add(k_SphereTag);
 
-            var sensor = perception.CreateSensor();
+            var sensor = perception.CreateSensors()[0];
             var expectedObs = (2 * perception.RaysPerDirection + 1) * (perception.DetectableTags.Count + 2);
             Assert.AreEqual(sensor.GetObservationSpec().Shape[0], expectedObs);
             var outputBuffer = new float[expectedObs];
                 }
                 perception.RayLayerMask = layerMask;
 
-                var sensor = perception.CreateSensor();
+                var sensor = perception.CreateSensors()[0];
                 var expectedObs = (2 * perception.RaysPerDirection + 1) * (perception.DetectableTags.Count + 2);
                 Assert.AreEqual(sensor.GetObservationSpec().Shape[0], expectedObs);
                 var outputBuffer = new float[expectedObs];
             foreach (var castRadius in radii)
             {
                 perception.SphereCastRadius = castRadius;
-                var sensor = perception.CreateSensor();
+                var sensor = perception.CreateSensors()[0];
 
                 var expectedObs = (2 * perception.RaysPerDirection + 1) * (perception.DetectableTags.Count + 2);
                 Assert.AreEqual(sensor.GetObservationSpec().Shape[0], expectedObs);
             {
                 // Set the layer mask to either the default, or one that ignores the close cube's layer
 
-                var sensor = perception.CreateSensor();
+                var sensor = perception.CreateSensors()[0];
                 var expectedObs = (2 * perception.RaysPerDirection + 1) * (perception.DetectableTags.Count + 2);
                 Assert.AreEqual(sensor.GetObservationSpec().Shape[0], expectedObs);
                 var outputBuffer = new float[expectedObs];

com.unity.ml-agents/Tests/Runtime/Sensor/RenderTextureSensorComponentTests.cs

 
                     var expectedShape = new InplaceArray<int>(height, width, grayscale ? 1 : 3);
 
-                    var sensor = renderTexComponent.CreateSensor();
+                    var sensor = renderTexComponent.CreateSensors()[0];
                     Assert.AreEqual(expectedShape, sensor.GetObservationSpec().Shape);
                     Assert.AreEqual(typeof(RenderTextureSensor), sensor.GetType());
                 }

docs/Migrating.md

 ```
 
 - The abstract method `SensorComponent.GetObservationShape()` was removed.
+- The abstract method `SensorComponent.CreateSensor()` was replaced with `CreateSensors()`, which returns an `ISensor[]`.
 
 ## Migrating to Release 13
 ### Implementing IHeuristic in your IActuator implementations

com.unity.ml-agents.extensions/Tests/Editor/Match3/match3obs_special1.png.meta

-fileFormatVersion: 2
-guid: fceb584222ca149cd984dad123c8ae25
-TextureImporter:
-  fileIDToRecycleName: {}
-  externalObjects: {}
-  serializedVersion: 9
-  mipmaps:
-    mipMapMode: 0
-    enableMipMap: 1
-    sRGBTexture: 1
-    linearTexture: 0
-    fadeOut: 0
-    borderMipMap: 0
-    mipMapsPreserveCoverage: 0
-    alphaTestReferenceValue: 0.5
-    mipMapFadeDistanceStart: 1
-    mipMapFadeDistanceEnd: 3
-  bumpmap:
-    convertToNormalMap: 0
-    externalNormalMap: 0
-    heightScale: 0.25
-    normalMapFilter: 0
-  isReadable: 0
-  streamingMipmaps: 0
-  streamingMipmapsPriority: 0
-  grayScaleToAlpha: 0
-  generateCubemap: 6
-  cubemapConvolution: 0
-  seamlessCubemap: 0
-  textureFormat: 1
-  maxTextureSize: 2048
-  textureSettings:
-    serializedVersion: 2
-    filterMode: -1
-    aniso: -1
-    mipBias: -100
-    wrapU: -1
-    wrapV: -1
-    wrapW: -1
-  nPOTScale: 1
-  lightmap: 0
-  compressionQuality: 50
-  spriteMode: 0
-  spriteExtrude: 1
-  spriteMeshType: 1
-  alignment: 0
-  spritePivot: {x: 0.5, y: 0.5}
-  spritePixelsToUnits: 100
-  spriteBorder: {x: 0, y: 0, z: 0, w: 0}
-  spriteGenerateFallbackPhysicsShape: 1
-  alphaUsage: 1
-  alphaIsTransparency: 0
-  spriteTessellationDetail: -1
-  textureType: 0
-  textureShape: 1
-  singleChannelComponent: 0
-  maxTextureSizeSet: 0
-  compressionQualitySet: 0
-  textureFormatSet: 0
-  platformSettings:
-  - serializedVersion: 2
-    buildTarget: DefaultTexturePlatform
-    maxTextureSize: 2048
-    resizeAlgorithm: 0
-    textureFormat: -1
-    textureCompression: 1
-    compressionQuality: 50
-    crunchedCompression: 0
-    allowsAlphaSplitting: 0
-    overridden: 0
-    androidETC2FallbackOverride: 0
-  spriteSheet:
-    serializedVersion: 2
-    sprites: []
-    outline: []
-    physicsShape: []
-    bones: []
-    spriteID: 
-    vertices: []
-    indices: 
-    edges: []
-    weights: []
-  spritePackingTag: 
-  pSDRemoveMatte: 0
-  pSDShowRemoveMatteOption: 0
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: