Tests for Demonstration loader and Model runner (#3510)

* Initial test code commit

* Fix typo

* Added more tests

* More tests

* Bumping test coverage

* Update com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs

Co-Authored-By: Chris Elion <chris.elion@unity3d.com>

Co-authored-by: Chris Elion <celion@gmail.com>

com.unity.ml-agents/Runtime/InferenceBrain/BarracudaModelParamLoader.cs

 
             // If there is no Vector Observation Input but the Brain Parameters expect one.
             if ((brainParameters.vectorObservationSize != 0) &&
-                (!tensorsNames.Contains(TensorNames.VectorObservationPlacholder)))
+                (!tensorsNames.Contains(TensorNames.VectorObservationPlaceholder)))
             {
                 failedModelChecks.Add(
                     "The model does not contain a Vector Observation  Placeholder Input. " +
             var tensorTester =
                 new Dictionary<string, Func<BrainParameters, TensorProxy, SensorComponent[], string>>()
             {
-                {TensorNames.VectorObservationPlacholder, CheckVectorObsShape},
+                {TensorNames.VectorObservationPlaceholder, CheckVectorObsShape},
                 {TensorNames.PreviousActionPlaceholder, CheckPreviousActionShape},
                 {TensorNames.RandomNormalEpsilonPlaceholder, ((bp, tensor, scs) => null)},
                 {TensorNames.ActionMaskPlaceholder, ((bp, tensor, scs) => null)},

com.unity.ml-agents/Runtime/InferenceBrain/TensorGenerator.cs

 
             if (vecObsGen != null)
             {
-                m_Dict[TensorNames.VectorObservationPlacholder] = vecObsGen;
+                m_Dict[TensorNames.VectorObservationPlaceholder] = vecObsGen;
             }
         }
 

com.unity.ml-agents/Runtime/InferenceBrain/TensorNames.cs

     {
         public const string BatchSizePlaceholder = "batch_size";
         public const string SequenceLengthPlaceholder = "sequence_length";
-        public const string VectorObservationPlacholder = "vector_observation";
+        public const string VectorObservationPlaceholder = "vector_observation";
         public const string RecurrentInPlaceholder = "recurrent_in";
         public const string recurrentInPlaceholderH = "recurrent_in_h";
         public const string recurrentInPlaceholderC = "recurrent_in_c";

com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs

 
         public override void AgentReset()
         {
-
             agentResetCalls += 1;
             collectObservationsCallsSinceLastReset = 0;
             agentActionCallsSinceLastReset = 0;
             for (var i = 0; i < 50; i++)
             {
                 expectedAgent1ActionSinceReset += 1;
-                if (expectedAgent1ActionSinceReset == agent1.maxStep || i == 0){
+                if (expectedAgent1ActionSinceReset == agent1.maxStep || i == 0)
+                {
                     expectedAgent1ActionSinceReset = 0;
                 }
                 agent2.RequestAction();
             agent1.LazyInitialize();
 
             var expectedAgentStepCount = 0;
-            var expectedResets= 0;
+            var expectedResets = 0;
             var expectedAgentAction = 0;
             var expectedAgentActionSinceReset = 0;
             var expectedCollectObsCalls = 0;
                 // If the next step will put the agent at maxSteps, we expect it to reset
                 if (agent1.StepCount == maxStep - 1 || (i == 0))
                 {
-                    expectedResets +=1;
+                    expectedResets += 1;
                 }
 
                 if (agent1.StepCount == maxStep - 1)

com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs

+using NUnit.Framework;
+using UnityEngine;
+using UnityEditor;
+using Barracuda;
+using MLAgents.InferenceBrain;
+using MLAgents.Sensors;
+using System.Linq;
+
+namespace MLAgents.Tests
+{
+    [TestFixture]
+    public class ModelRunnerTest
+    {
+        const string k_continuous2vis8vec2actionPath = "Packages/com.unity.ml-agents/Tests/Editor/Resources/continuous2vis8vec2action.nn";
+        const string k_discrete1vis0vec_2_3action_recurrModelPath = "Packages/com.unity.ml-agents/Tests/Editor/Resources/discrete1vis0vec_2_3action_recurr.nn";
+        NNModel continuous2vis8vec2actionModel;
+        NNModel discrete1vis0vec_2_3action_recurrModel;
+        Test3DSensorComponent sensor_21_20_3;
+        Test3DSensorComponent sensor_20_22_3;
+
+        private BrainParameters GetContinuous2vis8vec2actionBrainParameters()
+        {
+            var validBrainParameters = new BrainParameters();
+            validBrainParameters.vectorObservationSize = 8;
+            validBrainParameters.vectorActionSize = new int[] { 2 };
+            validBrainParameters.numStackedVectorObservations = 1;
+            validBrainParameters.vectorActionSpaceType = SpaceType.Continuous;
+            return validBrainParameters;
+        }
+
+        private BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters()
+        {
+            var validBrainParameters = new BrainParameters();
+            validBrainParameters.vectorObservationSize = 0;
+            validBrainParameters.vectorActionSize = new int[] { 2, 3 };
+            validBrainParameters.numStackedVectorObservations = 1;
+            validBrainParameters.vectorActionSpaceType = SpaceType.Discrete;
+            return validBrainParameters;
+        }
+
+        [SetUp]
+        public void SetUp()
+        {
+            continuous2vis8vec2actionModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuous2vis8vec2actionPath, typeof(NNModel));
+            discrete1vis0vec_2_3action_recurrModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_discrete1vis0vec_2_3action_recurrModelPath, typeof(NNModel));
+            var go = new GameObject("SensorA");
+            sensor_21_20_3 = go.AddComponent<Test3DSensorComponent>();
+            sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3);
+            sensor_20_22_3 = go.AddComponent<Test3DSensorComponent>();
+            sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3);
+        }
+
+        [Test]
+        public void TestModelExist()
+        {
+            Assert.IsNotNull(continuous2vis8vec2actionModel);
+            Assert.IsNotNull(discrete1vis0vec_2_3action_recurrModel);
+        }
+
+        [Test]
+        public void TestCreation()
+        {
+            var modelRunner = new ModelRunner(continuous2vis8vec2actionModel, GetContinuous2vis8vec2actionBrainParameters());
+            modelRunner.Dispose();
+            modelRunner = new ModelRunner(discrete1vis0vec_2_3action_recurrModel, GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters());
+            modelRunner.Dispose();
+        }
+
+        [Test]
+        public void TestHasModel()
+        {
+            var modelRunner = new ModelRunner(continuous2vis8vec2actionModel, GetContinuous2vis8vec2actionBrainParameters(), InferenceDevice.CPU);
+            Assert.True(modelRunner.HasModel(continuous2vis8vec2actionModel, InferenceDevice.CPU));
+            Assert.False(modelRunner.HasModel(continuous2vis8vec2actionModel, InferenceDevice.GPU));
+            Assert.False(modelRunner.HasModel(discrete1vis0vec_2_3action_recurrModel, InferenceDevice.CPU));
+            modelRunner.Dispose();
+        }
+
+        [Test]
+        public void TestRunModel()
+        {
+            var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
+            var modelRunner = new ModelRunner(discrete1vis0vec_2_3action_recurrModel, brainParameters);
+            var info1 = new AgentInfo();
+            info1.episodeId = 1;
+            modelRunner.PutObservations(info1, new ISensor[] { sensor_21_20_3.CreateSensor() }.ToList());
+            var info2 = new AgentInfo();
+            info2.episodeId = 2;
+            modelRunner.PutObservations(info2, new ISensor[] { sensor_21_20_3.CreateSensor() }.ToList());
+
+            modelRunner.DecideBatch();
+
+            Assert.IsNotNull(modelRunner.GetAction(1));
+            Assert.IsNotNull(modelRunner.GetAction(2));
+            Assert.IsNull(modelRunner.GetAction(3));
+            Assert.AreEqual(brainParameters.vectorActionSize.Count(), modelRunner.GetAction(1).Count());
+            modelRunner.Dispose();
+        }
+    }
+}

com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs.meta

+fileFormatVersion: 2
+guid: a2b924bac7d86467183ad9dc1436e550
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs

+using NUnit.Framework;
+using UnityEngine;
+using UnityEditor;
+using Barracuda;
+using MLAgents.InferenceBrain;
+using MLAgents.Sensors;
+using System.Linq;
+
+namespace MLAgents.Tests
+{
+    public class Test3DSensorComponent : SensorComponent
+    {
+        public ISensor Sensor;
+
+        public override ISensor CreateSensor()
+        {
+            return Sensor;
+        }
+
+        public override int[] GetObservationShape()
+        {
+            return Sensor.GetObservationShape();
+        }
+    }
+    public class Test3DSensor : ISensor
+    {
+        int m_Width;
+        int m_Height;
+        int m_Channels;
+        string m_Name;
+
+        public Test3DSensor(string name, int width, int height, int channels)
+        {
+            m_Width = width;
+            m_Height = height;
+            m_Channels = channels;
+            m_Name = name;
+        }
+
+        public int[] GetObservationShape()
+        {
+            return new int[] {m_Height, m_Width, m_Channels };
+        }
+
+        public int Write(WriteAdapter adapter)
+        {
+            for (int i = 0; i < m_Width * m_Height * m_Channels; i++)
+            {
+                adapter[i] = 0.0f;
+            }
+            return m_Width * m_Height * m_Channels;
+        }
+
+        public byte[] GetCompressedObservation()
+        {
+            return new byte[0];
+        }
+
+        public void Update() {}
+
+        public SensorCompressionType GetCompressionType()
+        {
+            return SensorCompressionType.None;
+        }
+
+        public string GetName()
+        {
+            return m_Name;
+        }
+    }
+
+    [TestFixture]
+    public class ParameterLoaderTest
+    {
+        const string k_continuous2vis8vec2actionPath = "Packages/com.unity.ml-agents/Tests/Editor/Resources/continuous2vis8vec2action.nn";
+        const string k_discrete1vis0vec_2_3action_recurrModelPath = "Packages/com.unity.ml-agents/Tests/Editor/Resources/discrete1vis0vec_2_3action_recurr.nn";
+        NNModel continuous2vis8vec2actionModel;
+        NNModel discrete1vis0vec_2_3action_recurrModel;
+        Test3DSensorComponent sensor_21_20_3;
+        Test3DSensorComponent sensor_20_22_3;
+
+        private BrainParameters GetContinuous2vis8vec2actionBrainParameters()
+        {
+            var validBrainParameters = new BrainParameters();
+            validBrainParameters.vectorObservationSize = 8;
+            validBrainParameters.vectorActionSize = new int[] { 2 };
+            validBrainParameters.numStackedVectorObservations = 1;
+            validBrainParameters.vectorActionSpaceType = SpaceType.Continuous;
+            return validBrainParameters;
+        }
+
+        private BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters()
+        {
+            var validBrainParameters = new BrainParameters();
+            validBrainParameters.vectorObservationSize = 0;
+            validBrainParameters.vectorActionSize = new int[] { 2, 3 };
+            validBrainParameters.numStackedVectorObservations = 1;
+            validBrainParameters.vectorActionSpaceType = SpaceType.Discrete;
+            return validBrainParameters;
+        }
+
+        [SetUp]
+        public void SetUp()
+        {
+            continuous2vis8vec2actionModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuous2vis8vec2actionPath, typeof(NNModel));
+            discrete1vis0vec_2_3action_recurrModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_discrete1vis0vec_2_3action_recurrModelPath, typeof(NNModel));
+            var go = new GameObject("SensorA");
+            sensor_21_20_3 = go.AddComponent<Test3DSensorComponent>();
+            sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3);
+            sensor_20_22_3 = go.AddComponent<Test3DSensorComponent>();
+            sensor_20_22_3.Sensor = new Test3DSensor("SensorA", 20, 22, 3);
+        }
+
+        [Test]
+        public void TestModelExist()
+        {
+            Assert.IsNotNull(continuous2vis8vec2actionModel);
+            Assert.IsNotNull(discrete1vis0vec_2_3action_recurrModel);
+        }
+
+        [Test]
+        public void TestGetInputTensors1()
+        {
+            var model = ModelLoader.Load(continuous2vis8vec2actionModel);
+            var inputTensors = BarracudaModelParamLoader.GetInputTensors(model);
+            var inputNames = inputTensors.Select(x => x.name).ToList();
+            // Model should contain 3 inputs : vector, visual 1 and visual 2
+            Assert.AreEqual(3, inputNames.Count);
+            Assert.Contains(TensorNames.VectorObservationPlaceholder, inputNames);
+            Assert.Contains(TensorNames.VisualObservationPlaceholderPrefix + "0", inputNames);
+            Assert.Contains(TensorNames.VisualObservationPlaceholderPrefix + "1", inputNames);
+
+            Assert.AreEqual(2, BarracudaModelParamLoader.GetNumVisualInputs(model));
+
+            // Test if the model is null
+            Assert.AreEqual(0, BarracudaModelParamLoader.GetInputTensors(null).Count);
+            Assert.AreEqual(0, BarracudaModelParamLoader.GetNumVisualInputs(null));
+        }
+
+        [Test]
+        public void TestGetInputTensors2()
+        {
+            var model = ModelLoader.Load(discrete1vis0vec_2_3action_recurrModel);
+            var inputTensors = BarracudaModelParamLoader.GetInputTensors(model);
+            var inputNames = inputTensors.Select(x => x.name).ToList();
+            // Model should contain 2 inputs : recurrent and visual 1
+
+            Assert.Contains(TensorNames.VisualObservationPlaceholderPrefix + "0", inputNames);
+            // TODO :There are some memory tensors as well
+        }
+
+        [Test]
+        public void TestGetOutputTensors1()
+        {
+            var model = ModelLoader.Load(continuous2vis8vec2actionModel);
+            var outputNames = BarracudaModelParamLoader.GetOutputNames(model);
+            Assert.Contains(TensorNames.ActionOutput, outputNames);
+            Assert.AreEqual(1, outputNames.Count());
+
+            Assert.AreEqual(0, BarracudaModelParamLoader.GetOutputNames(null).Count());
+        }
+
+        [Test]
+        public void TestGetOutputTensors2()
+        {
+            var model = ModelLoader.Load(discrete1vis0vec_2_3action_recurrModel);
+            var outputNames = BarracudaModelParamLoader.GetOutputNames(model);
+            Assert.Contains(TensorNames.ActionOutput, outputNames);
+            // TODO : There are some memory tensors as well
+        }
+
+        [Test]
+        public void TestCheckModelValid1()
+        {
+            var model = ModelLoader.Load(continuous2vis8vec2actionModel);
+            var validBrainParameters = GetContinuous2vis8vec2actionBrainParameters();
+
+            var errors = BarracudaModelParamLoader.CheckModel(model, validBrainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.AreEqual(0, errors.Count()); // There should not be any errors
+        }
+
+        [Test]
+        public void TestCheckModelValid2()
+        {
+            var model = ModelLoader.Load(discrete1vis0vec_2_3action_recurrModel);
+            var validBrainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
+
+            var errors = BarracudaModelParamLoader.CheckModel(model, validBrainParameters, new SensorComponent[] { sensor_21_20_3 });
+            Assert.AreEqual(0, errors.Count()); // There should not be any errors
+        }
+
+        [Test]
+        public void TestCheckModelThrowsVectorObservation1()
+        {
+            var model = ModelLoader.Load(continuous2vis8vec2actionModel);
+
+            var brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            brainParameters.vectorObservationSize = 9; // Invalid observation
+            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.Greater(errors.Count(), 0);
+
+            brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            brainParameters.numStackedVectorObservations = 2;// Invalid stacking
+            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.Greater(errors.Count(), 0);
+        }
+
+        [Test]
+        public void TestCheckModelThrowsVectorObservation2()
+        {
+            var model = ModelLoader.Load(discrete1vis0vec_2_3action_recurrModel);
+
+            var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
+            brainParameters.vectorObservationSize = 1; // Invalid observation
+            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 });
+            Assert.Greater(errors.Count(), 0);
+        }
+
+        [Test]
+        public void TestCheckModelThrowsAction1()
+        {
+            var model = ModelLoader.Load(continuous2vis8vec2actionModel);
+
+            var brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            brainParameters.vectorActionSize = new int[] { 3 }; // Invalid action
+            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.Greater(errors.Count(), 0);
+
+            brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            brainParameters.vectorActionSpaceType = SpaceType.Discrete;// Invalid SpaceType
+            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.Greater(errors.Count(), 0);
+        }
+
+        [Test]
+        public void TestCheckModelThrowsAction2()
+        {
+            var model = ModelLoader.Load(discrete1vis0vec_2_3action_recurrModel);
+
+            var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters();
+            brainParameters.vectorActionSize = new int[] { 3, 3 }; // Invalid action
+            var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 });
+            Assert.Greater(errors.Count(), 0);
+
+            brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            brainParameters.vectorActionSpaceType = SpaceType.Continuous;// Invalid SpaceType
+            errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 });
+            Assert.Greater(errors.Count(), 0);
+        }
+
+        [Test]
+        public void TestCheckModelThrowsNoModel()
+        {
+            var brainParameters = GetContinuous2vis8vec2actionBrainParameters();
+            var errors = BarracudaModelParamLoader.CheckModel(null, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 });
+            Assert.Greater(errors.Count(), 0);
+        }
+    }
+}

com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs.meta

+fileFormatVersion: 2
+guid: edd38d6ad78c8456d80f0a90bcb2e1b7
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Tests/Editor/Resources.meta

+fileFormatVersion: 2
+guid: 22f1c3c8541da48e480c6b921343c2ee
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Tests/Editor/Resources/continuous2vis8vec2action.nn.meta

+fileFormatVersion: 2
+guid: a75582ff670094ff2996c1c4ab9dfd15
+ScriptedImporter:
+  fileIDToRecycleName:
+    11400000: main obj
+    11400002: model data
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
+  script: {fileID: 11500000, guid: 19ed1486aa27d4903b34839f37b8f69f, type: 3}

com.unity.ml-agents/Tests/Editor/Resources/discrete1vis0vec_2_3action_recurr.nn.meta

+fileFormatVersion: 2
+guid: 8a92fbcd96caa4ef5a93dd55c0c36705
+ScriptedImporter:
+  fileIDToRecycleName:
+    11400000: main obj
+    11400002: model data
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
+  script: {fileID: 11500000, guid: 19ed1486aa27d4903b34839f37b8f69f, type: 3}