Add custom package settings (#5027)

.gitignore

 # Environemnt logfile
 *Project.log
 
+# Custom settings asset
+*.settings.asset*
+
 # Visual Studio 2015 cache directory
 /Project/.vs/
 

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/Editor.asmdef

 {
-    "name": "Unity.ML-Agents.Performance.Tests",
+    "name": "Unity.ML-Agents.DevTests.Editor",
     "references": [
         "Unity.ML-Agents.Editor",
         "Unity.ML-Agents",

DevProject/Assets/ML-Agents/Scripts/Tests/Runtime/AcademyTest/AcademyStepperTest.cs

         [SetUp]
         public void Setup()
         {
+            Academy.Instance.Dispose();
             SceneManager.LoadScene("ML-Agents/Scripts/Tests/Runtime/AcademyTest/AcademyStepperTestScene");
         }
 

DevProject/Assets/ML-Agents/Scripts/Tests/Runtime/Runtime.asmdef

 {
-    "name": "Runtime",
+    "name": "Unity.ML-Agents.DevTests.Runtime",
     "references": [
         "Unity.ML-Agents"
     ],

com.unity.ml-agents/CHANGELOG.md

 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
 - The `.onnx` models input names have changed. All input placeholders will now use the prefix `obs_` removing the distinction between visual and vector observations. Models created with this version will not be usable with previous versions of the package (#5080)
 - The `.onnx` models discrete action output now contains the discrete actions values and not the logits. Models created with this version will not be usable with previous versions of the package (#5080)
+- Added ML-Agents package settings. (#5027)
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 
 ### Bug Fixes

com.unity.ml-agents/Runtime/Academy.cs

                 // No arg passed, or malformed port number.
 #if UNITY_EDITOR
                 // Try connecting on the default editor port
-                return k_EditorTrainingPort;
+                return MLAgentsSettingsManager.Settings.ConnectTrainer ? MLAgentsSettingsManager.Settings.EditorPort : -1;
 #else
                 // This is an executable, so we don't try to connect.
                 return -1;

docs/Readme.md

 - [Designing a Learning Environment](Learning-Environment-Design.md)
   - [Designing Agents](Learning-Environment-Design-Agents.md)
 - [Using an Executable Environment](Learning-Environment-Executable.md)
+- [ML-Agents Package Settings](Package-Settings.md)
 
 ## Training & Inference
 

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/Editor.asmdef.meta

+fileFormatVersion: 2
+guid: 5b142e67c2d6b4b1e928e4d54f01a596
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/MLAgentsSettings.meta

+fileFormatVersion: 2
+guid: 1fc80f44976bc4177a9afaa0a38abab3
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Editor/MLAgentsSettingsBuildProvider.cs

+using System.Linq;
+using UnityEngine;
+using UnityEditor;
+using UnityEditor.Build;
+using UnityEditor.Build.Reporting;
+
+
+namespace Unity.MLAgents.Editor
+{
+    internal class MLAgentsSettingsBuildProvider : IPreprocessBuildWithReport, IPostprocessBuildWithReport
+    {
+        private MLAgentsSettings m_SettingsAddedToPreloadedAssets;
+
+        public int callbackOrder => 0;
+
+        public void OnPreprocessBuild(BuildReport report)
+        {
+            var wasDirty = IsPlayerSettingsDirty();
+            m_SettingsAddedToPreloadedAssets = null;
+
+            var preloadedAssets = PlayerSettings.GetPreloadedAssets().ToList();
+            if (!preloadedAssets.Contains(MLAgentsSettingsManager.Settings))
+            {
+                m_SettingsAddedToPreloadedAssets = MLAgentsSettingsManager.Settings;
+                preloadedAssets.Add(m_SettingsAddedToPreloadedAssets);
+                PlayerSettings.SetPreloadedAssets(preloadedAssets.ToArray());
+            }
+
+            if (!wasDirty)
+                ClearPlayerSettingsDirtyFlag();
+        }
+
+        public void OnPostprocessBuild(BuildReport report)
+        {
+            if (m_SettingsAddedToPreloadedAssets == null)
+                return;
+
+            var wasDirty = IsPlayerSettingsDirty();
+
+            var preloadedAssets = PlayerSettings.GetPreloadedAssets().ToList();
+            if (preloadedAssets.Contains(m_SettingsAddedToPreloadedAssets))
+            {
+                preloadedAssets.Remove(m_SettingsAddedToPreloadedAssets);
+                PlayerSettings.SetPreloadedAssets(preloadedAssets.ToArray());
+            }
+
+            m_SettingsAddedToPreloadedAssets = null;
+
+            if (!wasDirty)
+                ClearPlayerSettingsDirtyFlag();
+        }
+
+
+        private static bool IsPlayerSettingsDirty()
+        {
+#if UNITY_2019_OR_NEWER
+            var settings = Resources.FindObjectsOfTypeAll<PlayerSettings>();
+            if (settings != null && settings.Length > 0)
+                return EditorUtility.IsDirty(settings[0]);
+            return false;
+#else
+            return false;
+#endif
+        }
+
+        private static void ClearPlayerSettingsDirtyFlag()
+        {
+#if UNITY_2019_OR_NEWER
+            var settings = Resources.FindObjectsOfTypeAll<PlayerSettings>();
+            if (settings != null && settings.Length > 0)
+                EditorUtility.ClearDirty(settings[0]);
+#endif
+        }
+    }
+}

com.unity.ml-agents/Editor/MLAgentsSettingsBuildProvider.cs.meta

+fileFormatVersion: 2
+guid: bd59ff34305fa4259a2735e08afdb424
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Editor/MLAgentsSettingsProvider.cs

+using System;
+using System.Linq;
+using System.IO;
+using System.Runtime.CompilerServices;
+using UnityEngine;
+using UnityEditor;
+#if UNITY_2019_4_OR_NEWER
+using UnityEngine.UIElements;
+#else
+using UnityEngine.Experimental.UIElements;
+#endif
+
+[assembly: InternalsVisibleTo("Unity.ML-Agents.DevTests.Editor")]
+namespace Unity.MLAgents.Editor
+{
+    internal class MLAgentsSettingsProvider : SettingsProvider, IDisposable
+    {
+        const string k_SettingsPath = "Project/ML-Agents";
+        private static MLAgentsSettingsProvider s_Instance;
+        private string[] m_AvailableSettingsAssets;
+        private int m_CurrentSelectedSettingsAsset;
+        private SerializedObject m_SettingsObject;
+        [SerializeField]
+        private MLAgentsSettings m_Settings;
+
+
+        private MLAgentsSettingsProvider(string path, SettingsScope scope = SettingsScope.Project)
+            : base(path, scope)
+        {
+            s_Instance = this;
+        }
+
+        [SettingsProvider]
+        public static SettingsProvider CreateMLAgentsSettingsProvider()
+        {
+            return new MLAgentsSettingsProvider(k_SettingsPath, SettingsScope.Project);
+        }
+
+        public override void OnActivate(string searchContext, VisualElement rootElement)
+        {
+            base.OnActivate(searchContext, rootElement);
+            MLAgentsSettingsManager.OnSettingsChange += Reinitialize;
+        }
+
+        public override void OnDeactivate()
+        {
+            base.OnDeactivate();
+            MLAgentsSettingsManager.OnSettingsChange -= Reinitialize;
+        }
+
+        public void Dispose()
+        {
+            m_SettingsObject?.Dispose();
+        }
+
+        public override void OnTitleBarGUI()
+        {
+            if (EditorGUILayout.DropdownButton(EditorGUIUtility.IconContent("_Popup"), FocusType.Passive, EditorStyles.label))
+            {
+                var menu = new GenericMenu();
+                for (var i = 0; i < m_AvailableSettingsAssets.Length; i++)
+                {
+                    menu.AddItem(ExtractDisplayName(m_AvailableSettingsAssets[i]), m_CurrentSelectedSettingsAsset == i, (path) =>
+                    {
+                        MLAgentsSettingsManager.Settings = AssetDatabase.LoadAssetAtPath<MLAgentsSettings>((string)path);
+                    }, m_AvailableSettingsAssets[i]);
+                }
+                menu.AddSeparator("");
+                menu.AddItem(new GUIContent("New Settings Asset…"), false, CreateNewSettingsAsset);
+                menu.ShowAsContext();
+                Event.current.Use();
+            }
+        }
+
+        private GUIContent ExtractDisplayName(string name)
+        {
+            if (name.StartsWith("Assets/"))
+                name = name.Substring("Assets/".Length);
+            if (name.EndsWith(".asset"))
+                name = name.Substring(0, name.Length - ".asset".Length);
+            if (name.EndsWith(".mlagents.settings"))
+                name = name.Substring(0, name.Length - ".mlagents.settings".Length);
+
+            // Ugly hack: GenericMenu interprets "/" as a submenu path. But luckily, "/" is not the only slash we have in Unicode.
+            return new GUIContent(name.Replace("/", "\u29f8"));
+        }
+
+        private void CreateNewSettingsAsset()
+        {
+            // Asset database always use forward slashes. Use forward slashes for all the paths.
+            var projectName = PlayerSettings.productName;
+            var path = EditorUtility.SaveFilePanel("Create ML-Agents Settings File", "Assets",
+                projectName + ".mlagents.settings", "asset");
+            if (string.IsNullOrEmpty(path))
+            {
+                return;
+            }
+
+            path = path.Replace("\\", "/"); // Make sure we only get '/' separators.
+            var assetPath = Application.dataPath + "/";
+            if (!path.StartsWith(assetPath, StringComparison.CurrentCultureIgnoreCase))
+            {
+                Debug.LogError(string.Format(
+                    "Settings must be stored in Assets folder of the project (got: '{0}')", path));
+                return;
+            }
+
+            var extension = Path.GetExtension(path);
+            if (string.Compare(extension, ".asset", StringComparison.InvariantCultureIgnoreCase) != 0)
+            {
+                path += ".asset";
+            }
+            var relativePath = "Assets/" + path.Substring(assetPath.Length);
+            CreateNewSettingsAsset(relativePath);
+        }
+
+        private static void CreateNewSettingsAsset(string relativePath)
+        {
+            var settings = ScriptableObject.CreateInstance<MLAgentsSettings>();
+            AssetDatabase.CreateAsset(settings, relativePath);
+            EditorGUIUtility.PingObject(settings);
+            // Install the settings. This will lead to an MLAgentsManager.OnSettingsChange event
+            // which in turn will cause this Provider to reinitialize
+            MLAgentsSettingsManager.Settings = settings;
+        }
+
+        public override void OnGUI(string searchContext)
+        {
+            if (m_Settings == null)
+            {
+                InitializeWithCurrentSettings();
+            }
+
+            if (m_AvailableSettingsAssets.Length == 0)
+            {
+                EditorGUILayout.HelpBox(
+                    "Click the button below to create a settings asset you can edit.",
+                    MessageType.Info);
+                if (GUILayout.Button("Create settings asset", GUILayout.Height(30)))
+                    CreateNewSettingsAsset();
+                GUILayout.Space(20);
+            }
+
+            using (new EditorGUI.DisabledScope(m_AvailableSettingsAssets.Length == 0))
+            {
+                EditorGUI.BeginChangeCheck();
+                EditorGUILayout.LabelField("Trainer Settings", EditorStyles.boldLabel);
+                EditorGUI.indentLevel++;
+                EditorGUILayout.PropertyField(m_SettingsObject.FindProperty("m_ConnectTrainer"), new GUIContent("Connect to Trainer"));
+                EditorGUILayout.PropertyField(m_SettingsObject.FindProperty("m_EditorPort"), new GUIContent("Editor Training Port"));
+                EditorGUI.indentLevel--;
+                if (EditorGUI.EndChangeCheck())
+                    m_SettingsObject.ApplyModifiedProperties();
+            }
+        }
+
+        internal void InitializeWithCurrentSettings()
+        {
+            m_AvailableSettingsAssets = FindSettingsInProject();
+
+            m_Settings = MLAgentsSettingsManager.Settings;
+            var currentSettingsPath = AssetDatabase.GetAssetPath(m_Settings);
+            if (string.IsNullOrEmpty(currentSettingsPath))
+            {
+                if (m_AvailableSettingsAssets.Length > 0)
+                {
+                    m_CurrentSelectedSettingsAsset = 0;
+                    m_Settings = AssetDatabase.LoadAssetAtPath<MLAgentsSettings>(m_AvailableSettingsAssets[0]);
+                    MLAgentsSettingsManager.Settings = m_Settings;
+                }
+            }
+            else
+            {
+                var settingsList = m_AvailableSettingsAssets.ToList();
+                m_CurrentSelectedSettingsAsset = settingsList.IndexOf(currentSettingsPath);
+
+                EditorBuildSettings.AddConfigObject(MLAgentsSettingsManager.EditorBuildSettingsConfigKey, m_Settings, true);
+            }
+
+            m_SettingsObject = new SerializedObject(m_Settings);
+        }
+
+        private static string[] FindSettingsInProject()
+        {
+            var guids = AssetDatabase.FindAssets("t:MLAgentsSettings");
+            return guids.Select(guid => AssetDatabase.GUIDToAssetPath(guid)).ToArray();
+        }
+
+        private void Reinitialize()
+        {
+            if (m_Settings != null && MLAgentsSettingsManager.Settings != m_Settings)
+            {
+                InitializeWithCurrentSettings();
+            }
+            Repaint();
+        }
+    }
+}

com.unity.ml-agents/Editor/MLAgentsSettingsProvider.cs.meta

+fileFormatVersion: 2
+guid: 162489862d7f64a40990a0c06bb73bd0
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Runtime/MLAgentsSettings.cs

+using UnityEngine;
+using System.Runtime.CompilerServices;
+
+
+[assembly: InternalsVisibleTo("Unity.ML-Agents.DevTests.Editor")]
+namespace Unity.MLAgents
+{
+    internal class MLAgentsSettings : ScriptableObject
+    {
+        [SerializeField]
+        private bool m_ConnectTrainer = true;
+        [SerializeField]
+        private int m_EditorPort = 5004;
+
+        public bool ConnectTrainer
+        {
+            get { return m_ConnectTrainer; }
+            set
+            {
+                m_ConnectTrainer = value;
+                OnChange();
+            }
+        }
+
+        public int EditorPort
+        {
+            get { return m_EditorPort; }
+            set
+            {
+                m_EditorPort = value;
+                OnChange();
+            }
+        }
+
+        internal void OnChange()
+        {
+            if (MLAgentsSettingsManager.Settings == this)
+                MLAgentsSettingsManager.ApplySettings();
+        }
+    }
+}

com.unity.ml-agents/Runtime/MLAgentsSettings.cs.meta

+fileFormatVersion: 2
+guid: 71515ce028aaa4b4cb6bee13e96ef6f3
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Runtime/MLAgentsSettingsManager.cs

+using System;
+using System.Linq;
+using UnityEngine;
+#if UNITY_EDITOR
+using UnityEditor;
+#endif
+
+namespace Unity.MLAgents
+{
+#if UNITY_EDITOR
+    [InitializeOnLoad]
+#endif
+    internal static class MLAgentsSettingsManager
+    {
+        internal static event Action OnSettingsChange;
+        internal const string EditorBuildSettingsConfigKey = "com.unity.ml-agents.settings";
+        private static MLAgentsSettings s_Settings;
+
+
+        // setter will trigger callback for refreshing editor UI if using editor
+        public static MLAgentsSettings Settings
+        {
+            get
+            {
+                if (s_Settings == null)
+                {
+                    Initialize();
+                }
+                return s_Settings;
+            }
+            set
+            {
+                Debug.Assert(value != null);
+#if UNITY_EDITOR
+                if (!string.IsNullOrEmpty(AssetDatabase.GetAssetPath(value)))
+                {
+                    EditorBuildSettings.AddConfigObject(EditorBuildSettingsConfigKey, value, true);
+                }
+#endif
+                s_Settings = value;
+                ApplySettings();
+            }
+        }
+
+        static MLAgentsSettingsManager()
+        {
+            Initialize();
+        }
+
+        static void Initialize()
+        {
+#if UNITY_EDITOR
+            InitializeInEditor();
+#else
+            InitializeInPlayer();
+#endif
+        }
+
+#if UNITY_EDITOR
+        internal static void InitializeInEditor()
+        {
+            var settings = ScriptableObject.CreateInstance<MLAgentsSettings>();
+            if (EditorBuildSettings.TryGetConfigObject(EditorBuildSettingsConfigKey,
+                out MLAgentsSettings settingsAsset))
+            {
+                if (settingsAsset != null)
+                {
+                    settings = settingsAsset;
+                }
+            }
+            Settings = settings;
+        }
+#else
+        internal static void InitializeInPlayer()
+        {
+            Settings = Resources.FindObjectsOfTypeAll<MLAgentsSettings>().FirstOrDefault() ?? ScriptableObject.CreateInstance<MLAgentsSettings>();
+        }
+#endif
+
+        internal static void ApplySettings()
+        {
+            OnSettingsChange?.Invoke();
+        }
+
+        internal static void Destroy()
+        {
+            s_Settings = null;
+            OnSettingsChange = null;
+        }
+    }
+}

com.unity.ml-agents/Runtime/MLAgentsSettingsManager.cs.meta

+fileFormatVersion: 2
+guid: be40451993af54e3c84c7113140fdf2c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

docs/Package-Settings.md

+# ML-Agents Package Settings
+
+ML-Agents Package Settings contains settings that apply to the whole project.
+It allows you to configure ML-Agents-specific settings in the Editor.  These settings are available for use in both the Editor and Player.
+
+You can find them at `Edit` > `Project Settings...` > `ML-Agents`. It lists out all the available settings and their default values.
+
+
+## Create Custom Settings
+In order to to use you own settings for your project, you'll need to create an settings asset.
+
+You can do this by clicking the `Create Settings Asset` buttom or clicking the gear on the top right and select `New Settings Asset...`.
+The asset file can be placed anywhere in the `Asset/` folder in your project.
+After Creating the settings asset, you'll be able to modify the settings for your project and your settings will be saved in the asset.
+
+![Package Settings](images/package-settings.png)
+
+
+## Multiple Custom Settings for Different Scenarios
+You can create multiple settings assets in one project.
+
+By clicking the gear on the top right you'll see all available settings listed in the drop-down menu to choose from.
+
+This allows you to create different settings for different scenatios. For example, you can create two
+separate settings for training and inference, and specify which one you want to use according to what you're currently running.
+
+![Multiple Settings](images/multiple-settings.png)
+
+
+
+
+
+

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/MLAgentsSettings/MLAgentsSettingsTests.cs

+using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using NUnit.Framework;
+using UnityEditor;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Editor;
+
+
+namespace MLAgentsExamples.Tests.Settings
+{
+    [TestFixture]
+    public class MLAgentsSettingsTests
+    {
+        string EditorBuildSettingsConfigKey = MLAgentsSettingsManager.EditorBuildSettingsConfigKey;
+        string tempSettingsRootPath = "Assets/ML-Agents/Scripts/Tests/Editor/MLAgentsSettings";
+        MLAgentsSettings storedConfigObject;
+        [SetUp]
+        public void SetUp()
+        {
+            if (EditorBuildSettings.TryGetConfigObject(EditorBuildSettingsConfigKey,
+                out MLAgentsSettings settingsAsset))
+            {
+                if (settingsAsset != null)
+                {
+                    storedConfigObject = settingsAsset;
+                    EditorBuildSettings.RemoveConfigObject(EditorBuildSettingsConfigKey);
+                }
+            }
+            MLAgentsSettingsManager.Destroy();
+            ClearSettingsAssets();
+        }
+
+        [TearDown]
+        public void TearDown()
+        {
+            if (storedConfigObject != null)
+            {
+                EditorBuildSettings.AddConfigObject(EditorBuildSettingsConfigKey, storedConfigObject, true);
+                storedConfigObject = null;
+            }
+            MLAgentsSettingsManager.Destroy();
+            ClearSettingsAssets();
+        }
+
+        internal void ClearSettingsAssets()
+        {
+            var assetsGuids = AssetDatabase.FindAssets("t:MLAgentsSettings", new string[] { tempSettingsRootPath });
+            foreach (var guid in assetsGuids)
+            {
+                var path = AssetDatabase.GUIDToAssetPath(guid);
+                AssetDatabase.DeleteAsset(path);
+            }
+        }
+
+        [Test]
+        public void TestMLAgentsSettingsManager()
+        {
+            Assert.AreNotEqual(null, MLAgentsSettingsManager.Settings);
+            Assert.AreEqual(5004, MLAgentsSettingsManager.Settings.EditorPort); // default port
+            MLAgentsSettingsManager.Settings.EditorPort = 6000;
+            Assert.AreEqual(6000, MLAgentsSettingsManager.Settings.EditorPort);
+
+            var settingsObject = ScriptableObject.CreateInstance<MLAgentsSettings>();
+            settingsObject.EditorPort = 7000;
+            var tempSettingsAssetPath = tempSettingsRootPath + "/test.mlagents.settings.asset";
+            AssetDatabase.CreateAsset(settingsObject, tempSettingsAssetPath);
+            EditorBuildSettings.AddConfigObject(EditorBuildSettingsConfigKey, settingsObject, true);
+            // destroy manager instantiated as a side effect by accessing MLAgentsSettings directly without manager
+            MLAgentsSettingsManager.Destroy();
+            Assert.AreEqual(7000, MLAgentsSettingsManager.Settings.EditorPort);
+        }
+
+        // A mock class that can invoke private methods/fields in MLAgentsSettingsProvider
+        internal class MockSettingsProvider
+        {
+            public MLAgentsSettingsProvider Instance
+            {
+                get
+                {
+                    return (MLAgentsSettingsProvider)typeof(MLAgentsSettingsProvider).GetField("s_Instance",
+                        BindingFlags.Static | BindingFlags.NonPublic).GetValue(null);
+                }
+            }
+
+            public MLAgentsSettings Settings
+            {
+                get
+                {
+                    return (MLAgentsSettings)typeof(MLAgentsSettingsProvider).GetField("m_Settings",
+                        BindingFlags.Instance | BindingFlags.NonPublic).GetValue(Instance);
+                }
+            }
+
+            public void CreateMLAgentsSettingsProvider()
+            {
+                MLAgentsSettingsProvider.CreateMLAgentsSettingsProvider();
+            }
+
+            public void Reinitialize()
+            {
+                var method = typeof(MLAgentsSettingsProvider).GetMethod("Reinitialize",
+                    BindingFlags.Instance | BindingFlags.NonPublic);
+                method.Invoke(Instance, null);
+            }
+
+            public string[] FindSettingsInProject()
+            {
+                var method = typeof(MLAgentsSettingsProvider).GetMethod("FindSettingsInProject",
+                    BindingFlags.Static | BindingFlags.NonPublic);
+                return (string[])method.Invoke(null, null);
+            }
+
+            public void CreateNewSettingsAsset(string relativePath)
+            {
+                var method = typeof(MLAgentsSettingsProvider).GetMethod("CreateNewSettingsAsset",
+                    BindingFlags.Static | BindingFlags.NonPublic);
+                method.Invoke(null, new object[] { relativePath });
+            }
+        }
+
+        [Test]
+        public void TestMLAgentsSettingsProviderCreateAsset()
+        {
+            var mockProvider = new MockSettingsProvider();
+            mockProvider.CreateMLAgentsSettingsProvider();
+            Assert.AreNotEqual(null, mockProvider.Instance);
+
+            // mimic MLAgentsSettingsProvider.OnActivate()
+            MLAgentsSettingsManager.OnSettingsChange += mockProvider.Reinitialize;
+
+            mockProvider.Instance.InitializeWithCurrentSettings();
+            Assert.AreEqual(0, mockProvider.FindSettingsInProject().Length);
+
+            var tempSettingsAssetPath1 = tempSettingsRootPath + "/test.mlagents.settings.asset";
+            mockProvider.CreateNewSettingsAsset(tempSettingsAssetPath1);
+            Assert.AreEqual(1, mockProvider.FindSettingsInProject().Length);
+            Assert.AreEqual(5004, mockProvider.Settings.EditorPort);
+            MLAgentsSettingsManager.Settings.EditorPort = 6000; // change to something not default
+            // callback should update the field in provider
+            Assert.AreEqual(6000, mockProvider.Settings.EditorPort);
+
+            var tempSettingsAssetPath2 = tempSettingsRootPath + "/test2.mlagents.settings.asset";
+            mockProvider.CreateNewSettingsAsset(tempSettingsAssetPath2);
+            Assert.AreEqual(2, mockProvider.FindSettingsInProject().Length);
+            // manager should set to the new (default) one, not the previous modified one
+            Assert.AreEqual(5004, MLAgentsSettingsManager.Settings.EditorPort);
+
+            // mimic MLAgentsSettingsProvider.OnDeactivate()
+            MLAgentsSettingsManager.OnSettingsChange -= mockProvider.Reinitialize;
+            mockProvider.Instance.Dispose();
+        }
+
+        [Test]
+        public void TestMLAgentsSettingsProviderLoadAsset()
+        {
+            var mockProvider = new MockSettingsProvider();
+            var tempSettingsAssetPath1 = tempSettingsRootPath + "/test.mlagents.settings.asset";
+            mockProvider.CreateNewSettingsAsset(tempSettingsAssetPath1);
+            MLAgentsSettingsManager.Settings.EditorPort = 8000; // change to something not default
+
+            mockProvider.Instance?.Dispose();
+            MLAgentsSettingsManager.Destroy();
+
+            mockProvider.CreateMLAgentsSettingsProvider();
+            Assert.AreEqual(8000, MLAgentsSettingsManager.Settings.EditorPort);
+        }
+    }
+}

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/MLAgentsSettings/MLAgentsSettingsTests.cs.meta

+fileFormatVersion: 2
+guid: 44777c287385449678640ce8e4acc3ae
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

DevProject/Assets/ML-Agents/Scripts/Tests/Editor/Tests.asmdef.meta

-fileFormatVersion: 2
-guid: 1bad3ff1107ca4051af2c89caabd728c
-AssemblyDefinitionImporter:
-  externalObjects: {}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: