Automatically generate samples based on placement of mlagents-sample.json files in our examples. (#5077)

.yamato/com.unity.ml-agents-pack.yml

     image: package-ci/ubuntu:stable
     flavor: b1.small
   commands:
-    - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
-    - upm-ci project pack --project-path Project
+    - |
+      python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
+      unity-downloader-cli -u 2018.4 -c editor --wait --fast
+      ./.Editor/Unity -projectPath Project -batchMode -executeMethod Unity.MLAgents.SampleExporter.ExportCuratedSamples -logFile -
+      npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
+      upm-ci project pack --project-path Project
   artifacts:
     packages:
       paths:

Project/Assets/ML-Agents/Editor/Tests/SampleExporter.cs

 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
+using Newtonsoft.Json;
-using UnityEngine.iOS;
-        const string k_SceneFlag = "--mlagents-scene-path";
+        const string k_MLAgentsSampleFile = "mlagents-sample.json";
+        const string k_PackageSampleFile = ".sample.json";
+        const string k_MLAgentsDir = "ML-Agents";
+        const string k_MLAgentsExamplesDir = "Examples";
+        const string k_MLAgentsPackageName = "com.unity.ml-agents";
+        const string k_MLAgentsSamplesDirName = "Samples";
+        const string k_MLAgentsScriptsDirName = "Scripts";
+
+        struct MLAgentsSampleJson
+        {
+            public string displayName;
+            public string description;
+            public List<string> scenes;
+        }
+
+        struct PackageSampleJson
+        {
+            public string displayName;
+            public string description;
+        }
 
         public static void ExportCuratedSamples()
         {
             {
-                var args = Environment.GetCommandLineArgs();
-                var scenes = new List<string>();
-                for (var i = 0; i < args.Length - 1; i++)
+                // Path to Project/Assets
+                var assetsDir = Application.dataPath;
+                var repoRoot = Directory.GetParent(Directory.GetParent(assetsDir).FullName).FullName;
+
+                // Top level of where to store the samples
+                var samplesDir = Path.Combine(
+                    repoRoot,
+                    k_MLAgentsPackageName,
+                    k_MLAgentsSamplesDirName);
+
+                if (!Directory.Exists(samplesDir))
-                    if (args[i] == k_SceneFlag)
-                    {
-                        scenes.Add(args[i + 1]);
-                        Debug.Log($"Exporting Scene {scenes.Last()}");
-                    }
+                    Directory.CreateDirectory(samplesDir);
-                foreach (var scene in scenes)
+                // Path to the examples dir in the project
+                var examplesDir = Path.Combine(Application.dataPath, k_MLAgentsDir, k_MLAgentsExamplesDir);
+                foreach (var exampleDirectory in Directory.GetDirectories(examplesDir))
-                    var assets = new List<string> { scene };
-                    var exampleFolderToAdd = Directory.GetParent(Directory.GetParent(scene).FullName).FullName;
-                    Debug.Log($"Parent of Scene: {exampleFolderToAdd}");
-                    if (Directory.Exists(Path.Combine(exampleFolderToAdd, "Scripts")))
+                    var mlAgentsSamplePath = Path.Combine(exampleDirectory, k_MLAgentsSampleFile);
+                    if (File.Exists(mlAgentsSamplePath))
-                        exampleFolderToAdd = Path.Combine(exampleFolderToAdd, "Scripts");
-                    }
+                        var sampleJson = JsonConvert.DeserializeObject<MLAgentsSampleJson>(File.ReadAllText(mlAgentsSamplePath));
+                        Debug.Log(JsonConvert.SerializeObject(sampleJson));
+                        foreach (var scene in sampleJson.scenes)
+                        {
+                            var scenePath = Path.Combine(exampleDirectory, scene);
+                            if (File.Exists(scenePath))
+                            {
+                                // Create a Sample Directory
+                                var currentSampleDir = Directory.CreateDirectory(Path.Combine(samplesDir,
+                                    Path.GetFileNameWithoutExtension(scenePath)));
+
+
+                                var scriptsPath = Path.Combine(exampleDirectory, k_MLAgentsScriptsDirName);
+                                Debug.Log($"Scene Path: {scenePath}");
+                                var assets = new List<string> { scenePath.Substring(scenePath.IndexOf("Assets")) };
+                                if (!Directory.Exists(Path.Combine(scriptsPath)))
+                                {
+                                    scriptsPath = exampleDirectory;
+                                }
+
+                                scriptsPath = scriptsPath.Substring(scriptsPath.IndexOf("Assets"));
+                                foreach (var guid in AssetDatabase.FindAssets("t:Script", new[] { scriptsPath }))
+                                {
+                                    var path = AssetDatabase.GUIDToAssetPath(guid);
+                                    assets.Add(path);
+                                    Debug.Log($"Adding Asset: {path}");
+                                }
+
+                                var packageFilePath = Path.GetFileNameWithoutExtension(scenePath) + ".unitypackage";
+                                AssetDatabase.ExportPackage(assets.ToArray(),
+                                    Path.Combine(Application.dataPath, packageFilePath),
+                                    ExportPackageOptions.IncludeDependencies | ExportPackageOptions.Recurse);
+
+                                // Move the .unitypackage into the samples folder.
+                                var packageFileFullPath = Path.Combine(Application.dataPath, packageFilePath);
-                    exampleFolderToAdd = exampleFolderToAdd.Substring(exampleFolderToAdd.IndexOf("Assets"));
-                    foreach (var guid in AssetDatabase.FindAssets("t:Script", new[] { exampleFolderToAdd }))
-                    {
-                        var path = AssetDatabase.GUIDToAssetPath(guid);
-                        assets.Add(path);
-                        Debug.Log($"Adding Asset: {path}");
+                                var packageInSamplePath = Path.Combine(currentSampleDir.FullName, packageFilePath);
+                                Debug.Log($"Moving {packageFileFullPath} to {packageInSamplePath}");
+                                File.Move(packageFileFullPath, packageInSamplePath);
+
+                                // write the .sample.json file to the sample directory
+                                File.WriteAllText(Path.Combine(currentSampleDir.FullName, k_PackageSampleFile),
+                                    JsonConvert.SerializeObject(new PackageSampleJson
+                                    {
+                                        description = sampleJson.description,
+                                        displayName = sampleJson.displayName
+                                    }));
+                            }
+                        }
-                    AssetDatabase.ExportPackage(assets.ToArray(), Path.GetFileNameWithoutExtension(scene) + ".unitypackage", ExportPackageOptions.IncludeDependencies | ExportPackageOptions.Recurse);
                 }
             }
             catch (Exception e)

Project/Packages/manifest.json

     "com.unity.modules.video": "1.0.0",
     "com.unity.modules.vr": "1.0.0",
     "com.unity.modules.wind": "1.0.0",
-    "com.unity.modules.xr": "1.0.0"
+    "com.unity.modules.xr": "1.0.0",
+    "com.unity.nuget.newtonsoft-json": "2.0.0"
   },
   "testables": [
     "com.unity.ml-agents",

Project/Project.sln.DotSettings

 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=ML/@EntryIndexedValue">ML</s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Dont/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

com.unity.ml-agents/.gitignore

 /Assets/Plugins*
 /Assets/Demonstrations*
 /csharp_timers.json
+/Samples/
+/Samples.meta
-*.api.meta
+*.api.meta

ml-agents/tests/yamato/yamato_utils.py

-import glob
 import os
 import shutil
 import subprocess
 
     with open(dest_path, "w") as f:
         yaml.dump(configs, f)
-
-
-def create_samples(
-    scenes: List[str],
-    base_path: str,
-    log_output_path: Optional[str] = f"{get_base_output_path()}/sample_export.txt",
-) -> int:
-    unity_exe = get_unity_executable_path()
-    test_args = [
-        unity_exe,
-        "-projectPath",
-        f"{base_path}/Project",
-        "-batchmode",
-        "-executeMethod",
-        "Unity.MLAgents.SampleExporter.ExportCuratedSamples",
-    ]
-
-    if log_output_path:
-        os.makedirs(os.path.dirname(log_output_path), exist_ok=True)
-        subprocess.run(["touch", log_output_path])
-        test_args += ["-logfile", log_output_path]
-    else:
-        # Log to stdout
-        test_args += ["-logfile", "-"]
-
-    os.makedirs(os.path.join(get_base_output_path(), "Samples"), exist_ok=True)
-
-    for scene in scenes:
-        test_args += ["--mlagents-scene-path", scene]
-
-    timeout = 5 * 60  # 5 minutes for now
-    res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout)
-
-    if res.returncode == 0:
-        for file in glob.glob(os.path.join(base_path, "Project/*.unitypackage")):
-            print(
-                f"moving {file} to {os.path.join(get_base_output_path(), 'Samples', os.path.basename(file))}"
-            )
-            shutil.move(
-                file,
-                os.path.join(get_base_output_path(), "Samples", os.path.basename(file)),
-            )
-
-    # Print if we fail or want verbosity.
-    if res.returncode != 0:
-        if log_output_path:
-            subprocess.run(["cat", log_output_path])
-
-    return res.returncode

Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json

+{
+  "displayName": "3D Ball",
+  "description": "The 3D Ball sample is a simple environment that is a great for jumping into Ml-Agents to see how things work.",
+  "scenes": [
+      "Scenes/3DBall.unity"
+    ]
+}

Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json.meta

+fileFormatVersion: 2
+guid: 6b6f1c189dc84df391d1c3ccb13a54f7
+TextScriptImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

.yamato/ml-agents-sample-export.yml

-sample_export:
-    name: Samples Export 2021.2
-    agent:
-        type: Unity::VM
-        image: package-ci/ubuntu:stable
-        flavor: b1.large
-    variables:
-        UNITY_VERSION: 2021.2
-    commands:
-        - python3 -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
-        - python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
-        - unity-downloader-cli -u 2021.2 -c editor --wait --fast
-        - python3 -u -m ml-agents.tests.yamato.sample_curation  --scene "Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity" "Assets/ML-Agents/Examples/Match3/Scenes/Match3.unity" "Assets/ML-Agents/Examples/WallJump/Scenes/WallJump.unity" "Assets/ML-Agents/TestScenes/TestCompressedGrid/TestGridCompressed.unity" "Assets/ML-Agents/TestScenes/TestCompressedTexture/TestTextureCompressed.unity"
-    triggers:
-        cancel_old_ci: true
-    artifacts:
-        logs:
-            paths:
-                - "artifacts/sample_export.txt"
-        samples:
-            paths:
-                - "artifacts/Samples/**"

ml-agents/tests/yamato/sample_curation.py

-import sys
-import argparse
-
-from .yamato_utils import get_base_path, create_samples
-
-
-def main(scenes):
-    base_path = get_base_path()
-    print(f"Running in base path {base_path}")
-
-    returncode = create_samples(
-        scenes,
-        base_path,
-        log_output_path=None,  # Log to stdout so we get timestamps on the logs
-    )
-
-    if returncode == 0:
-        print("Test run SUCCEEDED!")
-    else:
-        print("Test run FAILED!")
-
-    sys.exit(returncode)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--scene", nargs="+", default=None, required=True)
-    args = parser.parse_args()
-    main(args.scene)