[CI] use downloader, update versions (#4562)

.yamato/compressed-sensor-test.yml

-test_editors:
-  - version: 2018.4
-  - version: 2019.3
+{% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
 test_compressed_obs_{{ editor.version }}:

.yamato/gym-interface-test.yml

-test_editors:
-  - version: 2018.4
-  - version: 2019.3
+{% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
 test_gym_interface_{{ editor.version }}:

.yamato/python-ll-api-test.yml

-test_editors:
-  - version: 2018.4
-  - version: 2019.3
+{% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
 test_mac_ll_api_{{ editor.version }}:

.yamato/standalone-build-test.yml

-test_editors:
-  - version: 2018.4
-  - version: 2019.3
+{% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
 test_mac_standalone_{{ editor.version }}:
     UNITY_VERSION: {{ editor.version }}
   commands:
     - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
+    # TODO remove the "--user" command and the path prefix when we can migrate away from the custom bokken image
+    - python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade --user
+    - /Users/bokken/Library/Python/3.7/bin/unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
     - python -u -m ml-agents.tests.yamato.standalone_build_tests
     - python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity
     - python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Bouncer/Scenes/Bouncer.unity

.yamato/training-int-tests.yml

-test_editors:
-  - version: 2018.4
-  - version: 2019.3
+{% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
 test_mac_training_int_{{ editor.version }}:
     UNITY_VERSION: {{ editor.version }}
   commands:
     - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
+    # TODO remove the "--user" command and the path prefix when we can migrate away from the custom bokken image
+    - python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade --user
+    - /Users/bokken/Library/Python/3.7/bin/unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
     - python -u -m ml-agents.tests.yamato.training_int_tests
     # Backwards-compatibility tests.
     # If we make a breaking change to the communication protocol, these will need

ml-agents/tests/yamato/training_int_tests.py

         "--mlagents-override-model-extension",
         model_extension,
     ]
-    res = subprocess.run(args)
+    print(f"Starting inference with args {' '.join(args)}")
+    timeout = 15 * 60  # 15 minutes for inference is more than enough
+    res = subprocess.run(args, timeout=timeout)
     end_time = time.time()
     if res.returncode != 0:
         print("Error running inference!")

ml-agents/tests/yamato/yamato_utils.py

 
 
 def get_unity_executable_path():
-    UNITY_VERSION = os.environ["UNITY_VERSION"]
-    BOKKEN_UNITY = f"/Users/bokken/{UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-    HUB_UNITY = (
-        f"/Applications/Unity/Hub/Editor/{UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-    )
-    if os.path.exists(BOKKEN_UNITY):
-        return BOKKEN_UNITY
-    if os.path.exists(HUB_UNITY):
-        return HUB_UNITY
-    raise FileNotFoundError("Can't find bokken or hub executables")
+    downloader_install_path = "./.Editor/Unity.app/Contents/MacOS/Unity"
+    if os.path.exists(downloader_install_path):
+        return downloader_install_path
+    raise FileNotFoundError("Can't find executable from unity-downloader-cli")
 
 
 def get_base_path():

.yamato/test_versions.metafile

+# List of editor versions for standalone-build-test and its dependencies.
+test_editors:
+  - version: 2018.4
+  - version: 2019.4
+  - version: 2020.1
+# Waiting on a barracuda fix, see https://jira.unity3d.com/browse/MLA-1464
+#  - version: 2020.2

ml-agents/tests/yamato/editmode_tests.py

-import os
-import sys
-import subprocess
-import shutil
-import xml.dom.minidom
-from typing import NamedTuple
-
-from .yamato_utils import get_base_path, get_unity_executable_path
-
-
-def clean_previous_results(base_path):
-    """
-    Clean up old results and make the artifacts path.
-    """
-    artifacts_path = os.path.join(base_path, "artifacts/")
-    results_xml_path = os.path.join(base_path, "results.xml")
-
-    if os.path.exists(results_xml_path):
-        os.remove(results_xml_path)
-
-    if os.path.exists(artifacts_path):
-        os.rmdir(artifacts_path)
-    os.mkdir(artifacts_path)
-
-
-class TestResults(NamedTuple):
-    total: str
-    passed: str
-    failed: str
-    duration: str
-
-
-def parse_results(results_xml):
-    """
-    Extract the test results from the xml file.
-    """
-    stats = {}
-    dom_tree = xml.dom.minidom.parse(results_xml)
-    collection = dom_tree.documentElement
-    for attribute in ["total", "passed", "failed", "duration"]:
-        stats[attribute] = collection.getAttribute(attribute)
-    return TestResults(**stats)
-
-
-def main():
-    base_path = get_base_path()
-    artifacts_path = os.path.join(base_path, "artifacts/")
-    results_xml_path = os.path.join(base_path, "results.xml")
-    print(f"Running in base path {base_path}")
-
-    print("Cleaning previous results")
-    clean_previous_results(base_path)
-
-    unity_exe = get_unity_executable_path()
-    print(f"Starting tests via {unity_exe}")
-
-    test_args = [
-        unity_exe,
-        "-batchmode",
-        "-runTests",
-        "-logfile",
-        "-",
-        "-projectPath",
-        f"{base_path}/Project",
-        "-testResults",
-        f"{base_path}/results.xml",
-        "-testPlatform",
-        "editmode",
-    ]
-    print(f"{' '.join(test_args)} ...")
-
-    timeout = 30 * 60  # 30 minutes, just in case
-    res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout)
-
-    stats = parse_results(results_xml_path)
-    print(
-        f"{stats.total} tests executed in {stats.duration}s: {stats.passed} passed, "
-        f"{stats.failed} failed. More details in results.xml"
-    )
-
-    try:
-        # copy results to artifacts dir
-        shutil.copy2(results_xml_path, artifacts_path)
-    except Exception:
-        pass
-
-    if res.returncode == 0 and os.path.exists(results_xml_path):
-        print("Test run SUCCEEDED!")
-    else:
-        print("Test run FAILED!")
-
-    sys.exit(res.returncode)
-
-
-if __name__ == "__main__":
-    main()