convert yamato tests to python (#3239)

* convert edit mode test runner to python

* unbuffered, strip slash in dir

* cleanup

* try to get artifacts

* parse xml instead of magic shell commands

* always copy results.xml

* convert standalone-build-test to python

* run as module

* os.getcwd()

.yamato/csharp-tests.yml

   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - ./run-tests-editmode-osx-editor.sh
+    - python -u -m ml-agents.tests.yamato.editmode_tests
   triggers:
     pull_requests:
       - targets:
             - "/hotfix-.*/"
+  artifacts:
+    unit:
+      paths:
+        - "artifacts/**"
 {% endfor %}

.yamato/standalone-build-test.yml

   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - ./run-standalone-build-osx.sh
+    - python -u -m ml-agents.tests.yamato.standalone_build_tests
   triggers:
     pull_requests:
       - targets:

ml-agents/tests/yamato/editmode_tests.py

+import os
+import sys
+import subprocess
+import shutil
+import xml.dom.minidom
+from typing import NamedTuple
+
+from .yamato_utils import get_base_path, get_unity_executable_path
+
+
+def clean_previous_results(base_path):
+    """
+    Clean up old results and make the artifacts path.
+    """
+    artifacts_path = os.path.join(base_path, "artifacts/")
+    results_xml_path = os.path.join(base_path, "results.xml")
+
+    if os.path.exists(results_xml_path):
+        os.remove(results_xml_path)
+
+    if os.path.exists(artifacts_path):
+        os.rmdir(artifacts_path)
+    os.mkdir(artifacts_path)
+
+
+class TestResults(NamedTuple):
+    total: str
+    passed: str
+    failed: str
+    duration: str
+
+
+def parse_results(results_xml):
+    """
+    Extract the test results from the xml file.
+    """
+    stats = {}
+    dom_tree = xml.dom.minidom.parse(results_xml)
+    collection = dom_tree.documentElement
+    for attribute in ["total", "passed", "failed", "duration"]:
+        stats[attribute] = collection.getAttribute(attribute)
+    return TestResults(**stats)
+
+
+def main():
+    base_path = get_base_path()
+    artifacts_path = os.path.join(base_path, "artifacts/")
+    results_xml_path = os.path.join(base_path, "results.xml")
+    print(f"Running in base path {base_path}")
+
+    print("Cleaning previous results")
+    clean_previous_results(base_path)
+
+    unity_exe = get_unity_executable_path()
+    print(f"Starting tests via {unity_exe}")
+
+    test_args = [
+        unity_exe,
+        "-batchmode",
+        "-runTests",
+        "-logfile",
+        "-",
+        "-projectPath",
+        f"{base_path}/UnitySDK",
+        "-testResults",
+        f"{base_path}/results.xml",
+        "-testPlatform",
+        "editmode",
+    ]
+    print(f"{' '.join(test_args)} ...")
+
+    timeout = 30 * 60  # 30 minutes, just in case
+    res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout)
+
+    stats = parse_results(results_xml_path)
+    print(
+        f"{stats.total} tests executed in {stats.duration}s: {stats.passed} passed, "
+        f"{stats.failed} failed. More details in results.xml"
+    )
+
+    try:
+        # copy results to artifacts dir
+        shutil.copy2(results_xml_path, artifacts_path)
+    except Exception:
+        pass
+
+    if res.returncode == 0 and os.path.exists(results_xml_path):
+        print("Test run SUCCEEDED!")
+    else:
+        print("Test run FAILED!")
+
+    sys.exit(res.returncode)
+
+
+if __name__ == "__main__":
+    main()

ml-agents/tests/yamato/standalone_build_tests.py

+import sys
+import subprocess
+
+from .yamato_utils import get_base_path, get_unity_executable_path
+
+
+def main():
+    base_path = get_base_path()
+    print(f"Running in base path {base_path}")
+
+    unity_exe = get_unity_executable_path()
+    print(f"Starting tests via {unity_exe}")
+
+    test_args = [
+        unity_exe,
+        "-projectPath",
+        f"{base_path}/UnitySDK",
+        "-logfile",
+        "-",
+        "-batchmode",
+        "-executeMethod",
+        "MLAgents.StandaloneBuildTest.BuildStandalonePlayerOSX",
+    ]
+    print(f"{' '.join(test_args)} ...")
+
+    timeout = 30 * 60  # 30 minutes, just in case
+    res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout)
+
+    if res.returncode == 0:
+        print("Test run SUCCEEDED!")
+    else:
+        print("Test run FAILED!")
+
+    sys.exit(res.returncode)
+
+
+if __name__ == "__main__":
+    main()

ml-agents/tests/yamato/yamato_utils.py

+import os
+
+
+def get_unity_executable_path():
+    UNITY_VERSION = os.environ["UNITY_VERSION"]
+    BOKKEN_UNITY = f"/Users/bokken/{UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
+    HUB_UNITY = (
+        f"/Applications/Unity/Hub/Editor/{UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
+    )
+    if os.path.exists(BOKKEN_UNITY):
+        return BOKKEN_UNITY
+    if os.path.exists(HUB_UNITY):
+        return HUB_UNITY
+    raise FileNotFoundError("Can't find bokken or hub executables")
+
+
+def get_base_path():
+    # We might need to do some more work here if the working directory ever changes
+    # E.g. take the full path and back out the main module main.
+    # But for now, this should work
+    return os.getcwd()

run-standalone-build-osx.sh

-#!/bin/bash
-
-set -eo pipefail
-
-if [[ -z "${UNITY_VERSION}" ]]; then
-
-    echo "Environment Variable UNITY_VERSION was not set"
-    exit 1
-
-else
-    BOKKEN_UNITY="/Users/bokken/${UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-    HUB_UNITY="/Applications/Unity/Hub/Editor/${UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-
-    if [[ -f ${BOKKEN_UNITY} ]]; then
-        UNITY=${BOKKEN_UNITY}
-    else
-        UNITY=${HUB_UNITY}
-    fi
-
-    pushd $(dirname "${0}") > /dev/null
-    BASETPATH=$(pwd -L)
-    popd > /dev/null
-
-    echo "Cleaning previous results"
-
-    echo "Starting tests via $UNITY"
-
-    CMD_LINE="$UNITY -projectPath $BASETPATH/UnitySDK -logfile - -batchmode -executeMethod MLAgents.StandaloneBuildTest.BuildStandalonePlayerOSX"
-
-    echo "$CMD_LINE ..."
-
-    ${CMD_LINE}
-    RES=$?
-
-    if [[ "${RES}" -eq "0" ]]; then
-        echo "Standalone build completed successfully.";
-        exit 0;
-    else
-        echo "Standalone build failed."
-        exit 1;
-    fi
-
-    exit ${RES}
-
-fi

run-tests-editmode-osx-editor.sh

-#!/bin/bash
-
-set -eo pipefail
-
-if [[ -z "${UNITY_VERSION}" ]]; then
-
-    echo "Environment Variable UNITY_VERSION was not set"
-    exit 1
-
-else
-    BOKKEN_UNITY="/Users/bokken/${UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-    HUB_UNITY="/Applications/Unity/Hub/Editor/${UNITY_VERSION}/Unity.app/Contents/MacOS/Unity"
-
-    if [[ -f ${BOKKEN_UNITY} ]]; then
-        UNITY=${BOKKEN_UNITY}
-    else
-        UNITY=${HUB_UNITY}
-    fi
-
-    pushd $(dirname "${0}") > /dev/null
-    BASETPATH=$(pwd -L)
-    popd > /dev/null
-
-    echo "Cleaning previous results"
-
-    if [[ -e ${BASETPATH}/results.xml ]]
-    then
-        rm ${BASETPATH}/results.xml
-    fi
-
-    echo "Starting tests via $UNITY"
-
-    CMD_LINE="$UNITY -batchmode -runTests -logfile - -projectPath $BASETPATH/UnitySDK -testResults $BASETPATH/results.xml -testPlatform editmode"
-
-    echo "$CMD_LINE ..."
-
-    $CMD_LINE
-    RES=$?
-
-    TOTAL=$(echo 'cat /test-run/test-suite/@total' | xmllint --shell results.xml | awk -F'[="]' '!/>/{print $(NF-1)}')
-    PASSED=$(echo 'cat /test-run/test-suite/@passed' | xmllint --shell results.xml | awk -F'[="]' '!/>/{print $(NF-1)}')
-    FAILED=$(echo 'cat /test-run/test-suite/@failed' | xmllint --shell results.xml | awk -F'[="]' '!/>/{print $(NF-1)}')
-    DURATION=$(echo 'cat /test-run/test-suite/@duration' | xmllint --shell results.xml | awk -F'[="]' '!/>/{print $(NF-1)}')
-
-    echo "$TOTAL tests executed in ${DURATION}s: $PASSED passed, $FAILED failed. More details in results.xml"
-
-    if [[ ${RES} -eq 0 ]] && [[ -e ${BASETPATH}/results.xml ]]; then
-        echo "Test run SUCCEEDED!"
-    else
-        echo "Test run FAILED!"
-    fi
-
-    rm "${BASETPATH}/results.xml"
-
-    exit ${RES}
-
-fi