ml-agents/ml-agents/tests/yamato/yamato_utils.py

import os
import shutil
import subprocess
import yaml
from typing import List, Optional
from sys import platform


def get_unity_executable_path():
    if platform == "darwin":
        downloader_install_path = "./.Editor/Unity.app/Contents/MacOS/Unity"
    else:  # if platform == "linux":
        downloader_install_path = "./.Editor/Unity"
    if os.path.exists(downloader_install_path):
        return downloader_install_path
    raise FileNotFoundError("Can't find executable from unity-downloader-cli")


def get_base_path():
    # We might need to do some more work here if the working directory ever changes
    # E.g. take the full path and back out the main module main.
    # But for now, this should work
    return os.getcwd()


def get_base_output_path():
    """"
    Returns the artifact folder to use for yamato jobs.
    """
    return os.path.join(get_base_path(), "artifacts")


def run_standalone_build(
    base_path: str,
    verbose: bool = False,
    output_path: str = None,
    scene_path: str = None,
    log_output_path: str = f"{get_base_output_path()}/standalone_build.txt",
) -> int:
    """
    Run BuildStandalonePlayerOSX test to produce a player. The location defaults to
    artifacts/standalone_build/testPlayer.
    """
    unity_exe = get_unity_executable_path()
    print(f"Running BuildStandalonePlayerOSX via {unity_exe}")

    test_args = [
        unity_exe,
        "-projectPath",
        f"{base_path}/Project",
        "-batchmode",
        "-executeMethod",
        "Unity.MLAgents.StandaloneBuildTest.BuildStandalonePlayerOSX",
    ]

    os.makedirs(os.path.dirname(log_output_path), exist_ok=True)
    subprocess.run(["touch", log_output_path])
    test_args += ["-logfile", log_output_path]

    if output_path is not None:
        output_path = os.path.join(get_base_output_path(), output_path)
        test_args += ["--mlagents-build-output-path", output_path]
        os.makedirs(os.path.dirname(output_path), exist_ok=True)
    if scene_path is not None:
        test_args += ["--mlagents-build-scene-path", scene_path]
    print(f"{' '.join(test_args)} ...")

    timeout = 30 * 60  # 30 minutes, just in case
    res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout)

    # Copy the default build name into the artifacts folder.
    if output_path is None and res.returncode == 0:
        shutil.move(
            os.path.join(base_path, "Project", "testPlayer.app"),
            os.path.join(get_base_output_path(), "testPlayer.app"),
        )

    # Print if we fail or want verbosity.
    if verbose or res.returncode != 0:
        subprocess.run(["cat", log_output_path])

    return res.returncode


def init_venv(
    mlagents_python_version: str = None, extra_packages: Optional[List[str]] = None
) -> str:
    """
    Set up the virtual environment, and return the venv path.
    :param mlagents_python_version: The version of mlagents python packcage to install.
        If None, will do a local install, otherwise will install from pypi
    :return:
    """
    # Use a different venv path for different versions
    venv_path = "venv"
    if mlagents_python_version:
        venv_path += "_" + mlagents_python_version

    # Set up the venv and install mlagents
    subprocess.check_call(f"python -m venv {venv_path}", shell=True)
    pip_commands = [
        "--upgrade pip",
        "--upgrade setuptools",
        # TODO build these and publish to internal pypi
        "~/tensorflow_pkg/tensorflow-2.0.0-cp37-cp37m-macosx_10_14_x86_64.whl",
    ]
    if mlagents_python_version:
        # install from pypi
        pip_commands += [
            f"mlagents=={mlagents_python_version}",
            f"gym-unity=={mlagents_python_version}",
        ]
    else:
        # Local install
        pip_commands += ["-e ./ml-agents-envs", "-e ./ml-agents", "-e ./gym-unity"]
    if extra_packages:
        pip_commands += extra_packages
    for cmd in pip_commands:
        pip_index_url = "--index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple"
        subprocess.check_call(
            f"source {venv_path}/bin/activate; python -m pip install -q {cmd} {pip_index_url}",
            shell=True,
        )
    return venv_path


def checkout_csharp_version(csharp_version):
    """
    Checks out the specific git revision (usually a tag) for the C# package and Project.
    If csharp_version is None, no changes are made.
    :param csharp_version:
    :return:
    """
    if csharp_version is None:
        return

    csharp_dirs = ["com.unity.ml-agents", "Project"]
    for csharp_dir in csharp_dirs:
        subprocess.check_call(f"rm -rf {csharp_dir}", shell=True)
        subprocess.check_call(
            f"git checkout {csharp_version} -- {csharp_dir}", shell=True
        )


def undo_git_checkout():
    """
    Clean up the git working directory.
    """
    subprocess.check_call("git reset HEAD .", shell=True)
    subprocess.check_call("git checkout -- .", shell=True)
    # Ensure the cache isn't polluted with old compiled assemblies.
    subprocess.check_call("rm -rf Project/Library", shell=True)


def override_config_file(src_path, dest_path, **kwargs):
    """
    Override settings in a trainer config file. For example,
        override_config_file(src_path, dest_path, max_steps=42)
    will copy the config file at src_path to dest_path, but override the max_steps field to 42 for all brains.
    """
    with open(src_path) as f:
        configs = yaml.safe_load(f)

    for config in configs.values():
        config.update(**kwargs)

    with open(dest_path, "w") as f:
        yaml.dump(configs, f)