浏览代码

[CI] move as much as possible to linux (#4767)

/MLA-1734-demo-provider
GitHub 4 年前
当前提交
6e3dafc8
共有 14 个文件被更改,包括 146 次插入109 次删除
  1. 4
      .yamato/com.unity.ml-agents-pack.yml
  2. 15
      .yamato/com.unity.ml-agents-performance.yml
  3. 4
      .yamato/com.unity.ml-agents-test.yml
  4. 19
      .yamato/compressed-sensor-test.yml
  5. 19
      .yamato/gym-interface-test.yml
  6. 4
      .yamato/protobuf-generation-test.yml
  7. 27
      .yamato/python-ll-api-test.yml
  8. 30
      .yamato/standalone-build-test.yml
  9. 31
      .yamato/training-int-tests.yml
  10. 8
      Project/Assets/ML-Agents/Editor/Tests/StandaloneBuildTest.cs
  11. 3
      ml-agents-envs/mlagents_envs/env_utils.py
  12. 12
      ml-agents/tests/yamato/standalone_build_tests.py
  13. 32
      ml-agents/tests/yamato/training_int_tests.py
  14. 47
      ml-agents/tests/yamato/yamato_utils.py

4
.yamato/com.unity.ml-agents-pack.yml


pack:
name: Pack
agent:
type: Unity::VM::osx
image: package-ci/mac:stable
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.small
commands:
- npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm

15
.yamato/com.unity.ml-agents-performance.yml


variables:
UNITY_VERSION: {{ editor.version }}
commands:
- python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
- python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
- unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
- curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr
- chmod +x ./utr

expression: |
(pull_request.target eq "master" OR
pull_request.target match "release.+") AND
NOT pull_request.draft AND
(pull_request.changes.any match "com.unity.ml-agents/**" OR
pull_request.changes.any match "DevProject/**" OR
pull_request.changes.any match "ml-agents/**" OR
pull_request.changes.any match "ml-agents-envs/**" OR
pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND
NOT pull_request.changes.all match "**/*.md"
recurring:
- branch: master
frequency: daily
artifacts:
logs:
paths:

4
.yamato/com.unity.ml-agents-test.yml


- .yamato/com.unity.ml-agents-pack.yml#pack
triggers:
cancel_old_ci: true
{% if platform.name == "mac" %}
{% if platform.name == "linux" %}
expression: |
(pull_request.target eq "master" OR
pull_request.target match "release.+") AND

image: {{ platform.image }}
flavor: {{ platform.flavor}}
commands:
- python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
- python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
- unity-downloader-cli -u trunk -c editor --wait --fast
- npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
- upm-ci project test -u {{ editor.version }} --project-path {{ editor.testProject }} --package-filter {{ package.name }} {{ coverageOptions }} --extra-create-project-arg="-upmNoDefaultPackages" --extra-utr-arg "reruncount=2"

19
.yamato/compressed-sensor-test.yml


test_compressed_obs_{{ editor.version }}:
name: Test Compressed Sensor Observation {{ editor.version }}
agent:
type: Unity::VM::osx
image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
flavor: b1.small
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.medium
- pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
- python -u -m ml-agents.tests.yamato.setup_venv
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestGridCompressed
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestTextureCompressed
- |
sudo apt-get update && sudo apt-get install -y python3-venv
python3 -m venv venv && source venv/bin/activate
python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python -u -m ml-agents.tests.yamato.setup_venv
python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestGridCompressed
python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestTextureCompressed
- .yamato/standalone-build-test.yml#test_mac_standalone_{{ editor.version }}
- .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
triggers:
cancel_old_ci: true
expression: |

19
.yamato/gym-interface-test.yml


---
{% for editor in test_editors %}
test_gym_interface_{{ editor.version }}:
name: Test Mac Gym Interface {{ editor.version }}
name: Test Linux Gym Interface {{ editor.version }}
type: Unity::VM::osx
image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
flavor: b1.small
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.medium
- pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
- python -u -m ml-agents.tests.yamato.setup_venv
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_gym.py --env=artifacts/testPlayer-Basic
- |
sudo apt-get update && sudo apt-get install -y python3-venv
python3 -m venv venv && source venv/bin/activate
python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python -u -m ml-agents.tests.yamato.setup_venv
python ml-agents/tests/yamato/scripts/run_gym.py --env=artifacts/testPlayer-Basic
- .yamato/standalone-build-test.yml#test_mac_standalone_{{ editor.version }}
- .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
triggers:
cancel_old_ci: true
expression: |

4
.yamato/protobuf-generation-test.yml


nuget install Grpc.Tools -Version $GRPC_VERSION -OutputDirectory protobuf-definitions/
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
pip install grpcio==1.28.1 grpcio-tools==1.13.0 protobuf==3.11.3 six==1.14.0 mypy-protobuf==1.16.0 --progress-bar=off --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python3 -m pip install --upgrade pip --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python3 -m pip install grpcio==1.28.1 grpcio-tools==1.13.0 protobuf==3.11.3 six==1.14.0 mypy-protobuf==1.16.0 --progress-bar=off --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
cd protobuf-definitions
chmod +x Grpc.Tools.$GRPC_VERSION/tools/macosx_x64/protoc
chmod +x Grpc.Tools.$GRPC_VERSION/tools/macosx_x64/grpc_csharp_plugin

27
.yamato/python-ll-api-test.yml


{% metadata_file .yamato/test_versions.metafile %}
---
{% for editor in test_editors %}
test_mac_ll_api_{{ editor.version }}:
name: Test Mac LL-API {{ editor.version }}
test_linux_ll_api_{{ editor.version }}:
name: Test Linux LL-API {{ editor.version }}
type: Unity::VM::osx
image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
flavor: b1.small
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.medium
- pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
- python -u -m ml-agents.tests.yamato.setup_venv
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Basic
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-WallJump
- ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Bouncer
- |
sudo apt-get update && sudo apt-get install -y python3-venv
python3 -m venv venv && source venv/bin/activate
python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python -u -m ml-agents.tests.yamato.setup_venv
python ml-agents/tests/yamato/scripts/run_llapi.py
python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Basic
python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-WallJump
python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Bouncer
- .yamato/standalone-build-test.yml#test_mac_standalone_{{ editor.version }}
- .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
triggers:
cancel_old_ci: true
expression: |

30
.yamato/standalone-build-test.yml


{% metadata_file .yamato/test_versions.metafile %}
---
{% for editor in test_editors %}
test_mac_standalone_{{ editor.version }}:
name: Test Mac Standalone {{ editor.version }}
test_linux_standalone_{{ editor.version }}:
name: Test Linux Standalone {{ editor.version }}
type: Unity::VM::osx
image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
flavor: i1.small
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.medium
- pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
# TODO remove the "--user" command and the path prefix when we can migrate away from the custom bokken image
- python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade --user
- /Users/bokken/Library/Python/3.7/bin/unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
- python -u -m ml-agents.tests.yamato.standalone_build_tests
- python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity
- python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Bouncer/Scenes/Bouncer.unity
- python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/WallJump/Scenes/WallJump.unity
- python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/TestScenes/TestCompressedGrid/TestGridCompressed.unity
- python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/TestScenes/TestCompressedTexture/TestTextureCompressed.unity
- python3 -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
- python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
- unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux --scene=Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux --scene=Assets/ML-Agents/Examples/Bouncer/Scenes/Bouncer.unity
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux --scene=Assets/ML-Agents/Examples/WallJump/Scenes/WallJump.unity
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux --scene=Assets/ML-Agents/TestScenes/TestCompressedGrid/TestGridCompressed.unity
- python3 -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=linux --scene=Assets/ML-Agents/TestScenes/TestCompressedTexture/TestTextureCompressed.unity
triggers:
cancel_old_ci: true
expression: |

standalonebuild:
paths:
- "artifacts/testPlayer*/**"
- "artifacts/**/UnityPlayer.so"
{% endfor %}

31
.yamato/training-int-tests.yml


{% metadata_file .yamato/test_versions.metafile %}
---
{% for editor in test_editors %}
test_mac_training_int_{{ editor.version }}:
name: Test Mac Fast Training {{ editor.version }}
test_linux_training_int_{{ editor.version }}:
name: Test Linux Fast Training {{ editor.version }}
type: Unity::VM::osx
image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
flavor: b1.small
type: Unity::VM
image: package-ci/ubuntu:stable
flavor: b1.medium
- pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
# TODO remove the "--user" command and the path prefix when we can migrate away from the custom bokken image
- python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade --user
- /Users/bokken/Library/Python/3.7/bin/unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
- python -u -m ml-agents.tests.yamato.training_int_tests
# Backwards-compatibility tests.
# If we make a breaking change to the communication protocol, these will need
# to be disabled until the next release.
- python -u -m ml-agents.tests.yamato.training_int_tests --python=0.16.0
- python -u -m ml-agents.tests.yamato.training_int_tests --csharp={{ editor.csharp_backcompat_version }}
- |
sudo apt-get update && sudo apt-get install -y python3-venv
python3 -m venv venv && source venv/bin/activate
python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
python -u -m ml-agents.tests.yamato.training_int_tests
- .yamato/standalone-build-test.yml#test_mac_standalone_{{ editor.version }}
- .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
triggers:
cancel_old_ci: true
expression: |

- "artifacts/inference.onnx.txt"
standalonebuild:
paths:
- "artifacts/testplayer*/**"
- "artifacts/testPlayer*/**"
- "artifacts/models/**"
{% endfor %}

8
Project/Assets/ML-Agents/Editor/Tests/StandaloneBuildTest.cs


{
const string k_OutputCommandLineFlag = "--mlagents-build-output-path";
const string k_SceneCommandLineFlag = "--mlagents-build-scene-path";
private const string k_BuildTargetFlag = "--mlagents-build-target";
public static void BuildStandalonePlayerOSX()
{

var buildTarget = BuildTarget.StandaloneOSX;
var args = Environment.GetCommandLineArgs();
for (var i = 0; i < args.Length - 1; i++)

{
scenePath = args[i + 1];
}
else if (args[i] == k_BuildTargetFlag)
{
buildTarget = (BuildTarget)Enum.Parse(typeof(BuildTarget), args[i + 1], ignoreCase: true);
}
}
string[] scenes = { scenePath };

BuildTarget.StandaloneOSX,
buildTarget,
BuildOptions.None
);
var isOk = buildResult.summary.result == BuildResult.Succeeded;

3
ml-agents-envs/mlagents_envs/env_utils.py


candidates = glob.glob(env_path + ".x86_64")
if len(candidates) == 0:
candidates = glob.glob(env_path + ".x86")
if len(candidates) == 0:
if os.path.isfile(env_path):
candidates = [env_path]
if len(candidates) > 0:
launch_string = candidates[0]

12
ml-agents/tests/yamato/standalone_build_tests.py


from .yamato_utils import get_base_path, run_standalone_build
def main(scene_path):
def main(scene_path, build_target):
executable_name = None
executable_name = "testPlayer"
if scene_path is not None:
executable_name = os.path.splitext(scene_path)[0] # Remove extension
executable_name = executable_name.split("/")[-1]

returncode = run_standalone_build(
base_path, output_path=executable_name, scene_path=scene_path
base_path,
output_path=executable_name,
scene_path=scene_path,
build_target=build_target,
)
if returncode == 0:

if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--scene", default=None)
parser.add_argument("--build-target", default="mac", choices=["mac", "linux"])
main(args.scene)
main(args.scene, args.build_target)

32
ml-agents/tests/yamato/training_int_tests.py


else:
standalone_player_path = "testPlayer"
venv_path = init_venv(python_version)
init_venv(python_version)
# Copy the default training config but override the max_steps parameter,
# and reduce the batch_size and buffer_size enough to ensure an update step happens.

}
override_config_file("config/ppo/3DBall.yaml", yaml_out, overrides)
env_path = os.path.join(get_base_output_path(), standalone_player_path + ".app")
mla_learn_cmd = (
f"mlagents-learn {yaml_out} --force --env={env_path} "
f"--run-id={run_id} --no-graphics --env-args -logFile -"
) # noqa
res = subprocess.run(
f"source {venv_path}/bin/activate; {mla_learn_cmd}", shell=True
)
log_output_path = f"{get_base_output_path()}/training.log"
env_path = os.path.join(get_base_output_path(), standalone_player_path)
mla_learn_cmd = [
"mlagents-learn",
yaml_out,
"--force",
"--env",
env_path,
"--run-id",
str(run_id),
"--no-graphics",
"--env-args",
"-logFile",
log_output_path,
]
res = subprocess.run(mla_learn_cmd)
shutil.copy(onnx_file_expected, model_artifacts_dir)
if os.path.exists(onnx_file_expected):
shutil.copy(onnx_file_expected, model_artifacts_dir)
print("Command line: " + " ".join(mla_learn_cmd))
subprocess.run(["cat", log_output_path])
return False
if csharp_version is None and python_version is None:

47
ml-agents/tests/yamato/yamato_utils.py


import shutil
import subprocess
import yaml
from sys import platform
downloader_install_path = "./.Editor/Unity.app/Contents/MacOS/Unity"
if platform == "darwin":
downloader_install_path = "./.Editor/Unity.app/Contents/MacOS/Unity"
else: # if platform == "linux":
downloader_install_path = "./.Editor/Unity"
if os.path.exists(downloader_install_path):
return downloader_install_path
raise FileNotFoundError("Can't find executable from unity-downloader-cli")

verbose: bool = False,
output_path: str = None,
scene_path: str = None,
build_target: str = None,
artifacts/standalone_build/testPlayer.
artifacts/standalonebuild/testPlayer.
print(f"Running BuildStandalonePlayerOSX via {unity_exe}")
print(f"Running BuildStandalonePlayer via {unity_exe}")
# enum values from https://docs.unity3d.com/2019.4/Documentation/ScriptReference/BuildTarget.html
build_target_to_enum = {
"mac": "StandaloneOSX",
"osx": "StandaloneOSX",
"linux": "StandaloneLinux64",
}
test_args = [
unity_exe,

os.makedirs(os.path.dirname(output_path), exist_ok=True)
if scene_path is not None:
test_args += ["--mlagents-build-scene-path", scene_path]
if build_target is not None:
test_args += ["--mlagents-build-target", build_target_to_enum[build_target]]
print(f"{' '.join(test_args)} ...")
timeout = 30 * 60 # 30 minutes, just in case

if output_path is None and res.returncode == 0:
exe_name = "testPlayer.app" if platform == "darwin" else "testPlayer"
os.path.join(base_path, "Project", "testPlayer.app"),
os.path.join(get_base_output_path(), "testPlayer.app"),
os.path.join(base_path, "Project", exe_name),
os.path.join(get_base_output_path(), exe_name),
)
# Print if we fail or want verbosity.

file_path = os.path.join(root, filename)
if os.access(file_path, os.X_OK):
exes.append(file_path)
# Also check the input path
if os.access(root_dir, os.X_OK):
exes.append(root_dir)
) -> str:
) -> None:
Set up the virtual environment, and return the venv path.
Install the necessary packages for the venv
# Use a different venv path for different versions
venv_path = "venv"
if mlagents_python_version:
venv_path += "_" + mlagents_python_version
# Set up the venv and install mlagents
subprocess.check_call(f"python -m venv {venv_path}", shell=True)
if platform != "darwin":
raise RuntimeError("Yamato can only run tensorflow on mac platforms!")
pip_commands += [
f"mlagents=={mlagents_python_version}",
f"gym-unity=={mlagents_python_version}",

pip_commands += ["-e ./ml-agents-envs", "-e ./ml-agents", "-e ./gym-unity"]
if extra_packages:
pip_commands += extra_packages
print(f'Running "python3 -m pip install -q {cmd} {pip_index_url}"')
f"source {venv_path}/bin/activate; python -m pip install -q {cmd} {pip_index_url}",
shell=True,
f"python3 -m pip install -q {cmd} {pip_index_url}", shell=True
return venv_path
def checkout_csharp_version(csharp_version):

正在加载...
取消
保存