- image : circleci/python:3.8.2
jobs:
build_python:
parameters:
executor:
type : executor
pyversion:
type : string
description : python version to being used (currently only affects caching).
pip_constraints:
type : string
description : Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
enforce_onnx_conversion:
type : integer
default : 0
description : Whether to raise an exception if ONNX models couldn't be saved.
executor : << parameters.executor >>
working_directory : ~/repo
# Run additional numpy checks on unit tests
environment:
TEST_ENFORCE_NUMPY_FLOAT32 : 1
TEST_ENFORCE_ONNX_CONVERSION : << parameters.enforce_onnx_conversion >>
steps:
- checkout
- run:
# Combine all the python dependencies into one file so that we can use that for the cache checksum
name : Combine pip dependencies for caching
command : cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt
- restore_cache:
keys:
# Parameterize the cache so that different python versions can get different versions of the packages
- v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
- run:
name : Install Dependencies
command : |
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip
pip install --upgrade setuptools
pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>
- save_cache:
paths:
- ./venv
key : v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
- run:
name : Run Tests for ml-agents and gym_unity
# This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
command : |
. venv/bin/activate
mkdir test-reports
pip freeze > test-reports/pip_versions.txt
pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings
- run:
name : Verify there are no hidden/missing metafiles.
# Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
command : |
. venv/bin/activate
python utils/validate_meta_files.py
- store_test_results:
path : test-reports
- store_artifacts:
path : test-reports
destination : test-reports
- store_artifacts:
path : htmlcov
destination : htmlcov
pre-commit:
docker:
- image : circleci/python:3.7.3
working_directory : ~/repo/
steps:
- checkout
- run:
name : Combine precommit config and python versions for caching
command : |
cat .pre-commit-config.yaml > pre-commit-deps.txt
python -VV >> pre-commit-deps.txt
- restore_cache:
keys:
- v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
- run:
name : Install Dependencies
command : |
# Need ruby for search-and-replace
sudo apt-get update
sudo apt-get install ruby-full
# install dotnet and the formatter - see https://docs.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-9-
pushd ~
wget -O - https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg
sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/
wget https://packages.microsoft.com/config/debian/9/prod.list
sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list
sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg
sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list
popd
sudo apt-get install -y apt-transport-https && \
sudo apt-get update && \
sudo apt-get install -y dotnet-sdk-3.1 && \
dotnet tool install -g dotnet-format --version 4.1.131201
echo "Setting up venv"
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip
pip install --upgrade setuptools
pip install pre-commit
# Install the hooks now so that they'll be cached
pre-commit install-hooks
- save_cache:
paths:
- ~/.cache/pre-commit
- ./venv
key : v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
- run:
name : Check Code Style using pre-commit
command : |
. venv/bin/activate
export PATH="$PATH:~/.dotnet/tools"
pre-commit run --show-diff-on-failure --all-files
markdown_link_check:
parameters:
precommit_command:
type : string
description : precommit hook to run
default : markdown-link-check
docker:
- image : circleci/node:12.6.0
working_directory : ~/repo
steps:
- checkout
- restore_cache:
keys:
- v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
# fallback to using the latest cache if no exact match is found
- v1-node-dependencies-
- run:
name : Install Dependencies
command : |
sudo apt-get install python3-venv
python3 -m venv venv
. venv/bin/activate
pip install pre-commit
- run : sudo npm install -g markdown-link-check
- save_cache:
paths:
- ./venv
key : v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
- run:
name : Run markdown-link-check via precommit
command : |
. venv/bin/activate
pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files
deploy:
parameters:
directory:
version : 2
workflow:
jobs:
- build_python:
name : python_3.6.1
executor : python361
pyversion : 3.6 .1
# Test python 3.6 with the oldest supported versions
pip_constraints : test_constraints_min_version.txt
- build_python:
name : python_3.7.3
executor : python373
pyversion : 3.7 .3
# Test python 3.7 with the newest supported versions
pip_constraints : test_constraints_max_tf1_version.txt
# Make sure ONNX conversion passes here (recent version of tensorflow 1.x)
enforce_onnx_conversion : 1
- build_python:
name : python_3.7.3+tf2
executor : python373
pyversion : 3.7 .3
# Test python 3.7 with the newest supported versions
pip_constraints : test_constraints_max_tf2_version.txt
- build_python:
name : python_3.8.2+tf2.2
executor : python382
pyversion : 3.8 .2
# Test python 3.8 with the newest edge versions
pip_constraints : test_constraints_max_tf2_version.txt
- markdown_link_check
- pre-commit
# The first deploy jobs are the "real" ones that upload to pypi
- deploy:
name : deploy ml-agents-envs
only : /^release_[0-9]+_test[0-9]+$/
branches:
ignore : /.*/
nightly:
triggers:
- schedule:
cron : "0 0 * * *"
filters:
branches:
only:
- develop
jobs:
- markdown_link_check:
name : markdown-link-check full
precommit_command : markdown-link-check-full
Ruo-Ping Dong
4 年前