version: 2.1 executors: python361: docker: - image: circleci/python:3.6.1 python373: docker: - image: circleci/python:3.7.3 python382: docker: - image: circleci/python:3.8.2 jobs: build_python: parameters: executor: type: executor pyversion: type: string description: python version to being used (currently only affects caching). pip_constraints: type: string description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility. enforce_onnx_conversion: type: integer default: 0 description: Whether to raise an exception if ONNX models couldn't be saved. executor: << parameters.executor >> working_directory: ~/repo # Run additional numpy checks on unit tests environment: TEST_ENFORCE_NUMPY_FLOAT32: 1 TEST_ENFORCE_ONNX_CONVERSION: << parameters.enforce_onnx_conversion >> steps: - checkout - run: # Combine all the python dependencies into one file so that we can use that for the cache checksum name: Combine pip dependencies for caching command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt - restore_cache: keys: # Parameterize the cache so that different python versions can get different versions of the packages - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} - run: name: Install Dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install --upgrade setuptools pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >> pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >> - save_cache: paths: - ./venv key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} - run: name: Run Tests for ml-agents and gym_unity # This also dumps the installed pip packages to a file, so we can see what versions are actually being used. command: | . venv/bin/activate mkdir test-reports pip freeze > test-reports/pip_versions.txt pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings - run: name: Verify there are no hidden/missing metafiles. # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy. command: | . venv/bin/activate python utils/validate_meta_files.py - store_test_results: path: test-reports - store_artifacts: path: test-reports destination: test-reports - store_artifacts: path: htmlcov destination: htmlcov pre-commit: docker: - image: circleci/python:3.7.3 working_directory: ~/repo/ steps: - checkout - run: name: Combine precommit config and python versions for caching command: | cat .pre-commit-config.yaml > pre-commit-deps.txt python -VV >> pre-commit-deps.txt - restore_cache: keys: - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Install Dependencies command: | # Need ruby for search-and-replace sudo apt-get update sudo apt-get install ruby-full # install dotnet and the formatter - see https://docs.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-9- pushd ~ wget -O - https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ wget https://packages.microsoft.com/config/debian/9/prod.list sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list popd sudo apt-get install -y apt-transport-https && \ sudo apt-get update && \ sudo apt-get install -y dotnet-sdk-3.1 && \ dotnet tool install -g dotnet-format --version 4.1.131201 echo "Setting up venv" python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install --upgrade setuptools pip install pre-commit # Install the hooks now so that they'll be cached pre-commit install-hooks - save_cache: paths: - ~/.cache/pre-commit - ./venv key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Check Code Style using pre-commit command: | . venv/bin/activate export PATH="$PATH:~/.dotnet/tools" pre-commit run --show-diff-on-failure --all-files markdown_link_check: parameters: precommit_command: type: string description: precommit hook to run default: markdown-link-check docker: - image: circleci/node:12.6.0 working_directory: ~/repo steps: - checkout - restore_cache: keys: - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} # fallback to using the latest cache if no exact match is found - v1-node-dependencies- - run: name: Install Dependencies command: | sudo apt-get install python3-venv python3 -m venv venv . venv/bin/activate pip install pre-commit - run: sudo npm install -g markdown-link-check - save_cache: paths: - ./venv key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} - run: name: Run markdown-link-check via precommit command: | . venv/bin/activate pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files deploy: parameters: directory: type: string description: Local directory to use for publishing (e.g. ml-agents) username: type: string description: pypi username default: mlagents test_repository_args: type: string description: Optional override repository URL. Only use for tests, leave blank for "real" default: "" docker: - image: circleci/python:3.6 steps: - checkout - run: name: install python dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install setuptools wheel twine - run: name: verify git tag vs. version command: | python3 -m venv venv . venv/bin/activate cd << parameters.directory >> python setup.py verify - run: name: create packages command: | . venv/bin/activate cd << parameters.directory >> python setup.py sdist python setup.py bdist_wheel - run: name: upload to pypi # To upload to test, just add the following flag to twine upload: # --repository-url https://test.pypi.org/legacy/ # and change the username to "mlagents-test" command: | . venv/bin/activate cd << parameters.directory >> twine upload -u << parameters.username >> -p $PYPI_PASSWORD << parameters.test_repository_args >> dist/* workflows: version: 2 workflow: jobs: - build_python: name: python_3.6.1 executor: python361 pyversion: 3.6.1 # Test python 3.6 with the oldest supported versions pip_constraints: test_constraints_min_version.txt - build_python: name: python_3.7.3 executor: python373 pyversion: 3.7.3 # Test python 3.7 with the newest supported versions pip_constraints: test_constraints_max_tf1_version.txt # Make sure ONNX conversion passes here (recent version of tensorflow 1.x) enforce_onnx_conversion: 1 - build_python: name: python_3.7.3+tf2 executor: python373 pyversion: 3.7.3 # Test python 3.7 with the newest supported versions pip_constraints: test_constraints_max_tf2_version.txt - build_python: name: python_3.8.2+tf2.2 executor: python382 pyversion: 3.8.2 # Test python 3.8 with the newest edge versions pip_constraints: test_constraints_max_tf2_version.txt - markdown_link_check - pre-commit # The first deploy jobs are the "real" ones that upload to pypi - deploy: name: deploy ml-agents-envs directory: ml-agents-envs filters: tags: # Matches e.g. "release_123" only: /^release_[0-9]+$/ branches: ignore: /.*/ - deploy: name: deploy ml-agents directory: ml-agents filters: tags: # Matches e.g. "release_123" only: /^release_[0-9]+$/ branches: ignore: /.*/ - deploy: name: deploy gym-unity directory: gym-unity filters: tags: # Matches e.g. "release_123" only: /^release_[0-9]+$/ branches: ignore: /.*/ # These deploy jobs upload to the pypi test repo. They have different tag triggers than the real ones. - deploy: name: test deploy ml-agents-envs directory: ml-agents-envs username: mlagents-test test_repository_args: --repository-url https://test.pypi.org/legacy/ filters: tags: # Matches e.g. "release_123_test456 only: /^release_[0-9]+_test[0-9]+$/ branches: ignore: /.*/ - deploy: name: test deploy ml-agents directory: ml-agents username: mlagents-test test_repository_args: --repository-url https://test.pypi.org/legacy/ filters: tags: # Matches e.g. "release_123_test456 only: /^release_[0-9]+_test[0-9]+$/ branches: ignore: /.*/ - deploy: name: test deploy gym-unity directory: gym-unity username: mlagents-test test_repository_args: --repository-url https://test.pypi.org/legacy/ filters: tags: # Matches e.g. "release_123_test456 only: /^release_[0-9]+_test[0-9]+$/ branches: ignore: /.*/ nightly: triggers: - schedule: cron: "0 0 * * *" filters: branches: only: - develop jobs: - markdown_link_check: name: markdown-link-check full precommit_command: markdown-link-check-full