version: 2.1 executors: python361: docker: - image: circleci/python:3.6.1 python373: docker: - image: circleci/python:3.7.3 jobs: build_python: parameters: executor: type: executor pyversion: type: string description: python version to being used (currently only affects caching). pip_constraints: type: string description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility. executor: << parameters.executor >> working_directory: ~/repo # Run additional numpy checks on unit tests environment: TEST_ENFORCE_NUMPY_FLOAT32: 1 steps: - checkout - run: # Combine all the python dependencies into one file so that we can use that for the cache checksum name: Combine pip dependencies for caching command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt - restore_cache: keys: # Parameterize the cache so that different python versions can get different versions of the packages - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} - run: name: Install Dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install --upgrade setuptools pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >> pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >> - save_cache: paths: - ./venv key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} - run: name: Run Tests for ml-agents and gym_unity # This also dumps the installed pip packages to a file, so we can see what versions are actually being used. command: | . venv/bin/activate mkdir test-reports pip freeze > test-reports/pip_versions.txt pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings - run: name: Verify there are no hidden/missing metafiles. # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy. command: | . venv/bin/activate python utils/validate_meta_files.py - store_test_results: path: test-reports - store_artifacts: path: test-reports destination: test-reports - store_artifacts: path: htmlcov destination: htmlcov pre-commit: docker: - image: circleci/python:3.7.3 working_directory: ~/repo/ steps: - checkout - run: name: Combine precommit config and python versions for caching command: | cat .pre-commit-config.yaml > pre-commit-deps.txt python -VV >> pre-commit-deps.txt - restore_cache: keys: - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Install Dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install --upgrade setuptools pip install pre-commit # Install the hooks now so that they'll be cached pre-commit install-hooks - save_cache: paths: - ~/.cache/pre-commit - ./venv key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Check Code Style using pre-commit command: | . venv/bin/activate pre-commit run --show-diff-on-failure --all-files markdown_link_check: parameters: precommit_command: type: string description: precommit hook to run default: markdown-link-check docker: - image: circleci/node:12.6.0 working_directory: ~/repo steps: - checkout - restore_cache: keys: - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} # fallback to using the latest cache if no exact match is found - v1-node-dependencies- - run: name: Install Dependencies command: | sudo apt-get install python3-venv python3 -m venv venv . venv/bin/activate pip install pre-commit - run: sudo npm install -g markdown-link-check - save_cache: paths: - ./venv key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} - run: name: Run markdown-link-check via precommit command: | . venv/bin/activate pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files protobuf_generation_check: docker: - image: circleci/python:3.7.3 working_directory: ~/repo/ steps: - checkout - run: name: Combine proto files for caching command: cat protobuf-definitions/proto/mlagents_envs/communicator_objects/*.proto > /tmp/proto_deps.txt - restore_cache: keys: - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }} - v1-protobuf-gen-dependencies- - run: name: Install Dependencies command: | sudo apt-get install nuget nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/ python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install grpcio-tools==1.13.0 --progress-bar=off pip install mypy-protobuf==1.16.0 --progress-bar=off - save_cache: paths: - ./venv key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }} - run: name: Generate protobufs command: | . venv/bin/activate cd protobuf-definitions chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh CS_PROTO_PATH=UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects git diff --exit-code --quiet -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" \ || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \ echo "Apply the patch with the command: git apply proto.patch"; \ git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > /tmp/proto.patch; \ exit $GIT_ERR; } - store_artifacts: path: /tmp/proto.patch destination: proto.patch deploy: parameters: directory: type: string description: Local directory to use for publishing (e.g. ml-agents) docker: - image: circleci/python:3.6 steps: - checkout - run: name: install python dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install setuptools wheel twine - run: name: verify git tag vs. version command: | python3 -m venv venv . venv/bin/activate cd << parameters.directory >> python setup.py verify - run: name: create packages command: | . venv/bin/activate cd << parameters.directory >> python setup.py sdist python setup.py bdist_wheel - run: name: upload to pypi # To upload to test, just add the following flag to twine upload: # --repository-url https://test.pypi.org/legacy/ # and change the username to "mlagents-test" command: | . venv/bin/activate cd << parameters.directory >> twine upload -u mlagents -p $PYPI_PASSWORD dist/* workflows: version: 2 workflow: jobs: - build_python: name: python_3.6.1 executor: python361 pyversion: 3.6.1 # Test python 3.6 with the oldest supported versions pip_constraints: test_constraints_min_version.txt - build_python: name: python_3.7.3 executor: python373 pyversion: 3.7.3 # Test python 3.7 with the newest supported versions pip_constraints: test_constraints_max_tf1_version.txt - build_python: name: python_3.7.3+tf2 executor: python373 pyversion: 3.7.3 # Test python 3.7 with the newest supported versions pip_constraints: test_constraints_max_tf2_version.txt - markdown_link_check - protobuf_generation_check - pre-commit - deploy: name: deploy ml-agents-envs directory: ml-agents-envs filters: tags: only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/ branches: ignore: /.*/ - deploy: name: deploy ml-agents directory: ml-agents filters: tags: only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/ branches: ignore: /.*/ - deploy: name: deploy gym-unity directory: gym-unity filters: tags: only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/ branches: ignore: /.*/ nightly: triggers: - schedule: cron: "0 0 * * *" filters: branches: only: - develop jobs: - markdown_link_check: name: markdown-link-check full precommit_command: markdown-link-check-full