version: 2.1 executors: python361: docker: - image: circleci/python:3.6.1 python373: docker: - image: circleci/python:3.7.3 jobs: build_python: parameters: executor: type: executor pyversion: type: string description: python version to being used (currently only affects caching). pip_constraints: type: string description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility. executor: << parameters.executor >> working_directory: ~/repo steps: - checkout - run: # Combine all the python dependencies into one file so that we can use that for the cache checksum name: Combine pip dependencies for caching command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt - restore_cache: keys: # Parameterize the cache so that different python versions can get different versions of the packages - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} # fallback to using the latest cache if no exact match is found - v1-dependencies-py<< parameters.pyversion >>- - run: name: Install Dependencies command: | python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install --upgrade setuptools pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >> pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >> pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >> - save_cache: paths: - ./venv key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }} - run: name: Run Tests for ml-agents and gym_unity # This also dumps the installed pip packages to a file, so we can see what versions are actually being used. command: | . venv/bin/activate mkdir test-reports pip freeze > test-reports/pip_versions.txt pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings - run: name: Check Code Style using pre-commit command: | . venv/bin/activate pre-commit run --show-diff-on-failure --all-files - run: name: Verify there are no hidden/missing metafiles. # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy. command: | . venv/bin/activate python utils/validate_meta_files.py - store_test_results: path: test-reports - store_artifacts: path: test-reports destination: test-reports - store_artifacts: path: htmlcov destination: htmlcov markdown_link_check: docker: - image: circleci/node:12.6.0 working_directory: ~/repo steps: - checkout - restore_cache: keys: - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} # fallback to using the latest cache if no exact match is found - v1-node-dependencies- - run: name: Install Dependencies command: | sudo apt-get install python3-venv python3 -m venv venv . venv/bin/activate pip install pre-commit - run: sudo npm install -g markdown-link-check - save_cache: paths: - ./venv key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }} - run: name: Run markdown-link-check via precommit command: | . venv/bin/activate pre-commit run --hook-stage manual markdown-link-check --all-files protobuf_generation_check: docker: - image: circleci/python:3.7.3 working_directory: ~/repo/ steps: - checkout - run: name: Combine proto files for caching command: cat protobuf-definitions/proto/mlagents/envs/communicator_objects/*.proto > /tmp/proto_deps.txt - restore_cache: keys: - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }} - v1-protobuf-gen-dependencies- - run: name: Install Dependencies command: | sudo apt-get install nuget nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/ python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install grpcio-tools==1.13.0 --progress-bar=off pip install mypy-protobuf --progress-bar=off - save_cache: paths: - ./venv key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }} - run: name: Generate protobufs command: | . venv/bin/activate cd protobuf-definitions chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh CS_PROTO_PATH=UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects git diff --exit-code --quiet -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" \ || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \ echo "Apply the patch with the command: git apply proto.patch"; \ git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > /tmp/proto.patch; \ exit $GIT_ERR; } - store_artifacts: path: /tmp/proto.patch destination: proto.patch workflows: workflow: jobs: - build_python: name: python_3.6.1 executor: python361 pyversion: 3.6.1 # Test python 3.6 with the oldest supported versions pip_constraints: test_constraints_min_version.txt - build_python: name: python_3.7.3 executor: python373 pyversion: 3.7.3 # Test python 3.7 with the newest supported versions pip_constraints: test_constraints_max_version.txt - markdown_link_check - protobuf_generation_check