浏览代码

get circleCI and GH action changes from master

/r2v-yamato-linux
Chris Elion 4 年前
当前提交
696212d7
共有 5 个文件被更改,包括 126 次插入201 次删除
  1. 201
      .circleci/config.yml
  2. 19
      .github/workflows/nightly.yml
  3. 41
      .github/workflows/pre-commit.yml
  4. 66
      .github/workflows/pytest.yml

201
.circleci/config.yml


- image: circleci/python:3.8.2
jobs:
build_python:
parameters:
executor:
type: executor
pyversion:
type: string
description: python version to being used (currently only affects caching).
pip_constraints:
type: string
description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
enforce_onnx_conversion:
type: integer
default: 0
description: Whether to raise an exception if ONNX models couldn't be saved.
executor: << parameters.executor >>
working_directory: ~/repo
# Run additional numpy checks on unit tests
environment:
TEST_ENFORCE_NUMPY_FLOAT32: 1
TEST_ENFORCE_ONNX_CONVERSION: << parameters.enforce_onnx_conversion >>
steps:
- checkout
- run:
# Combine all the python dependencies into one file so that we can use that for the cache checksum
name: Combine pip dependencies for caching
command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt
- restore_cache:
keys:
# Parameterize the cache so that different python versions can get different versions of the packages
- v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
- run:
name: Install Dependencies
command: |
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip
pip install --upgrade setuptools
pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>
- save_cache:
paths:
- ./venv
key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
- run:
name: Run Tests for ml-agents and gym_unity
# This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
command: |
. venv/bin/activate
mkdir test-reports
pip freeze > test-reports/pip_versions.txt
pytest -n 2 --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings
- run:
name: Verify there are no hidden/missing metafiles.
# Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
command: |
. venv/bin/activate
python utils/validate_meta_files.py
- store_test_results:
path: test-reports
- store_artifacts:
path: test-reports
destination: test-reports
- store_artifacts:
path: htmlcov
destination: htmlcov
pre-commit:
docker:
- image: circleci/python:3.7.3
working_directory: ~/repo/
steps:
- checkout
- run:
name: Combine precommit config and python versions for caching
command: |
cat .pre-commit-config.yaml > pre-commit-deps.txt
python -VV >> pre-commit-deps.txt
- restore_cache:
keys:
- v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
- run:
name: Install Dependencies
command: |
# Need ruby for search-and-replace
sudo apt-get update
sudo apt-get install ruby-full
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip
pip install --upgrade setuptools
pip install pre-commit
# Install the hooks now so that they'll be cached
pre-commit install-hooks
- save_cache:
paths:
- ~/.cache/pre-commit
- ./venv
key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
- run:
name: Check Code Style using pre-commit
command: |
. venv/bin/activate
pre-commit run --show-diff-on-failure --all-files
markdown_link_check:
parameters:
precommit_command:
type: string
description: precommit hook to run
default: markdown-link-check
docker:
- image: circleci/node:12.6.0
working_directory: ~/repo
steps:
- checkout
- restore_cache:
keys:
- v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
# fallback to using the latest cache if no exact match is found
- v1-node-dependencies-
- run:
name: Install Dependencies
command: |
sudo apt-get install python3-venv
python3 -m venv venv
. venv/bin/activate
pip install pre-commit
- run: sudo npm install -g markdown-link-check
- save_cache:
paths:
- ./venv
key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
- run:
name: Run markdown-link-check via precommit
command: |
. venv/bin/activate
pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files
deploy:
parameters:
directory:

version: 2
workflow:
jobs:
- build_python:
name: python_3.6.1
executor: python361
pyversion: 3.6.1
# Test python 3.6 with the oldest supported versions
pip_constraints: test_constraints_min_version.txt
- build_python:
name: python_3.7.3
executor: python373
pyversion: 3.7.3
# Test python 3.7 with the newest supported versions
pip_constraints: test_constraints_max_tf1_version.txt
# Make sure ONNX conversion passes here (recent version of tensorflow 1.x)
enforce_onnx_conversion: 1
- build_python:
name: python_3.7.3+tf2
executor: python373
pyversion: 3.7.3
# Test python 3.7 with the newest supported versions
pip_constraints: test_constraints_max_tf2_version.txt
- build_python:
name: python_3.8.2+tf2.2
executor: python382
pyversion: 3.8.2
# Test python 3.8 with the newest edge versions
pip_constraints: test_constraints_max_tf2_version.txt
- markdown_link_check
- pre-commit
# The first deploy jobs are the "real" ones that upload to pypi
- deploy:
name: deploy ml-agents-envs

only: /^release_[0-9]+_test[0-9]+$/
branches:
ignore: /.*/
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- develop
jobs:
- markdown_link_check:
name: markdown-link-check full
precommit_command: markdown-link-check-full

19
.github/workflows/nightly.yml


name: nightly
on:
schedule:
- cron: '0 7 * * *' # run at 7 AM UTC == midnight PST
jobs:
markdown-link-check-full:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
- uses: actions/setup-node@v2-beta
with:
node-version: '12'
- run: sudo npm install -g markdown-link-check
- uses: pre-commit/action@v2.0.0
with:
extra_args: --hook-stage manual markdown-link-check-full --all-files

41
.github/workflows/pre-commit.yml


name: pre-commit
on:
pull_request:
push:
branches: [master]
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
- uses: actions/setup-ruby@v1
with:
ruby-version: '2.6'
- uses: actions/setup-dotnet@v1
with:
dotnet-version: '3.1.x'
- run: dotnet tool install -g dotnet-format --version 4.1.131201
- uses: pre-commit/action@v2.0.0
markdown-link-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
- uses: actions/setup-node@v2-beta
with:
node-version: '12'
- run: sudo npm install -g markdown-link-check
- uses: pre-commit/action@v2.0.0
with:
extra_args: --hook-stage manual markdown-link-check --all-files
validate-meta-files:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
- run: python utils/validate_meta_files.py

66
.github/workflows/pytest.yml


name: pytest
on:
pull_request:
paths: # This action will only run if the PR modifies a file in one of these directories
- 'ml-agents/**'
- 'ml-agents-envs/**'
- 'gym-unity/**'
- 'test_constraints*.txt'
- 'test_requirements.txt'
push:
branches: [master]
jobs:
pytest:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6.x, 3.7.x, 3.8.x]
include:
- python-version: 3.6.x
pip_constraints: test_constraints_min_version.txt
- python-version: 3.7.x
pip_constraints: test_constraints_max_tf1_version.txt
- python-version: 3.8.x
pip_constraints: test_constraints_max_tf2_version.txt
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v2
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements file
key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'test_requirements.txt', matrix.pip_constraints) }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools
python -m pip install --progress-bar=off -e ./ml-agents-envs -c ${{ matrix.pip_constraints }}
python -m pip install --progress-bar=off -e ./ml-agents -c ${{ matrix.pip_constraints }}
python -m pip install --progress-bar=off -r test_requirements.txt -c ${{ matrix.pip_constraints }}
python -m pip install --progress-bar=off -e ./gym-unity -c ${{ matrix.pip_constraints }}
- name: Save python dependencies
run: pip freeze > pip_versions-${{ matrix.python-version }}.txt
- name: Run pytest
run: pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=junit/test-results-${{ matrix.python-version }}.xml -p no:warnings
- name: Upload pytest test results
uses: actions/upload-artifact@v2
with:
name: artifacts-${{ matrix.python-version }}
path: |
htmlcov
pip_versions-${{ matrix.python-version }}.txt
junit/test-results-${{ matrix.python-version }}.xml
# Use always() to always run this step to publish test results when there are test failures
if: ${{ always() }}
正在加载...
取消
保存