Merge pull request #4507 from Unity-Technologies/release_2_verified-CI

get circleCI and GH action changes from master

.circleci/config.yml

       - image: circleci/python:3.8.2
 
 jobs:
-  build_python:
-    parameters:
-      executor:
-        type: executor
-      pyversion:
-        type: string
-        description: python version to being used (currently only affects caching).
-      pip_constraints:
-        type: string
-        description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
-      enforce_onnx_conversion:
-        type: integer
-        default: 0
-        description: Whether to raise an exception if ONNX models couldn't be saved.
-    executor: << parameters.executor >>
-    working_directory: ~/repo
-
-    # Run additional numpy checks on unit tests
-    environment:
-      TEST_ENFORCE_NUMPY_FLOAT32: 1
-      TEST_ENFORCE_ONNX_CONVERSION: << parameters.enforce_onnx_conversion >>
-
-    steps:
-      - checkout
-
-      - run:
-          # Combine all the python dependencies into one file so that we can use that for the cache checksum
-          name: Combine pip dependencies for caching
-          command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt
-
-      - restore_cache:
-          keys:
-          # Parameterize the cache so that different python versions can get different versions of the packages
-          - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
-
-      - run:
-          name: Install Dependencies
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip
-            pip install --upgrade setuptools
-            pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
-            pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
-            pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
-            pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
-
-      - run:
-          name: Run Tests for ml-agents and gym_unity
-          # This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
-          command: |
-            . venv/bin/activate
-            mkdir test-reports
-            pip freeze > test-reports/pip_versions.txt
-            pytest -n 2 --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings
-
-      - run:
-          name: Verify there are no hidden/missing metafiles.
-          # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
-          command: |
-            . venv/bin/activate
-            python utils/validate_meta_files.py
-
-      - store_test_results:
-          path: test-reports
-
-      - store_artifacts:
-          path: test-reports
-          destination: test-reports
-
-      - store_artifacts:
-          path: htmlcov
-          destination: htmlcov
-
-  pre-commit:
-    docker:
-      - image: circleci/python:3.7.3
-    working_directory: ~/repo/
-
-    steps:
-      - checkout
-      - run:
-          name: Combine precommit config and python versions for caching
-          command: |
-            cat .pre-commit-config.yaml > pre-commit-deps.txt
-            python -VV >> pre-commit-deps.txt
-
-      - restore_cache:
-          keys:
-          - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
-
-      - run:
-          name: Install Dependencies
-          command: |
-            # Need ruby for search-and-replace
-            sudo apt-get update
-            sudo apt-get install ruby-full
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip
-            pip install --upgrade setuptools
-            pip install pre-commit
-            # Install the hooks now so that they'll be cached
-            pre-commit install-hooks
-
-      - save_cache:
-          paths:
-            - ~/.cache/pre-commit
-            - ./venv
-          key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
-
-      - run:
-          name: Check Code Style using pre-commit
-          command: |
-            . venv/bin/activate
-            pre-commit run --show-diff-on-failure --all-files
-
-  markdown_link_check:
-    parameters:
-      precommit_command:
-        type: string
-        description: precommit hook to run
-        default: markdown-link-check
-    docker:
-      - image: circleci/node:12.6.0
-    working_directory: ~/repo
-
-    steps:
-      - checkout
-
-      - restore_cache:
-          keys:
-          - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-node-dependencies-
-
-      - run:
-          name: Install Dependencies
-          command: |
-            sudo apt-get install python3-venv
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install pre-commit
-      - run: sudo npm install -g markdown-link-check
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
-
-      - run:
-          name: Run markdown-link-check via precommit
-          command: |
-            . venv/bin/activate
-            pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files
-
   deploy:
     parameters:
       directory:
   version: 2
   workflow:
     jobs:
-      - build_python:
-          name: python_3.6.1
-          executor: python361
-          pyversion: 3.6.1
-          # Test python 3.6 with the oldest supported versions
-          pip_constraints: test_constraints_min_version.txt
-      - build_python:
-          name: python_3.7.3
-          executor: python373
-          pyversion: 3.7.3
-          # Test python 3.7 with the newest supported versions
-          pip_constraints: test_constraints_max_tf1_version.txt
-          # Make sure ONNX conversion passes here (recent version of tensorflow 1.x)
-          enforce_onnx_conversion: 1
-      - build_python:
-          name: python_3.7.3+tf2
-          executor: python373
-          pyversion: 3.7.3
-          # Test python 3.7 with the newest supported versions
-          pip_constraints: test_constraints_max_tf2_version.txt
-      - build_python:
-          name: python_3.8.2+tf2.2
-          executor: python382
-          pyversion: 3.8.2
-          # Test python 3.8 with the newest edge versions
-          pip_constraints: test_constraints_max_tf2_version.txt
-      - markdown_link_check
-      - pre-commit
       # The first deploy jobs are the "real" ones that upload to pypi
       - deploy:
           name: deploy ml-agents-envs
               only: /^release_[0-9]+_test[0-9]+$/
             branches:
               ignore: /.*/
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - develop
-    jobs:
-      - markdown_link_check:
-          name: markdown-link-check full
-          precommit_command: markdown-link-check-full

.github/workflows/nightly.yml

+name: nightly
+
+on:
+  schedule:
+    - cron: '0 7 * * *' # run at 7 AM UTC == midnight PST
+
+jobs:
+  markdown-link-check-full:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: '12'
+    - run: sudo npm install -g markdown-link-check
+    - uses: pre-commit/action@v2.0.0
+      with:
+          extra_args: --hook-stage manual markdown-link-check-full --all-files

.github/workflows/pre-commit.yml

+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: '2.6'
+    - uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '3.1.x'
+    - run: dotnet tool install -g dotnet-format --version 4.1.131201
+    - uses: pre-commit/action@v2.0.0
+
+  markdown-link-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: '12'
+    - run: sudo npm install -g markdown-link-check
+    - uses: pre-commit/action@v2.0.0
+      with:
+          extra_args: --hook-stage manual markdown-link-check --all-files
+
+  validate-meta-files:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - run: python utils/validate_meta_files.py

.github/workflows/pytest.yml

+name: pytest
+
+on:
+  pull_request:
+    paths:  # This action will only run if the PR modifies a file in one of these directories
+    - 'ml-agents/**'
+    - 'ml-agents-envs/**'
+    - 'gym-unity/**'
+    - 'test_constraints*.txt'
+    - 'test_requirements.txt'
+  push:
+    branches: [master]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6.x, 3.7.x, 3.8.x]
+        include:
+          - python-version: 3.6.x
+            pip_constraints: test_constraints_min_version.txt
+          - python-version: 3.7.x
+            pip_constraints: test_constraints_max_tf1_version.txt
+          - python-version: 3.8.x
+            pip_constraints: test_constraints_max_tf2_version.txt
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'test_requirements.txt', matrix.pip_constraints) }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+          ${{ runner.os }}-
+    - name: Display Python version
+      run: python -c "import sys; print(sys.version)"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        python -m pip install --progress-bar=off -e ./ml-agents-envs -c ${{ matrix.pip_constraints }}
+        python -m pip install --progress-bar=off -e ./ml-agents -c ${{ matrix.pip_constraints }}
+        python -m pip install --progress-bar=off -r test_requirements.txt -c ${{ matrix.pip_constraints }}
+        python -m pip install --progress-bar=off -e ./gym-unity -c ${{ matrix.pip_constraints }}
+    - name: Save python dependencies
+      run: pip freeze > pip_versions-${{ matrix.python-version }}.txt
+    - name: Run pytest
+      run: pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=junit/test-results-${{ matrix.python-version }}.xml -p no:warnings
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: artifacts-${{ matrix.python-version }}
+        path: |
+          htmlcov
+          pip_versions-${{ matrix.python-version }}.txt
+          junit/test-results-${{ matrix.python-version }}.xml
+      # Use always() to always run this step to publish test results when there are test failures
+      if: ${{ always() }}