ml-agents/.circleci/config.yml

version: 2.1
executors:
  python361:
    docker:
      - image: circleci/python:3.6.1
  python373:
    docker:
      - image: circleci/python:3.7.3
  python382:
    docker:
      - image: circleci/python:3.8.2

jobs:
  build_python:
    parameters:
      executor:
        type: executor
      pyversion:
        type: string
        description: python version to being used (currently only affects caching).
      pip_constraints:
        type: string
        description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
      enforce_onnx_conversion:
        type: integer
        default: 0
        description: Whether to raise an exception if ONNX models couldn't be saved.
    executor: << parameters.executor >>
    working_directory: ~/repo

    # Run additional numpy checks on unit tests
    environment:
      TEST_ENFORCE_NUMPY_FLOAT32: 1
      TEST_ENFORCE_ONNX_CONVERSION: << parameters.enforce_onnx_conversion >>

    steps:
      - checkout

      - run:
          # Combine all the python dependencies into one file so that we can use that for the cache checksum
          name: Combine pip dependencies for caching
          command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt

      - restore_cache:
          keys:
          # Parameterize the cache so that different python versions can get different versions of the packages
          - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}

      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade setuptools
            pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
            pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>            

      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}

      - run:
          name: Run Tests for ml-agents and gym_unity
          # This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
          command: |
            . venv/bin/activate
            mkdir test-reports
            pip freeze > test-reports/pip_versions.txt
            pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings            

      - run:
          name: Verify there are no hidden/missing metafiles.
          # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
          command: |
            . venv/bin/activate
            python utils/validate_meta_files.py            

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports
          destination: test-reports

      - store_artifacts:
          path: htmlcov
          destination: htmlcov

  pre-commit:
    docker:
      - image: circleci/python:3.7.3
    working_directory: ~/repo/

    steps:
      - checkout
      - run:
          name: Combine precommit config and python versions for caching
          command: |
            cat .pre-commit-config.yaml > pre-commit-deps.txt
            python -VV >> pre-commit-deps.txt            

      - restore_cache:
          keys:
          - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}

      - run:
          name: Install Dependencies
          command: |
            # Need ruby for search-and-replace
            sudo apt-get update
            sudo apt-get install ruby-full

            # install dotnet and the formatter - see https://docs.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-9-
            pushd ~
            wget -O - https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg
            sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/
            wget https://packages.microsoft.com/config/debian/9/prod.list
            sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list
            sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg
            sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list
            popd

            sudo apt-get install -y apt-transport-https && \
              sudo apt-get update && \
              sudo apt-get install -y dotnet-sdk-3.1 && \
              dotnet tool install -g dotnet-format --version 4.1.131201

            echo "Setting up venv"
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade setuptools
            pip install pre-commit
            # Install the hooks now so that they'll be cached
            pre-commit install-hooks            

      - save_cache:
          paths:
            - ~/.cache/pre-commit
            - ./venv
          key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}

      - run:
          name: Check Code Style using pre-commit
          command: |
            . venv/bin/activate
            export PATH="$PATH:~/.dotnet/tools"
            pre-commit run --show-diff-on-failure --all-files            

  markdown_link_check:
    parameters:
      precommit_command:
        type: string
        description: precommit hook to run
        default: markdown-link-check
    docker:
      - image: circleci/node:12.6.0
    working_directory: ~/repo

    steps:
      - checkout

      - restore_cache:
          keys:
          - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
          # fallback to using the latest cache if no exact match is found
          - v1-node-dependencies-

      - run:
          name: Install Dependencies
          command: |
            sudo apt-get install python3-venv
            python3 -m venv venv
            . venv/bin/activate
            pip install pre-commit            
      - run: sudo npm install -g markdown-link-check

      - save_cache:
          paths:
            - ./venv
          key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}

      - run:
          name: Run markdown-link-check via precommit
          command: |
            . venv/bin/activate
            pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files            

  deploy:
    parameters:
      directory:
        type: string
        description: Local directory to use for publishing (e.g. ml-agents)
      username:
        type: string
        description: pypi username
        default: mlagents
      test_repository_args:
        type: string
        description: Optional override repository URL. Only use for tests, leave blank for "real"
        default: ""
    docker:
      - image: circleci/python:3.6
    steps:
      - checkout
      - run:
          name: install python dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install  --upgrade pip
            pip install setuptools wheel twine            
      - run:
          name: verify git tag vs. version
          command: |
            python3 -m venv venv
            . venv/bin/activate
            cd << parameters.directory >>
            python setup.py verify            
      - run:
          name: create packages
          command: |
            . venv/bin/activate
            cd << parameters.directory >>
            python setup.py sdist
            python setup.py bdist_wheel            
      - run:
          name: upload to pypi
          # To upload to test, just add the following flag to twine upload:
          # --repository-url https://test.pypi.org/legacy/
          # and change the username to "mlagents-test"
          command: |
            . venv/bin/activate
            cd << parameters.directory >>
            twine upload -u << parameters.username >> -p $PYPI_PASSWORD << parameters.test_repository_args >> dist/*            

workflows:
  version: 2
  workflow:
    jobs:
      - build_python:
          name: python_3.6.1
          executor: python361
          pyversion: 3.6.1
          # Test python 3.6 with the oldest supported versions
          pip_constraints: test_constraints_min_version.txt
      - build_python:
          name: python_3.7.3
          executor: python373
          pyversion: 3.7.3
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_tf1_version.txt
          # Make sure ONNX conversion passes here (recent version of tensorflow 1.x)
          enforce_onnx_conversion: 1
      - build_python:
          name: python_3.7.3+tf2
          executor: python373
          pyversion: 3.7.3
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_tf2_version.txt
      - build_python:
          name: python_3.8.2+tf2.2
          executor: python382
          pyversion: 3.8.2
          # Test python 3.8 with the newest edge versions
          pip_constraints: test_constraints_max_tf2_version.txt
      - markdown_link_check
      - pre-commit
      # The first deploy jobs are the "real" ones that upload to pypi
      - deploy:
          name: deploy ml-agents-envs
          directory: ml-agents-envs
          filters:
            tags:
              # Matches e.g. "release_123"
              only: /^release_[0-9]+$/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy ml-agents
          directory: ml-agents
          filters:
            tags:
              # Matches e.g. "release_123"
              only: /^release_[0-9]+$/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy gym-unity
          directory: gym-unity
          filters:
            tags:
              # Matches e.g. "release_123"
              only: /^release_[0-9]+$/
            branches:
              ignore: /.*/
      # These deploy jobs upload to the pypi test repo. They have different tag triggers than the real ones.
      - deploy:
          name: test deploy ml-agents-envs
          directory: ml-agents-envs
          username: mlagents-test
          test_repository_args: --repository-url https://test.pypi.org/legacy/
          filters:
            tags:
              # Matches e.g. "release_123_test456
              only: /^release_[0-9]+_test[0-9]+$/
            branches:
              ignore: /.*/
      - deploy:
          name: test deploy ml-agents
          directory: ml-agents
          username: mlagents-test
          test_repository_args: --repository-url https://test.pypi.org/legacy/
          filters:
            tags:
              # Matches e.g. "release_123_test456
              only: /^release_[0-9]+_test[0-9]+$/
            branches:
              ignore: /.*/
      - deploy:
          name: test deploy gym-unity
          directory: gym-unity
          username: mlagents-test
          test_repository_args: --repository-url https://test.pypi.org/legacy/
          filters:
            tags:
              # Matches e.g. "release_123_test456
              only: /^release_[0-9]+_test[0-9]+$/
            branches:
              ignore: /.*/
  nightly:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - develop
    jobs:
      - markdown_link_check:
          name: markdown-link-check full
          precommit_command: markdown-link-check-full