ml-agents/.circleci/config.yml

version: 2.1
executors:
  python361:
    docker:
      - image: circleci/python:3.6.1
  python373:
    docker:
      - image: circleci/python:3.7.3

jobs:
  build_python:
    parameters:
      executor:
        type: executor
      pyversion:
        type: string
        description: python version to being used (currently only affects caching).
      pip_constraints:
        type: string
        description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
    executor: << parameters.executor >>
    working_directory: ~/repo

    # Run additional numpy checks on unit tests
    environment:
      TEST_ENFORCE_NUMPY_FLOAT32: 1

    steps:
      - checkout

      - run:
          # Combine all the python dependencies into one file so that we can use that for the cache checksum
          name: Combine pip dependencies for caching
          command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt

      - restore_cache:
          keys:
          # Parameterize the cache so that different python versions can get different versions of the packages
          - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}

      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade setuptools
            pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
            pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>            

      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}

      - run:
          name: Run Tests for ml-agents and gym_unity
          # This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
          command: |
            . venv/bin/activate
            mkdir test-reports
            pip freeze > test-reports/pip_versions.txt
            pytest -n 2 --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings            

      - run:
          name: Verify there are no hidden/missing metafiles.
          # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
          command: |
            . venv/bin/activate
            python utils/validate_meta_files.py            

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports
          destination: test-reports

      - store_artifacts:
          path: htmlcov
          destination: htmlcov

  pre-commit:
    docker:
      - image: circleci/python:3.7.3
    working_directory: ~/repo/

    steps:
      - checkout
      - run:
          name: Combine precommit config and python versions for caching
          command: |
            cat .pre-commit-config.yaml > pre-commit-deps.txt
            python -VV >> pre-commit-deps.txt            

      - restore_cache:
          keys:
          - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}

      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade setuptools
            pip install pre-commit
            # Install the hooks now so that they'll be cached
            pre-commit install-hooks            

      - save_cache:
          paths:
            - ~/.cache/pre-commit
            - ./venv
          key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}

      - run:
          name: Check Code Style using pre-commit
          command: |
            . venv/bin/activate
            pre-commit run --show-diff-on-failure --all-files            

  markdown_link_check:
    parameters:
      precommit_command:
        type: string
        description: precommit hook to run
        default: markdown-link-check
    docker:
      - image: circleci/node:12.6.0
    working_directory: ~/repo

    steps:
      - checkout

      - restore_cache:
          keys:
          - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
          # fallback to using the latest cache if no exact match is found
          - v1-node-dependencies-

      - run:
          name: Install Dependencies
          command: |
            sudo apt-get install python3-venv
            python3 -m venv venv
            . venv/bin/activate
            pip install pre-commit            
      - run: sudo npm install -g markdown-link-check

      - save_cache:
          paths:
            - ./venv
          key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}

      - run:
          name: Run markdown-link-check via precommit
          command: |
            . venv/bin/activate
            pre-commit run --hook-stage manual << parameters.precommit_command >> --all-files            

  protobuf_generation_check:
    docker:
      - image: circleci/python:3.7.3
    working_directory: ~/repo/

    steps:
      - checkout
      - run:
          name: Combine proto files for caching
          command: cat protobuf-definitions/proto/mlagents_envs/communicator_objects/*.proto > /tmp/proto_deps.txt

      - restore_cache:
          keys:
          - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
          - v1-protobuf-gen-dependencies-

      - run:
          name: Install Dependencies
          command: |
            sudo apt-get install nuget
            nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install grpcio-tools==1.13.0  --progress-bar=off
            pip install mypy-protobuf==1.16.0 --progress-bar=off            
      - save_cache:
          paths:
            - ./venv
          key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}

      - run:
          name: Generate protobufs
          command: |
            . venv/bin/activate
            cd protobuf-definitions
            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc
            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin
            COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh
            CS_PROTO_PATH=UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects
            git diff --exit-code --quiet -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" \
              || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \
              echo "Apply the patch with the command: git apply proto.patch"; \
              git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > /tmp/proto.patch; \
              exit $GIT_ERR; }            
      - store_artifacts:
            path: /tmp/proto.patch
            destination: proto.patch

  deploy:
    parameters:
      directory:
        type: string
        description: Local directory to use for publishing (e.g. ml-agents)
    docker:
      - image: circleci/python:3.6
    steps:
      - checkout
      - run:
          name: install python dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install  --upgrade pip
            pip install setuptools wheel twine            
      - run:
          name: verify git tag vs. version
          command: |
            python3 -m venv venv
            . venv/bin/activate
            cd << parameters.directory >>
            python setup.py verify            
      - run:
          name: create packages
          command: |
            . venv/bin/activate
            cd << parameters.directory >>
            python setup.py sdist
            python setup.py bdist_wheel            
      - run:
          name: upload to pypi
          # To upload to test, just add the following flag to twine upload:
          # --repository-url https://test.pypi.org/legacy/
          # and change the username to "mlagents-test"
          command: |
            . venv/bin/activate
            cd << parameters.directory >>
            twine upload -u mlagents -p $PYPI_PASSWORD dist/*            

workflows:
  version: 2
  workflow:
    jobs:
      - build_python:
          name: python_3.6.1
          executor: python361
          pyversion: 3.6.1
          # Test python 3.6 with the oldest supported versions
          pip_constraints: test_constraints_min_version.txt
      - build_python:
          name: python_3.7.3
          executor: python373
          pyversion: 3.7.3
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_tf1_version.txt
      - build_python:
          name: python_3.7.3+tf2
          executor: python373
          pyversion: 3.7.3
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_tf2_version.txt
      - markdown_link_check
      - protobuf_generation_check
      - pre-commit
      - deploy:
          name: deploy ml-agents-envs
          directory: ml-agents-envs
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy ml-agents
          directory: ml-agents
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy gym-unity
          directory: gym-unity
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
            branches:
              ignore: /.*/
  nightly:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - develop
    jobs:
      - markdown_link_check:
          name: markdown-link-check full
          precommit_command: markdown-link-check-full