ml-agents/.circleci/config.yml

version: 2.1
executors:
  python361:
    docker:
      - image: circleci/python:3.6.1
  python373:
    docker:
      - image: circleci/python:3.7.3

jobs:
  build_python:
    parameters:
      executor:
        type: executor
      pyversion:
        type: string
        description: python version to being used (currently only affects caching).
      pip_constraints:
        type: string
        description: Constraints file that is passed to "pip install". We constraint older versions of libraries for older python runtime, in order to help ensure compatibility.
    executor: << parameters.executor >>
    working_directory: ~/repo

    steps:
      - checkout

      - run:
          # Combine all the python dependencies into one file so that we can use that for the cache checksum
          name: Combine pip dependencies for caching
          command: cat ml-agents/setup.py ml-agents-envs/setup.py gym-unity/setup.py test_requirements.txt << parameters.pip_constraints >> > python_deps.txt

      - restore_cache:
          keys:
          # Parameterize the cache so that different python versions can get different versions of the packages
          - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
          # fallback to using the latest cache if no exact match is found
          - v1-dependencies-py<< parameters.pyversion >>-

      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade setuptools
            pip install --progress-bar=off -e ./ml-agents-envs -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./ml-agents -c << parameters.pip_constraints >>
            pip install --progress-bar=off -r test_requirements.txt -c << parameters.pip_constraints >>
            pip install --progress-bar=off -e ./gym-unity -c << parameters.pip_constraints >>            

      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}

      - run:
          name: Run Tests for ml-agents and gym_unity
          # This also dumps the installed pip packages to a file, so we can see what versions are actually being used.
          command: |
            . venv/bin/activate
            mkdir test-reports
            pip freeze > test-reports/pip_versions.txt
            pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings            

      - run:
          name: Check Code Style using pre-commit
          command: |
            . venv/bin/activate
            pre-commit run --show-diff-on-failure --all-files            

      - run:
          name: Verify there are no hidden/missing metafiles.
          # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy.
          command: |
            . venv/bin/activate
            python utils/validate_meta_files.py            

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports
          destination: test-reports

      - store_artifacts:
          path: htmlcov
          destination: htmlcov


  markdown_link_check:
    docker:
      - image: circleci/node:12.6.0
    working_directory: ~/repo

    steps:
      - checkout

      - restore_cache:
          keys:
          - v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}
          # fallback to using the latest cache if no exact match is found
          - v1-node-dependencies-

      - run:
          name: Install Dependencies
          command: |
            sudo apt-get install python3-venv
            python3 -m venv venv
            . venv/bin/activate
            pip install pre-commit            
      - run: sudo npm install -g markdown-link-check

      - save_cache:
          paths:
            - ./venv
          key: v1-node-dependencies-{{ checksum ".pre-commit-config.yaml" }}

      - run:
          name: Run markdown-link-check via precommit
          command: |
            . venv/bin/activate
            pre-commit run --hook-stage manual markdown-link-check --all-files            

  protobuf_generation_check:
    docker:
      - image: circleci/python:3.7.3
    working_directory: ~/repo/

    steps:
      - checkout
      - run:
          name: Combine proto files for caching
          command: cat protobuf-definitions/proto/mlagents/envs/communicator_objects/*.proto > /tmp/proto_deps.txt

      - restore_cache:
          keys:
          - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
          - v1-protobuf-gen-dependencies-

      - run:
          name: Install Dependencies
          command: |
            sudo apt-get install nuget
            nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install grpcio-tools==1.13.0  --progress-bar=off
            pip install mypy-protobuf==1.16.0 --progress-bar=off            
      - save_cache:
          paths:
            - ./venv
          key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}

      - run:
          name: Generate protobufs
          command: |
            . venv/bin/activate
            cd protobuf-definitions
            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc
            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin
            COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh
            CS_PROTO_PATH=UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects
            git diff --exit-code --quiet -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" \
              || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \
              echo "Apply the patch with the command: git apply proto.patch"; \
              git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > /tmp/proto.patch; \
              exit $GIT_ERR; }            
      - store_artifacts:
            path: /tmp/proto.patch
            destination: proto.patch

deploy:
  parameters:
    version:
      type: directory
  docker:
    - image: circleci/python:3.6
  steps:
    - checkout
    - run:
        name: install python dependencies
        command: |
          python3 -m venv venv
          . venv/bin/activate
          pip install  --upgrade pip
          pip install setuptools wheel twine          
    - run:
        name: verify git tag vs. version
        command: |
          python3 -m venv venv
          . venv/bin/activate
          cd << parameters.directory >>
          python setup.py verify          
    - run:
        name: init .pypirc
        # TODO update username for final release
        command: |
          echo -e "[pypi]" >> ~/.pypirc
          echo -e "username = mlagents-test" >> ~/.pypirc
          echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc          
    - run:
        # TODO example doesn't use venv here but I think we need it
        name: create packages
        command: |
          . venv/bin/activate
          cd << parameters.directory >>
          python setup.py sdist
          python setup.py bdist_wheel          
    - run:
        name: upload to pypi
        # TODO remove --repository-url for final release
        command: |
          . venv/bin/activate
          cd << parameters.directory >>
          twine upload --repository-url https://test.pypi.org/legacy/ dist/*          

workflows:
  workflow:
    jobs:
      - build_python:
          name: python_3.6.1
          executor: python361
          pyversion: 3.6.1
          # Test python 3.6 with the oldest supported versions
          pip_constraints: test_constraints_min_version.txt
      - build_python:
          name: python_3.7.3
          executor: python373
          pyversion: 3.7.3
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_version.txt
      - markdown_link_check
      - protobuf_generation_check
      - deploy:
          name: deploy-ml-agents-envs
          directory: ml-agents-envs
          requires:
            - python_3.7.3
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy-ml-agents
          directory: ml-agents
          requires:
            - python_3.7.3
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*/
            branches:
              ignore: /.*/
      - deploy:
          name: deploy-gym-unity
          directory: gym-unity
          requires:
            - python_3.7.3
          filters:
            tags:
              only: /[0-9]+(\.[0-9]+)*/
            branches:
              ignore: /.*/