Deploy new versions to pypi (#2789)

* add VERSION variable to each setup.py

* update setups and config

* fix index

* fix indent for real

* fix parameter

* Bump version to 0.11.0.dev0

* Change CircleCI config to support dev releases

* Minor fix to deploy regex

* fix url in comments

* More circleCI tweaks

* Remove filters / 0.11.0.dev3

* Use test .pypirc

* Add config file flag to twine

* Manually pass user and pass to twine

* 0.11.0.dev0

* add precommit validation step

* remove todo

* Revert version to 0.10.1

* Docstring tweaks

* fix gym version

.circleci/config.yml

             path: /tmp/proto.patch
             destination: proto.patch
 
+  deploy:
+    parameters:
+      directory:
+        type: string
+        description: Local directory to use for publishing (e.g. ml-agents)
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - checkout
+      - run:
+          name: install python dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install  --upgrade pip
+            pip install setuptools wheel twine
+      - run:
+          name: verify git tag vs. version
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            cd << parameters.directory >>
+            python setup.py verify
+      - run:
+          name: create packages
+          command: |
+            . venv/bin/activate
+            cd << parameters.directory >>
+            python setup.py sdist
+            python setup.py bdist_wheel
+      - run:
+          name: upload to pypi
+          # To upload to test, just add the following flag to twine upload:
+          # --repository-url https://test.pypi.org/legacy/
+          # and change the username to "mlagents-test"
+          command: |
+            . venv/bin/activate
+            cd << parameters.directory >>
+            twine upload -u mlagents -p $PYPI_PASSWORD dist/*
+
+  version: 2
   workflow:
     jobs:
       - build_python:
           pip_constraints: test_constraints_max_version.txt
       - markdown_link_check
       - protobuf_generation_check
+      - deploy:
+          name: deploy ml-agents-envs
+          directory: ml-agents-envs
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: deploy ml-agents
+          directory: ml-agents
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: deploy gym-unity
+          directory: gym-unity
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/

.pre-commit-config.yaml

         exclude: ".*localized.*"
         # Only run manually, e.g. pre-commit run --hook-stage manual markdown-link-check
         stages: [manual]
+    -   id: validate-versions
+        name: validate library versions
+        language: script
+        entry: utils/validate_versions.py
+        files: ".*/setup.py"

gym-unity/setup.py

 #!/usr/bin/env python
 
+import os
+import sys
+from setuptools.command.install import install
+
+VERSION = "0.10.1"
+
+
+class VerifyVersionCommand(install):
+    """
+    Custom command to verify that the git tag matches our version
+    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    """
+
+    description = "verify that the git tag matches our version"
+
+    def run(self):
+        tag = os.getenv("CIRCLE_TAG")
+
+        if tag != VERSION:
+            info = "Git tag: {0} does not match the version of this app: {1}".format(
+                tag, VERSION
+            )
+            sys.exit(info)
+
-    version="0.4.8",
+    version=VERSION,
     description="Unity Machine Learning Agents Gym Interface",
     license="Apache License 2.0",
     author="Unity Technologies",
-    install_requires=["gym", "mlagents_envs==0.10.1"],
+    install_requires=["gym", "mlagents_envs=={}".format(VERSION)],
+    cmdclass={"verify": VerifyVersionCommand},
 )

ml-agents-envs/setup.py

+import os
+import sys
-from os import path
+from setuptools.command.install import install
+
+VERSION = "0.10.1"
+
+here = os.path.abspath(os.path.dirname(__file__))
+
-here = path.abspath(path.dirname(__file__))
+class VerifyVersionCommand(install):
+    """
+    Custom command to verify that the git tag matches our version
+    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    """
+
+    description = "verify that the git tag matches our version"
+
+    def run(self):
+        tag = os.getenv("CIRCLE_TAG")
+
+        if tag != VERSION:
+            info = "Git tag: {0} does not match the version of this app: {1}".format(
+                tag, VERSION
+            )
+            sys.exit(info)
+
-    version="0.10.1",
+    version=VERSION,
     description="Unity Machine Learning Agents Interface",
     url="https://github.com/Unity-Technologies/ml-agents",
     author="Unity Technologies",
         "protobuf>=3.6",
     ],
     python_requires=">=3.5",
+    cmdclass={"verify": VerifyVersionCommand},
 )

ml-agents/setup.py

+from io import open
+import os
+import sys
+
-from os import path
-from io import open
+from setuptools.command.install import install
+
+VERSION = "0.10.1"
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class VerifyVersionCommand(install):
+    """
+    Custom command to verify that the git tag matches our version
+    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    """
+
+    description = "verify that the git tag matches our version"
+
+    def run(self):
+        tag = os.getenv("CIRCLE_TAG")
-here = path.abspath(path.dirname(__file__))
+        if tag != VERSION:
+            info = "Git tag: {0} does not match the version of this app: {1}".format(
+                tag, VERSION
+            )
+            sys.exit(info)
+
-with open(path.join(here, "README.md"), encoding="utf-8") as f:
+with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
-    version="0.10.1",
+    version=VERSION,
     description="Unity Machine Learning Agents",
     long_description=long_description,
     long_description_content_type="text/markdown",
         "h5py>=2.9.0",
         "jupyter",
         "matplotlib",
-        "mlagents_envs==0.10.1",
+        "mlagents_envs=={}".format(VERSION),
         "numpy>=1.13.3,<2.0",
         "Pillow>=4.2.1",
         "protobuf>=3.6",
     ],
     python_requires=">=3.6.1",
     entry_points={"console_scripts": ["mlagents-learn=mlagents.trainers.learn:main"]},
+    cmdclass={"verify": VerifyVersionCommand},
 )

utils/validate_versions.py

+#!/usr/bin/env python
+
+import os
+import sys
+from typing import Dict
+
+VERSION_LINE_START = "VERSION = "
+
+DIRECTORIES = ["ml-agents", "ml-agents-envs", "gym-unity"]
+
+
+def extract_version_string(filename):
+    with open(filename) as f:
+        for l in f.readlines():
+            if l.startswith(VERSION_LINE_START):
+                return l.replace(VERSION_LINE_START, "").strip()
+    return None
+
+
+def check_versions() -> bool:
+    version_by_dir: Dict[str, str] = {}
+    for directory in DIRECTORIES:
+        path = os.path.join(directory, "setup.py")
+        version = extract_version_string(path)
+        print(f"Found version {version} for {directory}")
+        version_by_dir[directory] = version
+
+    # Make sure we have exactly one version, and it's not none
+    versions = set(version_by_dir.values())
+    if len(versions) != 1 or None in versions:
+        print("Each setup.py must have the same VERSION string.")
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    ok = check_versions()
+    return_code = 0 if ok else 1
+    sys.exit(return_code)