Release tooling (#3856)

* write release_tag too

* fix leading empty line

* update circle jobs and release tag logic

* set default to empty string

* set release tag, add sanity check

.circleci/config.yml

       directory:
         type: string
         description: Local directory to use for publishing (e.g. ml-agents)
+      username:
+        type: string
+        description: pypi username
+        default: mlagents
+      test_repository_args:
+        type: string
+        description: Optional override repository URL. Only use for tests, leave blank for "real"
+        default: ""
     docker:
       - image: circleci/python:3.6
     steps:
           command: |
             . venv/bin/activate
             cd << parameters.directory >>
-            twine upload -u mlagents -p $PYPI_PASSWORD dist/*
+            twine upload -u << parameters.username >> -p $PYPI_PASSWORD << parameters.test_repository_args >> dist/*
 
 workflows:
   version: 2
           pip_constraints: test_constraints_max_tf2_version.txt
       - markdown_link_check
       - pre-commit
+      # The first deploy jobs are the "real" ones that upload to pypi
-              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+              # Matches e.g. "release_123"
+              only: /^release_[0-9]+$/
             branches:
               ignore: /.*/
       - deploy:
             tags:
-              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+              # Matches e.g. "release_123"
+              only: /^release_[0-9]+$/
             branches:
               ignore: /.*/
       - deploy:
             tags:
-              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+              # Matches e.g. "release_123"
+              only: /^release_[0-9]+$/
+            branches:
+              ignore: /.*/
+      # These deploy jobs upload to the pypi test repo. They have different tag triggers than the real ones.
+      - deploy:
+          name: test deploy ml-agents-envs
+          directory: ml-agents-envs
+          username: mlagents-test
+          test_repository_args: --repository-url https://test.pypi.org/legacy/
+          filters:
+            tags:
+              # Matches e.g. "release_123_test456
+              only: /^release_[0-9]+_test[0-9]+$/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: test deploy ml-agents
+          directory: ml-agents
+          username: mlagents-test
+          test_repository_args: --repository-url https://test.pypi.org/legacy/
+          filters:
+            tags:
+              # Matches e.g. "release_123_test456
+              only: /^release_[0-9]+_test[0-9]+$/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: test deploy gym-unity
+          directory: gym-unity
+          username: mlagents-test
+          test_repository_args: --repository-url https://test.pypi.org/legacy/
+          filters:
+            tags:
+              # Matches e.g. "release_123_test456
+              only: /^release_[0-9]+_test[0-9]+$/
             branches:
               ignore: /.*/
   nightly:

gym-unity/gym_unity/__init__.py

+# Version of the library that will be used to upload to pypi
+
+# Git tag that will be checked to determine whether to trigger upload to pypi
+__release_tag__ = "release_1"

gym-unity/setup.py

 import gym_unity
 
 VERSION = gym_unity.__version__
+EXPECTED_TAG = gym_unity.__release_tag__
-    Custom command to verify that the git tag matches our version
-    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    Custom command to verify that the git tag is the expected one for the release.
+    Based on https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    This differs slightly because our tags and versions are different.
     """
 
     description = "verify that the git tag matches our version"
 
-        if tag != VERSION:
-            info = "Git tag: {0} does not match the version of this app: {1}".format(
-                tag, VERSION
+        if tag != EXPECTED_TAG:
+            info = "Git tag: {0} does not match the expected tag of this app: {1}".format(
+                tag, EXPECTED_TAG
             )
             sys.exit(info)
 

ml-agents-envs/mlagents_envs/__init__.py

+# Version of the library that will be used to upload to pypi
+
+# Git tag that will be checked to determine whether to trigger upload to pypi
+__release_tag__ = "release_1"

ml-agents-envs/setup.py

 import mlagents_envs
 
 VERSION = mlagents_envs.__version__
+EXPECTED_TAG = mlagents_envs.__release_tag__
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-    Custom command to verify that the git tag matches our version
-    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    Custom command to verify that the git tag is the expected one for the release.
+    Based on https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    This differs slightly because our tags and versions are different.
     """
 
     description = "verify that the git tag matches our version"
 
-        if tag != VERSION:
-            info = "Git tag: {0} does not match the version of this app: {1}".format(
-                tag, VERSION
+        if tag != EXPECTED_TAG:
+            info = "Git tag: {0} does not match the expected tag of this app: {1}".format(
+                tag, EXPECTED_TAG
             )
             sys.exit(info)
 

ml-agents/mlagents/trainers/__init__.py

+# Version of the library that will be used to upload to pypi
+
+# Git tag that will be checked to determine whether to trigger upload to pypi
+__release_tag__ = "release_1"

ml-agents/setup.py

 import mlagents.trainers
 
 VERSION = mlagents.trainers.__version__
+EXPECTED_TAG = mlagents.trainers.__release_tag__
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-    Custom command to verify that the git tag matches our version
-    See https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    Custom command to verify that the git tag is the expected one for the release.
+    Based on https://circleci.com/blog/continuously-deploying-python-packages-to-pypi-with-circleci/
+    This differs slightly because our tags and versions are different.
     """
 
     description = "verify that the git tag matches our version"
 
-        if tag != VERSION:
-            info = "Git tag: {0} does not match the version of this app: {1}".format(
-                tag, VERSION
+        if tag != EXPECTED_TAG:
+            info = "Git tag: {0} does not match the expected tag of this app: {1}".format(
+                tag, EXPECTED_TAG
             )
             sys.exit(info)
 

utils/validate_versions.py

 import os
 import json
 import sys
-from typing import Dict
+from typing import Dict, Optional
 import argparse
 
 VERSION_LINE_START = "__version__ = "
 UNITY_PACKAGE_JSON_PATH = "com.unity.ml-agents/package.json"
 ACADEMY_PATH = "com.unity.ml-agents/Runtime/Academy.cs"
 
+PYTHON_VERSION_FILE_TEMPLATE = """# Version of the library that will be used to upload to pypi
+__version__ = {version}
+
+# Git tag that will be checked to determine whether to trigger upload to pypi
+__release_tag__ = {release_tag}
+"""
+
+
+def _escape_non_none(s: Optional[str]) -> str:
+    """
+    Returns s escaped in quotes if it is non-None, else "None"
+    :param s:
+    :return:
+    """
+    if s is not None:
+        return f'"{s}"'
+    else:
+        return "None"
+
 
 def extract_version_string(filename):
     with open(filename) as f:
     return True
 
 
-def set_version(python_version: str, csharp_version: str) -> None:
-    new_contents = f'{VERSION_LINE_START}"{python_version}"\n'
+def set_version(
+    python_version: str, csharp_version: str, release_tag: Optional[str]
+) -> None:
+    # Sanity check - make sure test tags have a test or dev version
+    if release_tag and "test" in release_tag:
+        if not ("dev" in python_version or "test" in python_version):
+            raise RuntimeError('Test tags must use a "test" or "dev" version.')
+
+    new_contents = PYTHON_VERSION_FILE_TEMPLATE.format(
+        version=_escape_non_none(python_version),
+        release_tag=_escape_non_none(release_tag),
+    )
     for directory in DIRECTORIES:
         path = os.path.join(directory, "__init__.py")
         print(f"Setting {path} to version {python_version}")
     parser = argparse.ArgumentParser()
     parser.add_argument("--python-version", default=None)
     parser.add_argument("--csharp-version", default=None)
+    parser.add_argument("--release-tag", default=None)
     # unused, but allows precommit to pass filenames
     parser.add_argument("files", nargs="*")
     args = parser.parse_args()
         if args.csharp_version:
             print(f"Updating C# package to version {args.csharp_version}")
-        set_version(args.python_version, args.csharp_version)
+        set_version(args.python_version, args.csharp_version, args.release_tag)
     else:
         ok = check_versions()
         return_code = 0 if ok else 1