[ci] Add protobuf generation check as a CI job. (#2617)

5 年前 · b78abd36
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
          command: |
            . venv/bin/activate
            pre-commit run --hook-stage manual markdown-link-check --all-files
+  
+  protobuf_generation_check:
+    docker:
+      - image: circleci/python:3.7.3
+    working_directory: ~/repo/
+
+    steps:
+      - checkout
+      - run:
+          name: Combine proto files for caching
+          command: cat protobuf-definitions/proto/mlagents/envs/communicator_objects/*.proto > /tmp/proto_deps.txt
+
+      - restore_cache:
+          keys:
+          - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
+          - v1-protobuf-gen-dependencies-
+
+      - run:
+          name: Install Dependencies
+          command: |
+            sudo apt-get install nuget
+            nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install --upgrade pip
+            pip install grpcio-tools==1.13.0  --progress-bar=off
+            pip install mypy-protobuf --progress-bar=off
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
+
+      - run:
+          name: Generate protobufs
+          command: |
+            . venv/bin/activate
+            cd protobuf-definitions
+            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc
+            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin
+            COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh
+            git diff --exit-code --quiet -- :/ ':(exclude,top)UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects/*.meta' \
+              || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \
+              echo "Apply the patch with the command: git apply proto.patch"; \
+              git diff -- :/ ':(exclude,top)UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects/*.meta' > /tmp/proto.patch; \
+              exit $GIT_ERR; }
+      - store_artifacts:
+            path: /tmp/proto.patch
+            destination: proto.patch

 workflows:
  workflow:
          # Test python 3.7 with the newest supported versions
          pip_constraints: test_constraints_max_version.txt
      - markdown_link_check
+      - protobuf_generation_check
--- a/protobuf-definitions/README.md
+++ b/protobuf-definitions/README.md

 `pip install mypy-protobuf`

-If you don't have it already, download the latest version of [nuget](https://www.nuget.org/downloads).
+
+#### On Windows
+
+Download and install the latest version of [nuget](https://www.nuget.org/downloads).
+
+#### On Mac
+
+`brew install nuget`
+
+#### On Linux
+
+`sudo apt-get install nuget`
+
+
-
-### Installing Protobuf Compiler
-
-On Mac: `brew install protobuf`

 ## Running

-2. Un-comment line 7 in `make.bat` (for Windows, use `make_for_win.bat`), and set to correct Grpc.Tools sub-directory.
-3. Run the `.bat` from the terminal by navigating to `$MLAGENTS_ROOT\protobuf-definitions` and entering `make.bat` (for Windows, use `make_for_win.bat`)
+2. Un-comment line 7 in `make.sh` (for Windows, use `make_for_win.bat`), and set to correct Grpc.Tools sub-directory.
+3. Run the protobuf generation script from the terminal by navigating to `$MLAGENTS_ROOT\protobuf-definitions` and entering `make.sh` (for Windows, use `make_for_win.bat`)
-5. In the generated `UnityToExternalGrpc.cs` file in the `$MLAGENTS_ROOT/UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects` folder, you will need to add the following to the beginning of the file:
+5. In the generated `UnityToExternalGrpc.cs` file in the `$MLAGENTS_ROOT/UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects` folder, check to see if you need to add the following to the beginning of the file:

 ```csharp
 # if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
--- a/protobuf-definitions/make.sh
+++ b/protobuf-definitions/make.sh
+# variables
+
+# GRPC-TOOLS required. Install with `nuget install Grpc.Tools`. 
+# Then un-comment and replace [DIRECTORY] with location of files.
+# For example, on macOS, you might have something like:
+# COMPILER=Grpc.Tools.1.14.1/tools/macosx_x64
+# COMPILER=[DIRECTORY]
+
+SRC_DIR=proto/mlagents/envs/communicator_objects
+DST_DIR_C=../UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects
+DST_DIR_P=../ml-agents-envs
+PROTO_PATH=proto
+PYTHON_PACKAGE=mlagents/envs/communicator_objects
+
+# clean
+rm -rf $DST_DIR_C
+rm -rf $DST_DIR_P/$PYTHON_PACKAGE
+mkdir -p $DST_DIR_C
+mkdir -p $DST_DIR_P/$PYTHON_PACKAGE
+
+# generate proto objects in python and C#
+
+$COMPILER/protoc --proto_path=proto --csharp_out=$DST_DIR_C $SRC_DIR/*.proto
+$COMPILER/protoc --proto_path=proto --python_out=$DST_DIR_P --mypy_out=$DST_DIR_P $SRC_DIR/*.proto
+
+# grpc 
+
+GRPC=unity_to_external.proto
+
+$COMPILER/protoc --proto_path=proto --csharp_out $DST_DIR_C --grpc_out $DST_DIR_C $SRC_DIR/$GRPC --plugin=protoc-gen-grpc=$COMPILER/grpc_csharp_plugin
+python3 -m grpc_tools.protoc --proto_path=proto --python_out=$DST_DIR_P --grpc_python_out=$DST_DIR_P $SRC_DIR/$GRPC 
+
+
+# Generate the init file for the python module
+# rm -f $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
+for FILE in $DST_DIR_P/$PYTHON_PACKAGE/*.py
+do 
+FILE=${FILE##*/}
+# echo from .$(basename $FILE) import \* >> $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
+echo from .${FILE%.py} import \* >> $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
+done
+
+# Surround UnityToExternal.cs file with macro
+echo "#if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
+`cat $DST_DIR_C/UnityToExternalGrpc.cs`
+#endif" > $DST_DIR_C/UnityToExternalGrpc.cs
+
+# Remove the __init__.py file since it is not needed
+rm $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
+touch $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
--- a/protobuf-definitions/make.bat
+++ b/protobuf-definitions/make.bat
-# variables
-
-# GRPC-TOOLS required. Install with `nuget install Grpc.Tools`. 
-# Then un-comment and replace [DIRECTORY] with location of files.
-# For example, on macOS, you might have something like:
-# COMPILER=Grpc.Tools.1.14.1/tools/macosx_x64
-# COMPILER=[DIRECTORY]
-
-SRC_DIR=proto/mlagents/envs/communicator_objects
-DST_DIR_C=../UnitySDK/Assets/ML-Agents/Scripts/CommunicatorObjects
-DST_DIR_P=../ml-agents-envs
-PROTO_PATH=proto
-
-PYTHON_PACKAGE=mlagents/envs/communicator_objects
-
-# clean
-rm -rf $DST_DIR_C
-rm -rf $DST_DIR_P/$PYTHON_PACKAGE
-mkdir -p $DST_DIR_C
-mkdir -p $DST_DIR_P/$PYTHON_PACKAGE
-
-# generate proto objects in python and C#
-
-protoc --proto_path=proto --csharp_out=$DST_DIR_C $SRC_DIR/*.proto
-protoc --proto_path=proto --python_out=$DST_DIR_P --mypy_out=$DST_DIR_P $SRC_DIR/*.proto
-
-# grpc 
-
-GRPC=unity_to_external.proto
-
-$COMPILER/protoc --proto_path=proto --csharp_out $DST_DIR_C --grpc_out $DST_DIR_C $SRC_DIR/$GRPC --plugin=protoc-gen-grpc=$COMPILER/grpc_csharp_plugin
-python3 -m grpc_tools.protoc --proto_path=proto --python_out=$DST_DIR_P --grpc_python_out=$DST_DIR_P $SRC_DIR/$GRPC 
-
-
-# Generate the init file for the python module
-# rm -f $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
-for FILE in $DST_DIR_P/$PYTHON_PACKAGE/*.py
-do 
-FILE=${FILE##*/}
-# echo from .$(basename $FILE) import \* >> $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
-echo from .${FILE%.py} import \* >> $DST_DIR_P/$PYTHON_PACKAGE/__init__.py
-done
-