Merge commit 'f9c05a61d574305497789b5997f1ae3ea1b1ad3b' into develop-splitpolicyoptimizer

.github/ISSUE_TEMPLATE/config.yml

 blank_issues_enabled: false
 contact_links:
+  - name: Discussion / General Questions
+    url: https://forum.unity.com/forums/ml-agents.453/
+    about: Discussion about ML-Agents, RL algorithms, or game integrations.
+  - name: Installation / Setup
+    url: https://forum.unity.com/forums/ml-agents.453/
+    about: Questions about python installation, initial connection between Unity and training, etc.
-    about: Please ask Installation / Setup and Discussion / General Questions in the Unity Forum.
+    about: Please ask other questions in the ML-Agents Unity Forum.

.gitignore

 /envs
 
 # Environemnt logfile
-*UnitySDK.log
+*Project.log
-/UnitySDK/.vs/
+/Project/.vs/
 
 # Autogenerated VS/MD/Consulo solution and project files
 /com.unity.ml-agentsExportedObj/

.yamato/standalone-build-test.yml

   name: Test Mac Standalone {{ editor.version }}
   agent:
     type: Unity::VM::osx
-    image: ml-agents/ml-agents-bokken-mac:v0.1.3-475350
+    image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
+    - pip install pyyaml
     - python -u -m ml-agents.tests.yamato.standalone_build_tests
   triggers:
     pull_requests:

Project/.gitignore

 /Assets/AssetStoreTools*
 /Assets/Plugins*
 /Assets/Demonstrations*
+/Assets/ML-Agents/Timers*
 /csharp_timers.json
 
 # Environemnt logfile

Project/Assets/ML-Agents/Examples/3DBall/Prefabs/3DBall.prefab

   - component: {fileID: 114368073295828880}
   - component: {fileID: 114715123104194396}
   - component: {fileID: 1306725529891448089}
+  - component: {fileID: 1758424554059689351}
   m_Layer: 0
   m_Name: Agent
   m_TagString: Untagged
   DecisionPeriod: 5
   RepeatAction: 1
   offsetStep: 0
+--- !u!114 &1758424554059689351
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1424713891854676}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 3a6da8f78a394c6ab027688eab81e04d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
 --- !u!1 &1533320402322554
 GameObject:
   m_ObjectHideFlags: 0

Project/Assets/ML-Agents/Examples/3DBall/Scenes/3DBall.unity

       propertyPath: m_Name
       value: 3DBall (1)
       objectReference: {fileID: 0}
+    - target: {fileID: 1321468028730240, guid: cfa81c019162c4e3caf6e2999c6fdf48, type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
     - target: {fileID: 4679453577574622, guid: cfa81c019162c4e3caf6e2999c6fdf48, type: 3}
       propertyPath: m_LocalPosition.x
       value: 9

Project/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DAgent.cs

             Mathf.Abs(ball.transform.position.x - gameObject.transform.position.x) > 3f ||
             Mathf.Abs(ball.transform.position.z - gameObject.transform.position.z) > 3f)
         {
-            Done();
+            Done();
         }
         else
         {

Project/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DHardAgent.cs

             Mathf.Abs(ball.transform.position.x - gameObject.transform.position.x) > 3f ||
             Mathf.Abs(ball.transform.position.z - gameObject.transform.position.z) > 3f)
         {
-            Done();
+            Done();
         }
         else
         {

Project/Assets/ML-Agents/Examples/3DBall/TFModels/3DBall.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/3DBall/TFModels/3DBallHard.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Basic/Scripts/BasicAgent.cs

 
         if (m_Position == m_SmallGoalPosition)
         {
-            Done();
+            Done();
-            Done();
+            Done();
         }
     }
 

Project/Assets/ML-Agents/Examples/Basic/TFModels/Basic.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Bouncer/TFModels/Bouncer.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Crawler/Scripts/CrawlerAgent.cs

 using UnityEngine;
 using MLAgents;
+using MLAgentsExamples;
 
 [RequireComponent(typeof(JointDriveController))] // Required to set joint forces
 public class CrawlerAgent : Agent

Project/Assets/ML-Agents/Examples/Crawler/TFModels/CrawlerDynamic.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Crawler/TFModels/CrawlerStatic.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/FoodCollector/Scripts/FoodCollectorArea.cs

 using UnityEngine;
-using MLAgents;
+using MLAgentsExamples;
 
 public class FoodCollectorArea : Area
 {

Project/Assets/ML-Agents/Examples/FoodCollector/TFModels/FoodCollector.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs

 
             if (hit.Where(col => col.gameObject.CompareTag("goal")).ToArray().Length == 1)
             {
-                Done();
+                Done();
-            if (hit.Where(col => col.gameObject.CompareTag("pit")).ToArray().Length == 1)
+            else if (hit.Where(col => col.gameObject.CompareTag("pit")).ToArray().Length == 1)
+                SetReward(-1f);
-                SetReward(-1f);
             }
         }
     }

Project/Assets/ML-Agents/Examples/GridWorld/TFModels/GridWorld.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Hallway/TFModels/Hallway.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/PushBlock/TFModels/PushBlock.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Pyramids/Scripts/PyramidArea.cs

 using UnityEngine;
-using MLAgents;
+using MLAgentsExamples;
 
 public class PyramidArea : Area
 {

Project/Assets/ML-Agents/Examples/Pyramids/TFModels/Pyramids.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Reacher/TFModels/Reacher.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/AdjustTrainingTimescale.cs

 
 using UnityEngine;
 
-namespace  MLAgents
+namespace  MLAgentsExamples
 {
     public class AdjustTrainingTimescale : MonoBehaviour
     {

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/Area.cs

 using UnityEngine;
 
-namespace MLAgents
+namespace MLAgentsExamples
 {
     public class Area : MonoBehaviour
     {

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/CameraFollow.cs

 using UnityEngine;
 
-namespace MLAgents
+namespace MLAgentsExamples
 {
     public class CameraFollow : MonoBehaviour
     {

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/FlyCamera.cs

 using UnityEngine;
 
-namespace MLAgents
+namespace MLAgentsExamples
 {
     public class FlyCamera : MonoBehaviour
     {

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/GroundContact.cs

 using UnityEngine;
+using MLAgents;
-namespace MLAgents
+namespace MLAgentsExamples
 {
     /// <summary>
     /// This class contains logic for locomotion agents with joints which might make contact with the ground.

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/JointDriveController.cs

 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.Serialization;
+using MLAgents;
-namespace MLAgents
+namespace MLAgentsExamples
 {
     /// <summary>
     /// Used to store relevant information for acting and learning for each body part in agent.

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/ProjectSettingsOverrides.cs

 using UnityEngine;
 using MLAgents;
 
-public class ProjectSettingsOverrides : MonoBehaviour
+namespace MLAgentsExamples
-    // Original values
-    float m_OriginalMonitorVerticalOffset;
-    Vector3 m_OriginalGravity;
-    float m_OriginalFixedDeltaTime;
-    float m_OriginalMaximumDeltaTime;
-    int m_OriginalSolverIterations;
-    int m_OriginalSolverVelocityIterations;
+    public class ProjectSettingsOverrides : MonoBehaviour
+    {
+        // Original values
+        float m_OriginalMonitorVerticalOffset;
+        Vector3 m_OriginalGravity;
+        float m_OriginalFixedDeltaTime;
+        float m_OriginalMaximumDeltaTime;
+        int m_OriginalSolverIterations;
+        int m_OriginalSolverVelocityIterations;
-    [Tooltip("Increase or decrease the scene gravity. Use ~3x to make things less floaty")]
-    public float gravityMultiplier = 1.0f;
+        [Tooltip("Increase or decrease the scene gravity. Use ~3x to make things less floaty")]
+        public float gravityMultiplier = 1.0f;
-    [Header("Display Settings")]
-    public float monitorVerticalOffset;
+        [Header("Display Settings")]
+        public float monitorVerticalOffset;
-    [Header("Advanced physics settings")]
-    [Tooltip("The interval in seconds at which physics and other fixed frame rate updates (like MonoBehaviour's FixedUpdate) are performed.")]
-    public float fixedDeltaTime = .02f;
-    [Tooltip("The maximum time a frame can take. Physics and other fixed frame rate updates (like MonoBehaviour's FixedUpdate) will be performed only for this duration of time per frame.")]
-    public float maximumDeltaTime = 1.0f / 3.0f;
-    [Tooltip("Determines how accurately Rigidbody joints and collision contacts are resolved. (default 6). Must be positive.")]
-    public int solverIterations = 6;
-    [Tooltip("Affects how accurately the Rigidbody joints and collision contacts are resolved. (default 1). Must be positive.")]
-    public int solverVelocityIterations = 1;
+        [Header("Advanced physics settings")]
+        [Tooltip("The interval in seconds at which physics and other fixed frame rate updates (like MonoBehaviour's FixedUpdate) are performed.")]
+        public float fixedDeltaTime = .02f;
+        [Tooltip("The maximum time a frame can take. Physics and other fixed frame rate updates (like MonoBehaviour's FixedUpdate) will be performed only for this duration of time per frame.")]
+        public float maximumDeltaTime = 1.0f / 3.0f;
+        [Tooltip("Determines how accurately Rigidbody joints and collision contacts are resolved. (default 6). Must be positive.")]
+        public int solverIterations = 6;
+        [Tooltip("Affects how accurately the Rigidbody joints and collision contacts are resolved. (default 1). Must be positive.")]
+        public int solverVelocityIterations = 1;
-    public void Awake()
-    {
-        // Save the original values
-        m_OriginalMonitorVerticalOffset = Monitor.verticalOffset;
-        m_OriginalGravity = Physics.gravity;
-        m_OriginalFixedDeltaTime = Time.fixedDeltaTime;
-        m_OriginalMaximumDeltaTime = Time.maximumDeltaTime;
-        m_OriginalSolverIterations = Physics.defaultSolverIterations;
-        m_OriginalSolverVelocityIterations = Physics.defaultSolverVelocityIterations;
+        public void Awake()
+        {
+            // Save the original values
+            m_OriginalMonitorVerticalOffset = Monitor.verticalOffset;
+            m_OriginalGravity = Physics.gravity;
+            m_OriginalFixedDeltaTime = Time.fixedDeltaTime;
+            m_OriginalMaximumDeltaTime = Time.maximumDeltaTime;
+            m_OriginalSolverIterations = Physics.defaultSolverIterations;
+            m_OriginalSolverVelocityIterations = Physics.defaultSolverVelocityIterations;
-        // Override
-        Monitor.verticalOffset = monitorVerticalOffset;
-        Physics.gravity *= gravityMultiplier;
-        Time.fixedDeltaTime = fixedDeltaTime;
-        Time.maximumDeltaTime = maximumDeltaTime;
-        Physics.defaultSolverIterations = solverIterations;
-        Physics.defaultSolverVelocityIterations = solverVelocityIterations;
+            // Override
+            Monitor.verticalOffset = monitorVerticalOffset;
+            Physics.gravity *= gravityMultiplier;
+            Time.fixedDeltaTime = fixedDeltaTime;
+            Time.maximumDeltaTime = maximumDeltaTime;
+            Physics.defaultSolverIterations = solverIterations;
+            Physics.defaultSolverVelocityIterations = solverVelocityIterations;
-        Academy.Instance.FloatProperties.RegisterCallback("gravity", f => { Physics.gravity = new Vector3(0, -f, 0); });
-    }
+            Academy.Instance.FloatProperties.RegisterCallback("gravity", f => { Physics.gravity = new Vector3(0, -f, 0); });
+        }
-    public void OnDestroy()
-    {
-        Monitor.verticalOffset = m_OriginalMonitorVerticalOffset;
-        Physics.gravity = m_OriginalGravity;
-        Time.fixedDeltaTime = m_OriginalFixedDeltaTime;
-        Time.maximumDeltaTime = m_OriginalMaximumDeltaTime;
-        Physics.defaultSolverIterations = m_OriginalSolverIterations;
-        Physics.defaultSolverVelocityIterations = m_OriginalSolverVelocityIterations;
+        public void OnDestroy()
+        {
+            Monitor.verticalOffset = m_OriginalMonitorVerticalOffset;
+            Physics.gravity = m_OriginalGravity;
+            Time.fixedDeltaTime = m_OriginalFixedDeltaTime;
+            Time.maximumDeltaTime = m_OriginalMaximumDeltaTime;
+            Physics.defaultSolverIterations = m_OriginalSolverIterations;
+            Physics.defaultSolverVelocityIterations = m_OriginalSolverVelocityIterations;
+        }
-}
+}

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/TargetContact.cs

 using UnityEngine;
 
-namespace MLAgents
+namespace MLAgentsExamples
 {
     /// <summary>
     /// This class contains logic for locomotion agents with joints which might make contact with a target.

Project/Assets/ML-Agents/Examples/Soccer/Scripts/AgentSoccer.cs

+using System;
 using UnityEngine;
 using MLAgents;
 
     // * opposing player
     public enum Team
     {
-        Purple,
-        Blue
-    }
-    public enum AgentRole
-    {
-        Striker,
-        Goalie
+        Blue = 0,
+        Purple = 1
+    [HideInInspector]
-    public AgentRole agentRole;
     float m_KickPower;
     int m_PlayerIndex;
     public SoccerFieldArea area;
     SoccerSettings m_SoccerSettings;
     Renderer m_AgentRenderer;
+    BehaviorParameters m_BP;
+    Vector3 m_Transform;
-    public void ChooseRandomTeam()
+    public override void InitializeAgent()
-        team = (Team)Random.Range(0, 2);
-        if (team == Team.Purple)
+        base.InitializeAgent();
+        m_BP = gameObject.GetComponent<BehaviorParameters>();
+        if (m_BP.m_TeamID == (int)Team.Blue)
-            JoinPurpleTeam(agentRole);
+            team = Team.Blue;
+            m_Transform = new Vector3(transform.position.x - 4f, .5f, transform.position.z);
-            JoinBlueTeam(agentRole);
+            team = Team.Purple;
+            m_Transform = new Vector3(transform.position.x + 4f, .5f, transform.position.z);
-    }
-
-    public void JoinPurpleTeam(AgentRole role)
-    {
-        agentRole = role;
-        team = Team.Purple;
-        m_AgentRenderer.material = m_SoccerSettings.purpleMaterial;
-        tag = "purpleAgent";
-    }
-
-    public void JoinBlueTeam(AgentRole role)
-    {
-        agentRole = role;
-        team = Team.Blue;
-        m_AgentRenderer.material = m_SoccerSettings.blueMaterial;
-        tag = "blueAgent";
-    }
-
-    public override void InitializeAgent()
-    {
-        base.InitializeAgent();
         m_AgentRenderer = GetComponentInChildren<Renderer>();
         m_SoccerSettings = FindObjectOfType<SoccerSettings>();
         agentRb = GetComponent<Rigidbody>();
 
         var action = Mathf.FloorToInt(act[0]);
 
-        // Goalies and Strikers have slightly different action spaces.
-        if (agentRole == AgentRole.Goalie)
+        m_KickPower = 0f;
+
+        var forwardAxis = (int)act[0];
+        var rightAxis = (int)act[1];
+        var rotateAxis = (int)act[2];
+
+        switch (forwardAxis)
+        {
+            case 1:
+                dirToGo = transform.forward * 1f;
+                m_KickPower = 1f;
+                break;
+            case 2:
+                dirToGo = transform.forward * -1f;
+                break;
+        }
+
+        switch (rightAxis)
-            m_KickPower = 0f;
-            switch (action)
-            {
-                case 1:
-                    dirToGo = transform.forward * 1f;
-                    m_KickPower = 1f;
-                    break;
-                case 2:
-                    dirToGo = transform.forward * -1f;
-                    break;
-                case 4:
-                    dirToGo = transform.right * -1f;
-                    break;
-                case 3:
-                    dirToGo = transform.right * 1f;
-                    break;
-            }
+            case 1:
+                dirToGo = transform.right * 0.3f;
+                break;
+            case 2:
+                dirToGo = transform.right * -0.3f;
+                break;
-        else
+
+        switch (rotateAxis)
-            m_KickPower = 0f;
-            switch (action)
-            {
-                case 1:
-                    dirToGo = transform.forward * 1f;
-                    m_KickPower = 1f;
-                    break;
-                case 2:
-                    dirToGo = transform.forward * -1f;
-                    break;
-                case 3:
-                    rotateDir = transform.up * 1f;
-                    break;
-                case 4:
-                    rotateDir = transform.up * -1f;
-                    break;
-                case 5:
-                    dirToGo = transform.right * -0.75f;
-                    break;
-                case 6:
-                    dirToGo = transform.right * 0.75f;
-                    break;
-            }
+            case 1:
+                rotateDir = transform.up * -1f;
+                break;
+            case 2:
+                rotateDir = transform.up * 1f;
+                break;
+
         transform.Rotate(rotateDir, Time.deltaTime * 100f);
         agentRb.AddForce(dirToGo * m_SoccerSettings.agentRunSpeed,
             ForceMode.VelocityChange);
     {
         // Existential penalty for strikers.
-        if (agentRole == AgentRole.Striker)
+        AddReward(-1f / 3000f);
+        MoveAgent(vectorAction);
+    }
+
+    public override float[] Heuristic()
+    {
+        var action = new float[3];
+        //forward
+        if (Input.GetKey(KeyCode.W))
+        {
+            action[0] = 1f;
+        }
+        if (Input.GetKey(KeyCode.S))
-            AddReward(-1f / 3000f);
+            action[0] = 2f;
-        // Existential bonus for goalies.
-        if (agentRole == AgentRole.Goalie)
+        //rotate
+        if (Input.GetKey(KeyCode.A))
-            AddReward(1f / 3000f);
+            action[2] = 1f;
-        MoveAgent(vectorAction);
+        if (Input.GetKey(KeyCode.D))
+        {
+            action[2] = 2f;
+        }
+        //right
+        if (Input.GetKey(KeyCode.E))
+        {
+            action[1] = 1f;
+        }
+        if (Input.GetKey(KeyCode.Q))
+        {
+            action[1] = 2f;
+        }
+        return action;
-
     /// <summary>
     /// Used to provide a "kick" to the ball.
     /// </summary>
 
     public override void AgentReset()
     {
-        if (m_SoccerSettings.randomizePlayersTeamForTraining)
-        {
-            ChooseRandomTeam();
-        }
-
-            JoinPurpleTeam(agentRole);
-            JoinBlueTeam(agentRole);
-        transform.position = area.GetRandomSpawnPos(agentRole, team);
+        transform.position = m_Transform;
         agentRb.velocity = Vector3.zero;
         agentRb.angularVelocity = Vector3.zero;
         SetResetParameters();

Project/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerBallController.cs

 {
     [HideInInspector]
     public SoccerFieldArea area;
-    public AgentSoccer lastTouchedBy; //who was the last to touch the ball
-    public string agentTag; //will be used to check if collided with a agent
-    public string purpleGoalTag; //will be used to check if collided with red goal
+    public string purpleGoalTag; //will be used to check if collided with purple goal
-        if (col.gameObject.CompareTag(purpleGoalTag)) //ball touched red goal
+        if (col.gameObject.CompareTag(purpleGoalTag)) //ball touched purple goal
         {
             area.GoalTouched(AgentSoccer.Team.Blue);
         }

Project/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerFieldArea.cs

 
     SoccerSettings m_SoccerSettings;
 
-    public IEnumerator GoalScoredSwapGroundMaterial(Material mat, float time)
-    {
-        m_GroundRenderer.material = mat;
-        yield return new WaitForSeconds(time);
-        m_GroundRenderer.material = m_GroundMaterial;
-    }
-
     void Awake()
     {
         m_SoccerSettings = FindObjectOfType<SoccerSettings>();
         if (goalTextUI) goalTextUI.SetActive(false);
     }
 
-    public void AllPlayersDone(float reward)
-    {
-        foreach (var ps in playerStates)
-        {
-            if (ps.agentScript.gameObject.activeInHierarchy)
-            {
-                if (reward != 0)
-                {
-                    ps.agentScript.AddReward(reward);
-                }
-                ps.agentScript.Done();
-            }
-        }
-    }
-
     public void GoalTouched(AgentSoccer.Team scoredTeam)
     {
         foreach (var ps in playerStates)
-                RewardOrPunishPlayer(ps, m_SoccerSettings.strikerReward, m_SoccerSettings.goalieReward);
+                ps.agentScript.AddReward(1);
-                RewardOrPunishPlayer(ps, m_SoccerSettings.strikerPunish, m_SoccerSettings.goaliePunish);
-            }
-            if (m_SoccerSettings.randomizePlayersTeamForTraining)
-            {
-                ps.agentScript.ChooseRandomTeam();
+                ps.agentScript.AddReward(-1);
+            ps.agentScript.Done();  //all agents need to be reset
-            if (scoredTeam == AgentSoccer.Team.Purple)
-            {
-                StartCoroutine(GoalScoredSwapGroundMaterial(m_SoccerSettings.purpleMaterial, 1));
-            }
-            else
-            {
-                StartCoroutine(GoalScoredSwapGroundMaterial(m_SoccerSettings.blueMaterial, 1));
-            }
             if (goalTextUI)
             {
                 StartCoroutine(ShowGoalUI());
 
-    public void RewardOrPunishPlayer(PlayerState ps, float striker, float goalie)
-    {
-        if (ps.agentScript.agentRole == AgentSoccer.AgentRole.Striker)
-        {
-            ps.agentScript.AddReward(striker);
-        }
-        if (ps.agentScript.agentRole == AgentSoccer.AgentRole.Goalie)
-        {
-            ps.agentScript.AddReward(goalie);
-        }
-        ps.agentScript.Done();  //all agents need to be reset
-    }
-
-    public Vector3 GetRandomSpawnPos(AgentSoccer.AgentRole role, AgentSoccer.Team team)
-    {
-        var xOffset = 0f;
-        if (role == AgentSoccer.AgentRole.Goalie)
-        {
-            xOffset = 13f;
-        }
-        if (role == AgentSoccer.AgentRole.Striker)
-        {
-            xOffset = 7f;
-        }
-        if (team == AgentSoccer.Team.Blue)
-        {
-            xOffset = xOffset * -1f;
-        }
-        var randomSpawnPos = ground.transform.position +
-            new Vector3(xOffset, 0f, 0f)
-            + (Random.insideUnitSphere * 2);
-        randomSpawnPos.y = ground.transform.position.y + 2;
-        return randomSpawnPos;
-    }
-
-            new Vector3(0f, 0f, 0f)
-            + (Random.insideUnitSphere * 2);
-        randomSpawnPos.y = ground.transform.position.y + 2;
+            new Vector3(0f, 0f, 0f);
+        randomSpawnPos.y = ground.transform.position.y + .5f;
         return randomSpawnPos;
     }
 

Project/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerSettings.cs

     public Material purpleMaterial;
     public Material blueMaterial;
     public bool randomizePlayersTeamForTraining = true;
-
-
-    public float strikerPunish; //if opponents scores, the striker gets this neg reward (-1)
-    public float strikerReward; //if team scores a goal they get a reward (+1)
-    public float goaliePunish; //if opponents score, goalie gets this neg reward (-1)
-    public float goalieReward; //if team scores, goalie gets this reward (currently 0...no reward. can play with this later)
 }

Project/Assets/ML-Agents/Examples/Tennis/Scenes/Tennis.unity

   m_Father: {fileID: 1184319693}
   m_RootOrder: 1
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 1, y: 1}
-  m_AnchorMax: {x: 1, y: 1}
-  m_AnchoredPosition: {x: -20, y: -50}
+  m_AnchorMin: {x: 1, y: 0}
+  m_AnchorMax: {x: 1, y: 0}
+  m_AnchoredPosition: {x: -20, y: 50}
   m_SizeDelta: {x: 100, y: 50}
   m_Pivot: {x: 0.5, y: 0.5}
 --- !u!114 &1871669623
   m_Father: {fileID: 1184319693}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 0, y: 1}
-  m_AnchorMax: {x: 0, y: 1}
-  m_AnchoredPosition: {x: 100, y: -50}
+  m_AnchorMin: {x: 0, y: 0}
+  m_AnchorMax: {x: 0, y: 0}
+  m_AnchoredPosition: {x: 100, y: 50}
   m_SizeDelta: {x: 100, y: 50}
   m_Pivot: {x: 0.5, y: 0.5}
 --- !u!114 &2073469452

Project/Assets/ML-Agents/Examples/Tennis/TFModels/Tennis.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/Walker/Scripts/WalkerAgent.cs

 using UnityEngine;
 using MLAgents;
+using MLAgentsExamples;
 
 public class WalkerAgent : Agent
 {

Project/Assets/ML-Agents/Examples/Walker/TFModels/Walker.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/WallJump/Scripts/WallJumpAgent.cs

         if ((!Physics.Raycast(m_AgentRb.position, Vector3.down, 20))
             || (!Physics.Raycast(m_ShortBlockRb.position, Vector3.down, 20)))
         {
-            Done();
+            Done();
             ResetBlock(m_ShortBlockRb);
             StartCoroutine(
                 GoalScoredSwapGroundMaterial(m_WallJumpSettings.failMaterial, .5f));

Project/Assets/ML-Agents/Examples/WallJump/TFModels/BigWallJump.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/Assets/ML-Agents/Examples/WallJump/TFModels/SmallWallJump.nn.meta

 ScriptedImporter:
   fileIDToRecycleName:
     11400000: main obj
+    11400002: model data
   externalObjects: {}
   userData: 
   assetBundleName: 

Project/ProjectSettings/TagManager.asset

   - 
   - 
   - invisible
-  - ball
-  - goalieWall
-  - goalie
-  - strikerWall
-  - striker
+  - 
+  - 
+  - 
+  - 
+  - 
   - 
   - 
   - 

README.md

 
 In addition to our own documentation, here are some additional, relevant articles:
 
-* [Unity AI - Unity 3D Artificial Intelligence](https://www.youtube.com/watch?v=bqsfkGbBU6k)
+* [ML-Agents Penguins tutorial](https://learn.unity.com/project/ml-agents-penguins)
 
 ## Community and Feedback
 

com.unity.ml-agents/Editor/BehaviorParametersEditor.cs

             var brainParameters = behaviorParameters.brainParameters;
             if (model != null)
             {
-                barracudaModel = ModelLoader.Load(model.Value);
+                barracudaModel = ModelLoader.Load(model);
             }
             if (brainParameters != null)
             {

com.unity.ml-agents/Editor/Unity.ML-Agents.Editor.asmdef

 {
     "name": "Unity.ML-Agents.Editor",
     "references": [
-        "Unity.ML-Agents"
+        "Unity.ML-Agents",
+        "Barracuda"
     ],
     "includePlatforms": [
         "Editor"
     "defineConstraints": [],
     "versionDefines": [],
     "noEngineReferences": false
-}
+}

com.unity.ml-agents/Runtime/Academy.cs

     /// <summary>
     /// Helper class to step the Academy during FixedUpdate phase.
     /// </summary>
-    public class AcademyFixedUpdateStepper : MonoBehaviour
+    internal class AcademyFixedUpdateStepper : MonoBehaviour
     {
         void FixedUpdate()
         {
     /// attached to it.
     /// </remarks>
     [HelpURL("https://github.com/Unity-Technologies/ml-agents/blob/master/" +
-        "docs/Learning-Environment-Design-Academy.md")]
+        "docs/Learning-Environment-Design.md")]
-        const string k_ApiVersion = "API-14-dev0";
+        const string k_ApiVersion = "API-15-dev0";
         const int k_EditorTrainingPort = 5004;
 
         // Lazy initializer pattern, see https://csharpindepth.com/articles/singleton#lazy
         int m_TotalStepCount;
 
         /// Pointer to the communicator currently in use by the Academy.
-        public ICommunicator Communicator;
+        internal ICommunicator Communicator;
 
         bool m_Initialized;
         List<ModelRunner> m_ModelRunners = new List<ModelRunner>();
 
         // Signals to all the Agents at each environment step so they can use
         // their Policy to decide on their next action.
-        public event System.Action DecideAction;
+        internal event System.Action DecideAction;
-        public event System.Action DestroyAction;
+        internal event System.Action DestroyAction;
-        public event System.Action<int> AgentSetStatus;
+        internal event System.Action<int> AgentSetStatus;
-        public event System.Action AgentSendState;
+        internal event System.Action AgentSendState;
-        public event System.Action AgentAct;
+        internal event System.Action AgentAct;
-        public event System.Action AgentForceReset;
+        internal event System.Action AgentForceReset;
 
         // Signals that the Academy has been reset by the training process
         public event System.Action OnEnvironmentReset;
         /// Initialize the Academy if it hasn't already been initialized.
         /// This method is always safe to call; it will have no effect if the Academy is already initialized.
         /// </summary>
-        public void LazyInitialization()
+        internal void LazyInitialization()
         {
             if (!m_Initialized)
             {
         /// <param name="inferenceDevice"> The inference device (CPU or GPU)
         /// the ModelRunner will use </param>
         /// <returns> The ModelRunner compatible with the input settings</returns>
-        public ModelRunner GetOrCreateModelRunner(
+        internal ModelRunner GetOrCreateModelRunner(
             NNModel model, BrainParameters brainParameters, InferenceDevice inferenceDevice)
         {
             var modelRunner = m_ModelRunners.Find(x => x.HasModel(model, inferenceDevice));

com.unity.ml-agents/Runtime/ActionMasker.cs

 
 namespace MLAgents
 {
-    public class ActionMasker
+    internal class ActionMasker
     {
         /// When using discrete control, is the starting indices of the actions
         /// when all the branches are concatenated with each other.
 
         readonly BrainParameters m_BrainParameters;
 
-        public ActionMasker(BrainParameters brainParameters)
+        internal ActionMasker(BrainParameters brainParameters)
         {
             m_BrainParameters = brainParameters;
         }

com.unity.ml-agents/Runtime/Agent.cs

     /// Struct that contains the action information sent from the Brain to the
     /// Agent.
     /// </summary>
-    public struct AgentAction
+    internal struct AgentAction
     {
         public float[] vectorActions;
     }
         /// This Id will be changed every time the Agent resets.
         int m_EpisodeId;
 
+        /// Whether or not the Agent has been initialized already
+        bool m_Initialized;
+
         /// Keeps track of the actions that are masked at each step.
         ActionMasker m_ActionMasker;
 
         /// Currently generated from attached SensorComponents, and a legacy VectorSensor
         /// </summary>
         [FormerlySerializedAs("m_Sensors")]
-        public List<ISensor> sensors;
+        internal List<ISensor> sensors;
-        public VectorSensor collectObservationsSensor;
+        internal VectorSensor collectObservationsSensor;
-            m_EpisodeId = EpisodeIdCounter.GetEpisodeId();
-            OnEnableHelper();
-
-            m_Recorder = GetComponent<DemonstrationRecorder>();
+            LazyInitialize();
-        void OnEnableHelper()
+        public void LazyInitialize()
+            if (m_Initialized)
+            {
+                return;
+            }
+            m_Initialized = true;
+
+            // Grab the "static" properties for the Agent.
+            m_EpisodeId = EpisodeIdCounter.GetEpisodeId();
+            m_PolicyFactory = GetComponent<BehaviorParameters>();
+            m_Recorder = GetComponent<DemonstrationRecorder>();
+
+
             m_Info = new AgentInfo();
             m_Action = new AgentAction();
             sensors = new List<ISensor>();
             Academy.Instance.AgentAct += AgentStep;
             Academy.Instance.AgentForceReset += _AgentReset;
-            m_PolicyFactory = GetComponent<BehaviorParameters>();
+
         }
 
         /// Monobehavior function that is called when the attached GameObject
             }
             NotifyAgentDone();
             m_Brain?.Dispose();
+            m_Initialized = false;
         }
 
         void NotifyAgentDone(bool maxStepReached = false)
             m_Info.maxStepReached = maxStepReached;
             // Request the last decision with no callbacks
             // We request a decision so Python knows the Agent is done immediately
-            m_Brain?.RequestDecision(m_Info, sensors, (a) => {});
+            m_Brain?.RequestDecision(m_Info, sensors);
+
+            UpdateRewardStats();
+
             // The Agent is done, so we give it a new episode Id
             m_EpisodeId = EpisodeIdCounter.GetEpisodeId();
             m_Reward = 0f;
             return m_CumulativeReward;
         }
 
+        void UpdateRewardStats()
+        {
+            var gaugeName = $"{m_PolicyFactory.behaviorName}.CumulativeReward";
+            TimerStack.Instance.SetGauge(gaugeName, GetCumulativeReward());
+        }
+
         /// <summary>
         /// Sets the done flag to true.
         /// </summary>
         /// Set up the list of ISensors on the Agent. By default, this will select any
         /// SensorBase's attached to the Agent.
         /// </summary>
-        public void InitializeSensors()
+        internal void InitializeSensors()
         {
             // Get all attached sensor components
             SensorComponent[] attachedSensorComponents;
             m_Info.maxStepReached = false;
             m_Info.episodeId = m_EpisodeId;
 
-            m_Brain.RequestDecision(m_Info, sensors, UpdateAgentAction);
+            m_Brain.RequestDecision(m_Info, sensors);
 
             if (m_Recorder != null && m_Recorder.record && Application.isEditor)
             {
         }
 
         /// <summary>
+        /// Returns the last action that was decided on by the Agent (returns null if no decision has been made)
+        /// </summary>
+        public float[] GetAction()
+        {
+        	return m_Action.vectorActions;
+        }
+
+        /// <summary>
         /// This method will forcefully reset the agent and will also reset the hasAlreadyReset flag.
         /// This way, even if the agent was already in the process of reseting, it will be reset again
         /// and will not send a Done flag at the next step.
             AgentReset();
         }
 
-        public void UpdateAgentAction(AgentAction action)
-        {
-            m_Action = action;
-        }
-
-        /// <summary>
-        /// Updates the vector action.
-        /// </summary>
-        /// <param name="vectorActions">Vector actions.</param>
-        public void UpdateVectorAction(float[] vectorActions)
-        {
-            m_Action.vectorActions = vectorActions;
-        }
-
         /// <summary>
         /// Scales continuous action from [-1, 1] to arbitrary range.
         /// </summary>
             if ((m_RequestAction) && (m_Brain != null))
             {
                 m_RequestAction = false;
-                AgentAction(m_Action.vectorActions);
+                if (m_Action.vectorActions != null)
+                {
+                    AgentAction(m_Action.vectorActions);
+                }
-            m_Brain?.DecideAction();
+            m_Action.vectorActions = m_Brain?.DecideAction();
         }
     }
 }

com.unity.ml-agents/Runtime/Constants.cs

     /// <summary>
     /// Grouping for use in AddComponentMenu (instead of nesting the menus).
     /// </summary>
-    public enum MenuGroup
+    internal enum MenuGroup
     {
         Default = 0,
         Sensors = 50

com.unity.ml-agents/Runtime/DecisionRequester.cs

     /// A component that when attached to an Agent will automatically request decisions from it
     /// at regular intervals.
     /// </summary>
+    [AddComponentMenu("ML Agents/Decision Requester", (int)MenuGroup.Default)]
     public class DecisionRequester : MonoBehaviour
     {
         [Range(1, 20)]

com.unity.ml-agents/Runtime/DemonstrationRecorder.cs

             m_DemoStore.Initialize(
                 demonstrationName,
                 behaviorParams.brainParameters,
-                behaviorParams.behaviorName);
+                behaviorParams.fullyQualifiedBehaviorName);
             Monitor.Log("Recording Demonstration of Agent: ", m_RecordingAgent.name);
         }
 

com.unity.ml-agents/Runtime/EpisodeIdCounter.cs

 namespace MLAgents
 {
-    public static class EpisodeIdCounter
+    internal static class EpisodeIdCounter
     {
         private static int Counter;
         public static int GetEpisodeId()

com.unity.ml-agents/Runtime/Grpc/GrpcExtensions.cs

 
 namespace MLAgents
 {
-    public static class GrpcExtensions
+    internal static class GrpcExtensions
-        internal static AgentInfoActionPairProto ToInfoActionPairProto(this AgentInfo ai)
+        public static AgentInfoActionPairProto ToInfoActionPairProto(this AgentInfo ai)
         {
             var agentInfoProto = ai.ToAgentInfoProto();
 
         /// Converts a AgentInfo to a protobuf generated AgentInfoProto
         /// </summary>
         /// <returns>The protobuf version of the AgentInfo.</returns>
-        internal static AgentInfoProto ToAgentInfoProto(this AgentInfo ai)
+        public static AgentInfoProto ToAgentInfoProto(this AgentInfo ai)
         {
             var agentInfoProto = new AgentInfoProto
             {
         /// <param name="bp">The instance of BrainParameter to extend.</param>
         /// <param name="name">The name of the brain.</param>
         /// <param name="isTraining">Whether or not the Brain is training.</param>
-        internal static BrainParametersProto ToProto(this BrainParameters bp, string name, bool isTraining)
+        public static BrainParametersProto ToProto(this BrainParameters bp, string name, bool isTraining)
         {
             var brainParametersProto = new BrainParametersProto
             {
         /// <summary>
         /// Convert metadata object to proto object.
         /// </summary>
-        internal static DemonstrationMetaProto ToProto(this DemonstrationMetaData dm)
+        public static DemonstrationMetaProto ToProto(this DemonstrationMetaData dm)
         {
             var demoProto = new DemonstrationMetaProto
             {
         /// <summary>
         /// Initialize metadata values based on proto object.
         /// </summary>
-        internal static DemonstrationMetaData ToDemonstrationMetaData(this DemonstrationMetaProto demoProto)
+        public static DemonstrationMetaData ToDemonstrationMetaData(this DemonstrationMetaProto demoProto)
         {
             var dm = new DemonstrationMetaData
             {
         /// </summary>
         /// <param name="bpp">An instance of a brain parameters protobuf object.</param>
         /// <returns>A BrainParameters struct.</returns>
-        internal static BrainParameters ToBrainParameters(this BrainParametersProto bpp)
+        public static BrainParameters ToBrainParameters(this BrainParametersProto bpp)
         {
             var bp = new BrainParameters
             {
             return bp;
         }
 
-        internal static UnityRLInitParameters ToUnityRLInitParameters(this UnityRLInitializationInputProto inputProto)
+        public static UnityRLInitParameters ToUnityRLInitParameters(this UnityRLInitializationInputProto inputProto)
         {
             return new UnityRLInitParameters
             {
 
-        internal static AgentAction ToAgentAction(this AgentActionProto aap)
+        public static AgentAction ToAgentAction(this AgentActionProto aap)
         {
             return new AgentAction
             {
 
-        internal static List<AgentAction> ToAgentActionList(this UnityRLInputProto.Types.ListAgentActionProto proto)
+        public static List<AgentAction> ToAgentActionList(this UnityRLInputProto.Types.ListAgentActionProto proto)
         {
             var agentActions = new List<AgentAction>(proto.Value.Count);
             foreach (var ap in proto.Value)
             return agentActions;
         }
 
-        internal static ObservationProto ToProto(this Observation obs)
+        public static ObservationProto ToProto(this Observation obs)
         {
             ObservationProto obsProto = null;
 
         /// <param name="sensor"></param>
         /// <param name="writeAdapter"></param>
         /// <returns></returns>
-        internal static ObservationProto GetObservationProto(this ISensor sensor, WriteAdapter writeAdapter)
+        public static ObservationProto GetObservationProto(this ISensor sensor, WriteAdapter writeAdapter)
         {
             var shape = sensor.GetObservationShape();
             ObservationProto observationProto = null;

com.unity.ml-agents/Runtime/Grpc/RpcCommunicator.cs

 namespace MLAgents
 {
     /// Responsible for communication with External using gRPC.
-    public class RpcCommunicator : ICommunicator
+    internal class RpcCommunicator : ICommunicator
-        public struct IdCallbackPair
-        {
-            public int AgentId;