Merge branch 'master' into develop-add-inference-examples

.yamato/protobuf-generation-test.yml

         - "protobuf-definitions/*.md"
         - "protobuf-definitions/**/*.md"
   artifacts:
-    dist:
+    patch:
-        - "artifacts/*"
+        - "artifacts/*.*"

.yamato/standalone-build-test.yml

   commands:
     - pip install pyyaml
     - python -u -m ml-agents.tests.yamato.standalone_build_tests
+    - python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity
   triggers:
     cancel_old_ci: true
     changes:
         - "com.unity.ml-agents/*.md"
         - "com.unity.ml-agents/**/*.md"
   artifacts:
+    logs:
+      paths:
+        - "artifacts/standalone_build.txt"
-        - "Project/testPlayer*/**"
+        - "artifacts/testPlayer*/**"
 {% endfor %}

.yamato/training-int-tests.yml

     # Backwards-compatibility tests.
     # If we make a breaking change to the communication protocol, these will need
     # to be disabled until the next release.
-    - python -u -m ml-agents.tests.yamato.training_int_tests --python=0.15.0
-    - python -u -m ml-agents.tests.yamato.training_int_tests --csharp=0.15.0
+    # - python -u -m ml-agents.tests.yamato.training_int_tests --python=0.15.0
+    # - python -u -m ml-agents.tests.yamato.training_int_tests --csharp=0.15.0
   dependencies:
     - .yamato/standalone-build-test.yml#test_mac_standalone_{{ editor.version }}
   triggers:
         - "com.unity.ml-agents/*.md"
         - "com.unity.ml-agents/**/*.md"
   artifacts:
-    unit:
+    logs:
+      paths:
+        - "artifacts/standalone_build.txt"
+    standalonebuild:
-        - "artifacts/**"
+        - "artifacts/testplayer*/**"
 {% endfor %}

Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs

         public void Update() {}
 
         /// <inheritdoc/>
+        public void Reset() { }
+
+        /// <inheritdoc/>
         public virtual byte[] GetCompressedObservation()
         {
             return null;

Project/Assets/ML-Agents/Examples/Tennis/Scripts/TennisAgent.cs

 
     public override float[] Heuristic()
     {
-        var action = new float[2];
+        var action = new float[3];
-        action[0] = Input.GetAxis("Horizontal");
-        action[1] = Input.GetKey(KeyCode.Space) ? 1f : 0f;
+        action[0] = Input.GetAxis("Horizontal");    // Racket Movement
+        action[1] = Input.GetKey(KeyCode.Space) ? 1f : 0f;   // Racket Jumping
+        action[2] = Input.GetAxis("Vertical");   // Racket Rotation  
         return action;
     }
 

README.md

 |:-------:|:------:|:-------------:|:-------:|:------------:|
 | **master (unstable)** | -- | [source](https://github.com/Unity-Technologies/ml-agents/tree/master) | [docs](https://github.com/Unity-Technologies/ml-agents/tree/master/docs/Readme.md) | [download](https://github.com/Unity-Technologies/ml-agents/archive/master.zip) |
 | **0.15.1** | **March 30, 2020** | **[source](https://github.com/Unity-Technologies/ml-agents/tree/0.15.1)** | **[docs](https://github.com/Unity-Technologies/ml-agents/tree/0.15.1/docs/Readme.md)** | **[download](https://github.com/Unity-Technologies/ml-agents/archive/0.15.1.zip)** |
-| **0.15.0** | **March 18, 2020** | **[source](https://github.com/Unity-Technologies/ml-agents/tree/0.15.0)** | **[docs](https://github.com/Unity-Technologies/ml-agents/tree/0.15.0/docs/Readme.md)** | **[download](https://github.com/Unity-Technologies/ml-agents/archive/0.15.0.zip)** |
+| **0.15.0** | March 18, 2020 | [source](https://github.com/Unity-Technologies/ml-agents/tree/0.15.0) | [docs](https://github.com/Unity-Technologies/ml-agents/tree/0.15.0/docs/Readme.md) | [download](https://github.com/Unity-Technologies/ml-agents/archive/0.15.0.zip) |
 | **0.14.1** | February 26, 2020 | [source](https://github.com/Unity-Technologies/ml-agents/tree/0.14.1) | [docs](https://github.com/Unity-Technologies/ml-agents/tree/0.14.1/docs/Readme.md) | [download](https://github.com/Unity-Technologies/ml-agents/archive/0.14.1.zip) |
 | **0.14.0** | February 13, 2020 | [source](https://github.com/Unity-Technologies/ml-agents/tree/0.14.0) | [docs](https://github.com/Unity-Technologies/ml-agents/tree/0.14.0/docs/Readme.md) | [download](https://github.com/Unity-Technologies/ml-agents/archive/0.14.0.zip) |
 | **0.13.1** | January 21, 2020 | [source](https://github.com/Unity-Technologies/ml-agents/tree/0.13.1) | [docs](https://github.com/Unity-Technologies/ml-agents/tree/0.13.1/docs/Readme.md) | [download](https://github.com/Unity-Technologies/ml-agents/archive/0.13.1.zip) |

com.unity.ml-agents/CHANGELOG.md

  - The Jupyter notebooks have been removed from the repository.
  - Introduced the `SideChannelUtils` to register, unregister and access side channels.
  - `Academy.FloatProperties` was removed, please use `SideChannelUtils.GetSideChannel<FloatPropertiesChannel>()` instead.
+ - Removed the multi-agent gym option from the gym wrapper. For multi-agent scenarios, use the [Low Level Python API](Python-API.md).
+ - The low level Python API has changed. You can look at the document [Low Level Python API documentation](Python-API.md) for more information. If you use `mlagents-learn` for training, this should be a transparent change.
+ - Added ability to start training (initialize model weights) from a previous run ID. (#3710)
+ - The internal event `Academy.AgentSetStatus` was renamed to `Academy.AgentPreStep` and made public.
+ - The offset logic was removed from DecisionRequester.
+ - SideChannel IncomingMessages methods now take an optional default argument, which is used when trying to read more data than the message contains.
  - The way that UnityEnvironment decides the port was changed. If no port is specified, the behavior will depend on the `file_name` parameter. If it is `None`, 5004 (the editor port) will be used; otherwise 5005 (the base environment port) will be used.
  - Fixed an issue where exceptions from environments provided a returncode of 0. (#3680)
  - Running `mlagents-learn` with the same `--run-id` twice will no longer overwrite the existing files. (#3705)

com.unity.ml-agents/Runtime/Academy.cs

         /// on each side, although we may allow some flexibility in the future.
         /// This should be incremented whenever a change is made to the communication protocol.
         /// </summary>
-        const string k_ApiVersion = "0.15.0";
+        const string k_ApiVersion = "0.16.0";
-        internal const string k_PackageVersion = "0.15.0-preview";
+        internal const string k_PackageVersion = "0.15.1-preview";
 
         const int k_EditorTrainingPort = 5004;
 
         // This will mark the Agent as Done if it has reached its maxSteps.
         internal event Action AgentIncrementStep;
 
-        // Signals to all the agents at each environment step along with the
-        // Academy's maxStepReached, done and stepCount values. The agents rely
-        // on this event to update their own values of max step reached and done
-        // in addition to aligning on the step count of the global episode.
-        internal event Action<int> AgentSetStatus;
+
+        /// <summary>
+        /// Signals to all of the <see cref="Agent"/>s that their step is about to begin.
+        /// This is a good time for an <see cref="Agent"/> to decide if it would like to
+        /// call <see cref="Agent.RequestDecision"/> or <see cref="Agent.RequestAction"/>
+        /// for this step.  Any other pre-step setup could be done during this even as well.
+        /// </summary>
+        public event Action<int> AgentPreStep;
 
         // Signals to all the agents at each environment step so they can send
         // their state to their Policy if they have requested a decision.
         {
             DecideAction = () => {};
             DestroyAction = () => {};
-            AgentSetStatus = i => {};
+            AgentPreStep = i => {};
             AgentSendState = () => {};
             AgentAct = () => {};
             AgentForceReset = () => {};
                 ForcedFullReset();
             }
 
-            AgentSetStatus?.Invoke(m_StepCount);
+            AgentPreStep?.Invoke(m_StepCount);
 
             m_StepCount += 1;
             m_TotalStepCount += 1;

com.unity.ml-agents/Runtime/Agent.cs

             m_Info.reward = m_Reward;
             m_Info.done = true;
             m_Info.maxStepReached = doneReason == DoneReason.MaxStepReached;
+            if (collectObservationsSensor != null)
+            {
+                // Make sure the latest observations are being passed to training.
+                collectObservationsSensor.Reset();
+                CollectObservations(collectObservationsSensor);
+            }
+            ResetSensors();
 
             // We also have to write any to any DemonstationStores so that they get the "done" flag.
             foreach (var demoWriter in DemonstrationWriters)
                 UpdateRewardStats();
             }
 
-            // The Agent is done, so we give it a new episode Id
-            m_EpisodeId = EpisodeIdCounter.GetEpisodeId();
             m_Reward = 0f;
             m_CumulativeReward = 0f;
             m_RequestAction = false;
             foreach (var sensor in sensors)
             {
                 sensor.Update();
+            }
+        }
+
+        void ResetSensors()
+        {
+            foreach (var sensor in sensors)
+            {
+                sensor.Reset();
             }
         }
 

com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs

             {
                 m_OrderedAgentsRequestingDecisions[behaviorName] = new List<int>();
             }
-            m_OrderedAgentsRequestingDecisions[behaviorName].Add(info.episodeId);
+            if (!info.done)
+            {
+                m_OrderedAgentsRequestingDecisions[behaviorName].Add(info.episodeId);
+            }
             if (!m_LastActionsReceived.ContainsKey(behaviorName))
             {
                 m_LastActionsReceived[behaviorName] = new Dictionary<int, float[]>();

com.unity.ml-agents/Runtime/DecisionRequester.cs

+using System;
 using UnityEngine;
 using UnityEngine.Serialization;
 
     /// at regular intervals.
     /// </summary>
     [AddComponentMenu("ML Agents/Decision Requester", (int)MenuGroup.Default)]
-    internal class DecisionRequester : MonoBehaviour
+    [RequireComponent(typeof(Agent))]
+    public class DecisionRequester : MonoBehaviour
-        /// that the Agent will request a decision every 5 Academy steps.
-        /// </summary>
+        /// that the Agent will request a decision every 5 Academy steps. /// </summary>
         [Range(1, 20)]
         [Tooltip("The frequency with which the agent requests a decision. A DecisionPeriod " +
                  "of 5 means that the Agent will request a decision every 5 Academy steps.")]
         [FormerlySerializedAs("RepeatAction")]
         public bool TakeActionsBetweenDecisions = true;
 
-        /// <summary>
-        /// Whether or not the Agent decisions should start at an offset (different for each agent).
-        /// This does not affect <see cref="DecisionPeriod"/>. Turning this on will distribute
-        /// the decision-making computations for all the agents across multiple Academy steps.
-        /// This can be valuable in scenarios where you have many agents in the scene, particularly
-        /// during the inference phase.
-        /// </summary>
-        [Tooltip("Whether or not Agent decisions should start at an offset.")]
-        public bool offsetStep;
-
+        [NonSerialized]
-        int m_Offset;
-            m_Offset = offsetStep ? gameObject.GetInstanceID() : 0;
-            Academy.Instance.AgentSetStatus += MakeRequests;
+            Debug.Assert(m_Agent != null, "Agent component was not found on this gameObject and is required.");
+            Academy.Instance.AgentPreStep += MakeRequests;
         }
 
         void OnDestroy()
-                Academy.Instance.AgentSetStatus -= MakeRequests;
+                Academy.Instance.AgentPreStep -= MakeRequests;
-        void MakeRequests(int count)
+        /// <summary>
+        /// Method that hooks into the Academy in order inform the Agent on whether or not it should request a
+        /// decision, and whether or not it should take actions between decisions.
+        /// </summary>
+        /// <param name="academyStepCount">The current step count of the academy.</param>
+        void MakeRequests(int academyStepCount)
-            if ((count + m_Offset) % DecisionPeriod == 0)
+            if (academyStepCount % DecisionPeriod == 0)
             {
                 m_Agent?.RequestDecision();
             }

com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs

         public void Update() {}
 
         /// <inheritdoc/>
+        public void Reset() { }
+
+        /// <inheritdoc/>
         public SensorCompressionType GetCompressionType()
         {
             return m_CompressionType;

com.unity.ml-agents/Runtime/Sensors/ISensor.cs

         void Update();
 
         /// <summary>
+        /// Resets the internal states of the sensor. This is called at the end of an Agent's episode.
+        /// Most implementations can leave this empty.
+        /// </summary>
+        void Reset();
+
+        /// <summary>
         /// Return the compression type being used. If no compression is used, return
         /// <see cref="SensorCompressionType.None"/>.
         /// </summary>

com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs

         }
 
         /// <inheritdoc/>
+        public void Reset() { }
+
+        /// <inheritdoc/>
         public int[] GetObservationShape()
         {
             return m_Shape;

com.unity.ml-agents/Runtime/Sensors/RenderTextureSensor.cs

         public void Update() {}
 
         /// <inheritdoc/>
+        public void Reset() { }
+
+        /// <inheritdoc/>
         public SensorCompressionType GetCompressionType()
         {
             return m_CompressionType;

com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs

+using System;
+
 namespace MLAgents.Sensors
 {
     /// <summary>
         {
             m_WrappedSensor.Update();
             m_CurrentIndex = (m_CurrentIndex + 1) % m_NumStackedObservations;
+        }
+
+        /// <inheritdoc/>
+        public void Reset()
+        {
+            m_WrappedSensor.Reset();
+            // Zero out the buffer.
+            for (var i = 0; i < m_NumStackedObservations; i++)
+            {
+                Array.Clear(m_StackedObservations[i], 0, m_StackedObservations[i].Length);
+            }
         }
 
         /// <inheritdoc/>

com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs

         }
 
         /// <inheritdoc/>
+        public void Reset()
+        {
+            Clear();
+        }
+
+        /// <inheritdoc/>
         public int[] GetObservationShape()
         {
             return m_Shape;

com.unity.ml-agents/Runtime/SideChannels/IncomingMessage.cs

 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System;
 using System.IO;
 using System.Text;
         }
 
         /// <summary>
-        /// Read a boolan value from the message.
+        /// Read a boolean value from the message.
+        /// <param name="defaultValue">Default value to use if the end of the message is reached.</param>
-        public bool ReadBoolean()
+        public bool ReadBoolean(bool defaultValue = false)
-            return m_Reader.ReadBoolean();
+            return CanReadMore() ? m_Reader.ReadBoolean() : defaultValue;
+        /// <param name="defaultValue">Default value to use if the end of the message is reached.</param>
-        public int ReadInt32()
+        public int ReadInt32(int defaultValue = 0)
-            return m_Reader.ReadInt32();
+            return CanReadMore() ? m_Reader.ReadInt32() : defaultValue;
+        /// <param name="defaultValue">Default value to use if the end of the message is reached.</param>
-        public float ReadFloat32()
+        public float ReadFloat32(float defaultValue = 0.0f)
-            return m_Reader.ReadSingle();
+            return CanReadMore() ? m_Reader.ReadSingle() : defaultValue;
+        /// <param name="defaultValue">Default value to use if the end of the message is reached.</param>
-        public string ReadString()
+        public string ReadString(string defaultValue = default)
+            if (!CanReadMore())
+            {
+                return defaultValue;
+            }
+
             var strLength = ReadInt32();
             var str = Encoding.ASCII.GetString(m_Reader.ReadBytes(strLength));
             return str;
         /// Reads a list of floats from the message. The length of the list is stored in the message.
         /// </summary>
+        /// <param name="defaultValue">Default value to use if the end of the message is reached.</param>
-        public IList<float> ReadFloatList()
+        public IList<float> ReadFloatList(IList<float> defaultValue = default)
+            if (!CanReadMore())
+            {
+                return defaultValue;
+            }
+
             var len = ReadInt32();
             var output = new float[len];
             for (var i = 0; i < len; i++)
         {
             m_Reader?.Dispose();
             m_Stream?.Dispose();
+        }
+
+        /// <summary>
+        /// Whether or not there is more data left in the stream that can be read.
+        /// </summary>
+        /// <returns></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        bool CanReadMore()
+        {
+            return m_Stream.Position < m_Stream.Length;
         }
     }
 }

com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs

 using System.CodeDom;
+using System;
 using UnityEngine;
 using NUnit.Framework;
 using System.Reflection;
 {
     internal class TestPolicy : IPolicy
     {
-        public void RequestDecision(AgentInfo info, List<ISensor> sensors) {}
+        public Action OnRequestDecision;
+        private WriteAdapter m_Adapter = new WriteAdapter();
+        public void RequestDecision(AgentInfo info, List<ISensor> sensors) {
+            foreach(var sensor in sensors){
+                sensor.GetObservationProto(m_Adapter);
+            }
+            OnRequestDecision?.Invoke();
+        }
 
         public float[] DecideAction() { return new float[0]; }
 
         {
             collectObservationsCalls += 1;
             collectObservationsCallsForEpisode += 1;
-            sensor.AddObservation(0f);
+            sensor.AddObservation(collectObservationsCallsForEpisode);
         }
 
         public override void OnActionReceived(float[] vectorAction)
         public string sensorName;
         public int numWriteCalls;
         public int numCompressedCalls;
+        public int numResetCalls;
         public SensorCompressionType compressionType = SensorCompressionType.None;
 
         public TestSensor(string n)
         }
 
         public void Update() {}
+
+        public void Reset()
+        {
+            numResetCalls++;
+        }
     }
 
     [TestFixture]
                 aca.EnvironmentStep();
             }
         }
+
+        [Test]
+        public void AssertStackingReset()
+        {
+            var agentGo1 = new GameObject("TestAgent");
+            agentGo1.AddComponent<TestAgent>();
+            var behaviorParameters = agentGo1.GetComponent<BehaviorParameters>();
+            behaviorParameters.brainParameters.numStackedVectorObservations = 3;
+            var agent1 = agentGo1.GetComponent<TestAgent>();
+            var aca = Academy.Instance;
+            agent1.LazyInitialize();
+            var policy = new TestPolicy();
+            agent1.SetPolicy(policy);
+
+            StackingSensor sensor = null;
+            foreach(ISensor s in agent1.sensors){
+                if (s is  StackingSensor){
+                    sensor = s as StackingSensor;
+                }
+            }
+
+            Assert.NotNull(sensor);
+
+            for (int i = 0; i < 20; i++)
+            {
+                agent1.RequestDecision();
+                aca.EnvironmentStep();
+
+            }
+
+            policy.OnRequestDecision = () =>  SensorTestHelper.CompareObservation(sensor, new[] {18f, 19f, 21f});
+            agent1.EndEpisode();
+            SensorTestHelper.CompareObservation(sensor, new[] {0f, 0f, 0f});
+        }
     }
 
     [TestFixture]
             var expectedCollectObsCalls = 0;
             var expectedCollectObsCallsForEpisode = 0;
             var expectedCompletedEpisodes = 0;
+            var expectedSensorResetCalls = 0;
 
             for (var i = 0; i < 15; i++)
             {
                     expectedCollectObsCallsForEpisode = 0;
                     expectedAgentStepCount = 0;
                     expectedCompletedEpisodes++;
+                    expectedSensorResetCalls++;
+                    expectedCollectObsCalls += 1;
                 }
                 aca.EnvironmentStep();
 
                 Assert.AreEqual(expectedCollectObsCalls, agent1.collectObservationsCalls);
                 Assert.AreEqual(expectedCollectObsCallsForEpisode, agent1.collectObservationsCallsForEpisode);
                 Assert.AreEqual(expectedCompletedEpisodes, agent1.CompletedEpisodes);
+                Assert.AreEqual(expectedSensorResetCalls, agent1.sensor1.numResetCalls);
             }
         }
 

com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs

         }
 
         public void Update() {}
+        public void Reset() { }
 
         public SensorCompressionType GetCompressionType()
         {

com.unity.ml-agents/Tests/Editor/PublicAPI/PublicApiValidation.cs

 using MLAgents.Sensors;
 using NUnit.Framework;
 using UnityEngine;
+using UnityEngine.TestTools;
 
 namespace MLAgentsExamples
 {
             sensorComponent.observationStacks = 2;
 
             sensorComponent.CreateSensor();
-        }
-
-        class PublicApiAgent : Agent
-        {
-            public int numHeuristicCalls;
-
-            public override float[] Heuristic()
-            {
-                numHeuristicCalls++;
-                return base.Heuristic();
-            }
-        }
-
-        // Simple SensorComponent that sets up a StackingSensor
-        class StackingComponent : SensorComponent
-        {
-            public SensorComponent wrappedComponent;
-            public int numStacks;
-
-            public override ISensor CreateSensor()
-            {
-                var wrappedSensor = wrappedComponent.CreateSensor();
-                return new StackingSensor(wrappedSensor, numStacks);
-            }
-
-            public override int[] GetObservationShape()
-            {
-                int[] shape = (int[]) wrappedComponent.GetObservationShape().Clone();
-                for (var i = 0; i < shape.Length; i++)
-                {
-                    shape[i] *= numStacks;
-                }
-
-                return shape;
-            }
-        }
-
-
-        [Test]
-        public void CheckSetupAgent()
-        {
-            var gameObject = new GameObject();
-
-            var behaviorParams = gameObject.AddComponent<BehaviorParameters>();
-            behaviorParams.brainParameters.vectorObservationSize = 3;
-            behaviorParams.brainParameters.numStackedVectorObservations = 2;
-            behaviorParams.brainParameters.vectorActionDescriptions = new[] { "TestActionA", "TestActionB" };
-            behaviorParams.brainParameters.vectorActionSize = new[] { 2, 2 };
-            behaviorParams.brainParameters.vectorActionSpaceType = SpaceType.Discrete;
-            behaviorParams.behaviorName = "TestBehavior";
-            behaviorParams.TeamId = 42;
-            behaviorParams.useChildSensors = true;
-
-            var agent = gameObject.AddComponent<PublicApiAgent>();
-            // Make sure we can set the behavior type correctly after the agent is added
-            behaviorParams.behaviorType = BehaviorType.InferenceOnly;
-            // Can't actually create an Agent with InferenceOnly and no model, so change back
-            behaviorParams.behaviorType = BehaviorType.Default;
-
-            // TODO -  not internal yet
-            // var decisionRequester = gameObject.AddComponent<DecisionRequester>();
-            // decisionRequester.DecisionPeriod = 2;
-
-            var sensorComponent = gameObject.AddComponent<RayPerceptionSensorComponent3D>();
-            sensorComponent.sensorName = "ray3d";
-            sensorComponent.detectableTags = new List<string> { "Player", "Respawn" };
-            sensorComponent.raysPerDirection = 3;
-
-            // Make a StackingSensor that wraps the RayPerceptionSensorComponent3D
-            // This isn't necessarily practical, just to ensure that it can be done
-            var wrappingSensorComponent = gameObject.AddComponent<StackingComponent>();
-            wrappingSensorComponent.wrappedComponent = sensorComponent;
-            wrappingSensorComponent.numStacks = 3;
-
-            // ISensor isn't set up yet.
-            Assert.IsNull(sensorComponent.raySensor);
-
-            agent.LazyInitialize();
-            // Make sure we can set the behavior type correctly after the agent is initialized
-            // (this creates a new policy).
-            behaviorParams.behaviorType = BehaviorType.HeuristicOnly;
-
-            // Initialization should set up the sensors
-            Assert.IsNotNull(sensorComponent.raySensor);
-
-            // Let's change the inference device
-            var otherDevice = behaviorParams.inferenceDevice == InferenceDevice.CPU ? InferenceDevice.GPU : InferenceDevice.CPU;
-            agent.SetModel(behaviorParams.behaviorName, behaviorParams.model, otherDevice);
-
-            agent.AddReward(1.0f);
-
-            agent.RequestAction();
-            agent.RequestDecision();
-
-            Academy.Instance.AutomaticSteppingEnabled = false;
-            Academy.Instance.EnvironmentStep();
-
-            var actions = agent.GetAction();
-            // default Heuristic implementation should return zero actions.
-            Assert.AreEqual(new[] {0.0f, 0.0f}, actions);
-            Assert.AreEqual(1, agent.numHeuristicCalls);
         }
     }
 }

com.unity.ml-agents/Tests/Editor/Sensor/FloatVisualSensorTests.cs

         }
 
         public void Update() {}
+        public void Reset() { }
 
         public SensorCompressionType GetCompressionType()
         {

com.unity.ml-agents/Tests/Editor/Sensor/SensorShapeValidatorTests.cs

         }
 
         public void Update() { }
+        public void Reset() { }
 
         public SensorCompressionType GetCompressionType()
         {

com.unity.ml-agents/Tests/Editor/Sensor/StackingSensorTests.cs

             // Check that if we don't call Update(), the same observations are produced
             SensorTestHelper.CompareObservation(sensor, new[] {5f, 6f, 7f, 8f, 9f, 10f});
         }
+
+        [Test]
+        public void TestStackingReset()
+        {
+            VectorSensor wrapped = new VectorSensor(2);
+            ISensor sensor = new StackingSensor(wrapped, 3);
+
+            wrapped.AddObservation(new[] {1f, 2f});
+            SensorTestHelper.CompareObservation(sensor, new[] {0f, 0f, 0f, 0f, 1f, 2f});
+
+            sensor.Update();
+            wrapped.AddObservation(new[] {3f, 4f});
+            SensorTestHelper.CompareObservation(sensor, new[] {0f, 0f, 1f, 2f, 3f, 4f});
+
+            sensor.Reset();
+            wrapped.AddObservation(new[] {5f, 6f});
+            SensorTestHelper.CompareObservation(sensor, new[] {0f, 0f, 0f, 0f, 5f, 6f});
+        }
     }
 }

com.unity.ml-agents/Tests/Editor/SideChannelTests.cs

             Assert.AreEqual(stringVal, incomingMsg.ReadString());
             Assert.AreEqual(floatListVal, incomingMsg.ReadFloatList());
         }
+
+        [Test]
+        public void TestMessageReadDefaults()
+        {
+            // Make sure reading past the end of a message will apply defaults.
+            IncomingMessage incomingMsg;
+            using (var outgoingMsg = new OutgoingMessage())
+            {
+                incomingMsg = new IncomingMessage(outgoingMsg.ToByteArray());
+            }
+
+            Assert.AreEqual(false, incomingMsg.ReadBoolean());
+            Assert.AreEqual(true, incomingMsg.ReadBoolean(defaultValue: true));
+
+            Assert.AreEqual(0, incomingMsg.ReadInt32());
+            Assert.AreEqual(42, incomingMsg.ReadInt32(defaultValue: 42));
+
+            Assert.AreEqual(0.0f, incomingMsg.ReadFloat32());
+            Assert.AreEqual(1337.0f, incomingMsg.ReadFloat32(defaultValue: 1337.0f));
+
+            Assert.AreEqual(default(string), incomingMsg.ReadString());
+            Assert.AreEqual("foo", incomingMsg.ReadString(defaultValue: "foo"));
+
+            Assert.AreEqual(default(float[]), incomingMsg.ReadFloatList());
+            Assert.AreEqual(new float[] { 1001, 1002 }, incomingMsg.ReadFloatList(new float[] { 1001, 1002 }));
+        }
     }
 }

com.unity.ml-agents/package.json

 {
-	"name": "com.unity.ml-agents",
-	"displayName":"ML Agents",
-	"version": "0.15.0-preview",
-	"unity": "2018.4",
-	"description": "Add interactivity to your game with Machine Learning Agents trained using Deep Reinforcement Learning.",
-	"dependencies": {
-		"com.unity.barracuda": "0.6.1-preview"
-	}
+  "name": "com.unity.ml-agents",
+  "displayName": "ML Agents",
+  "version": "0.15.1-preview",
+  "unity": "2018.4",
+  "description": "Add interactivity to your game with Machine Learning Agents trained using Deep Reinforcement Learning.",
+  "dependencies": {
+    "com.unity.barracuda": "0.6.1-preview"
+  }
 }

config/trainer_config.yaml

     time_horizon: 1000
     self_play:
         window: 10
-        play_against_current_self_ratio: 0.5
+        play_against_latest_model_ratio: 0.5
+        team_change: 100000
 
 Soccer:
     normalize: false
     num_layers: 2
     self_play:
         window: 10
-        play_against_current_self_ratio: 0.5
+        play_against_latest_model_ratio: 0.5
+        team_change: 100000
 
 CrawlerStatic:
     normalize: true

docs/Getting-Started.md

 - For a "Hello World" introduction to creating your own Learning Environment,
   check out the [Making a New Learning
   Environment](Learning-Environment-Create-New.md) page.
-- For a series of YouTube video tutorials, checkout the
-  [Machine Learning Agents PlayList](https://www.youtube.com/playlist?list=PLX2vGYjWbI0R08eWQkO7nQkGiicHAX7IX)
-  page.

docs/Learning-Environment-Examples.md

 * Goal:
   * Get the ball into the opponent's goal while preventing
   the ball from entering own goal.
-  * Goalie:
 * Agents: The environment contains four agents, with the same
   Behavior Parameters : Soccer.
 * Agent Reward Function (dependent):

docs/ML-Agents-Overview.md

   [Training With Environment Parameter Randomization](Training-Environment-Parameter-Randomization.md)
   to learn more about this feature.
 
-- **Cloud Training on AWS** - To facilitate using the ML-Agents toolkit on
-  Amazon Web Services (AWS) machines, we provide a
-  [guide](Training-on-Amazon-Web-Service.md) on how to set-up EC2 instances in
-  addition to a public pre-configured Amazon Machine Image (AMI).
-
-- **Cloud Training on Microsoft Azure** - To facilitate using the ML-Agents
-  toolkit on Azure machines, we provide a
-  [guide](Training-on-Microsoft-Azure.md) on how to set-up virtual machine
-  instances in addition to a pre-configured data science image.
-
 ## Summary and Next Steps
 
 To briefly summarize: The ML-Agents toolkit enables games and simulations built

docs/Migrating.md

 ### Important changes
 * The `--load` and `--train` command-line flags have been deprecated and replaced with `--resume` and `--inference`.
 * Running with the same `--run-id` twice will now throw an error.
+* The `play_against_current_self_ratio` self-play trainer hyperparameter has been renamed to `play_against_latest_model_ratio`
+* Removed the multi-agent gym option from the gym wrapper. For multi-agent scenarios, use the [Low Level Python API](Python-API.md).
+* The low level Python API has changed. You can look at the document [Low Level Python API documentation](Python-API.md) for more information. If you use `mlagents-learn` for training, this should be a transparent change.
 
 ### Steps to Migrate
 * Replace the `--load` flag with `--resume` when calling `mlagents-learn`, and don't use the `--train` flag as training
 * Replace `Academy.FloatProperties` with `SideChannelUtils.GetSideChannel<FloatPropertiesChannel>()`.
 * Replace `Academy.RegisterSideChannel` with `SideChannelUtils.RegisterSideChannel()`.
 * Replace `Academy.UnregisterSideChannel` with `SideChannelUtils.UnregisterSideChannel`.
+
 
 ## Migrating from 0.14 to 0.15
 

docs/Python-API.md

 - **UnityEnvironment** — the main interface between the Unity application and
   your code. Use UnityEnvironment to start and control a simulation or training
   session.
-- **BatchedStepResult** — contains the data from Agents belonging to the same
-  "AgentGroup" in the simulation, such as observations and rewards.
-- **AgentGroupSpec** — describes the shape of the data inside a BatchedStepResult.
-  For example, provides the dimensions of the observations of a group.
+- **BehaviorName** - is a string that identifies a behavior in the simulation.
+- **AgentId** - is an `int` that serves as unique identifier for Agents in the
+  simulation.
+- **DecisionSteps** — contains the data from Agents belonging to the same
+  "Behavior" in the simulation, such as observations and rewards. Only Agents
+  that requested a decision since the last call to `env.step()` are in the
+  DecisionSteps object.
+- **TerminalSteps** — contains the data from Agents belonging to the same
+  "Behavior" in the simulation, such as observations and rewards. Only Agents
+  whose episode ended since the last call to `env.step()` are in the
+  TerminalSteps object.
+- **BehaviorSpec** — describes the shape of the observation data inside
+  DecisionSteps and TerminalSteps as well as the expected action shapes.
+
-An Agent Group is a group of Agents identified by a string name that share the same
-observations and action types. You can think about Agent Group as a group of agents
-that will share the same policy or behavior. All Agents in a group have the same goal
-and reward signals.
+An Agent "Behavior" is a group of Agents identified by a `BehaviorName` that share the same
+observations and action types (described in their `BehaviorSpec`). You can think about Agent
+Behavior as a group of agents that will share the same policy. All Agents with the same
+behavior have the same goal and reward signals.
-
-__Note__: The `Behavior Name` corresponds to the Agent Group name on Python.
 
 _Notice: Currently communication between Unity and Python takes place over an
 open socket without authentication. As such, please make sure that the network
    move forward until an Agent in the simulation needs a input from Python to act.
  - **Close : `env.close()`** Sends a shutdown signal to the environment and terminates
    the communication.
- - **Get Agent Group Names : `env.get_agent_groups()`** Returns a list of agent group ids.
+ - **Get Behavior Names : `env.get_behavior_names()`** Returns a list of `BehaviorName`.
-   agent groups are created in the simulation.
- - **Get Agent Group Spec : `env.get_agent_group_spec(agent_group: str)`** Returns
-   the `AgentGroupSpec` corresponding to the agent_group given as input. An
-   `AgentGroupSpec` contains information such as the observation shapes, the action
-   type (multi-discrete or continuous) and the action shape. Note that the `AgentGroupSpec`
+   Agent behaviors are created in the simulation.
+ - **Get Behavior Spec : `env.get_behavior_spec(behavior_name: str)`** Returns
+   the `BehaviorSpec` corresponding to the behavior_name given as input. A
+   `BehaviorSpec` contains information such as the observation shapes, the action
+   type (multi-discrete or continuous) and the action shape. Note that the `BehaviorSpec`
- - **Get Batched Step Result for Agent Group : `env.get_step_result(agent_group: str)`**
-   Returns a `BatchedStepResult` corresponding to the agent_group given as input.
-   A `BatchedStepResult` contains information about the state of the agents in a group
-   such as the observations, the rewards, the done flags and the agent identifiers. The
-   data is in `np.array` of which the first dimension is always the number of agents which
-   requested a decision in the simulation since the last call to `env.step()` note that the
-   number of agents is not guaranteed to remain constant during the simulation.
- - **Set Actions for Agent Group :`env.set_actions(agent_group: str, action: np.array)`**
+ - **Get Steps : `env.get_steps(behavior_name: str)`**
+   Returns a tuple `DecisionSteps, TerminalSteps` corresponding to the behavior_name
+   given as input.
+   The `DecisionSteps` contains information about the state of the agents
+   **that need an action this step** and have the behavior behavior_name.
+   The `TerminalSteps` contains information about the state of the agents
+   **whose episode ended** and have the behavior behavior_name.
+   Both `DecisionSteps` and `TerminalSteps` contain information such as
+   the observations, the rewards and the agent identifiers.
+   `DecisionSteps` also contains action masks for the next action while `TerminalSteps`
+   contains the reason for termination (did the Agent reach its maximum step and was
+   interrupted). The data is in `np.array` of which the first dimension is always the
+   number of agents note that the number of agents is not guaranteed to remain constant
+   during the simulation and it is not unusual to have either `DecisionSteps` or `TerminalSteps`
+   contain no Agents at all.
+ - **Set Actions :`env.set_actions(behavior_name: str, action: np.array)`**
    Sets the actions for a whole agent group. `action` is a 2D `np.array` of `dtype=np.int32`
    in the discrete action case and `dtype=np.float32` in the continuous action case.
    The first dimension of `action` is the number of agents that requested a decision
 
 __Note:__ If no action is provided for an agent group between two calls to `env.step()` then
 the default action will be all zeros (in either discrete or continuous action space)
-#### BathedStepResult and StepResult
+
+#### DecisionSteps and DecisionStep
+
+`DecisionSteps` (with `s`) contains information about a whole batch of Agents while
+`DecisionStep` (no `s`) only contains information about a single Agent.
-A `BatchedStepResult` has the following fields :
+A `DecisionSteps` has the following fields :
 
  - `obs` is a list of numpy arrays observations collected by the group of
  agent. The first dimension of the array corresponds to the batch size of
  rewards collected by each agent since the last simulation step.
  - `done` is an array of booleans of length batch size. Is true if the
  associated Agent was terminated during the last simulation step.
- - `max_step` is an array of booleans of length batch size. Is true if the
- associated Agent reached its maximum number of steps during the last
- simulation step.
  - `agent_id` is an int vector of length batch size containing unique
  identifier for the corresponding Agent. This is used to track Agents
  across simulation steps.
 
 It also has the two following methods:
 
- - `n_agents()` Returns the number of agents requesting a decision since
- the last call to `env.step()`
- - `get_agent_step_result(agent_id: int)` Returns a `StepResult`
+ - `len(DecisionSteps)` Returns the number of agents requesting a decision since
+ the last call to `env.step()`.
+ - `DecisionSteps[agent_id]` Returns a `DecisionStep`
-A `StepResult` has the following fields:
+A `DecisionStep` has the following fields:
- - `obs` is a list of numpy arrays observations collected by the group of
- agent. (Each array has one less dimension than the arrays in `BatchedStepResult`)
+ - `obs` is a list of numpy arrays observations collected by the agent.
+ (Each array has one less dimension than the arrays in `DecisionSteps`)
- - `max_step` is a bool. Is true if the Agent reached its maximum number of
- steps during the last simulation step.
  - `agent_id` is an int and an unique identifier for the corresponding Agent.
  - `action_mask` is an optional list of one dimensional array of booleans.
  Only available in multi-discrete action space type.
 
-#### AgentGroupSpec
+#### TerminalSteps and TerminalStep
+
+Similarly to `DecisionSteps` and `DecisionStep`,
+`TerminalSteps` (with `s`) contains information about a whole batch of Agents while
+`TerminalStep` (no `s`) only contains information about a single Agent.
-An Agent group can either have discrete or continuous actions. To check which type
+A `TerminalSteps` has the following fields :
+
+ - `obs` is a list of numpy arrays observations collected by the group of
+ agent. The first dimension of the array corresponds to the batch size of
+ the group (number of agents requesting a decision since the last call to
+ `env.step()`).
+ - `reward` is a float vector of length batch size. Corresponds to the
+ rewards collected by each agent since the last simulation step.
+ - `done` is an array of booleans of length batch size. Is true if the
+ associated Agent was terminated during the last simulation step.
+ - `agent_id` is an int vector of length batch size containing unique
+ identifier for the corresponding Agent. This is used to track Agents
+ across simulation steps.
+ - `max_step` is an array of booleans of length batch size. Is true if the
+ associated Agent reached its maximum number of steps during the last
+ simulation step.
+
+It also has the two following methods:
+
+ - `len(TerminalSteps)` Returns the number of agents requesting a decision since
+ the last call to `env.step()`.
+ - `TerminalSteps[agent_id]` Returns a `TerminalStep`
+ for the Agent with the `agent_id` unique identifier.
+
+A `TerminalStep` has the following fields:
+
+ - `obs` is a list of numpy arrays observations collected by the agent.
+ (Each array has one less dimension than the arrays in `TerminalSteps`)
+ - `reward` is a float. Corresponds to the rewards collected by the agent
+ since the last simulation step.
+ - `done` is a bool. Is true if the Agent was terminated during the last
+ simulation step.
+ - `agent_id` is an int and an unique identifier for the corresponding Agent.
+ - `max_step` is a bool. Is true if the Agent reached its maximum number of
+ steps during the last simulation step.
+
+
+#### BehaviorSpec
+
+An Agent behavior can either have discrete or continuous actions. To check which type
-An `AgentGroupSpec` has the following fields :
+A `BehaviorSpec` has the following fields :
- BatchedStepResult and StepResult.
+ DecisionSteps, DecisionStep, TerminalSteps and TerminalStep.
  - `action_type` is the type of data of the action. it can be discrete or
  continuous. If discrete, the action tensors are expected to be `np.int32`. If
  continuous, the actions are expected to be `np.float32`.
 
 
 ### Communicating additional information with the Environment
+
 In addition to the means of communicating between Unity and python described above,
 we also provide methods for sharing agent-agnostic information. These
 additional methods are referred to as side channels. ML-Agents includes two ready-made

docs/Training-ML-Agents.md

 specified, you will not be able to continue with training. Use `--force` to force ML-Agents to
 overwrite the existing data.
 
+Alternatively, you might want to start a new training run but _initialize_ it using an already-trained
+model. You may want to do this, for instance, if your environment changed and you want
+a new model, but the old behavior is still better than random. You can do this by specifying `--initialize-from=<run-identifier>`, where `<run-identifier>` is the old run ID.
+
 ### Command Line Training Options
 
 In addition to passing the path of the Unity executable containing your training
   as the current agents in your scene.
 * `--force`: Attempting to train a model with a run-id that has been used before will
   throw an error. Use `--force` to force-overwrite this run-id's summary and model data.
+* `--initialize-from=<run-identifier>`: Specify an old run-id here to initialize your model from
+  a previously trained model. Note that the previously saved models _must_ have the same behavior
+  parameters as your current environment.
 * `--no-graphics`: Specify this option to run the Unity executable in
   `-batchmode` and doesn't initialize the graphics driver. Use this only if your
   training doesn't involve visual observations (reading from Pixels). See
 * `--cpu`: Forces training using CPU only.
 * Engine Configuration :
-  * `--width' : The width of the executable window of the environment(s) in pixels
+  * `--width` : The width of the executable window of the environment(s) in pixels
   (ignored for editor training) (Default 84)
   * `--height` : The height of the executable window of the environment(s) in pixels
   (ignored for editor training). (Default 84)
 | train_interval       | How often to update the agent.                                                                                                                                                          | SAC                      |
 | num_update           | Number of mini-batches to update the agent with during each update.                                                                                                                     | SAC                      |
 | use_recurrent        | Train using a recurrent neural network. See [Using Recurrent Neural Networks](Feature-Memory.md).                                                                                       | PPO, SAC             |
+| init_path        | Initialize trainer from a previously saved model.                                                                                       | PPO, SAC             |
 
 \*PPO = Proximal Policy Optimization, SAC = Soft Actor-Critic, BC = Behavioral Cloning (Imitation), GAIL = Generative Adversarial Imitaiton Learning
 

docs/Training-PPO.md

 
 Typical Range: Approximately equal to PPO's `buffer_size`
 
+### (Optional) Advanced: Initialize Model Path
+
+`init_path` can be specified to initialize your model from a previous run before starting.
+Note that the prior run should have used the same trainer configurations as the current run,
+and have been saved with the same version of ML-Agents. You should provide the full path
+to the folder where the checkpoints were saved, e.g. `./models/{run-id}/{behavior_name}`.
+
+This option is provided in case you want to initialize different behaviors from different runs;
+in most cases, it is sufficient to use the `--initialize-from` CLI parameter to initialize
+all models from the same run.
+
 ## Training Statistics
 
 To view training statistics, use TensorBoard. For information on launching and

docs/Training-SAC.md

 
 Typical Range (Discrete): `32` - `512`
 
+### (Optional) Advanced: Initialize Model Path
+
+`init_path` can be specified to initialize your model from a previous run before starting.
+Note that the prior run should have used the same trainer configurations as the current run,
+and have been saved with the same version of ML-Agents. You should provide the full path
+to the folder where the checkpoints were saved, e.g. `./models/{run-id}/{behavior_name}`.
+
+This option is provided in case you want to initialize different behaviors from different runs;
+in most cases, it is sufficient to use the `--initialize-from` CLI parameter to initialize
+all models from the same run.
+
 ## Training Statistics
 
 To view training statistics, use TensorBoard. For information on launching and

docs/Training-Self-Play.md

 # Training with Self-Play
 
-ML-Agents provides the functionality to train symmetric, adversarial games with [Self-Play](https://openai.com/blog/competitive-self-play/).
-A symmetric game is one in which opposing agents are *equal* in form and function. In reinforcement learning,
-this means both agents have the same observation and action spaces.
-With self-play, an agent learns in adversarial games by competing against fixed, past versions of itself
-to provide a more stable, stationary learning environment. This is compared
-to competing against its current self in every episode, which is a constantly changing opponent.
+ML-Agents provides the functionality to train both symmetric and asymmetric adversarial games with
+[Self-Play](https://openai.com/blog/competitive-self-play/).
+A symmetric game is one in which opposing agents are equal in form, function and objective. Examples of symmetric games
+are our Tennis and Soccer example environments. In reinforcement learning, this means both agents have the same observation and
+action spaces and learn from the same reward function and so *they can share the same policy*. In asymmetric games,
+this is not the case. An example of an asymmetric games are Hide and Seek. Agents in these
+types of games do not always have the same observation or action spaces and so sharing policy networks is not
+necessarily ideal.
+
+With self-play, an agent learns in adversarial games by competing against fixed, past versions of its opponent
+(which could be itself as in symmetric games) to provide a more stable, stationary learning environment. This is compared
+to competing against the current, best opponent in every episode, which is constantly changing (because it's learning).
+However, from the perspective of an individual agent, these scenarios appear to have non-stationary dynamics because the opponent is often changing.
+This can cause significant issues in the experience replay mechanism used by SAC. Thus, we recommend that users use PPO. For further reading on
+this issue in particular, see the paper [Stabilising Experience Replay for Deep Multi-Agent Reinforcement Learning](https://arxiv.org/pdf/1702.08887.pdf).
 For more general information on training with ML-Agents, see [Training ML-Agents](Training-ML-Agents.md).
 For more algorithm specific instruction, please see the documentation for [PPO](Training-PPO.md) or [SAC](Training-SAC.md).
 
 
-See the trainer configuration and agent prefabs for our Tennis environment for an example.
+***Team ID must be 0 or an integer greater than 0.***
+
+In symmetric games, since all agents (even on opposing teams) will share the same policy, they should have the same 'Behavior Name' in their
+Behavior Parameters Script.  In asymmetric games, they should have a different Behavior Name in their Behavior Parameters script.
+Note, in asymmetric games, the agents must have both different Behavior Names *and* different team IDs! Then, specify the trainer configuration
+for each Behavior Name in your scene as you would normally, and remember to include the self-play hyperparameter hierarchy!
+
+For examples of how to use this feature, you can see the trainer configurations and agent prefabs for our Tennis and Soccer environments.
+Tennis and Soccer provide examples of symmetric games. To train an asymmetric game, specify trainer configurations for each of your behavior names
+and include the self-play hyperparameter hierarchy in both.
+
 
 ## Best Practices Training with Self-Play
 
 Training against a set of slowly or unchanging adversaries with low diversity
 results in a more stable learning process than training against a set of quickly
-changing adversaries with high diversity. With this context, this guide discusses the exposed self-play hyperparameters and intuitions for tuning them.
+changing adversaries with high diversity. With this context, this guide discusses
+the exposed self-play hyperparameters and intuitions for tuning them.
 
 
 ## Hyperparameters
 
 ### Save Steps
 
-The `save_steps` parameter corresponds to the number of *trainer steps* between snapshots.  For example, if `save_steps`=10000 then a snapshot of the current policy will be saved every 10000 trainer steps. Note, trainer steps are counted per agent. For more information, please see the [migration doc](Migrating.md) after v0.13.
+The `save_steps` parameter corresponds to the number of *trainer steps* between snapshots.  For example, if `save_steps=10000` then a snapshot of the current policy will be saved every `10000` trainer steps. Note, trainer steps are counted per agent. For more information, please see the [migration doc](Migrating.md) after v0.13.
+### Team Change
+
+The `team_change` parameter corresponds to the number of *trainer_steps* between switching the learning team.
+This is the number of trainer steps the teams associated with a specific ghost trainer will train before a different team
+becomes the new learning team. It is possible that, in asymmetric games, opposing teams require fewer trainer steps to make similar
+performance gains. This enables users to train a more complicated team of agents for more trainer steps than a simpler team of agents
+per team switch.
+
+A larger value of `team-change` will allow the agent to train longer against it's opponents.  The longer an agent trains against the same set of opponents
+the more able it will be to defeat them. However, training against them for too long may result in overfitting to the particular opponent strategies
+and so the agent may fail against the next batch of opponents.
+
+The value of `team-change` will determine how many snapshots of the agent's policy are saved to be used as opponents for the other team.  So, we
+recommend setting this value as a function of the `save_steps` parameter discussed previously.
+
+Recommended Range : 4x-10x where x=`save_steps`
+
+
-The `swap_steps` parameter corresponds to the number of *trainer steps* between swapping the opponents policy with a different snapshot. As in the `save_steps` discussion, note that trainer steps are counted per agent. For more information, please see the [migration doc](Migrating.md) after v0.13.
+The `swap_steps` parameter corresponds to the number of *ghost steps* (not trainer steps) between swapping the opponents policy with a different snapshot.
+A 'ghost step' refers to a step taken by an agent *that is following a fixed policy and not learning*. The reason for this distinction is that in asymmetric games,
+we may have teams with an unequal number of agents e.g. a 2v1 scenario. The team with two agents collects
+twice as many agent steps per environment step as the team with one agent.  Thus, these two values will need to be distinct to ensure that the same number
+of trainer steps corresponds to the same number of opponent swaps for each team. The formula for `swap_steps` if
+a user desires `x` swaps of a team with `num_agents` agents against an opponent team with `num_opponent_agents`
+agents during `team-change` total steps is:
+```
+swap_steps = (num_agents / num_opponent_agents) * (team_change / x)
+```
+
+As an example, in a 2v1 scenario, if we want the swap to occur `x=4` times during `team-change=200000` steps,
+the `swap_steps` for the team of one agent is:
+
+```
+swap_steps = (1 / 2) * (200000 / 4) = 25000
+```
+The `swap_steps` for the team of two agents is:
+```
+swap_steps = (2 / 1) * (200000 / 4) = 100000
+```
+Note, with equal team sizes, the first term is equal to 1 and `swap_steps` can be calculated by just dividing the total steps by the desired number of swaps.
-### Play against current self ratio
+### Play against latest model ratio
-The `play_against_current_self_ratio` parameter corresponds to the probability
-an agent will play against its ***current*** self. With probability
-1 - `play_against_current_self_ratio`, the agent will play against a snapshot of itself
-from a past iteration.
+The `play_against_latest_model_ratio` parameter corresponds to the probability
+an agent will play against the latest opponent policy. With probability
+1 - `play_against_latest_model_ratio`, the agent will play against a snapshot of its
+opponent from a past iteration.
-A larger value of `play_against_current_self_ratio` indicates that an agent will be playing against itself more often. Since the agent is updating it's policy, the opponent will be different from iteration to iteration.  This can lead to an unstable learning environment, but poses the agent with an [auto-curricula](https://openai.com/blog/emergent-tool-use/) of more increasingly challenging situations which may lead to a stronger final policy.
+A larger value of `play_against_latest_model_ratio` indicates that an agent will be playing against the current opponent more often. Since the agent is updating it's policy, the opponent will be different from iteration to iteration.  This can lead to an unstable learning environment, but poses the agent with an [auto-curricula](https://openai.com/blog/emergent-tool-use/) of more increasingly challenging situations which may lead to a stronger final policy.
-Recommended Range : 0.0 - 1.0
+Range : 0.0 - 1.0
 
 ### Window
 
 In adversarial games, the cumulative environment reward may not be a meaningful metric by which to track learning progress.  This is because cumulative reward is entirely dependent on the skill of the opponent. An agent at a particular skill level will get more or less reward against a worse or better agent, respectively.
 
 We provide an implementation of the ELO rating system, a method for calculating the relative skill level between two players from a given population in a zero-sum game. For more information on ELO, please see [the ELO wiki](https://en.wikipedia.org/wiki/Elo_rating_system).
+In a proper training run, the ELO of the agent should steadily increase. The absolute value of the ELO is less important than the change in ELO over training iterations.
-In a proper training run, the ELO of the agent should steadily increase. The absolute value of the ELO is less important than the change in ELO over training iterations.
+Note, this implementation will support any number of teams but ELO is only applicable to games with two teams.  It is ongoing work to implement
+a reliable metric for measuring progress in scenarios with three or more teams. These scenarios can still train, though as of now, reward and qualitative observations
+are the only metric by which we can judge performance.

gym-unity/README.md

 information on the gym interface, see [here](https://github.com/openai/gym).
 
 We provide a gym wrapper and instructions for using it with existing machine
-learning algorithms which utilize gyms. Both wrappers provide interfaces on top
+learning algorithms which utilize gym. Our wrapper provides interfaces on top
 of our `UnityEnvironment` class, which is the default way of interfacing with a
 Unity environment via Python.
 
 or by running the following from the `/gym-unity` directory of the repository:
 
 ```sh
-pip install .
+pip install -e .
 ```
 
 ## Using the Gym Wrapper
 ```python
 from gym_unity.envs import UnityEnv
 
-env = UnityEnv(environment_filename, worker_id, use_visual, uint8_visual, multiagent)
+env = UnityEnv(environment_filename, worker_id, use_visual, uint8_visual)
 ```
 
 *  `environment_filename` refers to the path to the Unity environment.
    (0-255). Many common Gym environments (e.g. Atari) do this. By default they
    will be floats (0.0-1.0). Defaults to `False`.
 
-*  `multiagent` refers to whether you intent to launch an environment which
-   contains more than one agent. Defaults to `False`.
-
 *  `flatten_branched` will flatten a branched discrete action space into a Gym Discrete.
    Otherwise, it will be converted into a MultiDiscrete. Defaults to `False`.
 
 
 ## Limitations
 
-* It is only possible to use an environment with a single Agent.
+* It is only possible to use an environment with a **single** Agent.
-* The `BatchedStepResult` output from the environment can still be accessed from the
-  `info` provided by `env.step(action)`.
+* The `TerminalSteps` or `DecisionSteps` output from the environment can still be
+accessed from the `info` provided by `env.step(action)`.
 * Stacked vector observations are not supported.
 * Environment registration for use with `gym.make()` is currently not supported.
 

gym-unity/gym_unity/envs/__init__.py

 from gym import error, spaces
 
 from mlagents_envs.environment import UnityEnvironment
-from mlagents_envs.base_env import BatchedStepResult
+from mlagents_envs.base_env import DecisionSteps, TerminalSteps
 from mlagents_envs import logging_util
 
 
 logger = logging_util.get_logger(__name__)
 logging_util.set_log_level(logging_util.INFO)
 
-GymSingleStepResult = Tuple[np.ndarray, float, bool, Dict]
-GymMultiStepResult = Tuple[List[np.ndarray], List[float], List[bool], Dict]
-GymStepResult = Union[GymSingleStepResult, GymMultiStepResult]
+GymStepResult = Tuple[np.ndarray, float, bool, Dict]
-    Multi-agent environments use lists for object types, as done here:
-    https://github.com/openai/multiagent-particle-envs
     """
 
     def __init__(
         use_visual: bool = False,
         uint8_visual: bool = False,
-        multiagent: bool = False,
         flatten_branched: bool = False,
         no_graphics: bool = False,
         allow_multiple_visual_obs: bool = False,
         :param worker_id: Worker number for environment.
         :param use_visual: Whether to use visual observation or vector observation.
         :param uint8_visual: Return visual observations as uint8 (0-255) matrices instead of float (0.0-1.0).
-        :param multiagent: Whether to run in multi-agent mode (lists of obs, reward, done).
         :param flatten_branched: If True, turn branched discrete action spaces into a Discrete space rather than
             MultiDiscrete.
         :param no_graphics: Whether to run the Unity simulator in no-graphics mode
         )
 
         # Take a single step so that the brain information will be sent over
-        if not self._env.get_agent_groups():
+        if not self._env.get_behavior_names():
-        self.agent_mapper = AgentIdIndexMapper()
-
-        self._previous_step_result: BatchedStepResult = None
-        self._multiagent = multiagent
+        self._previous_decision_step: DecisionSteps = None
         self._flattener = None
         # Hidden flag used by Atari environments to determine if the game is over
         self.game_over = False
-        if len(self._env.get_agent_groups()) != 1:
+        if len(self._env.get_behavior_names()) != 1:
-                "There can only be one brain in a UnityEnvironment "
+                "There can only be one behavior in a UnityEnvironment "
-        self.brain_name = self._env.get_agent_groups()[0]
-        self.name = self.brain_name
-        self.group_spec = self._env.get_agent_group_spec(self.brain_name)
+        self.name = self._env.get_behavior_names()[0]
+        self.group_spec