Clear ActionBuffers before Heuristic calls (#5227)

Project/Assets/ML-Agents/Examples/DungeonEscape/Scripts/PushAgentEscape.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[0] = 3;

Project/Assets/ML-Agents/Examples/FoodCollector/Scripts/FoodCollectorAgent.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var continuousActionsOut = actionsOut.ContinuousActions;
-        continuousActionsOut[0] = 0;
-        continuousActionsOut[1] = 0;
-        continuousActionsOut[2] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             continuousActionsOut[2] = 1;

Project/Assets/ML-Agents/Examples/Hallway/Scripts/HallwayAgent.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[0] = 3;

Project/Assets/ML-Agents/Examples/PushBlock/Scripts/PushAgentBasic.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[0] = 3;

Project/Assets/ML-Agents/Examples/PushBlock/Scripts/PushAgentCollab.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[0] = 3;

Project/Assets/ML-Agents/Examples/Pyramids/Scripts/PyramidAgent.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[0] = 3;

Project/Assets/ML-Agents/Examples/Soccer/Scripts/AgentSoccer.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut.Clear();
         //forward
         if (Input.GetKey(KeyCode.W))
         {

Project/Assets/ML-Agents/Examples/Sorter/Scripts/SorterAgent.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut.Clear();
         //forward
         if (Input.GetKey(KeyCode.W))
         {

Project/Assets/ML-Agents/Examples/WallJump/Scripts/WallJumpAgent.cs

     public override void Heuristic(in ActionBuffers actionsOut)
     {
         var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut.Clear();
         if (Input.GetKey(KeyCode.D))
         {
             discreteActionsOut[1] = 2;

com.unity.ml-agents/CHANGELOG.md

 - `DecisionRequester.ShouldRequestDecision()` and `ShouldRequestAction()`methods were added. These are used to
 determine whether `Agent.RequestDecision()` and `Agent.RequestAction()` are called (respectively). (#5223)
 - `RaycastPerceptionSensor` now caches its raycast results; they can be accessed via `RayPerceptionSensor.RayPerceptionOutput`. (#5222)
+- `ActionBuffers` are now reset to zero before being passed to `Agent.Heuristic()` and
+`IHeuristicProvider.Heuristic()`. (#5227)
 
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 - Some console output have been moved from `info` to `debug` and will not be printed by default. If you want all messages to be printed, you can run `mlagents-learn` with the `--debug` option or add the line `debug: true` at the top of the yaml config file. (#5211)

com.unity.ml-agents/Runtime/Policies/HeuristicPolicy.cs

         {
             if (!m_Done && m_DecisionRequested)
             {
+                m_ActionBuffers.Clear();
                 m_ActuatorManager.ApplyHeuristic(m_ActionBuffers);
             }
             m_DecisionRequested = false;

com.unity.ml-agents/Tests/Editor/Policies.meta

+fileFormatVersion: 2
+guid: df271cac120e4d6893b14599fa8eb64d
+timeCreated: 1617813392

com.unity.ml-agents/Tests/Editor/Policies/HeuristicPolicyTest.cs

+using NUnit.Framework;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Policies;
+using UnityEngine;
+
+namespace Unity.MLAgents.Tests.Policies
+{
+    [TestFixture]
+    public class HeuristicPolicyTest
+    {
+        [SetUp]
+        public void SetUp()
+        {
+            if (Academy.IsInitialized)
+            {
+                Academy.Instance.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Assert that the action buffers are initialized to zero, and then set them to non-zero values.
+        /// </summary>
+        /// <param name="actionsOut"></param>
+        static void CheckAndSetBuffer(in ActionBuffers actionsOut)
+        {
+            var continuousActions = actionsOut.ContinuousActions;
+            for (var continuousIndex = 0; continuousIndex < continuousActions.Length; continuousIndex++)
+            {
+                Assert.AreEqual(continuousActions[continuousIndex], 0.0f);
+                continuousActions[continuousIndex] = 1.0f;
+            }
+
+            var discreteActions = actionsOut.DiscreteActions;
+            for (var discreteIndex = 0; discreteIndex < discreteActions.Length; discreteIndex++)
+            {
+                Assert.AreEqual(discreteActions[discreteIndex], 0);
+                discreteActions[discreteIndex] = 1;
+            }
+        }
+
+
+        class ActionClearedAgent : Agent
+        {
+            public int HeuristicCalls = 0;
+            public override void Heuristic(in ActionBuffers actionsOut)
+            {
+                CheckAndSetBuffer(actionsOut);
+                HeuristicCalls++;
+            }
+        }
+
+        class ActionClearedActuator : IActuator
+        {
+            public int HeuristicCalls = 0;
+            public ActionClearedActuator(ActionSpec actionSpec)
+            {
+                ActionSpec = actionSpec;
+                Name = GetType().Name;
+            }
+
+            public void OnActionReceived(ActionBuffers actionBuffers)
+            {
+            }
+
+            public void WriteDiscreteActionMask(IDiscreteActionMask actionMask)
+            {
+            }
+
+            public void Heuristic(in ActionBuffers actionBuffersOut)
+            {
+                CheckAndSetBuffer(actionBuffersOut);
+                HeuristicCalls++;
+            }
+
+            public ActionSpec ActionSpec { get; }
+            public string Name { get; }
+
+            public void ResetData()
+            {
+
+            }
+        }
+
+        class ActionClearedActuatorComponent : ActuatorComponent
+        {
+            public ActionClearedActuator ActionClearedActuator;
+            public ActionClearedActuatorComponent()
+            {
+                ActionSpec = new ActionSpec(2, new[] { 3, 3 });
+            }
+
+            public override IActuator[] CreateActuators()
+            {
+                ActionClearedActuator = new ActionClearedActuator(ActionSpec);
+                return new IActuator[] { ActionClearedActuator };
+            }
+
+            public override ActionSpec ActionSpec { get; }
+        }
+
+        [Test]
+        public void TestActionsCleared()
+        {
+            var gameObj = new GameObject();
+            var agent = gameObj.AddComponent<ActionClearedAgent>();
+            var behaviorParameters = agent.GetComponent<BehaviorParameters>();
+            behaviorParameters.BrainParameters.ActionSpec = new ActionSpec(1, new[] { 4 });
+            behaviorParameters.BrainParameters.VectorObservationSize = 0;
+            behaviorParameters.BehaviorType = BehaviorType.HeuristicOnly;
+
+            var actuatorComponent = gameObj.AddComponent<ActionClearedActuatorComponent>();
+            agent.LazyInitialize();
+
+            const int k_NumSteps = 5;
+            for (var i = 0; i < k_NumSteps; i++)
+            {
+                agent.RequestDecision();
+                Academy.Instance.EnvironmentStep();
+            }
+
+            Assert.AreEqual(agent.HeuristicCalls, k_NumSteps);
+            Assert.AreEqual(actuatorComponent.ActionClearedActuator.HeuristicCalls, k_NumSteps);
+        }
+    }
+}

com.unity.ml-agents/Tests/Editor/Policies/HeuristicPolicyTest.cs.meta

+fileFormatVersion: 2
+guid: 5108e92f91a04ddab9d628c9bc57cadb
+timeCreated: 1617813411