refactor PushBlockTeamManager

3 年前 · a79d484d
--- a/Project/Assets/ML-Agents/Examples/PushBlock/Scenes/2ZombieVs3AgentsPushBlock.unity
+++ b/Project/Assets/ML-Agents/Examples/PushBlock/Scenes/2ZombieVs3AgentsPushBlock.unity
  m_Script: {fileID: 11500000, guid: 7df9fa4d08b7d4d198d5165b07ddd47e, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  MaxEnvironmentSteps: 25000
+  MaxEnvironmentSteps: 5000
  areaBounds:
    m_Center: {x: 0, y: 0, z: 0}
    m_Extent: {x: 0, y: 0, z: 0}
  UseRandomAgentPosition: 1
  UseRandomBlockRotation: 1
  UseRandomBlockPosition: 1
+  UseTeamManager: 1
+  UseTeamReward: 1
 --- !u!4 &35422107
 Transform:
  m_ObjectHideFlags: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 1
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 2
  m_BehaviorName: 
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 1
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 2
  m_BehaviorName: 
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 2
  m_BehaviorName: 
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 2
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 2
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 3
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 1
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_Script: {fileID: 11500000, guid: 7df9fa4d08b7d4d198d5165b07ddd47e, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  MaxEnvironmentSteps: 25000
+  MaxEnvironmentSteps: 5000
  areaBounds:
    m_Center: {x: 0, y: 0, z: 0}
    m_Extent: {x: 0, y: 0, z: 0}
  UseRandomAgentPosition: 1
  UseRandomBlockRotation: 1
  UseRandomBlockPosition: 1
+  UseTeamManager: 1
+  UseTeamReward: 1
 --- !u!4 &1056693890
 Transform:
  m_ObjectHideFlags: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 2
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 3
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 3
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_Script: {fileID: 11500000, guid: 7df9fa4d08b7d4d198d5165b07ddd47e, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  MaxEnvironmentSteps: 25000
+  MaxEnvironmentSteps: 5000
  areaBounds:
    m_Center: {x: 0, y: 0, z: 0}
    m_Extent: {x: 0, y: 0, z: 0}
  UseRandomAgentPosition: 1
  UseRandomBlockRotation: 1
  UseRandomBlockPosition: 1
+  UseTeamManager: 1
+  UseTeamReward: 1
 --- !u!4 &2124347356
 Transform:
  m_ObjectHideFlags: 0
  m_Script: {fileID: 11500000, guid: 7df9fa4d08b7d4d198d5165b07ddd47e, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  MaxEnvironmentSteps: 25000
+  MaxEnvironmentSteps: 5000
  areaBounds:
    m_Center: {x: 0, y: 0, z: 0}
    m_Extent: {x: 0, y: 0, z: 0}
  UseRandomAgentPosition: 1
  UseRandomBlockRotation: 1
  UseRandomBlockPosition: 1
+  UseTeamManager: 1
+  UseTeamReward: 1
 --- !u!1 &1257456948449766
 GameObject:
  m_ObjectHideFlags: 0
  m_BehaviorType: 0
  m_BehaviorName: PushBlock
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
  m_BehaviorType: 2
  m_BehaviorName: 
  TeamId: 0
-  GroupId: 0
  m_UseChildSensors: 1
  m_UseChildActuators: 1
  m_ObservableAttributeHandling: 0
--- a/Project/Assets/ML-Agents/Examples/PushBlock/Scripts/PushBlockTeamManager.cs
+++ b/Project/Assets/ML-Agents/Examples/PushBlock/Scripts/PushBlockTeamManager.cs
 using System.Collections.Generic;
 using Unity.MLAgents;
+using System.Linq;
+using UnityEngine;
 using Unity.MLAgents.Extensions.Teams;
 using Unity.MLAgents.Sensors;

    public override void OnAgentDone(Agent agent, Agent.DoneReason doneReason, List<ISensor> sensors)
    {
        m_AgentDoneState[agent] = true;
-        if (!m_AgentDoneState.ContainsValue(false))
+    }
+
+    public void OnTeamDone()
+    {
+        foreach (var agent in m_AgentDoneState.Keys.ToList())
-            foreach (var doneAgent in m_AgentDoneState.Keys)
+            if (m_AgentDoneState[agent])
-                doneAgent.SendDoneToTrainer();
-                m_AgentDoneState[doneAgent] = false;
+                agent.SendDoneToTrainer();
+                m_AgentDoneState[agent] = false;
-    public override void AddTeamReward(float reward)
+    public void AddTeamReward(float reward)
-            agent.AddReward(reward);
+            if (m_AgentDoneState[agent])
+            {
+                agent.AddRewardAfterDeath(reward);
+            }
+            else
+            {
+                agent.AddReward(reward);
+            }
        }
    }
 }
--- a/Project/Assets/ML-Agents/Examples/PushBlock/Scripts/ZombiePushBlockDeathEnvController.cs
+++ b/Project/Assets/ML-Agents/Examples/PushBlock/Scripts/ZombiePushBlockDeathEnvController.cs
    void FixedUpdate()
    {
        m_ResetTimer += 1;
-        if (m_ResetTimer > MaxEnvironmentSteps)
+        if (m_ResetTimer >= MaxEnvironmentSteps)
        {
            ResetScene();
        }
    public void KillAgent(Collision col, Transform t)
    {
        // print($"Zombie {t.gameObject.GetInstanceID()} ate Agent {col.gameObject.GetInstanceID()}");
-        //End Episode
-        foreach (var item in AgentsList)
-        {
-            if (!item.Agent)
-            {
-                return;
-            }
-            // item.Agent.EndEpisode();
-        }

        //Disable killed Agent
        foreach (var item in AgentsList)
        //Give Agent Rewards
        if (UseTeamManager && UseTeamReward)
        {
-            m_TeamManager.AddTeamReward(score);
+            var pushManager = (PushBlockTeamManager)m_TeamManager;
+            pushManager.AddTeamReward(score);
        }
        else
        {
        //Give Agent Rewards
        if (UseTeamManager && UseTeamReward)
        {
-            m_TeamManager.AddTeamReward(-1);
+            var pushManager = (PushBlockTeamManager)m_TeamManager;
+            pushManager.AddTeamReward(-1);
        }
        else
        {
            {
                return;
            }
-            item.Agent.EndEpisode();
+
+            // not disabling agent here might cause problem with the done state
+            // item.Agent.EndEpisode();
+            item.Agent.gameObject.SetActive(false);
+        }
+
+        // OnTeamDone has to be called after agents has called EndEpisode or been disabled.
+        if (UseTeamManager && UseTeamReward)
+        {
+            var pushManager = (PushBlockTeamManager)m_TeamManager;
+            pushManager.OnTeamDone();
+
        //Reset Agents
        foreach (var item in AgentsList)
        {
--- a/com.unity.ml-agents.extensions/Runtime/Teams/BaseTeamManager.cs
+++ b/com.unity.ml-agents.extensions/Runtime/Teams/BaseTeamManager.cs
            agent.SendDoneToTrainer();
        }

-        public virtual void AddTeamReward(float reward)
-        {
-
-        }
-
        public string GetId()
        {
            return m_Id;
--- a/com.unity.ml-agents/Runtime/Agent.cs
+++ b/com.unity.ml-agents/Runtime/Agent.cs

        public void SendDoneToTrainer()
        {
-            // m_Info.reward = m_Reward;
-            // m_Reward = 0f;
        }

        /// <summary>
 #endif
            m_Reward += increment;
            m_CumulativeReward += increment;
+        }
+
+        public void AddRewardAfterDeath(float increment)
+        {
+            m_Info.reward += increment;
        }

        /// <summary>