Make GridWorld a goal-based environment

3 年前 · 95fd8040
--- a/Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/Area.prefab
+++ b/Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/Area.prefab
  - component: {fileID: 114650561397225712}
  - component: {fileID: 114889700908650620}
  - component: {fileID: 7980686505185502968}
+  - component: {fileID: 8154757885281235403}
  m_Layer: 8
  m_Name: Agent
  m_TagString: agent
  m_Name: 
  m_EditorClassIdentifier: 
  m_BrainParameters:
-    vectorObservationSize: 0
-    numStackedVectorObservations: 1
-    vectorActionSize: 05000000
-    vectorActionDescriptions: []
-    vectorActionSpaceType: 0
+    VectorObservationSize: 0
+    NumStackedVectorObservations: 1
+    m_ActionSpec:
+      m_NumContinuousActions: 0
+      BranchSizes: 05000000
+    VectorActionSize: 05000000
+    VectorActionDescriptions: []
+    VectorActionSpaceType: 0
+    hasUpgradedBrainParametersWithActionSpec: 1
  m_Model: {fileID: 11400000, guid: a812f1ce7763a4a0c912717f3594fe20, type: 3}
  m_InferenceDevice: 0
  m_BehaviorType: 0
+  m_UseChildActuators: 1
+  m_ObservableAttributeHandling: 0
 --- !u!114 &114650561397225712
 MonoBehaviour:
  m_ObjectHideFlags: 0
  agentParameters:
    maxStep: 0
  hasUpgradedFromAgentParameters: 1
-  maxStep: 100
+  MaxStep: 100
+  gridGoal: 0
  maskActions: 1
 --- !u!114 &114889700908650620
 MonoBehaviour:
  m_Width: 84
  m_Height: 64
  m_Grayscale: 0
+  m_ObservationStacks: 1
  m_Compression: 1
 --- !u!114 &7980686505185502968
 MonoBehaviour:
  m_Script: {fileID: 11500000, guid: 3a6da8f78a394c6ab027688eab81e04d, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
+  debugCommandLineOverride: 
+--- !u!114 &8154757885281235403
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1488387672112076}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: fd48a95d788a348b0b7a8cefd39e7c27, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  observationSize: 2
 --- !u!1 &1625008366184734
 GameObject:
  m_ObjectHideFlags: 0
  trueAgent: {fileID: 1488387672112076}
  goalPref: {fileID: 1508142483324970, guid: 1ec4e4e96e7514d45b7ebc3ba5a9a481, type: 3}
  pitPref: {fileID: 1811317785436014, guid: d13ee2db77b3a4dcc8664d2fe2a0f219, type: 3}
+  numberOfObstacles: 1
 --- !u!1 &1656910849934022
 GameObject:
  m_ObjectHideFlags: 0
--- a/Project/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity
+++ b/Project/Assets/ML-Agents/Examples/GridWorld/Scenes/GridWorld.unity
  m_ReflectionIntensity: 1
  m_CustomReflection: {fileID: 0}
  m_Sun: {fileID: 0}
-  m_IndirectSpecularColor: {r: 0.44971228, g: 0.49977815, b: 0.57563734, a: 1}
+  m_IndirectSpecularColor: {r: 0.44971168, g: 0.4997775, b: 0.57563686, a: 1}
  m_UseRadianceAmbientProbe: 0
 --- !u!157 &3
 LightmapSettings:
  m_LocalScale: {x: 1, y: 1, z: 1}
  m_Children: []
  m_Father: {fileID: 0}
-  m_RootOrder: 5
+  m_RootOrder: 4
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &99095112
 GameObject:
  m_LocalScale: {x: 1, y: 1, z: 1}
  m_Children: []
  m_Father: {fileID: 0}
-  m_RootOrder: 4
+  m_RootOrder: 3
  m_LocalEulerAnglesHint: {x: 45, y: 45, z: 0}
 --- !u!1 &125487785
 GameObject:
  - component: {fileID: 125487790}
  - component: {fileID: 125487787}
  - component: {fileID: 125487791}
+  - component: {fileID: 125487792}
  m_Layer: 8
  m_Name: RenderTextureAgent
  m_TagString: agent
  area: {fileID: 1795599557}
  timeBetweenDecisionsAtInference: 0.15
  renderCamera: {fileID: 797520692}
+  gridGoal: 0
  maskActions: 1
 --- !u!65 &125487788
 BoxCollider:
  m_BrainParameters:
    VectorObservationSize: 0
    NumStackedVectorObservations: 1
+    m_ActionSpec:
+      m_NumContinuousActions: 0
+      BranchSizes: 05000000
+    hasUpgradedBrainParametersWithActionSpec: 1
  m_Model: {fileID: 11400000, guid: a812f1ce7763a4a0c912717f3594fe20, type: 3}
  m_InferenceDevice: 0
  m_BehaviorType: 0
  m_Grayscale: 0
  m_ObservationStacks: 1
  m_Compression: 1
+--- !u!114 &125487792
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 125487785}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: fd48a95d788a348b0b7a8cefd39e7c27, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  observationSize: 2
 --- !u!1 &260425459
 GameObject:
  m_ObjectHideFlags: 0
  - {fileID: 918893359}
  - {fileID: 1305247360}
  m_Father: {fileID: 0}
-  m_RootOrder: 3
+  m_RootOrder: 2
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
  m_AnchorMin: {x: 0, y: 0}
  m_AnchorMax: {x: 0, y: 0}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 13
+      value: 12
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 7
+      value: 6
      objectReference: {fileID: 0}
    - target: {fileID: 114650561397225712, guid: 5c2bd19e4bbda4991b74387ca5d28156,
        type: 3}
    - target: {fileID: 114889700908650620, guid: 5c2bd19e4bbda4991b74387ca5d28156,
        type: 3}
      propertyPath: compression
+      value: 0
+      objectReference: {fileID: 0}
+    - target: {fileID: 114889700908650620, guid: 5c2bd19e4bbda4991b74387ca5d28156,
+        type: 3}
+      propertyPath: m_Compression
      value: 0
      objectReference: {fileID: 0}
    m_RemovedComponents: []
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5889392e3f05b448a8a06c5def6c2dec, type: 3}
--- !u!1 &742849316
-GameObject:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  serializedVersion: 6
-  m_Component:
-  - component: {fileID: 742849319}
-  - component: {fileID: 742849318}
-  - component: {fileID: 742849317}
-  m_Layer: 0
-  m_Name: EventSystem
-  m_TagString: Untagged
-  m_Icon: {fileID: 0}
-  m_NavMeshLayer: 0
-  m_StaticEditorFlags: 0
-  m_IsActive: 1
--- !u!114 &742849317
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 742849316}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 1077351063, guid: f5f67c52d1564df4a8936ccd202a3bd8, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  m_HorizontalAxis: Horizontal
-  m_VerticalAxis: Vertical
-  m_SubmitButton: Submit
-  m_CancelButton: Cancel
-  m_InputActionsPerSecond: 10
-  m_RepeatDelay: 0.5
-  m_ForceModuleActive: 0
--- !u!114 &742849318
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 742849316}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: -619905303, guid: f5f67c52d1564df4a8936ccd202a3bd8, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  m_FirstSelected: {fileID: 0}
-  m_sendNavigationEvents: 1
-  m_DragThreshold: 5
--- !u!4 &742849319
-Transform:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 742849316}
-  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
-  m_LocalPosition: {x: 0, y: 0, z: 0}
-  m_LocalScale: {x: 1, y: 1, z: 1}
-  m_Children: []
-  m_Father: {fileID: 0}
-  m_RootOrder: 2
-  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1001 &790097508
 PrefabInstance:
  m_ObjectHideFlags: 0
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 9
+      value: 8
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 12
+      value: 11
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 11
+      value: 10
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 10
+      value: 9
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 8
+      value: 7
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
  - {fileID: 125487786}
  - {fileID: 797520693}
  m_Father: {fileID: 0}
-  m_RootOrder: 6
+  m_RootOrder: 5
  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &1892751309
 GameObject:
      objectReference: {fileID: 0}
    - target: {fileID: 4124767863011510, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
      propertyPath: m_RootOrder
-      value: 14
+      value: 13
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 5c2bd19e4bbda4991b74387ca5d28156, type: 3}
--- a/Project/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs
+++ b/Project/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs
 using UnityEngine;
 using System.Linq;
 using Unity.MLAgents;
+using Unity.MLAgents.Sensors;
 using Unity.MLAgents.Actuators;
 using UnityEngine.Serialization;

        "a camera to render before making a decision. Place the agentCam here if using " +
        "RenderTexture as observations.")]
    public Camera renderCamera;
+    GoalSensorComponent goalSensor;
+
+    public enum GridGoal
+    {
+        Plus,
+        Cross,
+    }
+
+    public GridGoal gridGoal;

    [Tooltip("Selecting will turn on action masking. Note that a model trained with action " +
        "masking turned on may not behave optimally when action masking is turned off.")]
    public override void Initialize()
    {
        m_ResetParams = Academy.Instance.EnvironmentParameters;
+    }
+
+    public override void CollectObservations(VectorSensor sensor)
+    {
+        Array values = Enum.GetValues(typeof(GridGoal));
+        int goalNum = (int)gridGoal;
+        goalSensor = this.GetComponent<GoalSensorComponent>();
+        goalSensor.AddOneHotGoal(goalNum, values.Length);
    }

    public override void WriteDiscreteActionMask(IDiscreteActionMask actionMask)

            if (hit.Where(col => col.gameObject.CompareTag("goal")).ToArray().Length == 1)
            {
-                SetReward(1f);
+                ProvideReward(GridGoal.Plus);
-                SetReward(-1f);
+                ProvideReward(GridGoal.Cross);
+        }
+    }
+
+    private void ProvideReward(GridGoal hitObject)
+    {
+        if (gridGoal == hitObject)
+        {
+            SetReward(1f);
+        }
+        else
+        {
+            SetReward(-1f);
        }
    }

--- a/com.unity.ml-agents/Runtime/Sensors/ITypedSensor.cs
+++ b/com.unity.ml-agents/Runtime/Sensors/ITypedSensor.cs
    /// <summary>
    /// The ObservationType enum of the Sensor.
    /// </summary>
-    internal enum ObservationType
+    public enum ObservationType
    {
        // Collected observations are generic.
        Default = 0,
    /// <summary>
    /// Sensor interface for sensors with variable types.
    /// </summary>
-    internal interface ITypedSensor
+    public interface ITypedSensor
    {
        /// <summary>
        /// Returns the ObservationType enum corresponding to the type of the sensor.
--- a/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs
+++ b/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs
    /// <summary>
    /// A sensor implementation for vector observations.
    /// </summary>
-    public class VectorSensor : ISensor
+    public class VectorSensor : ISensor, ITypedSensor
    {
        // TODO use float[] instead
        // TODO allow setting float[]
        public int[] GetObservationShape()
        {
            return m_Shape;
+        }
+
+        /// <inheritdoc/>
+        public virtual ObservationType GetObservationType()
+        {
+            return ObservationType.Default;
        }

        /// <inheritdoc/>
--- a/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/GoalSensorComponent.cs
+++ b/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/GoalSensorComponent.cs
+using Unity.MLAgents.Sensors;
+
+
+public class GoalSensorComponent : SensorComponent
+{
+    public int observationSize;
+    public GoalSensor goalSensor;
+    /// <summary>
+    /// Creates a GoalSensor.
+    /// </summary>
+    /// <returns></returns>
+    public override ISensor CreateSensor()
+    {
+        goalSensor = new GoalSensor(observationSize);
+        return goalSensor;
+    }
+
+    /// <inheritdoc/>
+    public override int[] GetObservationShape()
+    {
+        return new[] { observationSize };
+    }
+
+    public void AddGoal(float goal)
+    {
+        if (goalSensor != null)
+        {
+            goalSensor.AddObservation(goal);
+        }
+    }
+
+    public void AddOneHotGoal(int goal, int range)
+    {
+        if (goalSensor != null)
+        {
+            goalSensor.AddOneHotObservation(goal, range);
+        }
+    }
+}
+
+public class GoalSensor : VectorSensor
+{
+
+    public GoalSensor(int observationSize, string name = null) : base(observationSize)
+    {
+        if (name == null)
+        {
+            name = $"GoalSensor_size{observationSize}";
+        }
+    }
+
+    public override ObservationType GetObservationType()
+    {
+        return ObservationType.Goal;
+    }
+}
--- a/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/GoalSensorComponent.cs.meta
+++ b/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/GoalSensorComponent.cs.meta
+fileFormatVersion: 2
+guid: fd48a95d788a348b0b7a8cefd39e7c27
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: