sequence env

Project/Assets/ML-Agents/Examples/Arena-Sequence/Scripts/SequenceTile.cs

 public class SequenceTile : MonoBehaviour
 {
     public int NumberValue;
+    [HideInInspector]
+    public bool visited = false;
 
     // [HideInInspector]
     public MeshRenderer rend;

Project/Assets/ML-Agents/Examples/Arena-Sequence/Scripts/SequencerAgent.cs

     // }
     public bool SelectNewTiles;
 
-    public int NumberOfTilesToSpawn;
+    int m_NumberOfTilesToSpawn;
+    public int MaxNumberOfTiles;
+    BufferSensorComponent m_BufferSensor;
     public List<SequenceTile> SequenceTilesList = new List<SequenceTile>();
     public List<SequenceTile> CurrentlyVisibleTilesList = new List<SequenceTile>();
     private List<Transform> AlreadyTouchedList = new List<Transform>();
 
+    public GameObject area;
 
 
     // private SequenceTile m_NextExpectedTile;
     public override void Initialize()
     {
+        m_BufferSensor = GetComponent<BufferSensorComponent>();
         m_PushBlockSettings = FindObjectOfType<PushBlockSettings>();
         m_AgentRb = GetComponent<Rigidbody>();
         m_StartingPos = transform.position;
     public override void OnEpisodeBegin()
     {
 
+        m_NumberOfTilesToSpawn = Random.Range(1, MaxNumberOfTiles);
         SelectTilesToShow();
         SetTilePositions();
 
 
     private void Update()
     {
-        //DEBUG
-        if (SelectNewTiles)
-        {
-            SelectNewTiles = false;
-            SelectTilesToShow();
-            SetTilePositions();
-        }
+        ////DEBUG
+        //if (SelectNewTiles)
+        //{
+        //    SelectNewTiles = false;
+        //    SelectTilesToShow();
+        //    SetTilePositions();
+        //}
     }
 
 
         // {
         //     sensor.AddObservation(item.transform.localRotation.y / 360);
         // }
+        sensor.AddObservation((transform.position.x - area.transform.position.x) / 20f);
+        sensor.AddObservation((transform.position.z - area.transform.position.z) / 20f);
+
+        sensor.AddObservation(transform.forward.x);
+        sensor.AddObservation(transform.forward.z);
+
+        foreach (var item in CurrentlyVisibleTilesList)
+        {
+       
+            float[] listObservation = new float[MaxNumberOfTiles + 2];
+            listObservation[item.NumberValue] = 1.0f;
+            listObservation[MaxNumberOfTiles] = (item.transform.localRotation.eulerAngles.y / 360f);
+            listObservation[MaxNumberOfTiles + 1] = item.visited ? 1.0f : 0.0f; 
+            //Debug.Log(listObservation[20]);
+            //Debug.Log(listObservation[21]);
+            //Debug.Log(listObservation[22]);
+            m_BufferSensor.AppendObservation(listObservation);
+
+        };
+
     }
 
     private void OnCollisionEnter(Collision col)
             //failed
             AddReward(-1);
             EndEpisode();
-            print("no");
+            //print("no");
-            print("yes");
+            //print("yes");
+            tile.visited = true;
-            if (m_NextExpectedTileIndex == NumberOfTilesToSpawn)
+            if (m_NextExpectedTileIndex == m_NumberOfTilesToSpawn)
             {
                 EndEpisode();
             }
         //Disable all. We will enable the ones selected
         foreach (var item in SequenceTilesList)
         {
+            item.visited = false;
             item.gameObject.SetActive(false);
         }
 
         CurrentlyVisibleTilesList.Clear();
         AlreadyTouchedList.Clear();
 
-        int numLeft = NumberOfTilesToSpawn;
+        int numLeft = m_NumberOfTilesToSpawn;
-            int rndInt = Random.Range(0, SequenceTilesList.Count);
+            int rndInt = Random.Range(0, MaxNumberOfTiles);
             var tmp = SequenceTilesList[rndInt];
             if (!CurrentlyVisibleTilesList.Contains(tmp))
             {
 
         //Sort Ascending
         CurrentlyVisibleTilesList.Sort((x, y) => x.NumberValue.CompareTo(y.NumberValue));
+
         // m_NextExpectedTile = CurrentlyVisibleTilesList[0];
         m_NextExpectedTileIndex = 0;
     }
         var dirToGo = Vector3.zero;
         var rotateDir = Vector3.zero;
 
-        var action = act[0];
+        var forwardAxis = act[0];
+        var rightAxis = act[1];
+        var rotateAxis = act[2];
-        switch (action)
+        switch (forwardAxis)
         {
             case 1:
                 dirToGo = transform.forward * 1f;
                 break;
-            case 3:
-                rotateDir = transform.up * 1f;
+        }
+
+        switch (rightAxis)
+        {
+            case 1:
+                dirToGo = transform.right * 1f;
-            case 4:
-                rotateDir = transform.up * -1f;
+            case 2:
+                dirToGo = transform.right * -1f;
-            case 5:
-                dirToGo = transform.right * -0.75f;
+        }
+
+        switch (rotateAxis)
+        {
+            case 1:
+                rotateDir = transform.up * -1f;
-            case 6:
-                dirToGo = transform.right * 0.75f;
+            case 2:
+                rotateDir = transform.up * 1f;
-        transform.Rotate(rotateDir, Time.fixedDeltaTime * 200f);
+
+        transform.Rotate(rotateDir, Time.deltaTime * 200f);
-    }
+
+        }
 
     /// <summary>
     /// Called every step of the engine. Here the agent takes an action.
 
     public override void Heuristic(in ActionBuffers actionsOut)
     {
-        var discreteActionsOut = actionsOut.DiscreteActions;
-        discreteActionsOut[0] = 0;
-        if (Input.GetKey(KeyCode.D))
+            var discreteActionsOut = actionsOut.DiscreteActions;
+        discreteActionsOut.Clear();
+        //forward
+        if (Input.GetKey(KeyCode.W))
-            discreteActionsOut[0] = 3;
+            discreteActionsOut[0] = 1;
-        else if (Input.GetKey(KeyCode.W))
+        if (Input.GetKey(KeyCode.S))
-            discreteActionsOut[0] = 1;
+            discreteActionsOut[0] = 2;
-        else if (Input.GetKey(KeyCode.A))
+        //rotate
+        if (Input.GetKey(KeyCode.A))
-            discreteActionsOut[0] = 4;
+            discreteActionsOut[2] = 1;
+        }
+        if (Input.GetKey(KeyCode.D))
+        {
+            discreteActionsOut[2] = 2;
+        }
+        //right
+        if (Input.GetKey(KeyCode.E))
+        {
+            discreteActionsOut[1] = 1;
-        else if (Input.GetKey(KeyCode.S))
+        if (Input.GetKey(KeyCode.Q))
-            discreteActionsOut[0] = 2;
+            discreteActionsOut[1] = 2;
         }
     }
 }

config/ppo/Sorter.yaml

+behaviors:
+  Sorter:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 256
+      buffer_size: 10240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: False
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+    reward_signals:
+      extrinsic:
+        gamma: 0.99
+        strength: 1.0
+    keep_checkpoints: 5
+    max_steps: 50000000
+    time_horizon: 512
+    summary_freq: 10000
+    threaded: true
+env_settings:
+  num_envs: 8

Project/Assets/ML-Agents/Examples/Arena-Sequence/TFModels.meta

-fileFormatVersion: 2
-guid: bff21b61cb59d45ef929ade44ddd0a28
-folderAsset: yes
-DefaultImporter:
-  externalObjects: {}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: