still running with right leg in front

4 年前 · e88f3a32
--- a/Project/Assets/ML-Agents/Examples/Walker/Prefabs/WalkerPairDynamic.prefab
+++ b/Project/Assets/ML-Agents/Examples/Walker/Prefabs/WalkerPairDynamic.prefab
  m_Name: 
  m_EditorClassIdentifier: 
  m_BrainParameters:
-    VectorObservationSize: 224
+    VectorObservationSize: 223
    NumStackedVectorObservations: 1
    VectorActionSize: 27000000
    VectorActionDescriptions: []
  hasUpgradedFromAgentParameters: 1
  MaxStep: 5000
  target: {fileID: 8916348921871391354}
+  walkDirWorldspace: {x: 1, y: 0, z: 0}
  hips: {fileID: 8913670488009607195}
  chest: {fileID: 8913670489932920748}
  spine: {fileID: 8913670488103258779}
  m_Script: {fileID: 11500000, guid: 1b29724baddfa457da6eeab446fa49ca, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  maxJointSpring: 40000
-  jointDampen: 3000
+  maxJointSpring: 10000
+  jointDampen: 500
  maxJointForceLimit: 10000
  bodyPartsList: []
 --- !u!114 &8913670488868299308
--- a/Project/Assets/ML-Agents/Examples/Walker/Scenes/WalkerDynamic.unity
+++ b/Project/Assets/ML-Agents/Examples/Walker/Scenes/WalkerDynamic.unity
      propertyPath: m_Name
      value: WalkerPairDynamic (15)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
    - target: {fileID: 8913670488974397832, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: -0.5363668
+      value: -0.5363643
-      value: 0.5363668
+      value: 0.5363694
-      value: 0.000008424982
+      value: 0.000018183306
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489331594716, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489348644707, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.y
-      value: -0.7216031
+      value: -0.7216091
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489459620683, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489832487162, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: -0.000011091729
+      value: -0.00002085011
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489835536935, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (14)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
    - target: {fileID: 8913670488106135386, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.y
-      value: -1.2608812
+      value: -1.2608804
-      value: -1.2608887
+      value: -1.2608879
-      value: -0.7216148
+      value: -0.721614
      objectReference: {fileID: 0}
    - target: {fileID: 8913670488868299304, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670488974397832, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: -0.53636587
+      value: -0.53636557
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489135890572, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489222467606, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: 0.00001193188
+      value: 0.000013304144
-      value: 0.29360443
+      value: 0.2936039
-      value: -0.7216046
+      value: -0.7216031
-      value: 0.012145664
+      value: 0.012146643
-      value: 0.00000011626075
+      value: 0.00000011625982
-      value: -0.000012921426
+      value: -0.000011091729
-      value: -0.0000014377392
+      value: -0.0000006640945
-      value: -0.000000070177805
+      value: -0.000000070178544
-      value: -0.28813732
+      value: -0.2881377
      objectReference: {fileID: 0}
    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (6)
+      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (9)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
      propertyPath: m_Name
      value: WalkerPairDynamic (2)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
      propertyPath: m_Name
      value: WalkerPairDynamic (4)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
    - target: {fileID: 8913670488974397832, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: -0.5363668
+      value: -0.53636557
-      value: 0.5363668
+      value: 0.5363681
-      value: 0.000008424982
+      value: 0.000013304144
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489331594716, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (13)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (8)
+      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 012997b9fa39c4058978e9bd7c81f51d, type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (7)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
      propertyPath: m_Name
      value: WalkerPairDynamic (10)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
    - target: {fileID: 8913670488215205497, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.y
-      value: -0.72162604
+      value: -0.721614
      objectReference: {fileID: 0}
    - target: {fileID: 8913670488868299304, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489135890572, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: 0.5363643
+      value: 0.5363694
-      value: -0.0000013333412
+      value: 0.000018183306
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489331594716, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489348644707, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.y
-      value: -0.7215971
+      value: -0.7216091
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489459620683, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
    - target: {fileID: 8913670489832487162, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
-      value: -0.0000013333488
+      value: -0.00002085011
      objectReference: {fileID: 0}
    - target: {fileID: 8913670489835536935, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (12)
+      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (3)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
      propertyPath: m_Name
      value: WalkerPairDynamic (1)
      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
+      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
      propertyPath: m_Materials.Array.data[0]
        type: 3}
      propertyPath: m_Name
      value: WalkerPairDynamic (5)
+      objectReference: {fileID: 0}
+    - target: {fileID: 8914216747803489280, guid: 012997b9fa39c4058978e9bd7c81f51d,
+        type: 3}
+      propertyPath: m_IsActive
+      value: 1
      objectReference: {fileID: 0}
    - target: {fileID: 8928008596961116030, guid: 012997b9fa39c4058978e9bd7c81f51d,
        type: 3}
--- a/Project/Assets/ML-Agents/Examples/Walker/Scripts/WalkerAgentDynamic.cs
+++ b/Project/Assets/ML-Agents/Examples/Walker/Scripts/WalkerAgentDynamic.cs


        
-        //HIP RAYCAST FOR HEIGHT
-        RaycastHit hit;
-        if (Physics.Raycast(hips.position, Vector3.down, out hit, 10.0f))
-        {
-            sensor.AddObservation(hit.distance);
-        }
-        else
-            sensor.AddObservation(10.0f);
+//        //HIP RAYCAST FOR HEIGHT
+//        RaycastHit hit;
+//        if (Physics.Raycast(hips.position, Vector3.down, out hit, 10.0f))
+//        {
+//            sensor.AddObservation(hit.distance);
+//        }
+//        else
+//            sensor.AddObservation(10.0f);
+        
 //        //ORIENTATION MATRIX
 //        Vector3 worldPosMatrixPos = hips.position;
 //        worldPosMatrixPos.y = .5f;
        {
            CollectObservationBodyPart(bodyPart, sensor);
        }
-        
+
+//        print(m_OrientationCube.transform.rotation.eulerAngles);
+//        Debug.DrawRay(m_OrientationCube.transform.position, m_OrientationCube.transform.InverseTransformVector(m_JdController.bodyPartsDict[hips].rb.velocity), Color.green,Time.fixedDeltaTime * 5);
+

    }

        bpDict[forearmR].SetJointStrength(vectorAction[++i]);
    }

-    void FixedUpdate()
+    void UpdateOrientationCube()
    {
        //FACING DIR
 //        m_WalkDir = target.position - m_OrientationCube.transform.position;
        m_OrientationCube.transform.rotation = m_WalkDirLookRot;
        
        
+    }
+    
+    void FixedUpdate()
+    {
+        UpdateOrientationCube();
+        //reward looking at
+//        float facingReward = + 0.01f * Quaternion.Dot(m_OrientationCube.transform.rotation, hips.rotation)
+//                             + 0.01f * Quaternion.Dot(m_OrientationCube.transform.rotation, head.rotation);
+        float facingReward = + 0.1f * Quaternion.Dot(m_OrientationCube.transform.rotation, hips.rotation)
+                             + 0.1f * Quaternion.Dot(m_OrientationCube.transform.rotation, head.rotation);
+
+//        print($"FacingRewardDot {facingReward}");
+//        float velReward = +0.02f * Vector3.Dot(m_OrientationCube.transform.forward,m_OrientationCube.transform.InverseTransformVector(m_JdController.bodyPartsDict[hips].rb.velocity));
+        float velReward = +0.2f * Vector3.Dot(m_OrientationCube.transform.forward,m_JdController.bodyPartsDict[hips].rb.velocity); //because we are observing in local space???
+//        print($"VelRewardDot {velReward}");
+//        float velReward = +0.02f * Vector3.Dot(m_WalkDir.normalized, m_JdController.bodyPartsDict[hips].rb.velocity);
+
+        //Multiplying these amplifies the reward.
+        float runForwardTowardsTargetReward = facingReward * Mathf.Clamp(velReward, 0, 15);
+//        print($"Combined {runForwardTowardsTargetReward}");
+//        float runBackwardsTowardsTargetReward = facingReward * Mathf.Clamp(velReward, -1, 0);
        // Set reward for this step according to mixture of the following elements.
        // a. Velocity alignment with goal direction.
        // b. Rotation alignment with goal direction.
-            +0.02f * Vector3.Dot(m_WalkDir.normalized, m_JdController.bodyPartsDict[hips].rb.velocity)
+            runForwardTowardsTargetReward
+//            facingReward * velReward //max reward is moving towards while facing otherwise it is a penalty
+//            +0.02f * Vector3.Dot(m_WalkDir.normalized, m_JdController.bodyPartsDict[hips].rb.velocity)
-            + 0.01f * (Quaternion.Dot(m_OrientationCube.transform.rotation, hips.rotation) - 1) * .5f //penalize not looking at
-            + 0.01f * (Quaternion.Dot(m_OrientationCube.transform.rotation, head.rotation) - 1) * .5f //penalize not looking at
+//            + 0.01f * (Quaternion.Dot(m_OrientationCube.transform.rotation, hips.rotation) - 1) * .5f //penalize not looking at
+//            + 0.01f * (Quaternion.Dot(m_OrientationCube.transform.rotation, head.rotation) - 1) * .5f //penalize not looking at

            + 0.02f * (head.position.y - hips.position.y)
            - 0.01f * Vector3.Distance(m_JdController.bodyPartsDict[head].rb.velocity,
    /// </summary>
    public override void OnEpisodeBegin()
    {
+//        print("OnEpisodeBegin");
        foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
        {
            bodyPart.Reset(bodyPart);
 //            transform.rotation = Quaternion.LookRotation(m_WalkDir);
 //        }
        transform.rotation = Quaternion.Euler(0, Random.Range(0.0f, 360.0f), 0);
+        UpdateOrientationCube();
+
 //        transform.Rotate(Vector3.up, Random.Range(0.0f, 360.0f));