updated crawlerAgent code to match worm env

5 年前 · 2751b3a4
--- a/Project/Assets/ML-Agents/Examples/Crawler/Scenes/CrawlerDynamicTarget.unity
+++ b/Project/Assets/ML-Agents/Examples/Crawler/Scenes/CrawlerDynamicTarget.unity
      propertyPath: m_LocalPosition.z
      value: 100
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
      propertyPath: m_LocalPosition.z
      value: 100
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
      propertyPath: m_RootOrder
      value: 4
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    m_RemovedComponents: []
  m_SourcePrefab: {fileID: 100100000, guid: 0058b366f9d6d44a3ba35beb06b0174b, type: 3}
 --- !u!4 &986425173 stripped
    - target: {fileID: 4595975114078986, guid: 0058b366f9d6d44a3ba35beb06b0174b, type: 3}
      propertyPath: m_LocalPosition.z
      value: 100
+      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_LocalPosition.x
      value: 300
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
      propertyPath: m_LocalPosition.z
      value: 100
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
      propertyPath: m_LocalPosition.x
      value: 200
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
      propertyPath: m_LocalPosition.x
      value: 100
      objectReference: {fileID: 0}
+    - target: {fileID: 114060650647145362, guid: 0058b366f9d6d44a3ba35beb06b0174b,
+        type: 3}
+      propertyPath: m_BrainParameters.vectorObservationSize
+      value: 132
+      objectReference: {fileID: 0}
    - target: {fileID: 153054223423980232, guid: 0058b366f9d6d44a3ba35beb06b0174b,
        type: 3}
      propertyPath: m_ConnectedAnchor.x
  m_Script: {fileID: 11500000, guid: ec51f47c5ed0478080c449c74fd9c154, type: 3}
  m_Name: 
  m_EditorClassIdentifier: 
-  gravityMultiplier: 1
+  gravityMultiplier: 1.5
  fixedDeltaTime: 0.01333
  maximumDeltaTime: 0.15
  solverIterations: 12
--- a/Project/Assets/ML-Agents/Examples/Crawler/Scripts/CrawlerAgent.cs
+++ b/Project/Assets/ML-Agents/Examples/Crawler/Scripts/CrawlerAgent.cs
        m_JdController.SetupBodyPart(leg3Upper);
        m_JdController.SetupBodyPart(leg3Lower);
    }
+    
+    //Get Joint Rotation Relative to the Connected Rigidbody
+    //We want to collect this info because it is the actual rotation, not the "target rotation"
+    public Quaternion GetJointRotation(ConfigurableJoint joint)
+    {
+        return(Quaternion.FromToRotation(joint.axis, joint.connectedBody.transform.rotation.eulerAngles));
+    }

    /// <summary>
    /// Add relevant information on each body part to observations.
        {
            var localPosRelToBody = body.InverseTransformPoint(rb.position);
            sensor.AddObservation(localPosRelToBody);
-            sensor.AddObservation(bp.currentXNormalizedRot); // Current x rot
-            sensor.AddObservation(bp.currentYNormalizedRot); // Current y rot
-            sensor.AddObservation(bp.currentZNormalizedRot); // Current z rot
+            sensor.AddObservation(GetJointRotation(bp.joint));
+
+//            sensor.AddObservation(bp.currentXNormalizedRot); // Current x rot
+//            sensor.AddObservation(bp.currentYNormalizedRot); // Current y rot
+//            sensor.AddObservation(bp.currentZNormalizedRot); // Current z rot
            sensor.AddObservation(bp.currentStrength / m_JdController.maxJointForceLimit);
        }
    }
        m_TargetDirMatrix = Matrix4x4.TRS(Vector3.zero, m_LookRotation, Vector3.one);

        RaycastHit hit;
-        if (Physics.Raycast(body.position, Vector3.down, out hit, 10.0f))
+        float maxDist = 10;
+        if (Physics.Raycast(body.position, Vector3.down, out hit, maxDist))
-            sensor.AddObservation(hit.distance);
+            sensor.AddObservation(hit.distance/maxDist);
-            sensor.AddObservation(10.0f);
-
-        // Forward & up to help with orientation
-        var bodyForwardRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(body.forward);
-        sensor.AddObservation(bodyForwardRelativeToLookRotationToTarget);
-
-        var bodyUpRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(body.up);
-        sensor.AddObservation(bodyUpRelativeToLookRotationToTarget);
+            sensor.AddObservation(1);
+
+//        // Forward & up to help with orientation
+//        var bodyForwardRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(body.forward);
+//        sensor.AddObservation(bodyForwardRelativeToLookRotationToTarget);
+//
+//        var bodyUpRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(body.up);
+//        sensor.AddObservation(bodyUpRelativeToLookRotationToTarget);
+
+        //Rotation delta between the matrix and the head
+        Quaternion headRotationDeltaFromMatrixRot = Quaternion.Inverse(m_TargetDirMatrix.rotation) * body.rotation;
+        sensor.AddObservation(headRotationDeltaFromMatrixRot);
+        
+//        foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
+//        {
+//            CollectObservationBodyPart(bodyPart, sensor);
+//        }
    }

    /// <summary>
        bpDict[leg1Lower].SetJointStrength(vectorAction[++i]);
        bpDict[leg2Lower].SetJointStrength(vectorAction[++i]);
        bpDict[leg3Lower].SetJointStrength(vectorAction[++i]);
+        
+        // Set reward for this step according to mixture of the following elements.
+        if (rewardMovingTowardsTarget)
+        {
+            RewardFunctionMovingTowards();
+        }
+
+        if (rewardFacingTarget)
+        {
+            RewardFunctionFacingTarget();
+        }
+
+        if (rewardUseTimePenalty)
+        {
+            RewardFunctionTimePenalty();
+        }
+        
+        
    }

    void FixedUpdate()
                : unGroundedMaterial;
        }

-        // Set reward for this step according to mixture of the following elements.
-        if (rewardMovingTowardsTarget)
-        {
-            RewardFunctionMovingTowards();
-        }
-        if (rewardFacingTarget)
-        {
-            RewardFunctionFacingTarget();
-        }
-
-        if (rewardUseTimePenalty)
-        {
-            RewardFunctionTimePenalty();
-        }
+//    /// <summary>
+//    /// Reward moving towards target & Penalize moving away from target.
+//    /// </summary>
+//    void RewardFunctionMovingTowards()
+//    {
+//        m_MovingTowardsDot = Vector3.Dot(m_JdController.bodyPartsDict[body].rb.velocity, m_DirToTarget.normalized);
+//        AddReward(0.03f * m_MovingTowardsDot);
+//    }
+//
+//    /// <summary>
+//    /// Reward facing target & Penalize facing away from target
+//    /// </summary>
+//    void RewardFunctionFacingTarget()
+//    {
+//        m_FacingDot = Vector3.Dot(m_DirToTarget.normalized, body.forward);
+//        AddReward(0.01f * m_FacingDot);
+//    }
+    
+    
    /// <summary>
    /// Reward moving towards target & Penalize moving away from target.
    /// </summary>
-        AddReward(0.03f * m_MovingTowardsDot);
+        AddReward(0.01f * m_MovingTowardsDot);
    }

    /// <summary>
    {
-        m_FacingDot = Vector3.Dot(m_DirToTarget.normalized, body.forward);
-        AddReward(0.01f * m_FacingDot);
+        float bodyRotRelativeToMatrixDot = Quaternion.Dot(m_TargetDirMatrix.rotation, body.rotation);
+        AddReward(0.01f * bodyRotRelativeToMatrixDot);
+
+    
+    

    /// <summary>
    /// Existential penalty for time-contrained tasks.
    /// </summary>
    public override void OnEpisodeBegin()
    {
+        foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
+        {
+            bodyPart.Reset(bodyPart);
+        }
        if (m_DirToTarget != Vector3.zero)
        {
            transform.rotation = Quaternion.LookRotation(m_DirToTarget);
-        foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
-        {
-            bodyPart.Reset(bodyPart);
-        }
        if (!targetIsStatic)
        {
            GetRandomTargetPos();
--- a/config/trainer_config.yaml
+++ b/config/trainer_config.yaml
    time_horizon: 1000
    batch_size: 2024
    buffer_size: 20240
-    max_steps: 1e7
+    max_steps: 3.5e6
+    # max_steps: 1e7
    summary_freq: 30000
    num_layers: 3
    hidden_units: 512