Unity 机器学习代理工具包 (ML-Agents) 是一个开源项目,它使游戏和模拟能够作为训练智能代理的环境。
您最多选择25个主题 主题必须以中文或者字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符
 
 
 
 
 

247 行
8.8 KiB

using System;
using UnityEngine;
using Unity.MLAgents;
using Unity.MLAgents.Actuators;
using Unity.MLAgentsExamples;
using Unity.MLAgents.Sensors;
using Random = UnityEngine.Random;
[RequireComponent(typeof(JointDriveController))] // Required to set joint forces
public class CrawlerAgent : Agent
{
public float maximumWalkingSpeed = 999; //The max walk velocity magnitude an agent will be rewarded for
Vector3 m_WalkDir; //Direction to the target
Quaternion m_WalkDirLookRot; //Will hold the rotation to our target
[Header("Target To Walk Towards")]
[Space(10)]
public TargetController target; //Target the agent will walk towards.
[Header("Body Parts")] [Space(10)] public Transform body;
public Transform leg0Upper;
public Transform leg0Lower;
public Transform leg1Upper;
public Transform leg1Lower;
public Transform leg2Upper;
public Transform leg2Lower;
public Transform leg3Upper;
public Transform leg3Lower;
[Header("Orientation")]
[Space(10)]
//This will be used as a stabilized model space reference point for observations
//Because ragdolls can move erratically during training, using a stabilized reference transform improves learning
public OrientationCubeController orientationCube;
JointDriveController m_JdController;
[Header("Reward Functions To Use")]
[Space(10)]
public bool rewardMovingTowardsTarget; // Agent should move towards target
public bool rewardFacingTarget; // Agent should face the target
public bool rewardUseTimePenalty; // Hurry up
[Header("Foot Grounded Visualization")]
[Space(10)]
public bool useFootGroundedVisualization;
public MeshRenderer foot0;
public MeshRenderer foot1;
public MeshRenderer foot2;
public MeshRenderer foot3;
public Material groundedMaterial;
public Material unGroundedMaterial;
public override void Initialize()
{
orientationCube.UpdateOrientation(body, target.transform);
m_JdController = GetComponent<JointDriveController>();
//Setup each body part
m_JdController.SetupBodyPart(body);
m_JdController.SetupBodyPart(leg0Upper);
m_JdController.SetupBodyPart(leg0Lower);
m_JdController.SetupBodyPart(leg1Upper);
m_JdController.SetupBodyPart(leg1Lower);
m_JdController.SetupBodyPart(leg2Upper);
m_JdController.SetupBodyPart(leg2Lower);
m_JdController.SetupBodyPart(leg3Upper);
m_JdController.SetupBodyPart(leg3Lower);
}
/// <summary>
/// Loop over body parts and reset them to initial conditions.
/// </summary>
public override void OnEpisodeBegin()
{
foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
{
bodyPart.Reset(bodyPart);
}
//Random start rotation to help generalize
transform.rotation = Quaternion.Euler(0, Random.Range(0.0f, 360.0f), 0);
orientationCube.UpdateOrientation(body, target.transform);
}
/// <summary>
/// Add relevant information on each body part to observations.
/// </summary>
public void CollectObservationBodyPart(BodyPart bp, VectorSensor sensor)
{
//GROUND CHECK
sensor.AddObservation(bp.groundContact.touchingGround); // Is this bp touching the ground
if (bp.rb.transform != body)
{
sensor.AddObservation(bp.currentStrength / m_JdController.maxJointForceLimit);
}
}
/// <summary>
/// Loop over body parts to add them to observation.
/// </summary>
public override void CollectObservations(VectorSensor sensor)
{
//Add pos of target relative to orientation cube
sensor.AddObservation(orientationCube.transform.InverseTransformPoint(target.transform.position));
RaycastHit hit;
float maxRaycastDist = 10;
if (Physics.Raycast(body.position, Vector3.down, out hit, maxRaycastDist))
{
sensor.AddObservation(hit.distance / maxRaycastDist);
}
else
sensor.AddObservation(1);
foreach (var bodyPart in m_JdController.bodyPartsList)
{
CollectObservationBodyPart(bodyPart, sensor);
}
}
/// <summary>
/// Agent touched the target
/// </summary>
public void TouchedTarget()
{
AddReward(1f);
}
public override void OnActionReceived(ActionBuffers actionBuffers)
{
// The dictionary with all the body parts in it are in the jdController
var bpDict = m_JdController.bodyPartsDict;
var continuousActions = actionBuffers.ContinuousActions;
var i = -1;
// Pick a new target joint rotation
bpDict[leg0Upper].SetJointTargetRotation(continuousActions[++i], continuousActions[++i], 0);
bpDict[leg1Upper].SetJointTargetRotation(continuousActions[++i], continuousActions[++i], 0);
bpDict[leg2Upper].SetJointTargetRotation(continuousActions[++i], continuousActions[++i], 0);
bpDict[leg3Upper].SetJointTargetRotation(continuousActions[++i], continuousActions[++i], 0);
bpDict[leg0Lower].SetJointTargetRotation(continuousActions[++i], 0, 0);
bpDict[leg1Lower].SetJointTargetRotation(continuousActions[++i], 0, 0);
bpDict[leg2Lower].SetJointTargetRotation(continuousActions[++i], 0, 0);
bpDict[leg3Lower].SetJointTargetRotation(continuousActions[++i], 0, 0);
// Update joint strength
bpDict[leg0Upper].SetJointStrength(continuousActions[++i]);
bpDict[leg1Upper].SetJointStrength(continuousActions[++i]);
bpDict[leg2Upper].SetJointStrength(continuousActions[++i]);
bpDict[leg3Upper].SetJointStrength(continuousActions[++i]);
bpDict[leg0Lower].SetJointStrength(continuousActions[++i]);
bpDict[leg1Lower].SetJointStrength(continuousActions[++i]);
bpDict[leg2Lower].SetJointStrength(continuousActions[++i]);
bpDict[leg3Lower].SetJointStrength(continuousActions[++i]);
}
void FixedUpdate()
{
orientationCube.UpdateOrientation(body, target.transform);
// If enabled the feet will light up green when the foot is grounded.
// This is just a visualization and isn't necessary for function
if (useFootGroundedVisualization)
{
foot0.material = m_JdController.bodyPartsDict[leg0Lower].groundContact.touchingGround
? groundedMaterial
: unGroundedMaterial;
foot1.material = m_JdController.bodyPartsDict[leg1Lower].groundContact.touchingGround
? groundedMaterial
: unGroundedMaterial;
foot2.material = m_JdController.bodyPartsDict[leg2Lower].groundContact.touchingGround
? groundedMaterial
: unGroundedMaterial;
foot3.material = m_JdController.bodyPartsDict[leg3Lower].groundContact.touchingGround
? groundedMaterial
: unGroundedMaterial;
}
// Set reward for this step according to mixture of the following elements.
if (rewardMovingTowardsTarget)
{
RewardFunctionMovingTowards();
}
if (rewardFacingTarget)
{
RewardFunctionFacingTarget();
}
if (rewardUseTimePenalty)
{
RewardFunctionTimePenalty();
}
}
/// <summary>
/// Reward moving towards target & Penalize moving away from target.
/// </summary>
void RewardFunctionMovingTowards()
{
var movingTowardsDot = Vector3.Dot(orientationCube.transform.forward,
Vector3.ClampMagnitude(m_JdController.bodyPartsDict[body].rb.velocity, maximumWalkingSpeed));
if (float.IsNaN(movingTowardsDot))
{
throw new ArgumentException(
"NaN in movingTowardsDot.\n" +
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n" +
$" body.velocity: {m_JdController.bodyPartsDict[body].rb.velocity}\n" +
$" maximumWalkingSpeed: {maximumWalkingSpeed}"
);
}
AddReward(0.03f * movingTowardsDot);
}
/// <summary>
/// Reward facing target & Penalize facing away from target
/// </summary>
void RewardFunctionFacingTarget()
{
var facingReward = Vector3.Dot(orientationCube.transform.forward, body.forward);
if (float.IsNaN(facingReward))
{
throw new ArgumentException(
"NaN in movingTowardsDot.\n" +
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n" +
$" body.forward: {body.forward}"
);
}
AddReward(0.01f * facingReward);
}
/// <summary>
/// Existential penalty for time-contrained tasks.
/// </summary>
void RewardFunctionTimePenalty()
{
AddReward(-0.001f);
}
}