浏览代码

Merge remote-tracking branch 'origin/master' into release_6-to-master

/release_6_branch
Christopher Goy 4 年前
当前提交
061a6c43
共有 88 个文件被更改,包括 427 次插入321 次删除
  1. 18
      .circleci/config.yml
  2. 6
      .pre-commit-config.yaml
  3. 2
      DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs
  4. 2
      Project/Assets/ML-Agents/Editor/DisableBurstFromMenu.cs
  5. 18
      Project/Assets/ML-Agents/Examples/Crawler/Scripts/CrawlerAgent.cs
  6. 2
      Project/Assets/ML-Agents/Examples/FoodCollector/Scripts/FoodCollectorSettings.cs
  7. 10
      Project/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs
  8. 2
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/AdjustTrainingTimescale.cs
  9. 4
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/DirectionIndicator.cs
  10. 14
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/JointDriveController.cs
  11. 20
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/ModelOverrider.cs
  12. 6
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/Monitor.cs
  13. 2
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/OrientationCubeController.cs
  14. 2
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/ProjectSettingsOverrides.cs
  15. 2
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs
  16. 8
      Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/TargetController.cs
  17. 6
      Project/Assets/ML-Agents/Examples/Startup/Scripts/Startup.cs
  18. 14
      Project/Assets/ML-Agents/Examples/Tennis/Scripts/HitWall.cs
  19. 72
      Project/Assets/ML-Agents/Examples/Worm/Scripts/WormAgent.cs
  20. 2
      com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodyJointExtractor.cs
  21. 2
      com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodyPoseExtractor.cs
  22. 2
      com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodySensorComponent.cs
  23. 4
      com.unity.ml-agents.extensions/Runtime/Sensors/PhysicsBodySensor.cs
  24. 4
      com.unity.ml-agents.extensions/Runtime/Sensors/PhysicsSensorSettings.cs
  25. 10
      com.unity.ml-agents.extensions/Runtime/Sensors/PoseExtractor.cs
  26. 2
      com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodyJointExtractor.cs
  27. 4
      com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodyPoseExtractor.cs
  28. 4
      com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodySensorComponent.cs
  29. 2
      com.unity.ml-agents.extensions/Tests/Editor/Sensors/ArticulationBodyPoseExtractorTests.cs
  30. 2
      com.unity.ml-agents.extensions/Tests/Editor/Sensors/ArticulationBodySensorTests.cs
  31. 10
      com.unity.ml-agents.extensions/Tests/Editor/Sensors/PoseExtractorTests.cs
  32. 2
      com.unity.ml-agents.extensions/Tests/Editor/Sensors/RigidBodyPoseExtractorTests.cs
  33. 3
      com.unity.ml-agents.extensions/Tests/Runtime/RuntimeExampleTest.cs
  34. 1
      com.unity.ml-agents/CHANGELOG.md
  35. 7
      com.unity.ml-agents/CONTRIBUTING.md
  36. 2
      com.unity.ml-agents/Editor/DemonstrationImporter.cs
  37. 16
      com.unity.ml-agents/Runtime/Academy.cs
  38. 2
      com.unity.ml-agents/Runtime/Actuators/ActionSegment.cs
  39. 2
      com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs
  40. 6
      com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs
  41. 2
      com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs
  42. 10
      com.unity.ml-agents/Runtime/Agent.cs
  43. 2
      com.unity.ml-agents/Runtime/Agent.deprecated.cs
  44. 6
      com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs
  45. 36
      com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs
  46. 2
      com.unity.ml-agents/Runtime/Communicator/UnityRLCapabilities.cs
  47. 20
      com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
  48. 2
      com.unity.ml-agents/Runtime/Policies/BrainParameters.cs
  49. 2
      com.unity.ml-agents/Runtime/Policies/HeuristicPolicy.cs
  50. 14
      com.unity.ml-agents/Runtime/Sampler.cs
  51. 4
      com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs
  52. 14
      com.unity.ml-agents/Runtime/Sensors/CameraSensorComponent.cs
  53. 2
      com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs
  54. 4
      com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs
  55. 2
      com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensorComponentBase.cs
  56. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/BoolReflectionSensor.cs
  57. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/FloatReflectionSensor.cs
  58. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/IntReflectionSensor.cs
  59. 10
      com.unity.ml-agents/Runtime/Sensors/Reflection/ObservableAttribute.cs
  60. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/QuaternionReflectionSensor.cs
  61. 6
      com.unity.ml-agents/Runtime/Sensors/Reflection/ReflectionSensorBase.cs
  62. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/Vector2ReflectionSensor.cs
  63. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/Vector3ReflectionSensor.cs
  64. 2
      com.unity.ml-agents/Runtime/Sensors/Reflection/Vector4ReflectionSensor.cs
  65. 6
      com.unity.ml-agents/Runtime/Sensors/RenderTextureSensor.cs
  66. 10
      com.unity.ml-agents/Runtime/Sensors/RenderTextureSensorComponent.cs
  67. 3
      com.unity.ml-agents/Runtime/SideChannels/EnvironmentParametersChannel.cs
  68. 8
      com.unity.ml-agents/Runtime/Timer.cs
  69. 42
      com.unity.ml-agents/Tests/Editor/Actuators/ActuatorDiscreteActionMaskTests.cs
  70. 42
      com.unity.ml-agents/Tests/Editor/Actuators/ActuatorManagerTests.cs
  71. 12
      com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs
  72. 4
      com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs
  73. 8
      com.unity.ml-agents/Tests/Editor/DiscreteActionOutputApplierTest.cs
  74. 12
      com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs
  75. 8
      com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs
  76. 8
      com.unity.ml-agents/Tests/Editor/MultinomialTest.cs
  77. 12
      com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs
  78. 4
      com.unity.ml-agents/Tests/Editor/SamplerTests.cs
  79. 10
      com.unity.ml-agents/Tests/Editor/Sensor/CameraSensorTest.cs
  80. 2
      com.unity.ml-agents/Tests/Editor/Sensor/FloatVisualSensorTests.cs
  81. 2
      com.unity.ml-agents/Tests/Editor/Sensor/ObservableAttributeTests.cs
  82. 6
      com.unity.ml-agents/Tests/Editor/Sensor/ObservationWriterTests.cs
  83. 2
      com.unity.ml-agents/Tests/Editor/Sensor/SensorShapeValidatorTests.cs
  84. 36
      com.unity.ml-agents/Tests/Editor/Sensor/StackingSensorTests.cs
  85. 22
      com.unity.ml-agents/Tests/Editor/Sensor/VectorSensorTests.cs
  86. 14
      com.unity.ml-agents/Tests/Editor/UtilitiesTests.cs
  87. 24
      utils/run_dotnet_format.py
  88. 0
      /.editorconfig

18
.circleci/config.yml


# Need ruby for search-and-replace
sudo apt-get update
sudo apt-get install ruby-full
# install dotnet and the formatter - see https://docs.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-9-
pushd ~
wget -O - https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg
sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/
wget https://packages.microsoft.com/config/debian/9/prod.list
sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list
sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg
sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list
popd
sudo apt-get install -y apt-transport-https && \
sudo apt-get update && \
sudo apt-get install -y dotnet-sdk-3.1 && \
dotnet tool install -g dotnet-format --version 4.1.131201
echo "Setting up venv"
python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip

name: Check Code Style using pre-commit
command: |
. venv/bin/activate
export PATH="$PATH:~/.dotnet/tools"
pre-commit run --show-diff-on-failure --all-files
markdown_link_check:

6
.pre-commit-config.yaml


name: validate release links
language: script
entry: utils/validate_release_links.py
- id: dotnet-format
name: dotnet-format
language: script
entry: utils/run_dotnet_format.py
types: [c#]

2
DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs


{
using (Measure.ProfilerMarkers(s_Markers))
{
for(var i=0; i<k_MarkerTestSteps; i++)
for (var i = 0; i < k_MarkerTestSteps; i++)
{
RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore);
}

2
Project/Assets/ML-Agents/Editor/DisableBurstFromMenu.cs


#if UNITY_CLOUD_BUILD
#if UNITY_CLOUD_BUILD
using UnityEditor;
public class DisableBurstFromMenu

18
Project/Assets/ML-Agents/Examples/Crawler/Scripts/CrawlerAgent.cs


Vector3 m_WalkDir; //Direction to the target
Quaternion m_WalkDirLookRot; //Will hold the rotation to our target
[Header("Target To Walk Towards")] [Space(10)]
[Header("Target To Walk Towards")]
[Space(10)]
public TargetController target; //Target the agent will walk towards.
[Header("Body Parts")] [Space(10)] public Transform body;

public Transform leg3Lower;
[Header("Orientation")] [Space(10)]
[Header("Orientation")]
[Space(10)]
//This will be used as a stabilized model space reference point for observations
//Because ragdolls can move erratically during training, using a stabilized reference transform improves learning
public OrientationCubeController orientationCube;

[Header("Reward Functions To Use")] [Space(10)]
[Header("Reward Functions To Use")]
[Space(10)]
[Header("Foot Grounded Visualization")] [Space(10)]
[Header("Foot Grounded Visualization")]
[Space(10)]
public bool useFootGroundedVisualization;
public MeshRenderer foot0;

{
throw new ArgumentException(
"NaN in movingTowardsDot.\n" +
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n"+
$" body.velocity: {m_JdController.bodyPartsDict[body].rb.velocity}\n"+
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n" +
$" body.velocity: {m_JdController.bodyPartsDict[body].rb.velocity}\n" +
$" maximumWalkingSpeed: {maximumWalkingSpeed}"
);
}

{
throw new ArgumentException(
"NaN in movingTowardsDot.\n" +
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n"+
$" orientationCube.transform.forward: {orientationCube.transform.forward}\n" +
$" body.forward: {body.forward}"
);
}

2
Project/Assets/ML-Agents/Examples/FoodCollector/Scripts/FoodCollectorSettings.cs


// Send stats via SideChannel so that they'll appear in TensorBoard.
// These values get averaged every summary_frequency steps, so we don't
// need to send every Update() call.
if ((Time.frameCount % 100)== 0)
if ((Time.frameCount % 100) == 0)
{
m_Recorder.Add("TotalScore", totalScore);
}

10
Project/Assets/ML-Agents/Examples/GridWorld/Scripts/GridAgent.cs


// Mask the necessary actions if selected by the user.
if (maskActions)
{
// Prevents the agent from picking an action that would make it collide with a wall
// Prevents the agent from picking an action that would make it collide with a wall
var positionX = (int)transform.position.x;
var positionZ = (int)transform.position.z;
var maxPosition = (int)m_ResetParams.GetWithDefault("gridSize", 5f) - 1;

actionMask.WriteMask(0, new []{ k_Left});
actionMask.WriteMask(0, new[] { k_Left });
actionMask.WriteMask(0, new []{k_Right});
actionMask.WriteMask(0, new[] { k_Right });
actionMask.WriteMask(0, new []{k_Down});
actionMask.WriteMask(0, new[] { k_Down });
actionMask.WriteMask(0, new []{k_Up});
actionMask.WriteMask(0, new[] { k_Up });
}
}
}

2
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/AdjustTrainingTimescale.cs


using UnityEngine;
namespace MLAgentsExamples
namespace MLAgentsExamples
{
public class AdjustTrainingTimescale : MonoBehaviour
{

4
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/DirectionIndicator.cs


using UnityEngine;
using UnityEngine;
public bool updatedByAgent; //should this be updated by the agent? If not, it will use local settings
public Transform transformToFollow; //ex: hips or body
public Transform targetToLookAt; //target in the scene the indicator will point to

14
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/JointDriveController.cs


[System.Serializable]
public class BodyPart
{
[Header("Body Part Info")][Space(10)] public ConfigurableJoint joint;
[Header("Body Part Info")] [Space(10)] public ConfigurableJoint joint;
[Header("Ground & Target Contact")][Space(10)]
[Header("Ground & Target Contact")]
[Space(10)]
public GroundContact groundContact;
public TargetContact targetContact;

[Header("Current Joint Settings")][Space(10)]
[Header("Current Joint Settings")]
[Space(10)]
public Vector3 currentEularJointRotation;
[HideInInspector] public float currentStrength;

[Header("Other Debug Info")][Space(10)]
[Header("Other Debug Info")]
[Space(10)]
public Vector3 currentJointForce;
public float currentJointForceSqrMag;

public class JointDriveController : MonoBehaviour
{
[Header("Joint Drive Settings")][Space(10)]
[Header("Joint Drive Settings")]
[Space(10)]
public float maxJointSpring;
public float jointDampen;

20
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/ModelOverrider.cs


/// </summary>
public class ModelOverrider : MonoBehaviour
{
HashSet<string> k_SupportedExtensions = new HashSet<string>{"nn", "onnx"};
HashSet<string> k_SupportedExtensions = new HashSet<string> { "nn", "onnx" };
const string k_CommandLineModelOverrideFlag = "--mlagents-override-model";
const string k_CommandLineModelOverrideDirectoryFlag = "--mlagents-override-model-directory";
const string k_CommandLineModelOverrideExtensionFlag = "--mlagents-override-model-extension";

int TotalCompletedEpisodes
{
get { return m_PreviousAgentCompletedEpisodes + (m_Agent == null ? 0 : m_Agent.CompletedEpisodes); }
get { return m_PreviousAgentCompletedEpisodes + (m_Agent == null ? 0 : m_Agent.CompletedEpisodes); }
}
int TotalNumSteps

public bool HasOverrides
{
get { return m_BehaviorNameOverrides.Count > 0 || !string.IsNullOrEmpty(m_BehaviorNameOverrideDirectory); }
get { return m_BehaviorNameOverrides.Count > 0 || !string.IsNullOrEmpty(m_BehaviorNameOverrideDirectory); }
}
public static string GetOverrideBehaviorName(string originalBehaviorName)

var args = commandLineArgsOverride ?? Environment.GetCommandLineArgs();
for (var i = 0; i < args.Length; i++)
{
if (args[i] == k_CommandLineModelOverrideFlag && i < args.Length-2)
if (args[i] == k_CommandLineModelOverrideFlag && i < args.Length - 2)
else if (args[i] == k_CommandLineModelOverrideDirectoryFlag && i < args.Length-1)
else if (args[i] == k_CommandLineModelOverrideDirectoryFlag && i < args.Length - 1)
else if (args[i] == k_CommandLineModelOverrideExtensionFlag && i < args.Length-1)
else if (args[i] == k_CommandLineModelOverrideExtensionFlag && i < args.Length - 1)
{
m_OverrideExtension = args[i + 1].Trim().ToLower();
var isKnownExtension = k_SupportedExtensions.Contains(m_OverrideExtension);

#endif
}
}
else if (args[i] == k_CommandLineQuitAfterEpisodesFlag && i < args.Length-1)
else if (args[i] == k_CommandLineQuitAfterEpisodesFlag && i < args.Length - 1)
{
Int32.TryParse(args[i + 1], out maxEpisodes);
}

{
assetPath = m_BehaviorNameOverrides[behaviorName];
}
else if(!string.IsNullOrEmpty(m_BehaviorNameOverrideDirectory))
else if (!string.IsNullOrEmpty(m_BehaviorNameOverrideDirectory))
{
assetPath = Path.Combine(m_BehaviorNameOverrideDirectory, $"{behaviorName}.{m_OverrideExtension}");
}

{
model = File.ReadAllBytes(assetPath);
}
catch(IOException)
catch (IOException)
{
Debug.Log($"Couldn't load file {assetPath} at full path {Path.GetFullPath(assetPath)}", this);
// Cache the null so we don't repeatedly try to load a missing file

if (!overrideOk && m_QuitOnLoadFailure)
{
if(!string.IsNullOrEmpty(overrideError))
if (!string.IsNullOrEmpty(overrideError))
{
Debug.LogWarning(overrideError);
}

6
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/Monitor.cs


using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;

var displayValues = s_DisplayTransformValues[target];
var index = 0;
var orderedKeys = displayValues.Keys.OrderBy(x => - displayValues[x].time);
var orderedKeys = displayValues.Keys.OrderBy(x => -displayValues[x].time);
foreach (var key in orderedKeys)
{
s_KeyStyle.alignment = TextAnchor.MiddleRight;

s_RedStyle = s_ColorStyle[5];
}
}
}
}

2
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/OrientationCubeController.cs


using UnityEngine;
using UnityEngine;
namespace Unity.MLAgentsExamples
{

2
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/ProjectSettingsOverrides.cs


m_OriginalMaximumDeltaTime = Time.maximumDeltaTime;
m_OriginalSolverIterations = Physics.defaultSolverIterations;
m_OriginalSolverVelocityIterations = Physics.defaultSolverVelocityIterations;
m_OriginalReuseCollisionCallbacks = Physics.reuseCollisionCallbacks ;
m_OriginalReuseCollisionCallbacks = Physics.reuseCollisionCallbacks;
// Override
Physics.gravity *= gravityMultiplier;

2
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs


}
/// <inheritdoc/>
public void Update() {}
public void Update() { }
/// <inheritdoc/>
public void Reset() { }

8
Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/TargetController.cs


using UnityEngine;
using UnityEngine;
using Random = UnityEngine.Random;
using Unity.MLAgents;
using UnityEngine.Events;

/// </summary>
public class TargetController : MonoBehaviour
{
[Header("Target Fell Protection")]
public bool respawnIfFallsOffPlatform = true; //If the target falls off the platform, reset the position.
public float fallDistance = 5; //distance below the starting height that will trigger a respawn

6
Project/Assets/ML-Agents/Examples/Startup/Scripts/Startup.cs


// no scene environment variable is found.
var args = Environment.GetCommandLineArgs();
Console.WriteLine("Command line arguments passed: " + String.Join(" ", args));
for (int i = 0; i < args.Length; i++) {
if (args [i] == k_SceneCommandLineFlag && i < args.Length - 1) {
for (int i = 0; i < args.Length; i++)
{
if (args[i] == k_SceneCommandLineFlag && i < args.Length - 1)
{
sceneName = args[i + 1];
}
}

14
Project/Assets/ML-Agents/Examples/Tennis/Scripts/HitWall.cs


public bool net;
public enum FloorHit
{
Service,
FloorHitUnset,
FloorAHit,
FloorBHit
}
{
Service,
FloorHitUnset,
FloorAHit,
FloorBHit
}
public FloorHit lastFloorHit;

lastFloorHit = FloorHit.Service;
net = false;
}
void AgentAWins()
{
m_AgentA.SetReward(1);

72
Project/Assets/ML-Agents/Examples/Worm/Scripts/WormAgent.cs


[RequireComponent(typeof(JointDriveController))] // Required to set joint forces
public class WormAgent : Agent
{
[Header("Target To Walk Towards")] [Space(10)]
[Header("Target To Walk Towards")]
[Space(10)]
public bool detectTargets;
public bool targetIsStatic;
public bool respawnTargetWhenTouched;
public float targetSpawnRadius;
[Header("Body Parts")] [Space(10)] public Transform bodySegment0;
[Header("Body Parts")]
[Space(10)]
public Transform bodySegment0;
[Header("Joint Settings")] [Space(10)] JointDriveController m_JdController;
[Header("Joint Settings")]
[Space(10)]
JointDriveController m_JdController;
[Header("Reward Functions To Use")] [Space(10)]
[Header("Reward Functions To Use")]
[Space(10)]
public bool rewardMovingTowardsTarget; // Agent should move towards target
public bool rewardFacingTarget; // Agent should face the target

m_JdController.SetupBodyPart(bodySegment1);
m_JdController.SetupBodyPart(bodySegment2);
m_JdController.SetupBodyPart(bodySegment3);
//We only want the head to detect the target
//So we need to remove TargetContact from everything else
//This is a temp fix till we can redesign
DestroyImmediate(bodySegment1.GetComponent<TargetContact>());
DestroyImmediate(bodySegment2.GetComponent<TargetContact>());
DestroyImmediate(bodySegment3.GetComponent<TargetContact>());
}

var velocityRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(rb.velocity);
sensor.AddObservation(velocityRelativeToLookRotationToTarget);
var angularVelocityRelativeToLookRotationToTarget =
m_TargetDirMatrix.inverse.MultiplyVector(rb.angularVelocity);
var angularVelocityRelativeToLookRotationToTarget = m_TargetDirMatrix.inverse.MultiplyVector(rb.angularVelocity);
sensor.AddObservation(angularVelocityRelativeToLookRotationToTarget);
if (bp.rb.transform != bodySegment0)

else
sensor.AddObservation(1);
foreach (var bodyPart in m_JdController.bodyPartsList)
foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
Quaternion headRotationDeltaFromMatrixRot =
Quaternion.Inverse(m_TargetDirMatrix.rotation) * bodySegment0.rotation;
Quaternion headRotationDeltaFromMatrixRot = Quaternion.Inverse(m_TargetDirMatrix.rotation) * bodySegment0.rotation;
sensor.AddObservation(headRotationDeltaFromMatrixRot);
}

public void TouchedTarget()
{
AddReward(1f);
if (respawnTargetWhenTouched)
{
GetRandomTargetPos();
}
}
/// <summary>
/// Moves target to a random position within specified radius.
/// </summary>
public void GetRandomTargetPos()
{
var newTargetPos = Random.insideUnitSphere * targetSpawnRadius;
newTargetPos.y = 5;
target.position = newTargetPos + ground.position;
}
public override void OnActionReceived(ActionBuffers actionBuffers)

bpDict[bodySegment2].SetJointStrength(continuousActions[++i]);
bpDict[bodySegment3].SetJointStrength(continuousActions[++i]);
// Detect if worm fell off/through platform
if (bodySegment0.position.y < ground.position.y - 2)
{
EndEpisode();

void FixedUpdate()
{
if (detectTargets)
{
foreach (var bodyPart in m_JdController.bodyPartsDict.Values)
{
if (bodyPart.targetContact && bodyPart.targetContact.touchingTarget)
{
TouchedTarget();
}
}
}
// Set reward for this step according to mixture of the following elements.
if (rewardMovingTowardsTarget)
{

/// </summary>
void RewardFunctionMovingTowards()
{
m_MovingTowardsDot =
Vector3.Dot(m_JdController.bodyPartsDict[bodySegment0].rb.velocity, m_DirToTarget.normalized);
m_MovingTowardsDot = Vector3.Dot(m_JdController.bodyPartsDict[bodySegment0].rb.velocity, m_DirToTarget.normalized);
AddReward(0.01f * m_MovingTowardsDot);
}

}
/// <summary>
/// Existential penalty for time-constrained tasks.
/// Existential penalty for time-contrained tasks.
/// </summary>
void RewardFunctionTimePenalty()
{

{
bodyPart.Reset(bodyPart);
}
transform.Rotate(Vector3.up, Random.Range(0.0f, 360.0f));
transform.Rotate(Vector3.up, Random.Range(0.0f, 360.0f));
if (!targetIsStatic)
{
GetRandomTargetPos();
}
}
}

2
com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodyJointExtractor.cs


}
}
}
#endif
#endif

2
com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodyPoseExtractor.cs


}
}
}
#endif // UNITY_2020_1_OR_NEWER
#endif // UNITY_2020_1_OR_NEWER

2
com.unity.ml-agents.extensions/Runtime/Sensors/ArticulationBodySensorComponent.cs


}
}
#endif // UNITY_2020_1_OR_NEWER
#endif // UNITY_2020_1_OR_NEWER

4
com.unity.ml-agents.extensions/Runtime/Sensors/PhysicsBodySensor.cs


var numJointExtractorObservations = 0;
m_JointExtractors = new List<IJointExtractor>(poseExtractor.NumEnabledPoses);
foreach(var rb in poseExtractor.GetEnabledRigidbodies())
foreach (var rb in poseExtractor.GetEnabledRigidbodies())
{
var jointExtractor = new RigidBodyJointExtractor(rb);
numJointExtractorObservations += jointExtractor.NumObservations(settings);

}
/// <inheritdoc/>
public void Reset() {}
public void Reset() { }
/// <inheritdoc/>
public SensorCompressionType GetCompressionType()

4
com.unity.ml-agents.extensions/Runtime/Sensors/PhysicsSensorSettings.cs


}
}
foreach(var vel in poseExtractor.GetEnabledModelSpaceVelocities())
foreach (var vel in poseExtractor.GetEnabledModelSpaceVelocities())
{
if (settings.UseModelSpaceLinearVelocity)
{

}
}
foreach(var vel in poseExtractor.GetEnabledLocalSpaceVelocities())
foreach (var vel in poseExtractor.GetEnabledLocalSpaceVelocities())
{
if (settings.UseLocalSpaceLinearVelocity)
{

10
com.unity.ml-agents.extensions/Runtime/Sensors/PoseExtractor.cs


var localUp = localPose[i].rotation * Vector3.up;
var localFwd = localPose[i].rotation * Vector3.forward;
var localRight = localPose[i].rotation * Vector3.right;
Debug.DrawLine(current.position+offset, current.position+offset+.1f*localUp, Color.red);
Debug.DrawLine(current.position+offset, current.position+offset+.1f*localFwd, Color.green);
Debug.DrawLine(current.position+offset, current.position+offset+.1f*localRight, Color.blue);
Debug.DrawLine(current.position + offset, current.position + offset + .1f * localUp, Color.red);
Debug.DrawLine(current.position + offset, current.position + offset + .1f * localFwd, Color.green);
Debug.DrawLine(current.position + offset, current.position + offset + .1f * localRight, Color.blue);
}
}

{
// Push to the stack in reverse order
var children = tree[current];
for (var childIdx = children.Count-1; childIdx >= 0; childIdx--)
for (var childIdx = children.Count - 1; childIdx >= 0; childIdx--)
stack.Push((children[childIdx], depth+1));
stack.Push((children[childIdx], depth + 1));
}
}

2
com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodyJointExtractor.cs


public static int NumObservations(Rigidbody body, Joint joint, PhysicsSensorSettings settings)
{
if(body == null || joint == null)
if (body == null || joint == null)
{
return 0;
}

4
com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodyPoseExtractor.cs


if (rootGameObject == null)
{
rbs = rootBody.GetComponentsInChildren<Rigidbody>();
joints = rootBody.GetComponentsInChildren <Joint>();
joints = rootBody.GetComponentsInChildren<Joint>();
}
else
{

for (var i = 0; i < rbs.Length; i++)
{
if(rbs[i] != null)
if (rbs[i] != null)
{
bodyToIndex[rbs[i]] = i;
}

4
com.unity.ml-agents.extensions/Runtime/Sensors/RigidBodySensorComponent.cs


/// <summary>
/// Editor component that creates a PhysicsBodySensor for the Agent.
/// </summary>
public class RigidBodySensorComponent : SensorComponent
public class RigidBodySensorComponent : SensorComponent
{
/// <summary>
/// The root Rigidbody of the system.

var numPoseObservations = poseExtractor.GetNumPoseObservations(Settings);
var numJointObservations = 0;
foreach(var rb in poseExtractor.GetEnabledRigidbodies())
foreach (var rb in poseExtractor.GetEnabledRigidbodies())
{
var joint = rb.GetComponent<Joint>();
numJointObservations += RigidBodyJointExtractor.NumObservations(rb, joint, Settings);

2
com.unity.ml-agents.extensions/Tests/Editor/Sensors/ArticulationBodyPoseExtractorTests.cs


}
}
}
#endif
#endif

2
com.unity.ml-agents.extensions/Tests/Editor/Sensors/ArticulationBodySensorTests.cs


}
}
}
#endif // #if UNITY_2020_1_OR_NEWER
#endif // #if UNITY_2020_1_OR_NEWER

10
com.unity.ml-agents.extensions/Tests/Editor/Sensors/PoseExtractorTests.cs


return Pose.identity;
}
protected internal override Vector3 GetLinearVelocityAt(int index)
protected internal override Vector3 GetLinearVelocityAt(int index)
{
return Vector3.zero;
}

}
// Getting a parent index should throw an index exception
Assert.Throws <NullReferenceException>(
Assert.Throws<NullReferenceException>(
() => poseExtractor.GetParentIndex(0)
);

};
}
protected internal override Vector3 GetLinearVelocityAt(int index)
protected internal override Vector3 GetLinearVelocityAt(int index)
{
return Vector3.zero;
}

{
// Degenerate case with a loop
var poseExtractor = new UselessPoseExtractor();
poseExtractor.Init(new[] {-1, 2, 1});
poseExtractor.Init(new[] { -1, 2, 1 });
poseExtractor.Init(new[] {-1, 1});
poseExtractor.Init(new[] { -1, 1 });
// This just shouldn't blow up
poseExtractor.GetDisplayNodes();

2
com.unity.ml-agents.extensions/Tests/Editor/Sensors/RigidBodyPoseExtractorTests.cs


// body 1 has parent 0
Assert.AreEqual(0, poseExtractor.GetParentIndex(1));
var virtualRootPos = new Vector3(0,2,0);
var virtualRootPos = new Vector3(0, 2, 0);
var virtualRootRot = Quaternion.Euler(0, 42, 0);
virtualRoot.transform.position = virtualRootPos;
virtualRoot.transform.rotation = virtualRootRot;

3
com.unity.ml-agents.extensions/Tests/Runtime/RuntimeExampleTest.cs


namespace Unity.MLAgents.Extensions.Tests
{
internal class RuntimeExampleTest {
internal class RuntimeExampleTest
{
[Test]
public void RuntimeTestMath()

1
com.unity.ml-agents/CHANGELOG.md


### Minor Changes
#### com.unity.ml-agents (C#)
- Update Barracuda to 1.0.2.
- Enabled C# formatting using `dotnet-format`.
#### ml-agents / ml-agents-envs / gym-unity (Python)
### Bug Fixes

7
com.unity.ml-agents/CONTRIBUTING.md


### Code style
All python code should be formatted with
[`black`](https://github.com/ambv/black). Style and formatting for C# may be
enforced later.
[`black`](https://github.com/ambv/black).
C# code is formatted using [`dotnet-format`](https://github.com/dotnet/format).
You must have [dotnet](https://dotnet.microsoft.com/download) and
`dotnet-format` installed first.
### Python type annotations

2
com.unity.ml-agents/Editor/DemonstrationImporter.cs


/// <summary>
/// Asset Importer used to parse demonstration files.
/// </summary>
[ScriptedImporter(1, new[] {"demo"})]
[ScriptedImporter(1, new[] { "demo" })]
internal class DemonstrationImporter : ScriptedImporter
{
const string k_IconPath = "Packages/com.unity.ml-agents/Editor/Icons/DemoIcon.png";

16
com.unity.ml-agents/Runtime/Academy.cs


// This try-catch is because DontDestroyOnLoad cannot be used in Editor Tests
GameObject.DontDestroyOnLoad(m_StepperObject);
}
catch {}
catch { }
}
/// <summary>

void ResetActions()
{
DecideAction = () => {};
DestroyAction = () => {};
AgentPreStep = i => {};
AgentSendState = () => {};
AgentAct = () => {};
AgentForceReset = () => {};
OnEnvironmentReset = () => {};
DecideAction = () => { };
DestroyAction = () => { };
AgentPreStep = i => { };
AgentSendState = () => { };
AgentAct = () => { };
AgentForceReset = () => { };
OnEnvironmentReset = () => { };
}
static void OnQuitCommandReceived()

2
com.unity.ml-agents/Runtime/Actuators/ActionSegment.cs


/// </summary>
/// <param name="actionArray">The action array to use for the this segment.</param>
public ActionSegment(T[] actionArray)
: this(actionArray ?? System.Array.Empty<T>(), 0, actionArray?.Length ?? 0) {}
: this(actionArray ?? System.Array.Empty<T>(), 0, actionArray?.Length ?? 0) { }
/// <summary>
/// Construct an <see cref="ActionSegment{T}"/> with an underlying array

2
com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs


{
NumContinuousActions = numContinuousActions;
NumDiscreteActions = numDiscreteActions;
BranchSizes = branchSizes;
BranchSizes = branchSizes;
SumOfDiscreteBranchSizes = branchSizes?.Sum() ?? 0;
}
}

6
com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs


/// <see cref="ActionSegment{T}"/> initialized from a float array.</returns>
public static ActionBuffers FromDiscreteActions(float[] discreteActions)
{
return new ActionBuffers(ActionSegment<float>.Empty, discreteActions == null ? ActionSegment<int>.Empty
: new ActionSegment<int>(Array.ConvertAll(discreteActions,
x => (int)x)));
return new ActionBuffers(ActionSegment<float>.Empty, discreteActions == null ? ActionSegment<int>.Empty
: new ActionSegment<int>(Array.ConvertAll(discreteActions,
x => (int)x)));
}
public ActionBuffers(float[] continuousActions, int[] discreteActions)

2
com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs


public void WriteDiscreteActionMask(IDiscreteActionMask actionMask)
{
m_ActionReceiver.WriteDiscreteActionMask(actionMask);
m_ActionReceiver.WriteDiscreteActionMask(actionMask);
}
/// <summary>

10
com.unity.ml-agents/Runtime/Agent.cs


public int maxStep;
}
[SerializeField][HideInInspector]
[SerializeField]
[HideInInspector]
[SerializeField][HideInInspector]
[SerializeField]
[HideInInspector]
internal bool hasUpgradedFromAgentParameters;
/// <summary>

///
/// [GameObject]: https://docs.unity3d.com/Manual/GameObjects.html
/// </remarks>
public virtual void Initialize() {}
public virtual void Initialize() { }
/// <summary>
/// Implement `Heuristic()` to choose an action for this agent using a custom heuristic.

/// </summary>
/// <seealso cref="Initialize"/>
/// <seealso cref="EndEpisode"/>
public virtual void OnEpisodeBegin() {}
public virtual void OnEpisodeBegin() { }
/// <summary>
/// Gets the last ActionBuffer for this agent.

2
com.unity.ml-agents/Runtime/Agent.deprecated.cs


Array.Clear(actionsOut, 0, actionsOut.Length);
}
public virtual void OnActionReceived(float[] vectorAction) {}
public virtual void OnActionReceived(float[] vectorAction) { }
/// <summary>
/// Returns the last action that was decided on by the Agent.

6
com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs


var agentInfoProto = ai.ToAgentInfoProto();
var agentActionProto = new AgentActionProto();
if(ai.storedVectorActions != null)
if (ai.storedVectorActions != null)
{
agentActionProto.VectorActions.AddRange(ai.storedVectorActions);
}

var brainParametersProto = new BrainParametersProto
{
VectorActionSize = { bp.VectorActionSize },
VectorActionSpaceType = (SpaceTypeProto) bp.VectorActionSpaceType,
VectorActionSpaceType = (SpaceTypeProto)bp.VectorActionSpaceType,
if(bp.VectorActionDescriptions != null)
if (bp.VectorActionDescriptions != null)
{
brainParametersProto.VectorActionDescriptions.AddRange(bp.VectorActionDescriptions);
}

36
com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs


Dictionary<string, BrainParameters> m_UnsentBrainKeys = new Dictionary<string, BrainParameters>();
# if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
#if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
/// The Unity to External client.
UnityToExternalProto.UnityToExternalProtoClient m_Client;
#endif

UnityInputProto Initialize(UnityOutputProto unityOutput,
out UnityInputProto unityInput)
{
# if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
#if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
m_IsOpen = true;
var channel = new Channel(
"localhost:" + m_CommunicatorInitParameters.port,

/// </summary>
public void Dispose()
{
# if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
#if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
if (!m_IsOpen)
{
return;

switch (command)
{
case CommandProto.Quit:
{
QuitCommandReceived?.Invoke();
return;
}
{
QuitCommandReceived?.Invoke();
return;
}
{
foreach (var brainName in m_OrderedAgentsRequestingDecisions.Keys)
m_OrderedAgentsRequestingDecisions[brainName].Clear();
foreach (var brainName in m_OrderedAgentsRequestingDecisions.Keys)
{
m_OrderedAgentsRequestingDecisions[brainName].Clear();
}
ResetCommandReceived?.Invoke();
return;
ResetCommandReceived?.Invoke();
return;
}
{
return;
}
{
return;
}
}
}

/// <param name="sensors">Sensors that will produce the observations</param>
public void PutObservations(string behaviorName, AgentInfo info, List<ISensor> sensors)
{
# if DEBUG
#if DEBUG
if (!m_SensorShapeValidators.ContainsKey(behaviorName))
{
m_SensorShapeValidators[behaviorName] = new SensorShapeValidator();

/// <param name="unityOutput">The UnityOutput to be sent.</param>
UnityInputProto Exchange(UnityOutputProto unityOutput)
{
# if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
#if UNITY_EDITOR || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_STANDALONE_LINUX
if (!m_IsOpen)
{
return null;

2
com.unity.ml-agents/Runtime/Communicator/UnityRLCapabilities.cs


/// A class holding the capabilities flags for Reinforcement Learning across C# and the Trainer codebase. This
/// struct will be used to inform users if and when they are using C# / Trainer features that are mismatched.
/// </summary>
public UnityRLCapabilities(bool baseRlCapabilities=true)
public UnityRLCapabilities(bool baseRlCapabilities = true)
{
m_BaseRLCapabilities = baseRlCapabilities;
}

20
com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs


public InferenceDevice InferenceDevice
{
get { return m_InferenceDevice; }
set { m_InferenceDevice = value; UpdateAgentPolicy();}
set { m_InferenceDevice = value; UpdateAgentPolicy(); }
}
[HideInInspector, SerializeField]

case BehaviorType.HeuristicOnly:
return GenerateHeuristicPolicy(heuristic);
case BehaviorType.InferenceOnly:
{
if (m_Model == null)
var behaviorType = BehaviorType.InferenceOnly.ToString();
throw new UnityAgentsException(
$"Can't use Behavior Type {behaviorType} without a model. " +
"Either assign a model, or change to a different Behavior Type."
);
if (m_Model == null)
{
var behaviorType = BehaviorType.InferenceOnly.ToString();
throw new UnityAgentsException(
$"Can't use Behavior Type {behaviorType} without a model. " +
"Either assign a model, or change to a different Behavior Type."
);
}
return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice);
return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice);
}
case BehaviorType.Default:
if (Academy.Instance.IsCommunicatorOn)
{

2
com.unity.ml-agents/Runtime/Policies/BrainParameters.cs


/// For the discrete action space: the number of branches in the action space.
/// </value>
[FormerlySerializedAs("vectorActionSize")]
public int[] VectorActionSize = new[] {1};
public int[] VectorActionSize = new[] { 1 };
/// <summary>
/// The list of strings describing what the actions correspond to.

2
com.unity.ml-agents/Runtime/Policies/HeuristicPolicy.cs


public float this[int index]
{
get { return 0.0f; }
set {}
set { }
}
}

14
com.unity.ml-agents/Runtime/Sampler.cs


using System.Collections.Generic;
using Unity.MLAgents.Inference.Utils;
using UnityEngine;
using Random=System.Random;
using Random = System.Random;
namespace Unity.MLAgents
{

// Will be used to normalize intervalFuncs
float sumIntervalSizes = 0;
//The number of intervals
int numIntervals = (int)(intervals.Count/2);
int numIntervals = (int)(intervals.Count / 2);
// List that will store interval lengths
float[] intervalSizes = new float[numIntervals];
// List that will store uniform distributions

for(int i = 0; i < numIntervals; i++)
for (int i = 0; i < numIntervals; i++)
{
var min = intervals[2 * i];
var max = intervals[2 * i + 1];

intervalFuncs[i] = () => min + (float)distr.NextDouble() * intervalSize;
}
// Normalize interval lengths
for(int i = 0; i < numIntervals; i++)
for (int i = 0; i < numIntervals; i++)
}
}
for(int i = 1; i < numIntervals; i++)
for (int i = 1; i < numIntervals; i++)
}
}
Multinomial intervalDistr = new Multinomial(seed + 1);
float MultiRange()
{

4
com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs


/// </summary>
public SensorCompressionType CompressionType
{
get { return m_CompressionType; }
get { return m_CompressionType; }
set { m_CompressionType = value; }
}

}
/// <inheritdoc/>
public void Update() {}
public void Update() { }
/// <inheritdoc/>
public void Reset() { }

14
com.unity.ml-agents/Runtime/Sensors/CameraSensorComponent.cs


/// </summary>
public Camera Camera
{
get { return m_Camera; }
get { return m_Camera; }
set { m_Camera = value; UpdateSensor(); }
}

/// </summary>
public string SensorName
{
get { return m_SensorName; }
get { return m_SensorName; }
set { m_SensorName = value; }
}

/// </summary>
public int Width
{
get { return m_Width; }
get { return m_Width; }
set { m_Width = value; }
}

/// </summary>
public int Height
{
get { return m_Height; }
set { m_Height = value; }
get { return m_Height; }
set { m_Height = value; }
}
[HideInInspector, SerializeField, FormerlySerializedAs("grayscale")]

/// </summary>
public bool Grayscale
{
get { return m_Grayscale; }
get { return m_Grayscale; }
set { m_Grayscale = value; }
}

/// </summary>
public SensorCompressionType CompressionType
{
get { return m_Compression; }
get { return m_Compression; }
set { m_Compression = value; UpdateSensor(); }
}

2
com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs


TensorShape m_TensorShape;