浏览代码

Merge remote-tracking branch 'origin/develop' into enable-flake8

/develop-generalizationTraining-TrainerController
Chris Elion 5 年前
当前提交
5d07ca1f
共有 161 个文件被更改,包括 22523 次插入476 次删除
  1. 5
      .gitignore
  2. 1
      README.md
  3. 10
      UnitySDK/Assets/ML-Agents/Examples/3DBall/Scenes/3DBall.unity
  4. 10
      UnitySDK/Assets/ML-Agents/Examples/3DBall/Scenes/3DBallHard.unity
  5. 2
      UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DAcademy.cs
  6. 19
      UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DAgent.cs
  7. 19
      UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DHardAgent.cs
  8. 6
      UnitySDK/Assets/ML-Agents/Examples/Soccer/Scenes/SoccerTwos.unity
  9. 17
      UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/AgentSoccer.cs
  10. 2
      UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerAcademy.cs
  11. 25
      UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerFieldArea.cs
  12. 8
      UnitySDK/Assets/ML-Agents/Examples/Tennis/Scenes/Tennis.unity
  13. 18
      UnitySDK/Assets/ML-Agents/Examples/Tennis/Scenes/TennisIL.unity
  14. 2
      UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/HitWall.cs
  15. 1
      UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/TennisAcademy.cs
  16. 40
      UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/TennisAgent.cs
  17. 2
      docs/Background-Machine-Learning.md
  18. 19
      docs/Installation.md
  19. 38
      docs/Learning-Environment-Examples.md
  20. 125
      docs/Training-Imitation-Learning.md
  21. 70
      docs/Training-PPO.md
  22. 99
      docs/Training-RewardSignals.md
  23. 23
      docs/Using-Docker.md
  24. 6
      gym-unity/gym_unity/envs/unity_env.py
  25. 4
      ml-agents-envs/mlagents/envs/__init__.py
  26. 13
      ml-agents-envs/mlagents/envs/brain.py
  27. 12
      ml-agents-envs/mlagents/envs/environment.py
  28. 2
      ml-agents-envs/setup.py
  29. 3
      ml-agents/mlagents/trainers/__init__.py
  30. 4
      ml-agents/mlagents/trainers/bc/policy.py
  31. 14
      ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py
  32. 32
      ml-agents/mlagents/trainers/components/reward_signals/curiosity/signal.py
  33. 17
      ml-agents/mlagents/trainers/components/reward_signals/extrinsic/signal.py
  34. 22
      ml-agents/mlagents/trainers/components/reward_signals/reward_signal.py
  35. 6
      ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py
  36. 39
      ml-agents/mlagents/trainers/demo_loader.py
  37. 7
      ml-agents/mlagents/trainers/learn.py
  38. 37
      ml-agents/mlagents/trainers/models.py
  39. 39
      ml-agents/mlagents/trainers/ppo/policy.py
  40. 30
      ml-agents/mlagents/trainers/ppo/trainer.py
  41. 49
      ml-agents/mlagents/trainers/tests/mock_brain.py
  42. 13
      ml-agents/mlagents/trainers/tests/test_demo_loader.py
  43. 6
      ml-agents/mlagents/trainers/tests/test_learn.py
  44. 8
      ml-agents/mlagents/trainers/tests/test_policy.py
  45. 76
      ml-agents/mlagents/trainers/tests/test_ppo.py
  46. 154
      ml-agents/mlagents/trainers/tests/test_reward_signals.py
  47. 134
      ml-agents/mlagents/trainers/tests/test_trainer_controller.py
  48. 27
      ml-agents/mlagents/trainers/trainer.py
  49. 187
      ml-agents/mlagents/trainers/trainer_controller.py
  50. 14
      ml-agents/mlagents/trainers/tf_policy.py
  51. 6
      setup.cfg
  52. 92
      docs/Training-BehavioralCloning.md
  53. 80
      docs/images/mlagents-ImitationAndRL.png
  54. 38
      ml-agents-envs/mlagents/envs/env_manager.py
  55. 10
      ml-agents-envs/mlagents/envs/policy.py
  56. 187
      ml-agents-envs/mlagents/envs/subprocess_env_manager.py
  57. 110
      ml-agents-envs/mlagents/envs/tests/test_subprocess_env_manager.py
  58. 96
      ml-agents-envs/mlagents/envs/tests/test_timers.py
  59. 181
      ml-agents-envs/mlagents/envs/timers.py
  60. 158
      ml-agents/mlagents/trainers/tests/test_bcmodule.py
  61. 1001
      ml-agents/mlagents/trainers/tests/testdcvis.demo
  62. 442
      demos/Expert3DBall.demo
  63. 1001
      demos/Expert3DBallHard.demo
  64. 1001
      demos/ExpertBanana.demo
  65. 171
      demos/ExpertBasic.demo
  66. 198
      demos/ExpertBouncer.demo
  67. 1001
      demos/ExpertCrawlerSta.demo
  68. 1001
      demos/ExpertGrid.demo
  69. 1001
      demos/ExpertHallway.demo
  70. 1001
      demos/ExpertPush.demo
  71. 1001
      demos/ExpertPyramid.demo
  72. 1001
      demos/ExpertReacher.demo
  73. 1001
      demos/ExpertSoccerGoal.demo
  74. 1001
      demos/ExpertSoccerStri.demo
  75. 1001
      demos/ExpertTennis.demo
  76. 1001
      demos/ExpertWalker.demo
  77. 0
      docs/localized/KR/docs/Migrating.md
  78. 0
      docs/localized/KR/docs/Readme.md
  79. 115
      docs/localized/KR/docs/images/academy.png
  80. 70
      docs/localized/KR/docs/images/agent.png
  81. 664
      docs/localized/KR/docs/images/anaconda_default.PNG
  82. 635
      docs/localized/KR/docs/images/anaconda_install.PNG
  83. 1001
      docs/localized/KR/docs/images/balance.png
  84. 1001
      docs/localized/KR/docs/images/banana.png
  85. 611
      docs/localized/KR/docs/images/banner.png
  86. 69
      docs/localized/KR/docs/images/basic.png
  87. 51
      docs/localized/KR/docs/images/bc_teacher_helper.png
  88. 955
      docs/localized/KR/docs/images/bouncer.png
  89. 121
      docs/localized/KR/docs/images/brain.png
  90. 139
      docs/localized/KR/docs/images/broadcast.png
  91. 268
      docs/localized/KR/docs/images/conda_new.PNG
  92. 1001
      docs/localized/KR/docs/images/crawler.png

5
.gitignore


.DS_Store
.ipynb_checkpoints
# pytest cache
# pytest cache
*.pytest_cache/
# Ignore compiled protobuf files.

# Python virtual environment
venv/
.mypy_cache/
# Code coverage report
.coverage

1
README.md


we welcome any enhancements and improvements from the community.
* [Chinese](docs/localized/zh-CN/)
* [Korean](docs/localized/KR/)
## License

10
UnitySDK/Assets/ML-Agents/Examples/3DBall/Scenes/3DBall.unity


m_ReflectionIntensity: 1
m_CustomReflection: {fileID: 0}
m_Sun: {fileID: 0}
m_IndirectSpecularColor: {r: 0.44824862, g: 0.49827534, b: 0.57558274, a: 1}
m_IndirectSpecularColor: {r: 0.44824898, g: 0.49827564, b: 0.5755826, a: 1}
--- !u!157 &3
LightmapSettings:
m_ObjectHideFlags: 0

timeScale: 1
targetFrameRate: -1
resetParameters:
resetParameters: []
resetParameters:
- key: mass
value: 1
- key: gravity
value: 9.81
- key: scale
value: 1
--- !u!1 &1746325439
GameObject:
m_ObjectHideFlags: 0

10
UnitySDK/Assets/ML-Agents/Examples/3DBall/Scenes/3DBallHard.unity


m_ReflectionIntensity: 1
m_CustomReflection: {fileID: 0}
m_Sun: {fileID: 0}
m_IndirectSpecularColor: {r: 0.45096254, g: 0.5008292, b: 0.5744089, a: 1}
m_IndirectSpecularColor: {r: 0.45096314, g: 0.50082976, b: 0.57440954, a: 1}
--- !u!157 &3
LightmapSettings:
m_ObjectHideFlags: 0

timeScale: 1
targetFrameRate: -1
resetParameters:
resetParameters: []
resetParameters:
- key: mass
value: 1
- key: gravity
value: 9.81
- key: scale
value: 1
--- !u!1001 &1591880668
Prefab:
m_ObjectHideFlags: 0

2
UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DAcademy.cs


{
public override void AcademyReset()
{
Physics.gravity = new Vector3(0, -resetParameters["gravity"], 0);
}
public override void AcademyStep()

19
UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DAgent.cs


[Header("Specific to Ball3D")]
public GameObject ball;
private Rigidbody ballRb;
private ResetParameters resetParams;
var academy = Object.FindObjectOfType<Academy>() as Academy;
resetParams = academy.resetParameters;
SetResetParameters();
}
public override void CollectObservations()

public override void AgentAction(float[] vectorAction, string textAction)
{
if (brain.brainParameters.vectorActionSpaceType == SpaceType.continuous)
{
var actionZ = 2f * Mathf.Clamp(vectorAction[0], -1f, 1f);

ballRb.velocity = new Vector3(0f, 0f, 0f);
ball.transform.position = new Vector3(Random.Range(-1.5f, 1.5f), 4f, Random.Range(-1.5f, 1.5f))
+ gameObject.transform.position;
//Reset the parameters when the Agent is reset.
SetResetParameters();
}
public void SetBall()
{
//Set the attributes of the ball by fetching the information from the academy
ballRb.mass = resetParams["mass"];
var scale = resetParams["scale"];
ball.transform.localScale = new Vector3(scale, scale, scale);
public void SetResetParameters()
{
SetBall();
}
}

19
UnitySDK/Assets/ML-Agents/Examples/3DBall/Scripts/Ball3DHardAgent.cs


[Header("Specific to Ball3DHard")]
public GameObject ball;
private Rigidbody ballRb;
private ResetParameters resetParams;
var academy = Object.FindObjectOfType<Academy>() as Academy;
resetParams = academy.resetParameters;
SetResetParameters();
}
public override void CollectObservations()

public override void AgentAction(float[] vectorAction, string textAction)
{
if (brain.brainParameters.vectorActionSpaceType == SpaceType.continuous)
{
var actionZ = 2f * Mathf.Clamp(vectorAction[0], -1f, 1f);

}
public void SetBall()
{
//Set the attributes of the ball by fetching the information from the academy
ballRb.mass = resetParams["mass"];
var scale = resetParams["scale"];
ball.transform.localScale = new Vector3(scale, scale, scale);
}
public void SetResetParameters()
{
SetBall();
}
}

6
UnitySDK/Assets/ML-Agents/Examples/Soccer/Scenes/SoccerTwos.unity


timeScale: 2
targetFrameRate: 60
resetParameters:
resetParameters: []
resetParameters:
- key: ball_scale
value: 7.5
- key: gravity
value: 9.81
brainStriker: {fileID: 11400000, guid: 29ed78b3e8fef4340b3a1f6954b88f18, type: 2}
brainGoalie: {fileID: 11400000, guid: 090fa5a8588f5433bb7f878e6f5ac954, type: 2}
redMaterial: {fileID: 2100000, guid: 776dd8b57653342839c3fb5f46ce664e, type: 2}

17
UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/AgentSoccer.cs


public enum Team
{
Red,
Red,
Striker,
Striker,
[HideInInspector]
public Rigidbody agentRb;
SoccerAcademy academy;

var playerState = new PlayerState
{
agentRB = agentRb,
startingPos = transform.position,
agentRB = agentRb,
startingPos = transform.position,
agentScript = this,
};
area.playerStates.Add(playerState);

transform.position = area.GetRandomSpawnPos(agentRole, team);
agentRb.velocity = Vector3.zero;
agentRb.angularVelocity = Vector3.zero;
SetResetParameters();
}
public void SetResetParameters()
{
area.ResetBall();
}
}

2
UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerAcademy.cs


}
public override void AcademyReset()
{
Physics.gravity = new Vector3(0, -resetParameters["gravity"], 0);
}
public override void AcademyStep()

25
UnitySDK/Assets/ML-Agents/Examples/Soccer/Scripts/SoccerFieldArea.cs


[System.Serializable]
public class PlayerState
{
public int playerIndex;
public Rigidbody agentRB;
public Vector3 startingPos;
public AgentSoccer agentScript;
public int playerIndex;
public Rigidbody agentRB;
public Vector3 startingPos;
public AgentSoccer agentScript;
public float ballPosReward;
}

public GameObject ball;
[HideInInspector]
public Rigidbody ballRB;
public GameObject ground;
public GameObject ground;
public GameObject centerPitch;
SoccerBallController ballController;
public List<PlayerState> playerStates = new List<PlayerState>();

public IEnumerator GoalScoredSwapGroundMaterial(Material mat, float time)
{
groundRenderer.material = mat;
yield return new WaitForSeconds(time);
yield return new WaitForSeconds(time);
groundRenderer.material = groundMaterial;
}

academy = FindObjectOfType<SoccerAcademy>();
groundRenderer = centerPitch.GetComponent<Renderer>();
groundRenderer = centerPitch.GetComponent<Renderer>();
groundMaterial = groundRenderer.material;
canResetBall = true;
if (goalTextUI) { goalTextUI.SetActive(false); }

{
xOffset = xOffset * -1f;
}
var randomSpawnPos = ground.transform.position +
new Vector3(xOffset, 0f, 0f)
var randomSpawnPos = ground.transform.position +
new Vector3(xOffset, 0f, 0f)
+ (Random.insideUnitSphere * 2);
randomSpawnPos.y = ground.transform.position.y + 2;
return randomSpawnPos;

{
var randomSpawnPos = ground.transform.position +
new Vector3(0f, 0f, 0f)
var randomSpawnPos = ground.transform.position +
new Vector3(0f, 0f, 0f)
+ (Random.insideUnitSphere * 2);
randomSpawnPos.y = ground.transform.position.y + 2;
return randomSpawnPos;

ball.transform.position = GetBallSpawnPosition();
ballRB.velocity = Vector3.zero;
ballRB.angularVelocity = Vector3.zero;
var ballScale = academy.resetParameters["ball_scale"];
ballRB.transform.localScale = new Vector3(ballScale, ballScale, ballScale);
}
}

8
UnitySDK/Assets/ML-Agents/Examples/Tennis/Scenes/Tennis.unity


timeScale: 1
targetFrameRate: 60
resetParameters:
resetParameters: []
resetParameters:
- key: gravity
value: 9.81
- key: angle
value: 55
- key: scale
value: 1
--- !u!1001 &1065879750
Prefab:
m_ObjectHideFlags: 0

18
UnitySDK/Assets/ML-Agents/Examples/Tennis/Scenes/TennisIL.unity


brain: {fileID: 11400000, guid: 6bf6a586a645b471bb9bd1194ae0e229, type: 2}
agentParameters:
agentCameras: []
agentRenderTextures: []
maxStep: 5000
resetOnDone: 1
onDemandDecision: 0

score: 0
scoreText: {fileID: 2073469450}
opponent: {fileID: 1894084401}
angle: 0
scale: 0
--- !u!65 &348265184
BoxCollider:
m_ObjectHideFlags: 0

timeScale: 1
targetFrameRate: 60
resetParameters:
resetParameters: []
resetParameters:
- key: angle
value: 55
- key: scale
value: 1
- key: gravity
value: 9.81
--- !u!1 &1114726487
GameObject:
m_ObjectHideFlags: 0

brain: {fileID: 11400000, guid: 1674996276be448c2ad51fb139e21e05, type: 2}
agentParameters:
agentCameras: []
agentRenderTextures: []
maxStep: 5000
resetOnDone: 1
onDemandDecision: 0

score: 0
scoreText: {fileID: 1871669621}
opponent: {fileID: 348265181}
angle: 0
scale: 0
--- !u!65 &1894084404
BoxCollider:
m_ObjectHideFlags: 0

2
UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/HitWall.cs


lastAgentHit = collision.gameObject.name == "AgentA" ? 0 : 1;
}
}
}
}

1
UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/TennisAcademy.cs


public override void AcademyReset()
{
Physics.gravity = new Vector3(0, -resetParameters["gravity"], 0);
}
public override void AcademyStep()

40
UnitySDK/Assets/ML-Agents/Examples/Tennis/Scripts/TennisAgent.cs


public bool invertX;
public int score;
public GameObject myArea;
public float angle;
public float scale;
private ResetParameters resetParams;
// Looks for the scoreboard based on the name of the gameObjects.
// Do not modify the names of the Score GameObjects

ballRb = ball.GetComponent<Rigidbody>();
var canvas = GameObject.Find(CanvasName);
GameObject scoreBoard;
var academy = Object.FindObjectOfType<Academy>() as Academy;
resetParams = academy.resetParameters;
if (invertX)
{
scoreBoard = canvas.transform.Find(ScoreBoardBName).gameObject;

scoreBoard = canvas.transform.Find(ScoreBoardAName).gameObject;
}
textComponent = scoreBoard.GetComponent<Text>();
SetResetParameters();
}
public override void CollectObservations()

{
var moveX = Mathf.Clamp(vectorAction[0], -1f, 1f) * invertMult;
var moveY = Mathf.Clamp(vectorAction[1], -1f, 1f);
if (moveY > 0.5 && transform.position.y - transform.parent.transform.position.y < -1.5f)
{
agentRb.velocity = new Vector3(agentRb.velocity.x, 7f, 0f);

if (invertX && transform.position.x - transform.parent.transform.position.x < -invertMult ||
if (invertX && transform.position.x - transform.parent.transform.position.x < -invertMult ||
transform.position = new Vector3(-invertMult + transform.parent.transform.position.x,
transform.position.y,
transform.position.z);
transform.position = new Vector3(-invertMult + transform.parent.transform.position.x,
transform.position.y,
transform.position.z);
}
textComponent.text = score.ToString();

transform.position = new Vector3(-invertMult * Random.Range(6f, 8f), -1.5f, 0f) + transform.parent.transform.position;
agentRb.velocity = new Vector3(0f, 0f, 0f);
SetResetParameters();
}
public void SetRacket()
{
angle = resetParams["angle"];
gameObject.transform.eulerAngles = new Vector3(
gameObject.transform.eulerAngles.x,
gameObject.transform.eulerAngles.y,
invertMult * angle
);
}
public void SetBall()
{
scale = resetParams["scale"];
ball.transform.localScale = new Vector3(scale, scale, scale);
}
public void SetResetParameters()
{
SetRacket();
SetBall();
}
}

2
docs/Background-Machine-Learning.md


[Learning a policy](https://blogs.unity3d.com/2017/08/22/unity-ai-reinforcement-learning-with-q-learning/)
usually requires many trials and iterative policy updates. More specifically,
the robot is placed in several fire situations and over time learns an optimal
policy which allows it to put our fires more effectively. Obviously, we cannot
policy which allows it to put out fires more effectively. Obviously, we cannot
expect to train a robot repeatedly in the real world, particularly when fires
are involved. This is precisely why the use of
[Unity as a simulator](https://blogs.unity3d.com/2018/01/23/designing-safer-cities-through-simulations/)

19
docs/Installation.md


### Install Python and mlagents Package
In order to use ML-Agents toolkit, you need Python 3.6 along with the
dependencies listed in the [setup.py file](../ml-agents/setup.py).
Some of the primary dependencies include:
- [TensorFlow](Background-TensorFlow.md) (Requires a CPU w/ AVX support)
- [Jupyter](Background-Jupyter.md)
[Download](https://www.python.org/downloads/) and install Python 3.6 if you do not
already have it.
In order to use ML-Agents toolkit, you need Python 3.6.
[Download](https://www.python.org/downloads/) and install Python 3.6 if you do not already have it.
To install the dependencies and `mlagents` Python package, run from the command line:
To install the `mlagents` Python package, run from the command line:
```sh
pip3 install mlagents

If you installed this correctly, you should be able to run
`mlagents-learn --help`, after which you will see the Unity logo and the command line
parameters you can use with `mlagents-learn`.
By installing the `mlagents` package, its dependencies listed in the [setup.py file](../ml-agents/setup.py) are also installed.
Some of the primary dependencies include:
- [TensorFlow](Background-TensorFlow.md) (Requires a CPU w/ AVX support)
- [Jupyter](Background-Jupyter.md)
**Notes:**

38
docs/Learning-Environment-Examples.md


* Vector Action space: (Continuous) Size of 2, with one value corresponding to
X-rotation, and the other to Z-rotation.
* Visual Observations: None.
* Reset Parameters: None
* Reset Parameters: Three, corresponding to the following:
* scale: Specifies the scale of the ball in the 3 dimensions (equal across the three dimensions)
* Default: 1
* Recommended Minimum: 0.2
* Recommended Maximum: 5
* gravity: Magnitude of gravity
* Default: 9.81
* Recommended Minimum: 4
* Recommended Maximum: 105
* mass: Specifies mass of the ball
* Default: 1
* Recommended Minimum: 0.1
* Recommended Maximum: 20
* Benchmark Mean Reward: 100
## [GridWorld](https://youtu.be/gu8HE9WKEVI)

* Vector Action space: (Continuous) Size of 2, corresponding to movement
toward net or away from net, and jumping.
* Visual Observations: None.
* Reset Parameters: One, corresponding to size of ball.
* Reset Parameters: Three, corresponding to the following:
* angle: Angle of the racket from the vertical (Y) axis.
* Default: 55
* Recommended Minimum: 35
* Recommended Maximum: 65
* gravity: Magnitude of gravity
* Default: 9.81
* Recommended Minimum: 6
* Recommended Maximum: 20
* scale: Specifies the scale of the ball in the 3 dimensions (equal across the three dimensions)
* Default: 1
* Recommended Minimum: 0.2
* Recommended Maximum: 5
* Benchmark Mean Reward: 2.5
* Optional Imitation Learning scene: `TennisIL`.

as well as rotation.
* Goalie: 4 actions corresponding to forward, backward, sideways movement.
* Visual Observations: None.
* Reset Parameters: None
* Reset Parameters: Two, corresponding to the following:
* ball_scale: Specifies the scale of the ball in the 3 dimensions (equal across the three dimensions)
* Default: 7.5
* Recommended minimum: 4
* Recommended maximum: 10
* gravity: Magnitude of the gravity
* Default: 9.81
* Recommended minimum: 6
* Recommended maximum: 20
* Benchmark Mean Reward (Striker & Goalie Brain): 0 (the means will be inverse
of each other and criss crosses during training) __Note that our trainer is currently unable to consistently train this environment__

125
docs/Training-Imitation-Learning.md


Imitation Learning uses pairs of observations and actions from
from a demonstration to learn a policy. [Video Link](https://youtu.be/kpb8ZkMBFYs).
Imitation learning can also be used to help reinforcement learning. Especially in
environments with sparse (i.e., infrequent or rare) rewards, the agent may never see
the reward and thus not learn from it. Curiosity helps the agent explore, but in some cases
it is easier to just show the agent how to achieve the reward. In these cases,
imitation learning can dramatically reduce the time it takes to solve the environment.
For instance, on the [Pyramids environment](Learning-Environment-Examples.md#pyramids),
just 6 episodes of demonstrations can reduce training steps by more than 4 times.
<p align="center">
<img src="images/mlagents-ImitationAndRL.png"
alt="Using Demonstrations with Reinforcement Learning"
width="350" border="0" />
</p>
ML-Agents provides several ways to learn from demonstrations. For most situations,
[GAIL](Training-RewardSignals.md#the-gail-reward-signal) is the preferred approach.
* To train using GAIL (Generative Adversarial Imitaiton Learning) you can add the
[GAIL reward signal](Training-RewardSignals.md#the-gail-reward-signal). GAIL can be
used with or without environment rewards, and works well when there are a limited
number of demonstrations.
* To help bootstrap reinforcement learning, you can enable
[pretraining](Training-PPO.md#optional-pretraining-using-demonstrations)
on the PPO trainer, in addition to using a small GAIL reward signal.
* To train an agent to exactly mimic demonstrations, you can use the
[Behavioral Cloning](Training-BehavioralCloning.md) trainer. Behavioral Cloning can be
used offline and online (in-editor), and learns very quickly. However, it usually is ineffective
on more complex environments without a large number of demonstrations.
## Recording Demonstrations
It is possible to record demonstrations of agent behavior from the Unity Editor,

alt="BC Teacher Helper"
width="375" border="10" />
</p>
## Training with Behavioral Cloning
There are a variety of possible imitation learning algorithms which can
be used, the simplest one of them is Behavioral Cloning. It works by collecting
demonstrations from a teacher, and then simply uses them to directly learn a
policy, in the same way the supervised learning for image classification
or other traditional Machine Learning tasks work.
### Offline Training
With offline behavioral cloning, we can use demonstrations (`.demo` files)
generated using the `Demonstration Recorder` as the dataset used to train a behavior.
1. Choose an agent you would like to learn to imitate some set of demonstrations.
2. Record a set of demonstration using the `Demonstration Recorder` (see above).
For illustrative purposes we will refer to this file as `AgentRecording.demo`.
3. Build the scene, assigning the agent a Learning Brain, and set the Brain to
Control in the Broadcast Hub. For more information on Brains, see
[here](Learning-Environment-Design-Brains.md).
4. Open the `config/offline_bc_config.yaml` file.
5. Modify the `demo_path` parameter in the file to reference the path to the
demonstration file recorded in step 2. In our case this is:
`./UnitySDK/Assets/Demonstrations/AgentRecording.demo`
6. Launch `mlagent-learn`, providing `./config/offline_bc_config.yaml`
as the config parameter, and include the `--run-id` and `--train` as usual.
Provide your environment as the `--env` parameter if it has been compiled
as standalone, or omit to train in the editor.
7. (Optional) Observe training performance using TensorBoard.
This will use the demonstration file to train a neural network driven agent
to directly imitate the actions provided in the demonstration. The environment
will launch and be used for evaluating the agent's performance during training.
### Online Training
It is also possible to provide demonstrations in realtime during training,
without pre-recording a demonstration file. The steps to do this are as follows:
1. First create two Brains, one which will be the "Teacher," and the other which
will be the "Student." We will assume that the names of the Brain
Assets are "Teacher" and "Student" respectively.
2. The "Teacher" Brain must be a **Player Brain**. You must properly
configure the inputs to map to the corresponding actions.
3. The "Student" Brain must be a **Learning Brain**.
4. The Brain Parameters of both the "Teacher" and "Student" Brains must be
compatible with the agent.
5. Drag both the "Teacher" and "Student" Brain into the Academy's `Broadcast Hub`
and check the `Control` checkbox on the "Student" Brain.
6. Link the Brains to the desired Agents (one Agent as the teacher and at least
one Agent as a student).
7. In `config/online_bc_config.yaml`, add an entry for the "Student" Brain. Set
the `trainer` parameter of this entry to `online_bc`, and the
`brain_to_imitate` parameter to the name of the teacher Brain: "Teacher".
Additionally, set `batches_per_epoch`, which controls how much training to do
each moment. Increase the `max_steps` option if you'd like to keep training
the Agents for a longer period of time.
8. Launch the training process with `mlagents-learn config/online_bc_config.yaml
--train --slow`, and press the :arrow_forward: button in Unity when the
message _"Start training by pressing the Play button in the Unity Editor"_ is
displayed on the screen
9. From the Unity window, control the Agent with the Teacher Brain by providing
"teacher demonstrations" of the behavior you would like to see.
10. Watch as the Agent(s) with the student Brain attached begin to behave
similarly to the demonstrations.
11. Once the Student Agents are exhibiting the desired behavior, end the training
process with `CTL+C` from the command line.
12. Move the resulting `*.nn` file into the `TFModels` subdirectory of the
Assets folder (or a subdirectory within Assets of your choosing) , and use
with `Learning` Brain.
**BC Teacher Helper**
We provide a convenience utility, `BC Teacher Helper` component that you can add
to the Teacher Agent.
<p align="center">
<img src="images/bc_teacher_helper.png"
alt="BC Teacher Helper"
width="375" border="10" />
</p>
This utility enables you to use keyboard shortcuts to do the following:
1. To start and stop recording experiences. This is useful in case you'd like to
interact with the game _but not have the agents learn from these
interactions_. The default command to toggle this is to press `R` on the
keyboard.
2. Reset the training buffer. This enables you to instruct the agents to forget
their buffer of recent experiences. This is useful if you'd like to get them
to quickly learn a new behavior. The default command to reset the buffer is
to press `C` on the keyboard.

70
docs/Training-PPO.md


presented to an agent, see [Training with Curriculum
Learning](Training-Curriculum-Learning.md).
For information about imitation learning, which uses a different training
algorithm, see
For information about imitation learning from demonstrations, see
[Training with Imitation Learning](Training-Imitation-Learning.md).
## Best Practices when training with PPO

the agent will need to remember in order to successfully complete the task.
Typical Range: `64` - `512`
## (Optional) Pretraining Using Demonstrations
In some cases, you might want to bootstrap the agent's policy using behavior recorded
from a player. This can help guide the agent towards the reward. Pretraining adds
training operations that mimic a demonstration rather than attempting to maximize reward.
It is essentially equivalent to running [behavioral cloning](./Training-BehavioralCloning.md)
in-line with PPO.
To use pretraining, add a `pretraining` section to the trainer_config. For instance:
```
pretraining:
demo_path: ./demos/ExpertPyramid.demo
strength: 0.5
steps: 10000
```
Below are the avaliable hyperparameters for pretraining.
### Strength
`strength` corresponds to the learning rate of the imitation relative to the learning
rate of PPO, and roughly corresponds to how strongly we allow the behavioral cloning
to influence the policy.
Typical Range: `0.1` - `0.5`
### Demo Path
`demo_path` is the path to your `.demo` file or directory of `.demo` files.
See the [imitation learning guide](Training-ImitationLearning.md) for more on `.demo` files.
### Steps
During pretraining, it is often desirable to stop using demonstrations after the agent has
"seen" rewards, and allow it to optimize past the available demonstrations and/or generalize
outside of the provided demonstrations. `steps` corresponds to the training steps over which
pretraining is active. The learning rate of the pretrainer will anneal over the steps. Set
the steps to 0 for constant imitation over the entire training run.
### (Optional) Batch Size
`batch_size` is the number of demonstration experiences used for one iteration of a gradient
descent update. If not specified, it will default to the `batch_size` defined for PPO.
Typical Range (Continuous): `512` - `5120`
Typical Range (Discrete): `32` - `512`
### (Optional) Number of Epochs
`num_epoch` is the number of passes through the experience buffer during
gradient descent. If not specified, it will default to the number of epochs set for PPO.
Typical Range: `3` - `10`
### (Optional) Samples Per Update
`samples_per_update` is the maximum number of samples
to use during each imitation update. You may want to lower this if your demonstration
dataset is very large to avoid overfitting the policy on demonstrations. Set to 0
to train over all of the demonstrations at each update step.
Default Value: `0` (all)
Typical Range: Approximately equal to PPO's `buffer_size`
## Training Statistics

99
docs/Training-RewardSignals.md


observation, but also not too small to prevent it from learning to differentiate between
demonstrated and actual behavior.
Default Value: 64
Default Value: `64`
Typical Range: `64` - `256`
#### Learning Rate

Default Value: `3e-4`
### The GAIL Reward Signal
GAIL, or [Generative Adversarial Imitation Learning](https://arxiv.org/abs/1606.03476), is an
imitation learning algorithm that uses an adversarial approach, in a similar vein to GANs
(Generative Adversarial Networks). In this framework, a second neural network, the
discriminator, is taught to distinguish whether an observation/action is from a demonstration, or
produced by the agent. This discriminator can the examine a new observation/action and provide it a
reward based on how close it believes this new observation/action is to the provided demonstrations.
At each training step, the agent tries to learn how to maximize this reward. Then, the
discriminator is trained to better distinguish between demonstrations and agent state/actions.
In this way, while the agent gets better and better at mimicing the demonstrations, the
discriminator keeps getting stricter and stricter and the agent must try harder to "fool" it.
This approach, when compared to [Behavioral Cloning](Training-BehavioralCloning.md), requires
far fewer demonstrations to be provided. After all, we are still learning a policy that happens
to be similar to the demonstration, not directly copying the behavior of the demonstrations. It
is also especially effective when combined with an Extrinsic signal, but can also be used
independently to purely learn from demonstration.
Using GAIL requires recorded demonstrations from your Unity environment. See the
[imitation learning guide](Training-Imitation-Learning.md) to learn more about recording demonstrations.
#### Strength
`strength` is the factor by which to multiply the raw reward. Note that when using GAIL
with an Extrinsic Signal, this value should be set lower if your demonstrations are
suboptimal (e.g. from a human), so that a trained agent will focus on receiving extrinsic
rewards instead of exactly copying the demonstrations. Keep the strength below about 0.1 in those cases.
Typical Range: `0.01` - `1.0`
#### Gamma
`gamma` corresponds to the discount factor for future rewards.
Typical Range: `0.8` - `0.9`
#### Demo Path
`demo_path` is the path to your `.demo` file or directory of `.demo` files. See the [imitation learning guide]
(Training-ImitationLearning.md).
#### Encoding Size
`encoding_size` corresponds to the size of the hidden layer used by the discriminator.
This value should be small enough to encourage the discriminator to compress the original
observation, but also not too small to prevent it from learning to differentiate between
demonstrated and actual behavior. Dramatically increasing this size will also negatively affect
training times.
Default Value: `64`
Typical Range: `64` - `256`
#### Learning Rate
`learning_rate` is the learning rate used to update the discriminator.
This should typically be decreased if training is unstable, and the GAIL loss is unstable.
Default Value: `3e-4`
Typical Range: `1e-5` - `1e-3`
#### Use Actions
`use_actions` determines whether the discriminator should discriminate based on both
observations and actions, or just observations. Set to `True` if you want the agent to
mimic the actions from the demonstrations, and `False` if you'd rather have the agent
visit the same states as in the demonstrations but with possibly different actions.
Setting to `False` is more likely to be stable, especially with imperfect demonstrations,
but may learn slower.
Default Value: `false`
#### (Optional) Samples Per Update
`samples_per_update` is the maximum number of samples to use during each discriminator update. You may
want to lower this if your buffer size is very large to avoid overfitting the discriminator on current data.
If set to 0, we will use the minimum of buffer size and the number of demonstration samples.
Default Value: `0`
Typical Range: Approximately equal to [`buffer_size`](Training-PPO.md)
#### (Optional) Variational Discriminator Bottleneck
`use_vail` enables a [variational bottleneck](https://arxiv.org/abs/1810.00821) within the
GAIL discriminator. This forces the discriminator to learn a more general representation
and reduces its tendency to be "too good" at discriminating, making learning more stable.
However, it does increase training time. Enable this if you notice your imitation learning is
unstable, or unable to learn the task at hand.
Default Value: `false`

23
docs/Using-Docker.md


the repository:
```sh
docker run --name <container-name> \
docker run -it --name <container-name> \
-p 6006:6006 \
<image-name>:latest \
--docker-target-name=unity-volume \
<trainer-config-file> \

To train with a `3DBall` environment executable, the command would be:
```sh
docker run --name 3DBallContainer.first.trial \
docker run -it --name 3DBallContainer.first.trial \
-p 6006:6006 \
--env=3DBall
--env=3DBall \
--train \
--run-id=3dball_first_trial
```

**NOTE** If you are training using docker for environments that use visual observations, you may need to increase the default memory that Docker allocates for the container. For example, see [here](https://docs.docker.com/docker-for-mac/#advanced) for instructions for Docker for Mac.
### Running Tensorboard
You can run Tensorboard to monitor your training instance on http://localhost:6006:
```sh
docker exec -it <container-name> tensorboard --logdir=/unity-volume/summaries --host=0.0.0.0
```
With our previous 3DBall example, this command would look like this:
```sh
docker exec -it 3DBallContainer.first.trial tensorboard --logdir=/unity-volume/summaries --host=0.0.0.0
```
For more details on Tensorboard, check out the documentation about [Using Tensorboard](Using-Tensorboard.md).
### Stopping Container and Saving State

6
gym-unity/gym_unity/envs/unity_env.py


def _single_step(self, info):
if self.use_visual:
visual_obs = info.visual_observations
if isinstance(visual_obs, list):
visual_obs = np.array(visual_obs)
visual_obs_list.append(self._preprocess_single(obs[0, :, :, :]))
visual_obs_list.append(self._preprocess_single(obs[0]))
self.visual_obs = self._preprocess_single(visual_obs[0][0, :, :, :])
self.visual_obs = self._preprocess_single(visual_obs[0][0])
default_observation = self.visual_obs
else:

4
ml-agents-envs/mlagents/envs/__init__.py


from .brain import *
from .brain import AllBrainInfo, BrainInfo, BrainParameters
from .action_info import ActionInfo, ActionInfoOutputs
from .policy import Policy
from .environment import *
from .exception import *

13
ml-agents-envs/mlagents/envs/brain.py


return np.append(m1, m2, axis=0)
@staticmethod
def process_pixels(image_bytes, gray_scale):
def process_pixels(image_bytes: bytes, gray_scale: bool) -> np.ndarray:
"""
Converts byte array observation image into numpy array, re-sizes it,
and optionally converts it to grey scale

"""
s = bytearray(image_bytes)
image = Image.open(io.BytesIO(s))
image_bytearray = bytearray(image_bytes)
image = Image.open(io.BytesIO(image_bytearray))
s = np.array(image) / 255.0
if gray_scale:
s = np.mean(s, axis=2)

@staticmethod
def from_agent_proto(agent_info_list, brain_params):
def from_agent_proto(worker_id: int, agent_info_list, brain_params):
vis_obs = []
vis_obs: List[np.ndarray] = []
for i in range(brain_params.number_visual_observations):
obs = [
BrainInfo.process_pixels(

vector_obs = np.nan_to_num(
np.array([x.stacked_vector_observation for x in agent_info_list])
)
agents = [f"${worker_id}-{x.id}" for x in agent_info_list]
brain_info = BrainInfo(
visual_observation=vis_obs,
vector_observation=vector_obs,

agents=[x.id for x in agent_info_list],
agents=agents,
local_done=[x.done for x in agent_info_list],
vector_action=np.array([x.stored_vector_actions for x in agent_info_list]),
text_action=[list(x.stored_text_actions) for x in agent_info_list],

12
ml-agents-envs/mlagents/envs/environment.py


docker_training: bool = False,
no_graphics: bool = False,
timeout_wait: int = 30,
args: list = [],
):
"""
Starts a new unity environment and establishes a connection with the environment.

:bool no_graphics: Whether to run the Unity simulator in no-graphics mode
:int timeout_wait: Time (in seconds) to wait for connection from environment.
:bool train_mode: Whether to run in training mode, speeding up the simulation, by default.
:list args: Addition Unity command line arguments
"""
atexit.register(self._close)

None
) # The process that is started. If None, no process was started
self.communicator = self.get_communicator(worker_id, base_port, timeout_wait)
self.worker_id = worker_id
# If the environment name is None, a new environment will not be launched
# and the communicator will directly try to connect to an existing unity environment.

"the worker-id must be 0 in order to connect with the Editor."
)
if file_name is not None:
self.executable_launcher(file_name, docker_training, no_graphics)
self.executable_launcher(file_name, docker_training, no_graphics, args)
else:
logger.info(
"Start training by pressing the Play button in the Unity Editor."

def reset_parameters(self):
return self._resetParameters
def executable_launcher(self, file_name, docker_training, no_graphics):
def executable_launcher(self, file_name, docker_training, no_graphics, args):
cwd = os.getcwd()
file_name = (
file_name.strip()

"--port",
str(self.port),
]
+ args
[launch_string, "--port", str(self.port)]
[launch_string, "--port", str(self.port)] + args
)
else:
"""

for brain_name in output.agentInfos:
agent_info_list = output.agentInfos[brain_name].value
_data[brain_name] = BrainInfo.from_agent_proto(
agent_info_list, self.brains[brain_name]
self.worker_id, agent_info_list, self.brains[brain_name]
)
return _data, global_done

2
ml-agents-envs/setup.py


"pytest>=3.2.2,<4.0.0",
"protobuf>=3.6,<3.7",
"grpcio>=1.11.0,<1.12.0",
"cloudpickle==0.8.1",
"cloudpickle",
],
python_requires=">=3.5,<3.8",
)

3
ml-agents/mlagents/trainers/__init__.py


from .action_info import *
from .buffer import *
from .curriculum import *
from .meta_curriculum import *

from .policy import *
from .tf_policy import *
from .trainer_controller import *
from .bc.models import *
from .bc.offline_trainer import *

4
ml-agents/mlagents/trainers/bc/policy.py


import numpy as np
from mlagents.trainers.bc.models import BehavioralCloningModel
from mlagents.trainers.policy import Policy
from mlagents.trainers.tf_policy import TFPolicy
class BCPolicy(Policy):
class BCPolicy(TFPolicy):
def __init__(self, seed, brain, trainer_parameters, load):
"""
:param seed: Random seed.

14
ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py


from typing import List, Tuple
import tensorflow as tf
from mlagents.trainers.models import LearningModel

"""
self.encoding_size = encoding_size
self.policy_model = policy_model
self.next_visual_in: List[tf.Tensor] = []
def create_curiosity_encoders(self):
def create_curiosity_encoders(self) -> Tuple[tf.Tensor, tf.Tensor]:
"""
Creates state encoders for current and future observations.
Used for implementation of Curiosity-driven Exploration by Self-supervised Prediction

encoded_next_state = tf.concat(encoded_next_state_list, axis=1)
return encoded_state, encoded_next_state
def create_inverse_model(self, encoded_state, encoded_next_state):
def create_inverse_model(
self, encoded_state: tf.Tensor, encoded_next_state: tf.Tensor
) -> None:
"""
Creates inverse model TensorFlow ops for Curiosity module.
Predicts action taken given current and future encoded states.

tf.dynamic_partition(cross_entropy, self.policy_model.mask, 2)[1]
)
def create_forward_model(self, encoded_state, encoded_next_state):
def create_forward_model(
self, encoded_state: tf.Tensor, encoded_next_state: tf.Tensor
) -> None:
"""
Creates forward model TensorFlow ops for Curiosity module.
Predicts encoded future state based on encoded current state and given action.

tf.dynamic_partition(squared_difference, self.policy_model.mask, 2)[1]
)
def create_loss(self, learning_rate):
def create_loss(self, learning_rate: float) -> None:
"""
Creates the loss node of the model as well as the update_batch optimizer to update the model.
:param learning_rate: The learning rate for the optimizer.

32
ml-agents/mlagents/trainers/components/reward_signals/curiosity/signal.py


from typing import Any, Dict, List
from mlagents.envs.brain import BrainInfo
from mlagents.trainers.buffer import Buffer
from mlagents.trainers.policy import Policy
from mlagents.trainers.tf_policy import TFPolicy
policy: Policy,
policy: TFPolicy,
strength: float,
gamma: float,
encoding_size: int = 128,

"""
Creates the Curiosity reward generator
:param policy: The Learning Policy
:param encoding_size: The size of the Curiosity encoding
:param signal_strength: The scaling parameter for the reward. The scaled reward will be the unscaled
:param strength: The scaling parameter for the reward. The scaled reward will be the unscaled
:param gamma: The time discounting factor used for this reward.
:param encoding_size: The size of the hidden encoding layer for the ICM
:param learning_rate: The learning rate for the ICM.
:param num_epoch: The number of epochs to train over the training buffer for the ICM.
"""
super().__init__(policy, strength, gamma)
self.model = CuriosityModel(

}
self.has_updated = False
def evaluate(self, current_info, next_info):
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
) -> RewardSignalResult:
"""
Evaluates the reward for the agents present in current_info given the next_info
:param current_info: The current BrainInfo.

return RewardSignalResult(scaled_reward, unscaled_reward)
@classmethod
def check_config(cls, config_dict):
def check_config(
cls, config_dict: Dict[str, Any], param_keys: List[str] = None
) -> None:
"""
Checks the config and throw an exception if a hyperparameter is missing. Curiosity requires strength,
gamma, and encoding size at minimum.

def update(self, update_buffer, num_sequences):
def update(self, update_buffer: Buffer, num_sequences: int) -> Dict[str, float]:
"""
Updates Curiosity model using training buffer. Divides training buffer into mini batches and performs
gradient descent.

"""
forward_total, inverse_total = [], []
forward_total: List[float] = []
inverse_total: List[float] = []
for _ in range(self.num_epoch):
update_buffer.shuffle()
buffer = update_buffer

}
return update_stats
def _update_batch(self, mini_batch, num_sequences):
def _update_batch(
self, mini_batch: Dict[str, np.ndarray], num_sequences: int
) -> Dict[str, float]:
"""
Updates model using buffer.
:param num_sequences: Number of trajectories in batch.

17
ml-agents/mlagents/trainers/components/reward_signals/extrinsic/signal.py


from typing import Any, Dict, List
from mlagents.envs.brain import BrainInfo
from mlagents.trainers.buffer import Buffer
from mlagents.trainers.policy import Policy
from mlagents.trainers.tf_policy import TFPolicy
def __init__(self, policy: Policy, strength: float, gamma: float):
def __init__(self, policy: TFPolicy, strength: float, gamma: float):
"""
The extrinsic reward generator. Returns the reward received by the environment
:param policy: The Policy object (e.g. PPOPolicy) that this Reward Signal will apply to.

super().__init__(policy, strength, gamma)
@classmethod
def check_config(cls, config_dict):
def check_config(
cls, config_dict: Dict[str, Any], param_keys: List[str] = None
) -> None:
"""
Checks the config and throw an exception if a hyperparameter is missing. Extrinsic requires strength and gamma
at minimum.

def evaluate(self, current_info, next_info):
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
) -> RewardSignalResult:
"""
Evaluates the reward for the agents present in current_info given the next_info
:param current_info: The current BrainInfo.

scaled_reward = self.strength * unscaled_reward
return RewardSignalResult(scaled_reward, unscaled_reward)
def update(self, update_buffer, num_sequences):
def update(self, update_buffer: Buffer, num_sequences: int) -> Dict[str, float]:
"""
This method does nothing, as there is nothing to update.
"""

22
ml-agents/mlagents/trainers/components/reward_signals/reward_signal.py


import logging
from mlagents.trainers.trainer import UnityTrainerException
from mlagents.trainers.policy import Policy
from typing import Any, Dict, List
from collections import namedtuple
import numpy as np
import abc

from mlagents.envs.brain import BrainInfo
from mlagents.trainers.trainer import UnityTrainerException
from mlagents.trainers.tf_policy import TFPolicy
from mlagents.trainers.buffer import Buffer
logger = logging.getLogger("mlagents.trainers")
RewardSignalResult = namedtuple(

class RewardSignal(abc.ABC):
def __init__(self, policy: Policy, strength: float, gamma: float):
def __init__(self, policy: TFPolicy, strength: float, gamma: float):
"""
Initializes a reward signal. At minimum, you must pass in the policy it is being applied to,
the reward strength, and the gamma (discount factor.)

self.policy = policy
self.strength = strength
def evaluate(self, current_info, next_info):
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
) -> RewardSignalResult:
"""
Evaluates the reward for the agents present in current_info given the next_info
:param current_info: The current BrainInfo.

return (
return RewardSignalResult(
def update(self, update_buffer, n_sequences):
def update(self, update_buffer: Buffer, num_sequences: int) -> Dict[str, float]:
"""
If the reward signal has an internal model (e.g. GAIL or Curiosity), update that model.
:param update_buffer: An AgentBuffer that contains the live data from which to update.

return {}
@classmethod
def check_config(cls, config_dict, param_keys=None):
def check_config(
cls, config_dict: Dict[str, Any], param_keys: List[str] = None
) -> None:
"""
Check the config dict, and throw an error if there are missing hyperparameters.
"""

6
ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py


from mlagents.trainers.components.reward_signals.extrinsic.signal import (
ExtrinsicRewardSignal,
)
from mlagents.trainers.components.reward_signals.gail.signal import GAILRewardSignal
from mlagents.trainers.policy import Policy
from mlagents.trainers.tf_policy import TFPolicy
logger = logging.getLogger("mlagents.trainers")

"curiosity": CuriosityRewardSignal,
"gail": GAILRewardSignal,
policy: Policy, name: str, config_entry: Dict[str, Any]
policy: TFPolicy, name: str, config_entry: Dict[str, Any]
) -> RewardSignal:
"""
Creates a reward signal class based on the name and config entry provided as a dict.

39
ml-agents/mlagents/trainers/demo_loader.py


import pathlib
import logging
import os
from typing import List, Tuple
from mlagents.envs.communicator_objects import *
from mlagents.envs.communicator_objects import (
AgentInfoProto,
BrainParametersProto,