浏览代码

Develop remove past action communication (#2913)

* Modifying the .proto files

* attempt 1 at refactoring Python

* works for ppo hallway

* changing the documentation

* now works with both sac and ppo both training and inference

* Ned to fix the tests

* TODOs :
 - Fix the demonstration recorder
 - Fix the demonstration loader
 - verify the intrinsic reward signals work
 - Fix the tests on Python
 - Fix the C# tests

* Regenerating the protos

* fix proto typo

* protos and modifying the C# demo recorder

* modified the demo loader

* Demos are loading

* IMPORTANT : THESE ARE THE FILES USED FOR CONVERSION FROM OLD TO NEW FORMAT

* Modified all the demo files

* Fixing all the tests

* fixing ci

* addressing comments

* removing reference to memories in the ll-api
/develop-newnormalization
GitHub 5 年前
当前提交
69d1a033
共有 53 个文件被更改,包括 6486 次插入5507 次删除
  1. 4
      UnitySDK/Assets/ML-Agents/Editor/Tests/DemonstrationTests.cs
  2. 4
      UnitySDK/Assets/ML-Agents/Scripts/DemonstrationStore.cs
  3. 38
      UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/AgentInfo.cs
  4. 24
      UnitySDK/Assets/ML-Agents/Scripts/Grpc/GrpcExtensions.cs
  5. 2
      UnitySDK/Assets/ML-Agents/Scripts/Grpc/RpcCommunicator.cs
  6. 18
      UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/GeneratorImpl.cs
  7. 1001
      demos/Expert3DBall.demo
  8. 1001
      demos/Expert3DBallHard.demo
  9. 366
      demos/ExpertBasic.demo
  10. 510
      demos/ExpertBouncer.demo
  11. 995
      demos/ExpertCrawlerDyn.demo
  12. 1000
      demos/ExpertFood.demo
  13. 1000
      demos/ExpertGrid.demo
  14. 1001
      demos/ExpertHallway.demo
  15. 1001
      demos/ExpertPyramid.demo
  16. 1001
      demos/ExpertReacher.demo
  17. 1001
      demos/ExpertTennis.demo
  18. 5
      docs/Python-API.md
  19. 3
      ml-agents-envs/mlagents/envs/brain.py
  20. 23
      ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.py
  21. 6
      ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.pyi
  22. 5
      ml-agents-envs/mlagents/envs/mock_communicator.py
  23. 3
      ml-agents/mlagents/trainers/components/reward_signals/__init__.py
  24. 4
      ml-agents/mlagents/trainers/components/reward_signals/curiosity/signal.py
  25. 2
      ml-agents/mlagents/trainers/components/reward_signals/extrinsic/signal.py
  26. 4
      ml-agents/mlagents/trainers/components/reward_signals/gail/signal.py
  27. 58
      ml-agents/mlagents/trainers/demo_loader.py
  28. 10
      ml-agents/mlagents/trainers/ppo/policy.py
  29. 17
      ml-agents/mlagents/trainers/rl_trainer.py
  30. 6
      ml-agents/mlagents/trainers/sac/policy.py
  31. 7
      ml-agents/mlagents/trainers/tests/mock_brain.py
  32. 344
      ml-agents/mlagents/trainers/tests/test.demo
  33. 16
      ml-agents/mlagents/trainers/tests/test_demo_loader.py
  34. 4
      ml-agents/mlagents/trainers/tests/test_reward_signals.py
  35. 2
      ml-agents/mlagents/trainers/tests/test_rl_trainer.py
  36. 2
      ml-agents/mlagents/trainers/tests/test_simple_rl.py
  37. 153
      ml-agents/mlagents/trainers/tests/testdcvis.demo
  38. 50
      ml-agents/mlagents/trainers/tf_policy.py
  39. 2
      protobuf-definitions/proto/mlagents/envs/communicator_objects/agent_info.proto
  40. 219
      UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/AgentInfoActionPair.cs
  41. 11
      UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/AgentInfoActionPair.cs.meta
  42. 83
      ml-agents-envs/mlagents/envs/communicator_objects/agent_info_action_pair_pb2.py
  43. 57
      ml-agents-envs/mlagents/envs/communicator_objects/agent_info_action_pair_pb2.pyi
  44. 130
      ml-agents/mlagents/trainers/tests/test_demo_dir/test4.demo
  45. 130
      ml-agents/mlagents/trainers/tests/test_demo_dir/test5.demo
  46. 130
      ml-agents/mlagents/trainers/tests/test_demo_dir/test6.demo
  47. 12
      protobuf-definitions/proto/mlagents/envs/communicator_objects/agent_info_action_pair.proto
  48. 176
      ml-agents/mlagents/trainers/tests/test_demo_dir/test.demo
  49. 176
      ml-agents/mlagents/trainers/tests/test_demo_dir/test2.demo
  50. 176
      ml-agents/mlagents/trainers/tests/test_demo_dir/test3.demo

4
UnitySDK/Assets/ML-Agents/Editor/Tests/DemonstrationTests.cs


reader.Seek(DemonstrationStore.MetaDataBytes + 1, 0);
BrainParametersProto.Parser.ParseDelimitedFrom(reader);
var agentInfoProto = AgentInfoProto.Parser.ParseDelimitedFrom(reader);
var agentInfoProto = AgentInfoActionPairProto.Parser.ParseDelimitedFrom(reader).AgentInfo;
var obs = agentInfoProto.Observations[2]; // skip dummy sensors
{
var vecObs = obs.FloatData.Data;

Assert.AreEqual((float) i+1, vecObs[i]);
Assert.AreEqual((float)i + 1, vecObs[i]);
}
}

4
UnitySDK/Assets/ML-Agents/Scripts/DemonstrationStore.cs


}
m_Writer = m_FileSystem.File.Create(m_FilePath);
m_MetaData = new DemonstrationMetaData {demonstrationName = demonstrationName};
m_MetaData = new DemonstrationMetaData { demonstrationName = demonstrationName };
var metaProto = m_MetaData.ToProto();
metaProto.WriteDelimitedTo(m_Writer);
}

}
// Write AgentInfo to file.
var agentProto = info.ToProto();
var agentProto = info.ToInfoActionPairProto();
agentProto.WriteDelimitedTo(m_Writer);
}

38
UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects/AgentInfo.cs


string.Concat(
"CjNtbGFnZW50cy9lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL2FnZW50X2lu",
"Zm8ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGjRtbGFnZW50cy9lbnZz",
"L2NvbW11bmljYXRvcl9vYmplY3RzL29ic2VydmF0aW9uLnByb3RvIuoBCg5B",
"Z2VudEluZm9Qcm90bxIdChVzdG9yZWRfdmVjdG9yX2FjdGlvbnMYBCADKAIS",
"DgoGcmV3YXJkGAcgASgCEgwKBGRvbmUYCCABKAgSGAoQbWF4X3N0ZXBfcmVh",
"Y2hlZBgJIAEoCBIKCgJpZBgKIAEoBRITCgthY3Rpb25fbWFzaxgLIAMoCBI8",
"CgxvYnNlcnZhdGlvbnMYDSADKAsyJi5jb21tdW5pY2F0b3Jfb2JqZWN0cy5P",
"YnNlcnZhdGlvblByb3RvSgQIARACSgQIAhADSgQIAxAESgQIBRAGSgQIBhAH",
"SgQIDBANQh+qAhxNTEFnZW50cy5Db21tdW5pY2F0b3JPYmplY3RzYgZwcm90",
"bzM="));
"L2NvbW11bmljYXRvcl9vYmplY3RzL29ic2VydmF0aW9uLnByb3RvItEBCg5B",
"Z2VudEluZm9Qcm90bxIOCgZyZXdhcmQYByABKAISDAoEZG9uZRgIIAEoCBIY",
"ChBtYXhfc3RlcF9yZWFjaGVkGAkgASgIEgoKAmlkGAogASgFEhMKC2FjdGlv",
"bl9tYXNrGAsgAygIEjwKDG9ic2VydmF0aW9ucxgNIAMoCzImLmNvbW11bmlj",
"YXRvcl9vYmplY3RzLk9ic2VydmF0aW9uUHJvdG9KBAgBEAJKBAgCEANKBAgD",
"EARKBAgEEAVKBAgFEAZKBAgGEAdKBAgMEA1CH6oCHE1MQWdlbnRzLkNvbW11",
"bmljYXRvck9iamVjdHNiBnByb3RvMw=="));
new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.AgentInfoProto), global::MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "StoredVectorActions", "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "Observations" }, null, null, null)
new pbr::GeneratedClrTypeInfo(typeof(global::MLAgents.CommunicatorObjects.AgentInfoProto), global::MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "Observations" }, null, null, null)
}));
}
#endregion

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public AgentInfoProto(AgentInfoProto other) : this() {
storedVectorActions_ = other.storedVectorActions_.Clone();
reward_ = other.reward_;
done_ = other.done_;
maxStepReached_ = other.maxStepReached_;

return new AgentInfoProto(this);
}
/// <summary>Field number for the "stored_vector_actions" field.</summary>
public const int StoredVectorActionsFieldNumber = 4;
private static readonly pb::FieldCodec<float> _repeated_storedVectorActions_codec
= pb::FieldCodec.ForFloat(34);
private readonly pbc::RepeatedField<float> storedVectorActions_ = new pbc::RepeatedField<float>();
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public pbc::RepeatedField<float> StoredVectorActions {
get { return storedVectorActions_; }
}
/// <summary>Field number for the "reward" field.</summary>
public const int RewardFieldNumber = 7;
private float reward_;

if (ReferenceEquals(other, this)) {
return true;
}
if(!storedVectorActions_.Equals(other.storedVectorActions_)) return false;
if (!pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.Equals(Reward, other.Reward)) return false;
if (Done != other.Done) return false;
if (MaxStepReached != other.MaxStepReached) return false;

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public override int GetHashCode() {
int hash = 1;
hash ^= storedVectorActions_.GetHashCode();
if (Reward != 0F) hash ^= pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.GetHashCode(Reward);
if (Done != false) hash ^= Done.GetHashCode();
if (MaxStepReached != false) hash ^= MaxStepReached.GetHashCode();

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public void WriteTo(pb::CodedOutputStream output) {
storedVectorActions_.WriteTo(output, _repeated_storedVectorActions_codec);
if (Reward != 0F) {
output.WriteRawTag(61);
output.WriteFloat(Reward);

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public int CalculateSize() {
int size = 0;
size += storedVectorActions_.CalculateSize(_repeated_storedVectorActions_codec);
if (Reward != 0F) {
size += 1 + 4;
}

if (other == null) {
return;
}
storedVectorActions_.Add(other.storedVectorActions_);
if (other.Reward != 0F) {
Reward = other.Reward;
}

default:
_unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input);
break;
case 34:
case 37: {
storedVectorActions_.AddEntriesFrom(input, _repeated_storedVectorActions_codec);
break;
}
case 61: {
Reward = input.ReadFloat();
break;

24
UnitySDK/Assets/ML-Agents/Scripts/Grpc/GrpcExtensions.cs


{
public static class GrpcExtensions
{
/// <summary>
/// Converts a AgentInfo to a protobuf generated AgentInfoActionPairProto
/// </summary>
/// <returns>The protobuf version of the AgentInfoActionPairProto.</returns>
public static AgentInfoActionPairProto ToInfoActionPairProto(this AgentInfo ai)
{
var agentInfoProto = ai.ToAgentInfoProto();
var agentActionProto = new AgentActionProto
{
VectorActions = { ai.storedVectorActions }
};
return new AgentInfoActionPairProto
{
AgentInfo = agentInfoProto,
ActionInfo = agentActionProto
};
}
public static AgentInfoProto ToProto(this AgentInfo ai)
public static AgentInfoProto ToAgentInfoProto(this AgentInfo ai)
StoredVectorActions = { ai.storedVectorActions },
Reward = ai.reward,
MaxStepReached = ai.maxStepReached,
Done = ai.done,

2
UnitySDK/Assets/ML-Agents/Scripts/Grpc/RpcCommunicator.cs


{
// Update the sensor data on the AgentInfo
agent.GenerateSensorData();
var agentInfoProto = agent.Info.ToProto();
var agentInfoProto = agent.Info.ToAgentInfoProto();
m_CurrentUnityRlOutput.AgentInfos[brainKey].Value.Add(agentInfoProto);
}

18
UnitySDK/Assets/ML-Agents/Scripts/InferenceBrain/GeneratorImpl.cs


{
var info = agent.Info;
List<float> memory;
if (agent.Info.done)
{
m_Memories.Remove(agent.Info.id);
}
for (var j = 0; j < memorySize; j++)
{
tensorProxy.data[agentIndex, j] = 0;
}
agentIndex++;
continue;
}

{
var offset = memorySize * m_MemoryIndex;
List<float> memory;
if (agent.Info.done)
{
m_Memories.Remove(agent.Info.id);
}
for (var j = 0; j < memorySize; j++)
{
tensorProxy.data[agentIndex, j] = 0;
}
agentIndex++;
continue;
}

1001
demos/Expert3DBall.demo
文件差异内容过多而无法显示
查看文件

1001
demos/Expert3DBallHard.demo
文件差异内容过多而无法显示
查看文件

366
demos/ExpertBasic.demo


 ExpertBasicI -��W?*: BasicLearning�
P�?"P���������jW
 ExpertBasicI -��W?*: BasicLearningn
dP���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=�p}?@P���������jW
P�?
@u
k=�p}?@P���������jW
P�?�
P�?"@=
P�?
@s
i=
P�?�
P�?"@=
P�?
@m
i=
P�?
P�?

510
demos/ExpertBouncer.demo


 ExpertBouncer` -�ćA ***0:BouncerLearning�
H~� ��
o?>�K@Dp�?l`@6��?" P���������jO
 ExpertBouncer` -�ćA ***0:BouncerLearningn
\P���������jO
o?>�K@Dp�?l`@6��?�
H~� ��
o?>�K@Dp�?l`@6��?^ @��t?xXO?���@.5��" ��>C���X�=6�|?P���������jO
o?>�K@Dp�?l`@6��?
��>C���X�s
a=6�|?P���������jO
o?>�K@Dp�?l`@6��?^ @��t?xXO?���@.5���
H~� ��
o?>�K@Dp�?l`@6��?^ @��t?xXO?��о��@.5���� ���G?&����*�W��@��" �*�����1�о=�i}?P���������jO
o?>�K@Dp�?l`@6��?^ @��t?xXO?���@.5��
�*�����1�оs
a=�i}?P���������jO
o?>�K@Dp�?l`@6��?^ @��t?xXO?���@.5���� ���G?&����*�W��@���
H^ @��t?xXO?��о��@.5���� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@���" �o&�ո>|>=��?P���������jO
o?>�K@Dp�?l`@6��?^ @��t?xXO?���@.5���� ���G?&����*�W��@��
�o&�ո>|>s
a=��?P���������jO
H^ @��t?xXO?��о��@.5���� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@����
H�� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?" y�>/�+��.þ=��|?P���������jO
H^ @��t?xXO?��о��@.5���� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@���
y�>/�+��.þs
a=��|?P���������jO
H�� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?�
HK�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@" �f��!о�%?=Qj|?P���������jO
H�� ���G?&����*�W��@��K�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?
�f��!о�%?s
a=Qj|?P���������jO
HK�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@�
H���?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?" ^��>[#����>=Lp}?P���������jO
HK�6�B�6?W鿶�X?�]@������?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@
^��>[#����>s
a=Lp}?P���������jO
H���?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?�
Hn��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@" ��>���_��=��}?P���������jO
H���?PXQ?���'#+���%@���?n��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?
��>���_��s
a=��}?P���������jO
Hn��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@�
HIڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@���" q%��Z��+V?=��y?P���������jO
Hn��:�[?�@�-#�V�>@�ӕ@Iڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@
q%��Z��+V?s
a=��y?P���������jO
HIڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@����
H��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?" l����B��S�=l�{?P���������jO
HIڼ�]�?���@LPA@=;�@Fe~?��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@���
l����B��S�s
a=l�{?P���������jO
H��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?�
Hr��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>" +Xp>��(>���>=E�~?P���������jO
H��a@�m?�{�"��?k1@Rc�@r��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?
+Xp>��(>���>s
a=E�~?P���������jO
Hr��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>�
HDl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@" 7vR>.��}�@�=g+?P���������jO
Hr��?H0c?Yc�@�c⿏��@���Dl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>
7vR>.��}�@�s
a=g+?P���������jO
HDl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@�
H���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?" �i�]qZ::��>=b�}?P���������jO
HDl����L?8(�� 3��]�@ƭV?���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@
�i�]qZ::��>s
a=b�}?P���������jO
H���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?�
H��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@" ��>ސ��)�6�=�p|?P���������jO
H���}�7?+-�? j�?�>�@�>��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?
��>ސ��)�6�s
a=�p|?P���������jO
H��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@�
HJ���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#�" � > �˾��?=�~?P���������jO
H��T?v#m?���S_O��ֵ@ks�@J���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@
� > �˾��?s
a=�~?P���������jO
HJ���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#��
H(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD��" ���>1�!�~ �=]\|?P���������jO
HJ���u�J?٥�@W��x��@��z?(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#�
���>1�!�~ �s
a=]\|?P���������jO
H(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD���
H�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?" ��W��$�9�>=�!{?P���������jO
H(c&�$�M?���N�;�,k@>b�@�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD��
��W��$�9�>s
a=�!{?P���������jO
H�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?�
H)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@���" ��>iI)�<A?=��|?P���������jO
H�f���e?&Y�@T@=G@%�#�)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?
��>iI)�<A?s
a=��|?P���������jO
H)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@����
HJa��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@���ñ��f8R?w�!�]^�K$@�4 >" �?�i�q���H�=%ky?P���������jO
H)X�@ڵ}?d{�J��4T@GD��Ja��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@���
�?�i�q���H�s
a=%ky?P���������jO
HJa��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@���ñ��f8R?w�!�]^�K$@�4 >�
Hδ�@@x��?�@{��@��j�" ��n=`��چ�>=~?@P���������jO
HJa��c�I?�S뾤Qk���Q@J��?E�>��Rp? ��@r�s��@���ñ��f8R?w�!�]^�K$@�4 >
��n=`��چ�>u
c=~?@P���������jO
Hδ�@@x��?�@{��@��j��
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@" �+^� �m�bѾ=~cx?P���������jO
Hδ�@@x��?�@{��@��j�
�+^� �m�bѾs
a=~cx?P���������jO
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@�
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@" ��i?�"��(?=y?P���������jO
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@
��i?�"��(?s
a=y?P���������jO
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@�
H*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@��" ҄�i�p�Y܊�=�I�?P���������jO
Hδ�@@x��?�@{��@��j�*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@
҄�i�p�Y܊�s
a=�I�?P���������jO
H*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@Ό
H���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'�" Z�.>ZJ�H�L�=��?P���������jO
H*�����W?�.�S @FԀ@���@���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@��
Z�.>ZJ�H�L�s
a=��?P���������jO
H���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'��
H����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@����" ��_>��g�l�X>=|?P���������jO
H���@��U?4&�@��8@�(�@��\@����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'�
��_>��g�l�X>s
a=|?P���������jO
H����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@�����
H$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@���������R?����%@�Zn@H�j�" �*�
4�<j��=\n}?P���������jO
H����=�a?�ޏ?ȓ}>��@��$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@����
�*�
4�<j��s
a=\n}?P���������jO
H$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@���������R?����%@�Zn@H�j��
H��;@ʋV?��>�� ����@���������R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j�" �-�>=\о�l�>=C �P���������jO
H$?>�u?^�����.@�� @f�'���;@ʋV?��>�� ����@���������R?����%@�Zn@H�j�
�-�>=\о�l�>s
a=C �P���������jO
H��;@ʋV?��>�� ����@���������R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j��
H�����R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@" �t�:8��&��=��~?P���������jO
H��;@ʋV?��>�� ����@���������R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j�
�t�:8��&��s
a=��~?P���������jO
H�����R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@�
H��t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@" �H/�K�u=��^?=�ڧ�P���������jO
H�����R?����%@�Zn@H�j���t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@
�H/�K�u=��^?s
a=�ڧ�P���������jO
H��t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@�
H�z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z��" �?�����1)�=�{?P���������jO
H��t?0�\?��^��%@�Zn@H�j��z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@
�?�����1)�s
a=�{?P���������jO
H�z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z���
H �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r�" 1W��#^���׾=�{?P���������jO
H�z?B1m?��t��3���M�@�)@ �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z��
1W��#^���׾s
a=�{?P���������jO
H �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r��
H�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG�" ��?�~��)>=�F}?P���������jO
H �D�q?c?A�3���M�@�)@�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r�
��?�~��)>s
a=�F}?P���������jO
H�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG��
HXd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@" q^�����,D�=�[?P���������jO
H�����s?i"�?�$�5�@�z��Xd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG�
q^�����,D�s
a=�[?P���������jO
HXd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@�
H ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@" �����=�_^?=�N�P���������jO
HXd���R?P�[�X�>5ja@�r� ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@
�����=�_^?s
a=�N�P���������jO
H ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@�
HlX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ��" � 2�]dc�ne��=��{?P���������jO
H ��?� s?�鸿��(�z��@ƨG�lX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@
� 2�]dc�ne��s
a=��{?P���������jO
HlX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ���
H�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?" w����ž�p
�=�~?P���������jO
HlX���u? /���`�1C@�v�@�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ��
w����ž�p
�s
a=�~?P���������jO
H�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?�
H�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?" �/5?;�׾3�>=RM\�P���������jO
H�l}�J'J?~�A�`�1C@�v�@�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?
�/5?;�׾3�>s
a=RM\�P���������jO
H�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?�
H,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@���" ����"��)�>=�2?P���������jO
H�-����w?��@i6����@� ��,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?
����"��)�>s
a=�2?P���������jO
H,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@�꿹
Hܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@�����@l��?�چ�F�?���@����" �>C��C;�=�D}?P���������jO
H,$��pb?RKI��~?vk�@���?ܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@���
�>C��C;�s
a=�D}?P���������jO
Hܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@�����@l��?�چ�F�?���@�����
H]ۙ�@B�Q�H-m@�9�@����" ��|�%���X�ȼ=�~?@P���������jO
Hܒt@S?��?�~?vk�@���?�V����z?�i@�o�?��@�����@l��?�چ�F�?���@����
��|�%���X�ȼu
c=�~?@P���������jO
H]ۙ�@B�Q�H-m@�9�@�����
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2�" ϓ[? r��$��=�q|?P���������jO
H]ۙ�@B�Q�H-m@�9�@����
ϓ[? r��$��s
a=�q|?P���������jO
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2��
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@����" �5d�N��tx>=%U{?P���������jO
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2�
�5d�N��tx>s
a=%U{?P���������jO
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@�����
H�a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@������(�!}?�U�������Y�@k� @" ?�=-��'���=0�?P���������jO
H]ۙ�@B�Q�H-m@�9�@�����a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@����
?�=-��'���s
a=0�?P���������jO
H�a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@������(�!}?�U�������Y�@k� @�
H�l���P?+���cQ%����@������(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S��" k�J�������\?=�|?P���������jO
H�a�@x~O?)���&`5�u�V@u�2��l���P?+���cQ%����@������(�!}?�U�������Y�@k� @
k�J�������\?s
a=�|?P���������jO
H�l���P?+���cQ%����@������(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S⿹
H��(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?" �;>?[;����Y�="z?P���������jO
H�l���P?+���cQ%����@������(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S��
�;>?[;����Y�s
a="z?P���������jO
H��(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?�
Hil��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?�����p?�~@,�þjZ�@���" �����Z���"?=z|?P���������jO
H��(�!}?�U�������Y�@k� @il��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?
�����Z���"?s
a=z|?P���������jO
Hil��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?�����p?�~@,�þjZ�@����
H�u�@�k?����+����@n�?�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@" Ī�>�/J=.D��=�~?P���������jO
Hil��r>M?�X�@��5@��@3S���u�@�k?����+����@n�?�����p?�~@,�þjZ�@���
Ī�>�/J=.D��s
a=�~?P���������jO
H�u�@�k?����+����@n�?�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@�
H�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@(���f?B��@O�>@�N�@� ��" �������?=b7}?P���������jO
H�u�@�k?����+����@n�?�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@
�������?s
a=b7}?P���������jO
H�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@(���f?B��@O�>@�N�@� ���
H�]G@^�`?����:��x��@�T�@(���f?B��@O�>@�N�@� ���ƽ@�2g?
�����4��.�@��D@" CyR?+� |�=��x?P���������jO
H�����p?�~@,�þjZ�@����]G@^�`?����:��x��@�T�@(���f?B��@O�>@�N�@� ��
CyR?+Ж� |�s
a=��x?P���������jO
�����4��.�@��D@�
H(���f?B��@O�>@�N�@� ���ƽ@�2g?
�����4��.�@��D@)>��(RE?���@:�����;@4,�?" �tj��� �%o?=ӗx?P���������jO
�����4��.�@��D@
�tj��� �%o?s
a=ӗx?P���������jO
�����4��.�@��D@)>��(RE?���@:�����;@4,�?�
H�ƽ@�2g?
�����4��.�@��D@)>��(RE?���@:�����;@4,�?�Ӟ���c?rw�?Jo��P@��" S�U>��H��X�=(|?P���������jO
�����4��.�@��D@)>��(RE?���@:�����;@4,�?
S�U>��H��X�s
a=(|?P���������jO
�����4��.�@��D@)>��(RE?���@:�����;@4,�?�Ӟ���c?rw�?Jo��P@���
H)>��(RE?���@:�����;@4,�?�Ӟ���c?rw�?Jo��P@��
���V�]?=�g�\U@@V��@�8�?" �[4=-+Y���k�=��|?P���������jO
�����4��.�@��D@)>��(RE?���@:�����;@4,�?�Ӟ���c?rw�?Jo��P@��
�[4=-+Y���k�s
a=��|?P���������jO
���V�]?=�g�\U@@V��@�8�?�
H�Ӟ���c?rw�?Jo��P@��
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?" ��P?H��rt�>=yX|?P���������jO
���V�]?=�g�\U@@V��@�8�?
��P?H��rt�>s
a=yX|?P���������jO
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?�
H
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿" ��|�Ԃ�>��9�=o{?P���������jO
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?
��|�Ԃ�>��9�s
a=o{?P���������jO
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿�
H(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@" KG^?׭�>U�'�=y�{?P���������jO
���V�]?=�g�\U@@V��@�8�?(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿
KG^?׭�>U�'�s
a=y�{?P���������jO
H(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@�
HD5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@��" s:ᄍ�����,?=��|?P���������jO
H(T�@��Z?��]@U��Wj�@�"�?D5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@
s:ᄍ�����,?s
a=��|?P���������jO
HD5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@���
H�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@��" �<C���&���
�=\;N�P���������jO
HD5�f_Y?���>�Ǔ�J�@w�޿�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@��
�<C���&���
�s
a=\;N�P���������jO
H�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@���
H�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@" �M=s�-���T�=�~?P���������jO
H�{�@|�N?x_�I@ ��@)L�@�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@��
�M=s�-���T�s
a=�~?P���������jO
H�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@�
H0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@�E�@�^?P��@X:@,^�@��0�" c3?���� �?=��{?P���������jO
H�f���M?A�@.�?����@��0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@
c3?���� �?s
a=��{?P���������jO
H0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@�E�@�^?P��@X:@,^�@��0��
Ha�<�@���?B@�@� �?" W���E*���;t�=T���@P���������jO
H0JG�2��?�s�>.�?����@�� }!�J�R?3!O���@���@�O�@�E�@�^?P��@X:@,^�@��0�
W���E*���;t�u
c=T���@P���������jO
Ha�<�@���?B@�@� �?�
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?" �.,?�.r�6v�>=a�#�P���������jO
Ha�<�@���?B@�@� �?
�.,?�.r�6v�>s
a=a�#�P���������jO
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?�
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��Ŀ"
1����O����==X�~?P���������jO
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?
1����O����=s
a=X�~?P���������jO
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��Ŀ�
H~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{�" ��5?�~.�%�ܾ=�}?P���������jO
Ha�<�@���?B@�@� �?~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��Ŀ
��5?�~.�%�ܾs
a=�}?P���������jO
H~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{��
H��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{�" 0�v��㪾�*B==fM��P���������jO
H~�q@jI�@n��??B@�@� �?��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{�
0�v��㪾�*B=s
a=fM��P���������jO
H��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{��
HVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H��" �۽<��r���==��?P���������jO
H��l���p?O@�k@��@��ĿVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{�
�۽<��r���=s
a=��?P���������jO
HVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H���
H9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:�" c�B?��μL1��=^u}?P���������jO
HVO�@�m�?4����<�@r{�9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H��
c�B?��μL1��s
a=^u}?P���������jO
H9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:��
H>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@" l�W����>���='i|?P���������jO
H9���?��?)gW���<�@r{�>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:�
l�W����>���s
a='i|?P���������jO
H>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@�
H���@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@" ��w?�^�>2~R?=���P���������jO
H>���S�?LA���.�?R��@H�����@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@
��w?�^�>2~R?s
a=���P���������jO
H���@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@�
H]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?" �3��4���=${?P���������jO
H���@v(x?�_s����B?�@gj:�]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@
�3��4���s
a=${?P���������jO
H]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?�
H�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-�" *q(�3�V�wD==��}?P���������jO
H]g���K?�/���k@ ��@LlC@�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?
*q(�3�V�wD=s
a=��}?P���������jO
H�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-��
H�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@" ���>ۉ+� ���=�S~?P���������jO
H�t�@�"E?��A�k@ ��@LlC@�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-�
���>ۉ+� ���s
a=�S~?P���������jO
H�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@�
H�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾" ���>��o��o?=���?P���������jO
H�E�?�c?���?����`��@jZ.?�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@
���>��o��o?s
a=���?P���������jO
H�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾�
H����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ�" T�*�'��<�E��=|5}?P���������jO
H�����tp?�< @m�l�HK�@S-�����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾
T�*�'��<�E��s
a=|5}?P���������jO
H����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ��
H9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ�T|Af�N?</W��,!�c��@�:��" e�~?]{>;..==փ{?P���������jO
H����)G�?`������?�7�@C�@9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ�
e�~?]{>;..=s
a=փ{?P���������jO
H9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ�T|Af�N?</W��,!�c��@�:���
HU��zm`?����0��@r2�@�T|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:��" a=q�=q�<K�z�=E���P���������jO
H9�Z@��a?2L9@|M ��Z�@��˾U��zm`?����0��@r2�@Κ�T|Af�N?</W��,!�c��@�:��
a=q�=q�<K�z�s
a=E���P���������jO
HU��zm`?����0��@r2�@�T|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:���
HT|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:��" � _=�O�9���==�xH�P���������jO
HU��zm`?����0��@r2�@�T|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:��
� _=�O�9���=s
a=�xH�P���������jO
HT|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:���
HYb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@" �D>����6��==-N?P���������jO
HT|Af�N?</W��,!�c��@�:��Yb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:��
�D>����6��=s
a=-N?P���������jO
HYb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@�
Hi�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@/��?K?5$�@d��>JZ�@֫�?" �8Y��[=Ē/?=��}?P���������jO
HYb���]?�����,!�c��@�:��i�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@
�8Y��[=Ē/?s
a=��}?P���������jO
Hi�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@/��?K?5$�@d��>JZ�@֫�?�
H������r?P֊�� !��{�@:�n@/��?K?5$�@d��>JZ�@֫�?`�@v\�?)+�
�W����@�'�" ��>�we�� %�=9�|?P���������jO
Hi�y���9?�ݵ��,!�c��@�:��������r?P֊�� !��{�@:�n@/��?K?5$�@d��>JZ�@֫�?
��>�we�� %�s
a=9�|?P���������jO
�W����@�'��
H�5 �@�6�?6D�@r�<@���" 0O6�Sa�d[�>=�}?@P���������jO
�W����@�'�
0O6�Sa�d[�>u
c=�}?@P���������jO
H�5 �@�6�?6D�@r�<@����
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@���" ;Q�>�T� ���=�G�P���������jO
H�5 �@�6�?6D�@r�<@���
;Q�>�T� ���s
a=�G�P���������jO
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@����
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:�" �5�������=Ŧ?P���������jO
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@���
�5�������s
a=Ŧ?P���������jO
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:��
H��@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:�l)>@�9O?D�l��<p>���@���@" G\"�����7H>=*�?P���������jO
H�5 �@�6�?6D�@r�<@�����@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:�
G\"�����7H>s
a=*�?P���������jO
H��@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:�l)>@�9O?D�l��<p>���@���@�
H�$�@�@?�ho��/�@U_�@:�l)>@�9O?D�l��<p>���@���@z�8��
:?c1A-��?'ט@ᘿ" �fž (�;{ 5?=D;}?P���������jO
H��@a�h?ܿ7�6D�@r�<@����$�@�@?�ho��/�@U_�@:�l)>@�9O?D�l��<p>���@���@
�fž (�;{ 5?s
a=D;}?P���������jO
:?c1A-��?'ט@ᘿ�
Hl)>@�9O?D�l��<p>���@���@z�8��
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?" Z��>���w�=�a{?P���������jO
:?c1A-��?'ט@ᘿ
Z��>���w�s
a=�a{?P���������jO
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?�
Hz�8��
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?" �0!�D�
�%�$?=^H{?P���������jO
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?
�0!�D�
�%�$?s
a=^H{?P���������jO
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?�
H� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?" #�*>�:���|�=�z?P���������jO
:?c1A-��?'ט@ᘿ� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?
#�*>�:���|�s
a=�z?P���������jO
H� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?�
H;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?���@��s?�=��.�?�/�@ƈ�" 3tL?�0H�����=`�z?P���������jO
H� @��m?�`x��C���q@�k�?;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?
3tL?�0H�����s
a=`�z?P���������jO
H;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?���@��s?�=��.�?�/�@ƈ��
H%O���S?�<?�_{@�K@��?���@��s?�=��.�?�/�@ƈ�_{�=v
W?J��Ą�@[�8@�Ք�" *����s �K��=��|?P���������jO
H;f���G[?���@/'`��B�@|�?%O���S?�<?�_{@�K@��?���@��s?�=��.�?�/�@ƈ�
*����s �K��s
a=��|?P���������jO
W?J��Ą�@[�8@�Ք��
H���@��s?�=��.�?�/�@ƈ�_{�=v
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ" ��?e�K��:>=�
W?J��Ą�@[�8@�Ք�
��?e�K��:>s
a=�
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ�
H_{�=v
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>" �����&2�
�>=$o�?P���������jO
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ
�����&2�
�>s
a=$o�?P���������jO
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>�
HV�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@���" :'���˻[�==W�~?P���������jO
W?J��Ą�@[�8@�Ք�V�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>
:'���˻[�=s
a=W�~?P���������jO
HV�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@����
H'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@����?��G?����n<?�!@b� �" ?��>c�n>_Z��=D�}?P���������jO
HV�@�Nc?���@�N@QQտ'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@���
?��>c�n>_Z��s
a=D�}?P���������jO
H'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@����?��G?����n<?�!@b� ��
H���� ]r?�z�?��H����@����?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@" N<y{ �!��>=��}?P���������jO
H'?�?�`V?�)����%���@���>���� ]r?�z�?��H����@����?��G?����n<?�!@b� �
N<y{ �!��>s
a=��}?P���������jO
H���� ]r?�z�?��H����@����?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@�
H�?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@s�b�)S?0�@�'�uy�@��@" ���� ���+?=��|?P���������jO
H���� ]r?�z�?��H����@����?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@
���� ���+?s
a=��|?P���������jO
H�?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@s�b�)S?0�@�'�uy�@��@�
H� +?��z?\#���*R�㲒@8�@s�b�)S?0�@�'�uy�@��@Z���
!�?�G @hI_��j�@�@" � =�O*==˰�=�z?P���������jO
H�?��G?����n<?�!@b� �� +?��z?\#���*R�㲒@8�@s�b�)S?0�@�'�uy�@��@
� =�O*==˰�e
a=�z?P���������jO
!�?�G @hI_��j�@�@
!�?�G @hI_��j�@�@

995
demos/ExpertCrawlerDyn.demo
文件差异内容过多而无法显示
查看文件

1000
demos/ExpertFood.demo
文件差异内容过多而无法显示
查看文件

1000
demos/ExpertGrid.demo
文件差异内容过多而无法显示
查看文件

1001
demos/ExpertHallway.demo
文件差异内容过多而无法显示
查看文件

1001
demos/ExpertPyramid.demo
文件差异内容过多而无法显示
查看文件

1001
demos/ExpertReacher.demo
文件差异内容过多而无法显示
查看文件

1001
demos/ExpertTennis.demo
文件差异内容过多而无法显示
查看文件

5
docs/Python-API.md


the list corresponds to the n<sup>th</sup> observation of the Brain.
- **`vector_observations`** : A two dimensional numpy array of dimension `(batch
size, vector observation size)`.
- **`memories`** : A two dimensional numpy array of dimension `(batch size,
memory size)` which corresponds to the memories sent at the previous step.
- **`rewards`** : A list as long as the number of Agents using the Brain
containing the rewards they each obtained at the previous step.
- **`local_done`** : A list as long as the number of Agents using the Brain

- **`agents`** : A list of the unique ids of the Agents using the Brain.
- **`previous_actions`** : A two dimensional numpy array of dimension `(batch
size, vector action size)` if the vector action space is continuous and
`(batch size, number of branches)` if the vector action space is discrete.
Once loaded, you can use your UnityEnvironment object, which referenced by a
variable named `env` in this example, can be used in the following way:

3
ml-agents-envs/mlagents/envs/brain.py


reward=None,
agents=None,
local_done=None,
vector_action=None,
max_reached=None,
action_mask=None,
):

self.local_done = local_done
self.max_reached = max_reached
self.agents = agents
self.previous_vector_actions = vector_action
self.action_masks = action_mask
@staticmethod

reward=[x.reward if not np.isnan(x.reward) else 0 for x in agent_info_list],
agents=agents,
local_done=[x.done for x in agent_info_list],
vector_action=np.array([x.stored_vector_actions for x in agent_info_list]),
max_reached=[x.max_step_reached for x in agent_info_list],
action_mask=mask_actions,
)

23
ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.py


name='mlagents/envs/communicator_objects/agent_info.proto',
package='communicator_objects',
syntax='proto3',
serialized_pb=_b('\n3mlagents/envs/communicator_objects/agent_info.proto\x12\x14\x63ommunicator_objects\x1a\x34mlagents/envs/communicator_objects/observation.proto\"\xea\x01\n\x0e\x41gentInfoProto\x12\x1d\n\x15stored_vector_actions\x18\x04 \x03(\x02\x12\x0e\n\x06reward\x18\x07 \x01(\x02\x12\x0c\n\x04\x64one\x18\x08 \x01(\x08\x12\x18\n\x10max_step_reached\x18\t \x01(\x08\x12\n\n\x02id\x18\n \x01(\x05\x12\x13\n\x0b\x61\x63tion_mask\x18\x0b \x03(\x08\x12<\n\x0cobservations\x18\r \x03(\x0b\x32&.communicator_objects.ObservationProtoJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x0c\x10\rB\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
serialized_pb=_b('\n3mlagents/envs/communicator_objects/agent_info.proto\x12\x14\x63ommunicator_objects\x1a\x34mlagents/envs/communicator_objects/observation.proto\"\xd1\x01\n\x0e\x41gentInfoProto\x12\x0e\n\x06reward\x18\x07 \x01(\x02\x12\x0c\n\x04\x64one\x18\x08 \x01(\x08\x12\x18\n\x10max_step_reached\x18\t \x01(\x08\x12\n\n\x02id\x18\n \x01(\x05\x12\x13\n\x0b\x61\x63tion_mask\x18\x0b \x03(\x08\x12<\n\x0cobservations\x18\r \x03(\x0b\x32&.communicator_objects.ObservationProtoJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x0c\x10\rB\x1f\xaa\x02\x1cMLAgents.CommunicatorObjectsb\x06proto3')
,
dependencies=[mlagents_dot_envs_dot_communicator__objects_dot_observation__pb2.DESCRIPTOR,])

containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='stored_vector_actions', full_name='communicator_objects.AgentInfoProto.stored_vector_actions', index=0,
number=4, type=2, cpp_type=6, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='reward', full_name='communicator_objects.AgentInfoProto.reward', index=1,
name='reward', full_name='communicator_objects.AgentInfoProto.reward', index=0,
number=7, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,

name='done', full_name='communicator_objects.AgentInfoProto.done', index=2,
name='done', full_name='communicator_objects.AgentInfoProto.done', index=1,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,

name='max_step_reached', full_name='communicator_objects.AgentInfoProto.max_step_reached', index=3,
name='max_step_reached', full_name='communicator_objects.AgentInfoProto.max_step_reached', index=2,
number=9, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,

name='id', full_name='communicator_objects.AgentInfoProto.id', index=4,
name='id', full_name='communicator_objects.AgentInfoProto.id', index=3,
number=10, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,

name='action_mask', full_name='communicator_objects.AgentInfoProto.action_mask', index=5,
name='action_mask', full_name='communicator_objects.AgentInfoProto.action_mask', index=4,
number=11, type=8, cpp_type=7, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,

name='observations', full_name='communicator_objects.AgentInfoProto.observations', index=6,
name='observations', full_name='communicator_objects.AgentInfoProto.observations', index=5,
number=13, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,

oneofs=[
],
serialized_start=132,
serialized_end=366,
serialized_end=341,
)
_AGENTINFOPROTO.fields_by_name['observations'].message_type = mlagents_dot_envs_dot_communicator__objects_dot_observation__pb2._OBSERVATIONPROTO

6
ml-agents-envs/mlagents/envs/communicator_objects/agent_info_pb2.pyi


class AgentInfoProto(google___protobuf___message___Message):
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
stored_vector_actions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float]
reward = ... # type: builtin___float
done = ... # type: builtin___bool
max_step_reached = ... # type: builtin___bool

def __init__(self,
*,
stored_vector_actions : typing___Optional[typing___Iterable[builtin___float]] = None,
reward : typing___Optional[builtin___float] = None,
done : typing___Optional[builtin___bool] = None,
max_step_reached : typing___Optional[builtin___bool] = None,

def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ...
if sys.version_info >= (3,):
def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"done",u"id",u"max_step_reached",u"observations",u"reward",u"stored_vector_actions"]) -> None: ...
def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"done",u"id",u"max_step_reached",u"observations",u"reward"]) -> None: ...
def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"observations",b"observations",u"reward",b"reward",u"stored_vector_actions",b"stored_vector_actions"]) -> None: ...
def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"done",b"done",u"id",b"id",u"max_step_reached",b"max_step_reached",u"observations",b"observations",u"reward",b"reward"]) -> None: ...

5
ml-agents-envs/mlagents/envs/mock_communicator.py


def _get_agent_infos(self):
dict_agent_info = {}
if self.is_discrete:
vector_action = [1]
else:
vector_action = [1, 2]
list_agent_info = []
vector_obs = [1, 2, 3]

list_agent_info.append(
AgentInfoProto(
reward=1,
stored_vector_actions=vector_action,
done=(i == 2),
max_step_reached=False,
id=i,

3
ml-agents/mlagents/trainers/components/reward_signals/__init__.py


self.stats_name_to_update_name: Dict[str, str] = {}
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
self, current_info: BrainInfo, action: np.array, next_info: BrainInfo
:param action: the action that was taken between the two infos
:param next_info: The BrainInfo from the next timestep.
:return: a RewardSignalResult of (scaled intrinsic reward, unscaled intrinsic reward) provided by the generator
"""

4
ml-agents/mlagents/trainers/components/reward_signals/curiosity/signal.py


self.has_updated = False
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
self, current_info: BrainInfo, action: np.array, next_info: BrainInfo
) -> RewardSignalResult:
"""
Evaluates the reward for the agents present in current_info given the next_info

return RewardSignalResult([], [])
mini_batch: Dict[str, np.array] = {}
# Construct the batch and use evaluate_batch
mini_batch["actions"] = next_info.previous_vector_actions
mini_batch["actions"] = action
mini_batch["done"] = np.reshape(next_info.local_done, [-1, 1])
for i in range(len(current_info.visual_observations)):
mini_batch["visual_obs%d" % i] = current_info.visual_observations[i]

2
ml-agents/mlagents/trainers/components/reward_signals/extrinsic/signal.py


super().check_config(config_dict, param_keys)
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
self, current_info: BrainInfo, action: np.array, next_info: BrainInfo
) -> RewardSignalResult:
"""
Evaluates the reward for the agents present in current_info given the next_info

4
ml-agents/mlagents/trainers/components/reward_signals/gail/signal.py


}
def evaluate(
self, current_info: BrainInfo, next_info: BrainInfo
self, current_info: BrainInfo, action: np.array, next_info: BrainInfo
mini_batch["actions"] = next_info.previous_vector_actions
mini_batch["actions"] = action
mini_batch["done"] = np.reshape(next_info.local_done, [-1, 1])
for i, obs in enumerate(current_info.visual_observations):
mini_batch["visual_obs%d" % i] = obs

58
ml-agents/mlagents/trainers/demo_loader.py


import logging
import os
from typing import List, Tuple
import numpy as np
from mlagents.envs.communicator_objects.agent_info_pb2 import AgentInfoProto
from mlagents.envs.communicator_objects.agent_info_action_pair_pb2 import (
AgentInfoActionPairProto,
)
from mlagents.envs.communicator_objects.brain_parameters_pb2 import BrainParametersProto
from mlagents.envs.communicator_objects.demonstration_meta_pb2 import (
DemonstrationMetaProto,

def make_demo_buffer(
brain_infos: List[BrainInfo], brain_params: BrainParameters, sequence_length: int
pair_infos: List[AgentInfoActionPairProto],
brain_params: BrainParameters,
sequence_length: int,
for idx, experience in enumerate(brain_infos):
if idx > len(brain_infos) - 2:
for idx, experience in enumerate(pair_infos):
if idx > len(pair_infos) - 2:
current_brain_info = brain_infos[idx]
next_brain_info = brain_infos[idx + 1]
current_pair_info = pair_infos[idx]
next_pair_info = pair_infos[idx + 1]
current_brain_info = BrainInfo.from_agent_proto(
0, [current_pair_info.agent_info], brain_params
)
next_brain_info = BrainInfo.from_agent_proto(
0, [next_pair_info.agent_info], brain_params
)
previous_action = np.array(pair_infos[idx].action_info.vector_actions) * 0
if idx > 0:
previous_action = np.array(pair_infos[idx - 1].action_info.vector_actions)
demo_buffer[0].last_brain_info = current_brain_info
demo_buffer[0]["done"].append(next_brain_info.local_done[0])
demo_buffer[0]["rewards"].append(next_brain_info.rewards[0])

demo_buffer[0]["vector_obs"].append(
current_brain_info.vector_observations[0]
)
demo_buffer[0]["actions"].append(next_brain_info.previous_vector_actions[0])
demo_buffer[0]["prev_action"].append(
current_brain_info.previous_vector_actions[0]
)
demo_buffer[0]["actions"].append(current_pair_info.action_info.vector_actions)
demo_buffer[0]["prev_action"].append(previous_action)
if next_brain_info.local_done[0]:
demo_buffer.append_update_buffer(
0, batch_size=None, training_length=sequence_length

:param sequence_length: Length of trajectories to fill buffer.
:return:
"""
brain_params, brain_infos, _ = load_demonstration(file_path)
demo_buffer = make_demo_buffer(brain_infos, brain_params, sequence_length)
brain_params, info_action_pair, _ = load_demonstration(file_path)
demo_buffer = make_demo_buffer(info_action_pair, brain_params, sequence_length)
def load_demonstration(file_path: str) -> Tuple[BrainParameters, List[BrainInfo], int]:
def load_demonstration(
file_path: str
) -> Tuple[BrainParameters, List[AgentInfoActionPairProto], int]:
:return: BrainParameter and list of BrainInfos containing demonstration data.
:return: BrainParameter and list of AgentInfoActionPairProto containing demonstration data.
"""
# First 32 bytes of file dedicated to meta-data.

brain_params = None
brain_param_proto = None
brain_infos = []
info_action_pairs = []
total_expected = 0
for _file_path in file_paths:
data = open(_file_path, "rb").read()

if obs_decoded == 1:
brain_param_proto = BrainParametersProto()
brain_param_proto.ParseFromString(data[pos : pos + next_pos])
agent_info = AgentInfoProto()
agent_info.ParseFromString(data[pos : pos + next_pos])
agent_info_action = AgentInfoActionPairProto()
agent_info_action.ParseFromString(data[pos : pos + next_pos])
brain_param_proto, agent_info
brain_param_proto, agent_info_action.agent_info
brain_info = BrainInfo.from_agent_proto(0, [agent_info], brain_params)
brain_infos.append(brain_info)
if len(brain_infos) == total_expected:
info_action_pairs.append(agent_info_action)
if len(info_action_pairs) == total_expected:
return brain_params, brain_infos, total_expected
return brain_params, info_action_pairs, total_expected

10
ml-agents/mlagents/trainers/ppo/policy.py


epsilon = None
if self.use_recurrent:
if not self.use_continuous_act:
feed_dict[
self.model.prev_action
] = brain_info.previous_vector_actions.reshape(
[-1, len(self.model.act_size)]
feed_dict[self.model.prev_action] = self.retrieve_previous_action(
brain_info.agents
)
feed_dict[self.model.memory_in] = self.retrieve_memories(brain_info.agents)
if self.use_continuous_act:

if self.use_recurrent:
feed_dict[self.model.memory_in] = self.retrieve_memories([idx])
if not self.use_continuous_act and self.use_recurrent:
feed_dict[self.model.prev_action] = [
brain_info.previous_vector_actions[idx]
]
feed_dict[self.model.prev_action] = self.retrieve_previous_action([idx])
value_estimates = self.sess.run(self.model.value_heads, feed_dict)
value_estimates = {k: float(v) for k, v in value_estimates.items()}

17
ml-agents/mlagents/trainers/rl_trainer.py


local_dones = []
max_reacheds = []
agents = []
prev_vector_actions = []
action_masks = []
for agent_id in next_info.agents:
agent_brain_info = self.training_buffer[agent_id].last_brain_info

local_dones.append(agent_brain_info.local_done[agent_index])
max_reacheds.append(agent_brain_info.max_reached[agent_index])
agents.append(agent_brain_info.agents[agent_index])
prev_vector_actions.append(
agent_brain_info.previous_vector_actions[agent_index]
)
action_masks.append(agent_brain_info.action_masks[agent_index])
curr_info = BrainInfo(
visual_observations,

local_dones,
prev_vector_actions,
max_reacheds,
action_masks,
)

# Evaluate and store the reward signals