浏览代码
[MLA-1009] observable performance tests (#4031)
[MLA-1009] observable performance tests (#4031)
* WIP perf tests * WIP perf test * add marker tests too * move to devproject * yamato first pass * chmod * fix trigger, fix meta files * fix utr command * fix artifact paths * Update com.unity.ml-agents-performance.yml * test properties, reduce some noise * timer around RequestDecision * actually set ObservableAttributeHandling * undo asmdef changes/MLA-1734-demo-provider
GitHub
5 年前
当前提交
af9abb6c
共有 15 个文件被更改,包括 314 次插入 和 1 次删除
-
1DevProject/Packages/manifest.json
-
5com.unity.ml-agents/Runtime/Agent.cs
-
37.yamato/com.unity.ml-agents-performance.yml
-
8DevProject/Assets/ML-Agents.meta
-
8DevProject/Assets/ML-Agents/Scripts.meta
-
8DevProject/Assets/ML-Agents/Scripts/Tests.meta
-
8DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta
-
194DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs
-
11DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta
-
28DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef
-
7DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta
|
|||
test_editors: |
|||
- version: 2019.3 |
|||
- version: 2020.1 |
|||
--- |
|||
{% for editor in test_editors %} |
|||
Run_Mac_Perfomance_Tests{{ editor.version }}: |
|||
name: Run Mac Performance Tests {{ editor.version }} |
|||
agent: |
|||
type: Unity::VM::osx |
|||
image: package-ci/mac:stable |
|||
flavor: b1.small |
|||
variables: |
|||
UNITY_VERSION: {{ editor.version }} |
|||
commands: |
|||
- python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple |
|||
- unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast |
|||
- curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr |
|||
- chmod +x ./utr |
|||
- ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=com.unity.ml-agents --zero-tests-are-ok=1 |
|||
triggers: |
|||
cancel_old_ci: true |
|||
expression: | |
|||
(pull_request.target eq "master" OR |
|||
pull_request.target match "release.+") AND |
|||
NOT pull_request.draft AND |
|||
(pull_request.changes.any match "com.unity.ml-agents/**" OR |
|||
pull_request.changes.any match "DevProject/**" OR |
|||
pull_request.changes.any match "ml-agents/**" OR |
|||
pull_request.changes.any match "ml-agents-envs/**" OR |
|||
pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND |
|||
NOT pull_request.changes.all match "**/*.md" |
|||
artifacts: |
|||
logs: |
|||
paths: |
|||
- "build/test-results/**" |
|||
- "*.log" |
|||
{% endfor %} |
|
|||
fileFormatVersion: 2 |
|||
guid: 8498a7556d0b348cbaff70bd8635db8b |
|||
folderAsset: yes |
|||
DefaultImporter: |
|||
externalObjects: {} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
|
|||
fileFormatVersion: 2 |
|||
guid: 45f48c01abd5d47048bc3edcc6db1dd0 |
|||
folderAsset: yes |
|||
DefaultImporter: |
|||
externalObjects: {} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
|
|||
fileFormatVersion: 2 |
|||
guid: eb96b88fc6ae14a41949d1bd75e68d38 |
|||
folderAsset: yes |
|||
DefaultImporter: |
|||
externalObjects: {} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
|
|||
fileFormatVersion: 2 |
|||
guid: fb5960f38a3fc42e8a6b52c9bf47d83d |
|||
folderAsset: yes |
|||
DefaultImporter: |
|||
externalObjects: {} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
|
|||
using NUnit.Framework; |
|||
using Unity.MLAgents; |
|||
using Unity.MLAgents.Policies; |
|||
using Unity.MLAgents.Sensors; |
|||
using Unity.MLAgents.Sensors.Reflection; |
|||
using Unity.PerformanceTesting; |
|||
using UnityEngine; |
|||
|
|||
namespace MLAgentsExamples.Tests.Performance |
|||
{ |
|||
[TestFixture] |
|||
public class SensorPerformanceTests |
|||
{ |
|||
string[] s_Markers = |
|||
{ |
|||
"root.InitializeSensors", |
|||
"root.AgentSendState.CollectObservations", |
|||
"root.AgentSendState.RequestDecision" |
|||
}; |
|||
const int k_NumAgentSteps = 10; |
|||
const int k_MeasurementCount = 25; |
|||
const int k_MarkerTestSteps = 10; |
|||
|
|||
[SetUp] |
|||
public void SetUp() |
|||
{ |
|||
// Step a dummy agent here, so that we don't time the Academy initialization connection attempt and
|
|||
// any other static setup costs.
|
|||
RunAgent<DummyAgent>(1, 0, ObservableAttributeOptions.ExamineAll); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Simple Agent just used for "burning in" the Academy for testing.
|
|||
/// </summary>
|
|||
class DummyAgent : Agent |
|||
{ |
|||
public override void CollectObservations(VectorSensor sensor) |
|||
{ |
|||
} |
|||
|
|||
public override void Heuristic(float[] actionsOut) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Agent used for performance testing that uses the CollectObservations interface.
|
|||
/// </summary>
|
|||
class CollectObservationsAgent : Agent |
|||
{ |
|||
public override void CollectObservations(VectorSensor sensor) |
|||
{ |
|||
sensor.AddObservation(new Vector3(1, 2, 3)); |
|||
sensor.AddObservation(new Quaternion(1, 2, 3, 4)); |
|||
} |
|||
|
|||
public override void Heuristic(float[] actionsOut) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Agent used for performance testing that uses the ObservableAttributes on fields.
|
|||
/// </summary>
|
|||
class ObservableFieldAgent : Agent |
|||
{ |
|||
[Observable] |
|||
public Vector3 Vector3Field = new Vector3(1, 2, 3); |
|||
|
|||
[Observable] |
|||
public Quaternion QuaternionField = new Quaternion(1, 2, 3, 4); |
|||
|
|||
public override void Heuristic(float[] actionsOut) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Agent used for performance testing that uses the ObservableAttributes on properties.
|
|||
/// </summary>
|
|||
class ObservablePropertyAgent : Agent |
|||
{ |
|||
Vector3 m_Vector3Field = new Vector3(1, 2, 3); |
|||
|
|||
[Observable] |
|||
Vector3 Vector3Property |
|||
{ |
|||
get { return m_Vector3Field; } |
|||
} |
|||
|
|||
Quaternion m_QuaternionField = new Quaternion(1, 2, 3, 4); |
|||
|
|||
[Observable] |
|||
Quaternion QuaternionProperty |
|||
{ |
|||
get { return m_QuaternionField; } |
|||
} |
|||
|
|||
public override void Heuristic(float[] actionsOut) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
void RunAgent<T>(int numSteps, int obsSize, ObservableAttributeOptions obsOptions) where T : Agent |
|||
{ |
|||
var agentGameObj = new GameObject(); |
|||
var agent = agentGameObj.AddComponent<T>(); |
|||
|
|||
var behaviorParams = agent.GetComponent<BehaviorParameters>(); |
|||
behaviorParams.BrainParameters.VectorObservationSize = obsSize; |
|||
behaviorParams.ObservableAttributeHandling = obsOptions; |
|||
|
|||
agent.LazyInitialize(); |
|||
for (var i = 0; i < numSteps; i++) |
|||
{ |
|||
agent.RequestDecision(); |
|||
Academy.Instance.EnvironmentStep(); |
|||
} |
|||
Object.DestroyImmediate(agentGameObj); |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestCollectObservationsAgent() |
|||
{ |
|||
Measure.Method(() => |
|||
{ |
|||
RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); |
|||
}) |
|||
.MeasurementCount(k_MeasurementCount) |
|||
.GC() |
|||
.Run(); |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestObservableFieldAgent() |
|||
{ |
|||
Measure.Method(() => |
|||
{ |
|||
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); |
|||
}) |
|||
.MeasurementCount(k_MeasurementCount) |
|||
.GC() |
|||
.Run(); |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestObservablePropertyAgent() |
|||
{ |
|||
Measure.Method(() => |
|||
{ |
|||
RunAgent<ObservablePropertyAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); |
|||
}) |
|||
.MeasurementCount(k_MeasurementCount) |
|||
.GC() |
|||
.Run(); |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestCollectObservationsAgentMarkers() |
|||
{ |
|||
using (Measure.ProfilerMarkers(s_Markers)) |
|||
{ |
|||
for(var i=0; i<k_MarkerTestSteps; i++) |
|||
{ |
|||
RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); |
|||
} |
|||
} |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestObservableFieldAgentMarkers() |
|||
{ |
|||
using (Measure.ProfilerMarkers(s_Markers)) |
|||
{ |
|||
for (var i = 0; i < k_MarkerTestSteps; i++) |
|||
{ |
|||
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); |
|||
} |
|||
} |
|||
} |
|||
|
|||
[Test, Performance] |
|||
public void TestObservablePropertyAgentMarkers() |
|||
{ |
|||
using (Measure.ProfilerMarkers(s_Markers)) |
|||
{ |
|||
for (var i = 0; i < k_MarkerTestSteps; i++) |
|||
{ |
|||
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|
|||
fileFormatVersion: 2 |
|||
guid: c11a031f0b2d94ec3a90c1bd54713399 |
|||
MonoImporter: |
|||
externalObjects: {} |
|||
serializedVersion: 2 |
|||
defaultReferences: [] |
|||
executionOrder: 0 |
|||
icon: {instanceID: 0} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
|
|||
{ |
|||
"name": "Unity.ML-Agents.Performance.Tests", |
|||
"references": [ |
|||
"Unity.ML-Agents.Editor", |
|||
"Unity.ML-Agents", |
|||
"Unity.Barracuda", |
|||
"Unity.ML-Agents.CommunicatorObjects", |
|||
"Unity.PerformanceTesting" |
|||
], |
|||
"optionalUnityReferences": [ |
|||
"TestAssemblies" |
|||
], |
|||
"includePlatforms": [ |
|||
"Editor" |
|||
], |
|||
"excludePlatforms": [], |
|||
"allowUnsafeCode": false, |
|||
"overrideReferences": true, |
|||
"precompiledReferences": [ |
|||
"System.IO.Abstractions.dll", |
|||
"System.IO.Abstractions.TestingHelpers.dll", |
|||
"Google.Protobuf.dll" |
|||
], |
|||
"autoReferenced": false, |
|||
"defineConstraints": [ |
|||
"UNITY_INCLUDE_TESTS" |
|||
] |
|||
} |
|
|||
fileFormatVersion: 2 |
|||
guid: 1bad3ff1107ca4051af2c89caabd728c |
|||
AssemblyDefinitionImporter: |
|||
externalObjects: {} |
|||
userData: |
|||
assetBundleName: |
|||
assetBundleVariant: |
撰写
预览
正在加载...
取消
保存
Reference in new issue