浏览代码

[MLA-1009] observable performance tests (#4031)

* WIP perf tests

* WIP perf test

* add marker tests too

* move to devproject

* yamato first pass

* chmod

* fix trigger, fix meta files

* fix utr command

* fix artifact paths

* Update com.unity.ml-agents-performance.yml

* test properties, reduce some noise

* timer around RequestDecision

* actually set ObservableAttributeHandling

* undo asmdef changes
/MLA-1734-demo-provider
GitHub 5 年前
当前提交
af9abb6c
共有 15 个文件被更改,包括 314 次插入1 次删除
  1. 1
      DevProject/Packages/manifest.json
  2. 5
      com.unity.ml-agents/Runtime/Agent.cs
  3. 37
      .yamato/com.unity.ml-agents-performance.yml
  4. 8
      DevProject/Assets/ML-Agents.meta
  5. 8
      DevProject/Assets/ML-Agents/Scripts.meta
  6. 8
      DevProject/Assets/ML-Agents/Scripts/Tests.meta
  7. 8
      DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta
  8. 194
      DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs
  9. 11
      DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta
  10. 28
      DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef
  11. 7
      DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta

1
DevProject/Packages/manifest.json


"com.unity.package-validation-suite": "0.11.0-preview",
"com.unity.purchasing": "2.0.6",
"com.unity.test-framework": "1.1.13",
"com.unity.test-framework.performance": "2.2.0-preview",
"com.unity.testtools.codecoverage": "0.2.2-preview",
"com.unity.textmeshpro": "2.0.1",
"com.unity.timeline": "1.2.12",

5
com.unity.ml-agents/Runtime/Agent.cs


m_Info.maxStepReached = false;
m_Info.episodeId = m_EpisodeId;
m_Brain.RequestDecision(m_Info, sensors);
using (TimerStack.Instance.Scoped("RequestDecision"))
{
m_Brain.RequestDecision(m_Info, sensors);
}
// If we have any DemonstrationWriters, write the AgentInfo and sensors to them.
foreach (var demoWriter in DemonstrationWriters)

37
.yamato/com.unity.ml-agents-performance.yml


test_editors:
- version: 2019.3
- version: 2020.1
---
{% for editor in test_editors %}
Run_Mac_Perfomance_Tests{{ editor.version }}:
name: Run Mac Performance Tests {{ editor.version }}
agent:
type: Unity::VM::osx
image: package-ci/mac:stable
flavor: b1.small
variables:
UNITY_VERSION: {{ editor.version }}
commands:
- python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple
- unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
- curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr
- chmod +x ./utr
- ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=com.unity.ml-agents --zero-tests-are-ok=1
triggers:
cancel_old_ci: true
expression: |
(pull_request.target eq "master" OR
pull_request.target match "release.+") AND
NOT pull_request.draft AND
(pull_request.changes.any match "com.unity.ml-agents/**" OR
pull_request.changes.any match "DevProject/**" OR
pull_request.changes.any match "ml-agents/**" OR
pull_request.changes.any match "ml-agents-envs/**" OR
pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND
NOT pull_request.changes.all match "**/*.md"
artifacts:
logs:
paths:
- "build/test-results/**"
- "*.log"
{% endfor %}

8
DevProject/Assets/ML-Agents.meta


fileFormatVersion: 2
guid: 8498a7556d0b348cbaff70bd8635db8b
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
DevProject/Assets/ML-Agents/Scripts.meta


fileFormatVersion: 2
guid: 45f48c01abd5d47048bc3edcc6db1dd0
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
DevProject/Assets/ML-Agents/Scripts/Tests.meta


fileFormatVersion: 2
guid: eb96b88fc6ae14a41949d1bd75e68d38
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta


fileFormatVersion: 2
guid: fb5960f38a3fc42e8a6b52c9bf47d83d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

194
DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs


using NUnit.Framework;
using Unity.MLAgents;
using Unity.MLAgents.Policies;
using Unity.MLAgents.Sensors;
using Unity.MLAgents.Sensors.Reflection;
using Unity.PerformanceTesting;
using UnityEngine;
namespace MLAgentsExamples.Tests.Performance
{
[TestFixture]
public class SensorPerformanceTests
{
string[] s_Markers =
{
"root.InitializeSensors",
"root.AgentSendState.CollectObservations",
"root.AgentSendState.RequestDecision"
};
const int k_NumAgentSteps = 10;
const int k_MeasurementCount = 25;
const int k_MarkerTestSteps = 10;
[SetUp]
public void SetUp()
{
// Step a dummy agent here, so that we don't time the Academy initialization connection attempt and
// any other static setup costs.
RunAgent<DummyAgent>(1, 0, ObservableAttributeOptions.ExamineAll);
}
/// <summary>
/// Simple Agent just used for "burning in" the Academy for testing.
/// </summary>
class DummyAgent : Agent
{
public override void CollectObservations(VectorSensor sensor)
{
}
public override void Heuristic(float[] actionsOut)
{
}
}
/// <summary>
/// Agent used for performance testing that uses the CollectObservations interface.
/// </summary>
class CollectObservationsAgent : Agent
{
public override void CollectObservations(VectorSensor sensor)
{
sensor.AddObservation(new Vector3(1, 2, 3));
sensor.AddObservation(new Quaternion(1, 2, 3, 4));
}
public override void Heuristic(float[] actionsOut)
{
}
}
/// <summary>
/// Agent used for performance testing that uses the ObservableAttributes on fields.
/// </summary>
class ObservableFieldAgent : Agent
{
[Observable]
public Vector3 Vector3Field = new Vector3(1, 2, 3);
[Observable]
public Quaternion QuaternionField = new Quaternion(1, 2, 3, 4);
public override void Heuristic(float[] actionsOut)
{
}
}
/// <summary>
/// Agent used for performance testing that uses the ObservableAttributes on properties.
/// </summary>
class ObservablePropertyAgent : Agent
{
Vector3 m_Vector3Field = new Vector3(1, 2, 3);
[Observable]
Vector3 Vector3Property
{
get { return m_Vector3Field; }
}
Quaternion m_QuaternionField = new Quaternion(1, 2, 3, 4);
[Observable]
Quaternion QuaternionProperty
{
get { return m_QuaternionField; }
}
public override void Heuristic(float[] actionsOut)
{
}
}
void RunAgent<T>(int numSteps, int obsSize, ObservableAttributeOptions obsOptions) where T : Agent
{
var agentGameObj = new GameObject();
var agent = agentGameObj.AddComponent<T>();
var behaviorParams = agent.GetComponent<BehaviorParameters>();
behaviorParams.BrainParameters.VectorObservationSize = obsSize;
behaviorParams.ObservableAttributeHandling = obsOptions;
agent.LazyInitialize();
for (var i = 0; i < numSteps; i++)
{
agent.RequestDecision();
Academy.Instance.EnvironmentStep();
}
Object.DestroyImmediate(agentGameObj);
}
[Test, Performance]
public void TestCollectObservationsAgent()
{
Measure.Method(() =>
{
RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore);
})
.MeasurementCount(k_MeasurementCount)
.GC()
.Run();
}
[Test, Performance]
public void TestObservableFieldAgent()
{
Measure.Method(() =>
{
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
})
.MeasurementCount(k_MeasurementCount)
.GC()
.Run();
}
[Test, Performance]
public void TestObservablePropertyAgent()
{
Measure.Method(() =>
{
RunAgent<ObservablePropertyAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
})
.MeasurementCount(k_MeasurementCount)
.GC()
.Run();
}
[Test, Performance]
public void TestCollectObservationsAgentMarkers()
{
using (Measure.ProfilerMarkers(s_Markers))
{
for(var i=0; i<k_MarkerTestSteps; i++)
{
RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore);
}
}
}
[Test, Performance]
public void TestObservableFieldAgentMarkers()
{
using (Measure.ProfilerMarkers(s_Markers))
{
for (var i = 0; i < k_MarkerTestSteps; i++)
{
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
}
}
}
[Test, Performance]
public void TestObservablePropertyAgentMarkers()
{
using (Measure.ProfilerMarkers(s_Markers))
{
for (var i = 0; i < k_MarkerTestSteps; i++)
{
RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
}
}
}
}
}

11
DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta


fileFormatVersion: 2
guid: c11a031f0b2d94ec3a90c1bd54713399
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

28
DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef


{
"name": "Unity.ML-Agents.Performance.Tests",
"references": [
"Unity.ML-Agents.Editor",
"Unity.ML-Agents",
"Unity.Barracuda",
"Unity.ML-Agents.CommunicatorObjects",
"Unity.PerformanceTesting"
],
"optionalUnityReferences": [
"TestAssemblies"
],
"includePlatforms": [
"Editor"
],
"excludePlatforms": [],
"allowUnsafeCode": false,
"overrideReferences": true,
"precompiledReferences": [
"System.IO.Abstractions.dll",
"System.IO.Abstractions.TestingHelpers.dll",
"Google.Protobuf.dll"
],
"autoReferenced": false,
"defineConstraints": [
"UNITY_INCLUDE_TESTS"
]
}

7
DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta


fileFormatVersion: 2
guid: 1bad3ff1107ca4051af2c89caabd728c
AssemblyDefinitionImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:
正在加载...
取消
保存