GitHub
4 年前
当前提交
c0bea621
共有 16 个文件被更改，包括 486 次插入 和 4 次删除
-
6README.md
-
6com.unity.ml-agents/CHANGELOG.md
-
4com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
-
10com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
-
24com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
-
5com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
-
3com.unity.ml-agents/Runtime/Analytics.meta
-
3com.unity.ml-agents/Tests/Editor/Analytics.meta
-
89com.unity.ml-agents/Runtime/Analytics/Events.cs
-
3com.unity.ml-agents/Runtime/Analytics/Events.cs.meta
-
263com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
-
3com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta
-
68com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
-
3com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta
|
|||
fileFormatVersion: 2 |
|||
guid: 8b12ac54c5224758af88c67e2af4a01e |
|||
timeCreated: 1604359666 |
|||
|
|||
fileFormatVersion: 2 |
|||
guid: adbf291ff40848a296523d69a5be65a5 |
|||
timeCreated: 1607379470 |
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using Unity.MLAgents.Actuators; |
|||
using Unity.MLAgents.Sensors; |
|||
|
|||
namespace Unity.MLAgents.Analytics |
|||
{ |
|||
internal struct InferenceEvent |
|||
{ |
|||
/// <summary>
|
|||
/// Hash of the BehaviorName.
|
|||
/// </summary>
|
|||
public string BehaviorName; |
|||
public string BarracudaModelSource; |
|||
public string BarracudaModelVersion; |
|||
public string BarracudaModelProducer; |
|||
public string BarracudaPackageVersion; |
|||
/// <summary>
|
|||
/// Whether inference is performed on CPU (0) or GPU (1).
|
|||
/// </summary>
|
|||
public int InferenceDevice; |
|||
public List<EventObservationSpec> ObservationSpecs; |
|||
public EventActionSpec ActionSpec; |
|||
public int MemorySize; |
|||
public long TotalWeightSizeBytes; |
|||
public string ModelHash; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Simplified version of ActionSpec struct for use in analytics
|
|||
/// </summary>
|
|||
[Serializable] |
|||
internal struct EventActionSpec |
|||
{ |
|||
public int NumContinuousActions; |
|||
public int NumDiscreteActions; |
|||
public int[] BranchSizes; |
|||
|
|||
public static EventActionSpec FromActionSpec(ActionSpec actionSpec) |
|||
{ |
|||
var branchSizes = actionSpec.BranchSizes ?? Array.Empty<int>(); |
|||
return new EventActionSpec |
|||
{ |
|||
NumContinuousActions = actionSpec.NumContinuousActions, |
|||
NumDiscreteActions = actionSpec.NumDiscreteActions, |
|||
BranchSizes = branchSizes, |
|||
}; |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Information about one dimension of an observation.
|
|||
/// </summary>
|
|||
[Serializable] |
|||
internal struct EventObservationDimensionInfo |
|||
{ |
|||
public int Size; |
|||
public int Flags; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Simplified summary of Agent observations for use in analytics
|
|||
/// </summary>
|
|||
[Serializable] |
|||
internal struct EventObservationSpec |
|||
{ |
|||
public string SensorName; |
|||
public string CompressionType; |
|||
public EventObservationDimensionInfo[] DimensionInfos; |
|||
|
|||
public static EventObservationSpec FromSensor(ISensor sensor) |
|||
{ |
|||
var shape = sensor.GetObservationShape(); |
|||
var dimInfos = new EventObservationDimensionInfo[shape.Length]; |
|||
for (var i = 0; i < shape.Length; i++) |
|||
{ |
|||
dimInfos[i].Size = shape[i]; |
|||
// TODO copy flags when we have them
|
|||
} |
|||
|
|||
return new EventObservationSpec |
|||
{ |
|||
SensorName = sensor.GetName(), |
|||
CompressionType = sensor.GetCompressionType().ToString(), |
|||
DimensionInfos = dimInfos, |
|||
}; |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileFormatVersion: 2 |
|||
guid: 0a0d7cda6d74425a80775769a9283ba6 |
|||
timeCreated: 1604359798 |
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using Unity.Barracuda; |
|||
using Unity.MLAgents.Actuators; |
|||
using Unity.MLAgents.Inference; |
|||
using Unity.MLAgents.Policies; |
|||
using Unity.MLAgents.Sensors; |
|||
using UnityEngine; |
|||
using UnityEngine.Analytics; |
|||
|
|||
#if UNITY_EDITOR
|
|||
using UnityEditor; |
|||
using UnityEditor.Analytics; |
|||
#endif
|
|||
|
|||
|
|||
namespace Unity.MLAgents.Analytics |
|||
{ |
|||
internal class InferenceAnalytics |
|||
{ |
|||
const string k_VendorKey = "unity.ml-agents"; |
|||
const string k_EventName = "ml_agents_inferencemodelset"; |
|||
|
|||
/// <summary>
|
|||
/// Whether or not we've registered this particular event yet
|
|||
/// </summary>
|
|||
static bool s_EventRegistered = false; |
|||
|
|||
/// <summary>
|
|||
/// Hourly limit for this event name
|
|||
/// </summary>
|
|||
const int k_MaxEventsPerHour = 1000; |
|||
|
|||
/// <summary>
|
|||
/// Maximum number of items in this event.
|
|||
/// </summary>
|
|||
const int k_MaxNumberOfElements = 1000; |
|||
|
|||
/// <summary>
|
|||
/// Models that we've already sent events for.
|
|||
/// </summary>
|
|||
private static HashSet<NNModel> s_SentModels; |
|||
|
|||
static bool EnableAnalytics() |
|||
{ |
|||
if (s_EventRegistered) |
|||
{ |
|||
return true; |
|||
} |
|||
|
|||
#if UNITY_EDITOR
|
|||
AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); |
|||
#else
|
|||
AnalyticsResult result = AnalyticsResult.UnsupportedPlatform; |
|||
#endif
|
|||
if (result == AnalyticsResult.Ok) |
|||
{ |
|||
s_EventRegistered = true; |
|||
} |
|||
|
|||
if (s_EventRegistered && s_SentModels == null) |
|||
{ |
|||
s_SentModels = new HashSet<NNModel>(); |
|||
} |
|||
|
|||
return s_EventRegistered; |
|||
} |
|||
|
|||
public static bool IsAnalyticsEnabled() |
|||
{ |
|||
#if UNITY_EDITOR
|
|||
return EditorAnalytics.enabled; |
|||
#else
|
|||
return false; |
|||
#endif
|
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Send an analytics event for the NNModel when it is set up for inference.
|
|||
/// No events will be sent if analytics are disabled, and at most one event
|
|||
/// will be sent per model instance.
|
|||
/// </summary>
|
|||
/// <param name="nnModel">The NNModel being used for inference.</param>
|
|||
/// <param name="behaviorName">The BehaviorName of the Agent using the model</param>
|
|||
/// <param name="inferenceDevice">Whether inference is being performed on the CPU or GPU</param>
|
|||
/// <param name="sensors">List of ISensors for the Agent. Used to generate information about the observation space.</param>
|
|||
/// <param name="actionSpec">ActionSpec for the Agent. Used to generate information about the action space.</param>
|
|||
/// <returns></returns>
|
|||
public static void InferenceModelSet( |
|||
NNModel nnModel, |
|||
string behaviorName, |
|||
InferenceDevice inferenceDevice, |
|||
IList<ISensor> sensors, |
|||
ActionSpec actionSpec |
|||
) |
|||
{ |
|||
// The event shouldn't be able to report if this is disabled but if we know we're not going to report
|
|||
// Lets early out and not waste time gathering all the data
|
|||
if (!IsAnalyticsEnabled()) |
|||
return; |
|||
|
|||
if (!EnableAnalytics()) |
|||
return; |
|||
|
|||
var added = s_SentModels.Add(nnModel); |
|||
|
|||
if (!added) |
|||
{ |
|||
// We previously added this model. Exit so we don't resend.
|
|||
return; |
|||
} |
|||
|
|||
var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); |
|||
// Note - to debug, use JsonUtility.ToJson on the event.
|
|||
// Debug.Log(JsonUtility.ToJson(data, true));
|
|||
#if UNITY_EDITOR
|
|||
EditorAnalytics.SendEventWithLimit(k_EventName, data); |
|||
#else
|
|||
return; |
|||
#endif
|
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Generate an InferenceEvent for the model.
|
|||
/// </summary>
|
|||
/// <param name="nnModel"></param>
|
|||
/// <param name="behaviorName"></param>
|
|||
/// <param name="inferenceDevice"></param>
|
|||
/// <param name="sensors"></param>
|
|||
/// <param name="actionSpec"></param>
|
|||
/// <returns></returns>
|
|||
internal static InferenceEvent GetEventForModel( |
|||
NNModel nnModel, |
|||
string behaviorName, |
|||
InferenceDevice inferenceDevice, |
|||
IList<ISensor> sensors, |
|||
ActionSpec actionSpec |
|||
) |
|||
{ |
|||
var barracudaModel = ModelLoader.Load(nnModel); |
|||
var inferenceEvent = new InferenceEvent(); |
|||
|
|||
// Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
|
|||
var behaviorNameHash = Hash128.Compute(behaviorName); |
|||
inferenceEvent.BehaviorName = behaviorNameHash.ToString(); |
|||
|
|||
inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; |
|||
inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; |
|||
inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; |
|||
inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0]; |
|||
inferenceEvent.InferenceDevice = (int)inferenceDevice; |
|||
|
|||
if (barracudaModel.ProducerName == "Script") |
|||
{ |
|||
// .nn files don't have these fields set correctly. Assign some placeholder values.
|
|||
inferenceEvent.BarracudaModelSource = "NN"; |
|||
inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py"; |
|||
} |
|||
|
|||
#if UNITY_2019_3_OR_NEWER && UNITY_EDITOR
|
|||
var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); |
|||
inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version; |
|||
#else
|
|||
inferenceEvent.BarracudaPackageVersion = null; |
|||
#endif
|
|||
|
|||
inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); |
|||
inferenceEvent.ObservationSpecs = new List<EventObservationSpec>(sensors.Count); |
|||
foreach (var sensor in sensors) |
|||
{ |
|||
inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); |
|||
} |
|||
|
|||
inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel); |
|||
inferenceEvent.ModelHash = GetModelHash(barracudaModel); |
|||
return inferenceEvent; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compute the total model weight size in bytes.
|
|||
/// This corresponds to the "Total weight size" display in the Barracuda inspector,
|
|||
/// and the calculations are the same.
|
|||
/// </summary>
|
|||
/// <param name="barracudaModel"></param>
|
|||
/// <returns></returns>
|
|||
static long GetModelWeightSize(Model barracudaModel) |
|||
{ |
|||
long totalWeightsSizeInBytes = 0; |
|||
for (var l = 0; l < barracudaModel.layers.Count; ++l) |
|||
{ |
|||
for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) |
|||
{ |
|||
totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; |
|||
} |
|||
} |
|||
return totalWeightsSizeInBytes; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Wrapper around Hash128 that supports Append(float[], int, int)
|
|||
/// </summary>
|
|||
struct MLAgentsHash128 |
|||
{ |
|||
private Hash128 m_Hash; |
|||
|
|||
public void Append(float[] values, int count) |
|||
{ |
|||
if (values == null) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
// Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars)
|
|||
// For these versions, we'll hash element by element.
|
|||
#if UNITY_2020_1_OR_NEWER
|
|||
m_Hash.Append(values, 0, count); |
|||
#else
|
|||
for (var i = 0; i < count; i++) |
|||
{ |
|||
var tempHash = new Hash128(); |
|||
HashUtilities.ComputeHash128(ref values[i], ref tempHash); |
|||
HashUtilities.AppendHash(ref tempHash, ref m_Hash); |
|||
} |
|||
#endif
|
|||
} |
|||
|
|||
public void Append(string value) |
|||
{ |
|||
var tempHash = Hash128.Compute(value); |
|||
HashUtilities.AppendHash(ref tempHash, ref m_Hash); |
|||
} |
|||
|
|||
public override string ToString() |
|||
{ |
|||
return m_Hash.ToString(); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compute a hash of the model's layer data and return it as a string.
|
|||
/// A subset of the layer weights are used for performance.
|
|||
/// This increases the chance of a collision, but this should still be extremely rare.
|
|||
/// </summary>
|
|||
/// <param name="barracudaModel"></param>
|
|||
/// <returns></returns>
|
|||
static string GetModelHash(Model barracudaModel) |
|||
{ |
|||
var hash = new MLAgentsHash128(); |
|||
|
|||
// Limit the max number of float bytes that we hash for performance.
|
|||
const int kMaxFloats = 256; |
|||
|
|||
foreach (var layer in barracudaModel.layers) |
|||
{ |
|||
hash.Append(layer.name); |
|||
var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats); |
|||
hash.Append(layer.weights, numFloatsToHash); |
|||
} |
|||
|
|||
return hash.ToString(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileFormatVersion: 2 |
|||
guid: ac4c40c2394d481ebf602caa600a32f3 |
|||
timeCreated: 1604359787 |
|||
|
|||
using System.Collections.Generic; |
|||
using NUnit.Framework; |
|||
using Unity.MLAgents.Sensors; |
|||
using UnityEngine; |
|||
using Unity.Barracuda; |
|||
using Unity.MLAgents.Actuators; |
|||
using Unity.MLAgents.Analytics; |
|||
using Unity.MLAgents.Policies; |
|||
using UnityEditor; |
|||
|
|||
namespace Unity.MLAgents.Tests.Analytics |
|||
{ |
|||
[TestFixture] |
|||
public class InferenceAnalyticsTests |
|||
{ |
|||
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx"; |
|||
NNModel continuousONNXModel; |
|||
Test3DSensorComponent sensor_21_20_3; |
|||
Test3DSensorComponent sensor_20_22_3; |
|||
|
|||
ActionSpec GetContinuous2vis8vec2actionActionSpec() |
|||
{ |
|||
return ActionSpec.MakeContinuous(2); |
|||
} |
|||
|
|||
[SetUp] |
|||
public void SetUp() |
|||
{ |
|||
continuousONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuousONNXPath, typeof(NNModel)); |
|||
var go = new GameObject("SensorA"); |
|||
sensor_21_20_3 = go.AddComponent<Test3DSensorComponent>(); |
|||
sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3); |
|||
sensor_20_22_3 = go.AddComponent<Test3DSensorComponent>(); |
|||
sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3); |
|||
} |
|||
|
|||
[Test] |
|||
public void TestModelEvent() |
|||
{ |
|||
var sensors = new List<ISensor> { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor }; |
|||
var behaviorName = "continuousModel"; |
|||
|
|||
var continuousEvent = InferenceAnalytics.GetEventForModel( |
|||
continuousONNXModel, behaviorName, |
|||
InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec() |
|||
); |
|||
|
|||
// The behavior name should be hashed, not pass-through.
|
|||
Assert.AreNotEqual(behaviorName, continuousEvent.BehaviorName); |
|||
|
|||
Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions); |
|||
Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions); |
|||
Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count); |
|||
Assert.AreEqual(3, continuousEvent.ObservationSpecs[0].DimensionInfos.Length); |
|||
Assert.AreEqual(20, continuousEvent.ObservationSpecs[0].DimensionInfos[0].Size); |
|||
Assert.AreEqual("None", continuousEvent.ObservationSpecs[0].CompressionType); |
|||
Assert.AreNotEqual(null, continuousEvent.ModelHash); |
|||
|
|||
// Make sure nested fields get serialized
|
|||
var jsonString = JsonUtility.ToJson(continuousEvent, true); |
|||
Assert.IsTrue(jsonString.Contains("ObservationSpecs")); |
|||
Assert.IsTrue(jsonString.Contains("ActionSpec")); |
|||
Assert.IsTrue(jsonString.Contains("NumDiscreteActions")); |
|||
Assert.IsTrue(jsonString.Contains("SensorName")); |
|||
Assert.IsTrue(jsonString.Contains("Flags")); |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileFormatVersion: 2 |
|||
guid: 9f054f620b8b468bbd8ccf7d2cc14ccd |
|||
timeCreated: 1607379491 |
|||
撰写
预览
正在加载...
取消
保存
Reference in new issue