using System; using System.Collections.Generic; using Unity.Barracuda; using Unity.MLAgents.Actuators; using Unity.MLAgents.Inference; using Unity.MLAgents.Policies; using Unity.MLAgents.Sensors; using UnityEngine; using UnityEngine.Analytics; #if UNITY_EDITOR using UnityEditor; using UnityEditor.Analytics; #endif namespace Unity.MLAgents.Analytics { internal class InferenceAnalytics { const string k_VendorKey = "unity.ml-agents"; const string k_EventName = "ml_agents_inferencemodelset"; const int k_EventVersion = 1; /// /// Whether or not we've registered this particular event yet /// static bool s_EventRegistered = false; /// /// Hourly limit for this event name /// const int k_MaxEventsPerHour = 1000; /// /// Maximum number of items in this event. /// const int k_MaxNumberOfElements = 1000; /// /// Models that we've already sent events for. /// private static HashSet s_SentModels; static bool EnableAnalytics() { if (s_EventRegistered) { return true; } #if UNITY_EDITOR AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey, k_EventVersion); #else AnalyticsResult result = AnalyticsResult.UnsupportedPlatform; #endif if (result == AnalyticsResult.Ok) { s_EventRegistered = true; } if (s_EventRegistered && s_SentModels == null) { s_SentModels = new HashSet(); } return s_EventRegistered; } public static bool IsAnalyticsEnabled() { #if UNITY_EDITOR return EditorAnalytics.enabled; #else return false; #endif } /// /// Send an analytics event for the NNModel when it is set up for inference. /// No events will be sent if analytics are disabled, and at most one event /// will be sent per model instance. /// /// The NNModel being used for inference. /// The BehaviorName of the Agent using the model /// Whether inference is being performed on the CPU or GPU /// List of ISensors for the Agent. Used to generate information about the observation space. /// ActionSpec for the Agent. Used to generate information about the action space. /// List of IActuators for the Agent. Used to generate information about the action space. /// public static void InferenceModelSet( NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice, IList sensors, ActionSpec actionSpec, IList actuators ) { // The event shouldn't be able to report if this is disabled but if we know we're not going to report // Lets early out and not waste time gathering all the data if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) return; var added = s_SentModels.Add(nnModel); if (!added) { // We previously added this model. Exit so we don't resend. return; } var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec, actuators); // Note - to debug, use JsonUtility.ToJson on the event. // Debug.Log(JsonUtility.ToJson(data, true)); #if UNITY_EDITOR if (AnalyticsUtils.s_SendEditorAnalytics) { EditorAnalytics.SendEventWithLimit(k_EventName, data, k_EventVersion); } #else return; #endif } /// /// Generate an InferenceEvent for the model. /// /// /// /// /// /// /// /// internal static InferenceEvent GetEventForModel( NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice, IList sensors, ActionSpec actionSpec, IList actuators ) { var barracudaModel = ModelLoader.Load(nnModel); var inferenceEvent = new InferenceEvent(); // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. inferenceEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName); inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0]; inferenceEvent.InferenceDevice = (int)inferenceDevice; if (barracudaModel.ProducerName == "Script") { // .nn files don't have these fields set correctly. Assign some placeholder values. inferenceEvent.BarracudaModelSource = "NN"; inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py"; } #if UNITY_2019_3_OR_NEWER && UNITY_EDITOR var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version; #else inferenceEvent.BarracudaPackageVersion = null; #endif inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); inferenceEvent.ObservationSpecs = new List(sensors.Count); foreach (var sensor in sensors) { inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } inferenceEvent.ActuatorInfos = new List(actuators.Count); foreach (var actuator in actuators) { inferenceEvent.ActuatorInfos.Add(EventActuatorInfo.FromActuator(actuator)); } inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel); inferenceEvent.ModelHash = GetModelHash(barracudaModel); return inferenceEvent; } /// /// Compute the total model weight size in bytes. /// This corresponds to the "Total weight size" display in the Barracuda inspector, /// and the calculations are the same. /// /// /// static long GetModelWeightSize(Model barracudaModel) { long totalWeightsSizeInBytes = 0; for (var l = 0; l < barracudaModel.layers.Count; ++l) { for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) { totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; } } return totalWeightsSizeInBytes; } /// /// Wrapper around Hash128 that supports Append(float[], int, int) /// struct MLAgentsHash128 { private Hash128 m_Hash; public void Append(float[] values, int count) { if (values == null) { return; } // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars) // For these versions, we'll hash element by element. #if UNITY_2020_1_OR_NEWER m_Hash.Append(values, 0, count); #else for (var i = 0; i < count; i++) { var tempHash = new Hash128(); HashUtilities.ComputeHash128(ref values[i], ref tempHash); HashUtilities.AppendHash(ref tempHash, ref m_Hash); } #endif } public void Append(string value) { var tempHash = Hash128.Compute(value); HashUtilities.AppendHash(ref tempHash, ref m_Hash); } public override string ToString() { return m_Hash.ToString(); } } /// /// Compute a hash of the model's layer data and return it as a string. /// A subset of the layer weights are used for performance. /// This increases the chance of a collision, but this should still be extremely rare. /// /// /// static string GetModelHash(Model barracudaModel) { var hash = new MLAgentsHash128(); // Limit the max number of float bytes that we hash for performance. const int kMaxFloats = 256; foreach (var layer in barracudaModel.layers) { hash.Append(layer.name); var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats); hash.Append(layer.weights, numFloatsToHash); } return hash.ToString(); } } }