using System; using System.Collections.Generic; using System.Diagnostics; using Unity.MLAgents.Actuators; using Unity.MLAgents.Sensors; using UnityEngine; #if MLA_UNITY_ANALYTICS_MODULE using UnityEngine.Analytics; #if UNITY_EDITOR using UnityEditor.Analytics; #endif #endif #if UNITY_EDITOR using UnityEditor; #endif namespace Unity.MLAgents.Analytics { internal static class TrainingAnalytics { const string k_VendorKey = "unity.ml-agents"; const string k_TrainingEnvironmentInitializedEventName = "ml_agents_training_environment_initialized"; const string k_TrainingBehaviorInitializedEventName = "ml_agents_training_behavior_initialized"; const string k_RemotePolicyInitializedEventName = "ml_agents_remote_policy_initialized"; private static readonly string[] s_EventNames = { k_TrainingEnvironmentInitializedEventName, k_TrainingBehaviorInitializedEventName, k_RemotePolicyInitializedEventName }; /// /// Hourly limit for this event name /// const int k_MaxEventsPerHour = 1000; /// /// Maximum number of items in this event. /// const int k_MaxNumberOfElements = 1000; private static bool s_SentEnvironmentInitialized; #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE /// /// Whether or not we've registered this particular event yet /// static bool s_EventsRegistered; /// /// Behaviors that we've already sent events for. /// private static HashSet s_SentRemotePolicyInitialized; private static HashSet s_SentTrainingBehaviorInitialized; #endif private static Guid s_TrainingSessionGuid; // These are set when the RpcCommunicator connects private static string s_TrainerPackageVersion = ""; private static string s_TrainerCommunicationVersion = ""; internal static bool EnableAnalytics() { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE if (s_EventsRegistered) { return true; } foreach (var eventName in s_EventNames) { AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(eventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); if (result != AnalyticsResult.Ok) { return false; } } s_EventsRegistered = true; if (s_SentRemotePolicyInitialized == null) { s_SentRemotePolicyInitialized = new HashSet(); s_SentTrainingBehaviorInitialized = new HashSet(); s_TrainingSessionGuid = Guid.NewGuid(); } return s_EventsRegistered; #else return false; #endif // MLA_UNITY_ANALYTICS_MODULE } /// /// Cache information about the trainer when it becomes available in the RpcCommunicator. /// /// /// [Conditional("MLA_UNITY_ANALYTICS_MODULE")] public static void SetTrainerInformation(string packageVersion, string communicationVersion) { s_TrainerPackageVersion = packageVersion; s_TrainerCommunicationVersion = communicationVersion; } public static bool IsAnalyticsEnabled() { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE return EditorAnalytics.enabled; #else return false; #endif } [Conditional("MLA_UNITY_ANALYTICS_MODULE")] public static void TrainingEnvironmentInitialized(TrainingEnvironmentInitializedEvent tbiEvent) { if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) return; if (s_SentEnvironmentInitialized) { // We already sent an TrainingEnvironmentInitializedEvent. Exit so we don't resend. return; } s_SentEnvironmentInitialized = true; tbiEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString(); // Note - to debug, use JsonUtility.ToJson on the event. // Debug.Log( // $"Would send event {k_TrainingEnvironmentInitializedEventName} with body {JsonUtility.ToJson(tbiEvent, true)}" // ); #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE if (AnalyticsUtils.s_SendEditorAnalytics) { EditorAnalytics.SendEventWithLimit(k_TrainingEnvironmentInitializedEventName, tbiEvent); } #endif } [Conditional("MLA_UNITY_ANALYTICS_MODULE")] public static void RemotePolicyInitialized( string fullyQualifiedBehaviorName, IList sensors, ActionSpec actionSpec, IList actuators ) { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) return; // Extract base behavior name (no team ID) var behaviorName = ParseBehaviorName(fullyQualifiedBehaviorName); var added = s_SentRemotePolicyInitialized.Add(behaviorName); if (!added) { // We previously added this model. Exit so we don't resend. return; } var data = GetEventForRemotePolicy(behaviorName, sensors, actionSpec, actuators); // Note - to debug, use JsonUtility.ToJson on the event. // Debug.Log( // $"Would send event {k_RemotePolicyInitializedEventName} with body {JsonUtility.ToJson(data, true)}" // ); if (AnalyticsUtils.s_SendEditorAnalytics) { EditorAnalytics.SendEventWithLimit(k_RemotePolicyInitializedEventName, data); } #endif } internal static string ParseBehaviorName(string fullyQualifiedBehaviorName) { var lastQuestionIndex = fullyQualifiedBehaviorName.LastIndexOf("?"); if (lastQuestionIndex < 0) { // Nothing to remove return fullyQualifiedBehaviorName; } return fullyQualifiedBehaviorName.Substring(0, lastQuestionIndex); } [Conditional("MLA_UNITY_ANALYTICS_MODULE")] public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent tbiEvent) { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) return; var behaviorName = tbiEvent.BehaviorName; var added = s_SentTrainingBehaviorInitialized.Add(behaviorName); if (!added) { // We previously added this model. Exit so we don't resend. return; } // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. tbiEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString(); tbiEvent.BehaviorName = AnalyticsUtils.Hash(tbiEvent.BehaviorName); // Note - to debug, use JsonUtility.ToJson on the event. // Debug.Log( // $"Would send event {k_TrainingBehaviorInitializedEventName} with body {JsonUtility.ToJson(tbiEvent, true)}" // ); if (AnalyticsUtils.s_SendEditorAnalytics) { EditorAnalytics.SendEventWithLimit(k_TrainingBehaviorInitializedEventName, tbiEvent); } #endif } internal static RemotePolicyInitializedEvent GetEventForRemotePolicy( string behaviorName, IList sensors, ActionSpec actionSpec, IList actuators ) { var remotePolicyEvent = new RemotePolicyInitializedEvent(); // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName); remotePolicyEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString(); remotePolicyEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); remotePolicyEvent.ObservationSpecs = new List(sensors.Count); foreach (var sensor in sensors) { remotePolicyEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } remotePolicyEvent.ActuatorInfos = new List(actuators.Count); foreach (var actuator in actuators) { remotePolicyEvent.ActuatorInfos.Add(EventActuatorInfo.FromActuator(actuator)); } remotePolicyEvent.MLAgentsEnvsVersion = s_TrainerPackageVersion; remotePolicyEvent.TrainerCommunicationVersion = s_TrainerCommunicationVersion; return remotePolicyEvent; } } }