using System.Collections.Generic; using UnityEngine; using System.IO; using System.Linq; #if UNITY_EDITOR using UnityEditor; #endif /** * Welcome to Unity Machine Learning Agents (ML-Agents). * * The ML-Agents toolkit contains five entities: Academy, Brain, Agent, Communicator and * Python API. The academy, and all its brains and connected agents live within * a learning environment (herin called Environment), while the communicator * manages the communication between the learning environment and the Python * API. For more information on each of these entities, in addition to how to * set-up a learning environment and train the behavior of characters in a * Unity scene, please browse our documentation pages on GitHub: * https://github.com/Unity-Technologies/ml-agents/blob/master/docs/ */ namespace MLAgents { /// /// Wraps the environment-level parameters that are provided within the /// Editor. These parameters can be provided for training and inference /// modes separately and represent screen resolution, rendering quality and /// frame rate. /// [System.Serializable] public class EnvironmentConfiguration { [Tooltip("Width of the environment window in pixels.")] public int width; [Tooltip("Height of the environment window in pixels.")] public int height; [Tooltip("Rendering quality of environment. (Higher is better quality.)")] [Range(0, 5)] public int qualityLevel; [Tooltip("Speed at which environment is run. (Higher is faster.)")] [Range(1f, 100f)] public float timeScale; [Tooltip("Frames per second (FPS) engine attempts to maintain.")] public int targetFrameRate; /// Initializes a new instance of the /// class. /// Width of environment window (pixels). /// Height of environment window (pixels). /// /// Rendering quality of environment. Ranges from 0 to 5, with higher. /// /// /// Speed at which environment is run. Ranges from 1 to 100, with higher /// values representing faster speed. /// /// /// Target frame rate (per second) that the engine tries to maintain. /// public EnvironmentConfiguration( int width, int height, int qualityLevel, float timeScale, int targetFrameRate) { this.width = width; this.height = height; this.qualityLevel = qualityLevel; this.timeScale = timeScale; this.targetFrameRate = targetFrameRate; } } /// /// An Academy is where Agent objects go to train their behaviors. More /// specifically, an academy is a collection of Brain objects and each agent /// in a scene is attached to one brain (a single brain may be attached to /// multiple agents). Currently, this class is expected to be extended to /// implement the desired academy behavior. /// /// /// When an academy is run, it can either be in inference or training mode. /// The mode is determined by the presence or absence of a Communicator. In /// the presence of a communicator, the academy is run in training mode where /// the states and observations of each agent are sent through the /// communicator. In the absence of a communciator, the academy is run in /// inference mode where the agent behavior is determined by the brain /// attached to it (which may be internal, heuristic or player). /// [HelpURL("https://github.com/Unity-Technologies/ml-agents/blob/master/" + "docs/Learning-Environment-Design-Academy.md")] public abstract class Academy : MonoBehaviour { private const string kApiVersion = "API-4"; // Fields provided in the Inspector [SerializeField] [Tooltip("Total number of steps per global episode.\nNon-positive " + "values correspond to episodes without a maximum number of \n" + "steps. Once the step counter reaches this maximum value, the " + "environment will reset.")] int maxSteps; [SerializeField] [Tooltip("The engine-level settings which correspond to rendering " + "quality and engine speed during Training.")] EnvironmentConfiguration trainingConfiguration = new EnvironmentConfiguration(80, 80, 1, 100.0f, -1); [SerializeField] [Tooltip("The engine-level settings which correspond to rendering " + "quality and engine speed during Inference.")] EnvironmentConfiguration inferenceConfiguration = new EnvironmentConfiguration(1280, 720, 5, 1.0f, 60); /// /// Contains a mapping from parameter names to float values. They are /// used in and /// to modify elements in the environment at reset time. /// /// /// Default reset parameters are specified in the academy Editor, and can /// be modified when training with an external Brain by passinga config /// dictionary at reset. /// [SerializeField] [Tooltip("List of custom parameters that can be changed in the " + "environment when it resets.")] public ResetParameters resetParameters; // Fields not provided in the Inspector. /// Boolean flag indicating whether a communicator is accessible by the /// environment. This also specifies whether the environment is in /// Training or Inference mode. bool isCommunicatorOn; /// Keeps track of the id of the last communicator message received. /// Remains 0 if there are no communicators. Is used to ensure that /// the same message is not used multiple times. private ulong lastCommunicatorMessageNumber; /// If true, the Academy will use inference settings. This field is /// initialized in depending on the presence /// or absence of a communicator. Furthermore, it can be modified by an /// external Brain during reset via . bool isInference = true; /// The done flag of the academy. When set to true, the academy will /// call instead of /// at step time. If true, all agents done flags will be set to true. bool done; /// Whether the academy has reached the maximum number of steps for the /// current episode. bool maxStepReached; /// The number of episodes completed by the environment. Incremented /// each time the environment is reset. int episodeCount; /// The number of steps completed within the current episide. Incremented /// each time a step is taken in the environment. Is reset to 0 during /// . int stepCount; /// Flag that indicates whether the inference/training mode of the /// environment was switched by the external Brain. This impacts the /// engine settings at the next environment step. bool modeSwitched; /// Pointer to the batcher currently in use by the Academy. MLAgents.Batcher brainBatcher; /// Used to write error messages. StreamWriter logWriter; /// The path to where the log should be written. string logPath; // Flag used to keep track of the first time the Academy is reset. bool firstAcademyReset; // The Academy uses a series of events to communicate with agents and // brains to facilitate synchronization. More specifically, it ensure // that all the agents performs their steps in a consistent order (i.e. no // agent can act based on a decision before another agent has had a chance // to request a decision). // Signals to all the Brains at each environment step so they can decide // actions for their agents. public event System.Action BrainDecideAction; // Signals to all the agents at each environment step along with the // Academy's maxStepReached, done and stepCount values. The agents rely // on this event to update their own values of max step reached and done // in addition to aligning on the step count of the global episode. public event System.Action AgentSetStatus; // Signals to all the agents at each environment step so they can reset // if their flag has been set to done (assuming the agent has requested a // decision). public event System.Action AgentResetIfDone; // Signals to all the agents at each environment step so they can send // their state to their Brain if they have requested a decision. public event System.Action AgentSendState; // Signals to all the agents at each environment step so they can act if // they have requested a decision. public event System.Action AgentAct; // Sigals to all the agents each time the Academy force resets. public event System.Action AgentForceReset; /// /// Monobehavior function called at the very beginning of environment /// creation. Academy uses this time to initialize internal data /// structures, initialize the environment and check for the existence /// of a communicator. /// void Awake() { InitializeEnvironment(); } // Used to read Python-provided environment parameters private int ReadArgs() { var args = System.Environment.GetCommandLineArgs(); var inputPort = ""; for (var i = 0; i < args.Length; i++) { if (args[i] == "--port") { inputPort = args[i + 1]; } } return int.Parse(inputPort); } /// /// Initializes the environment, configures it and initialized the Academy. /// private void InitializeEnvironment() { // Retrieve Brain and initialize Academy var brains = GetBrains(gameObject); InitializeAcademy(); MLAgents.Communicator communicator = null; // Try to launch the communicator by usig the arguments passed at launch try { communicator = new MLAgents.RPCCommunicator( new MLAgents.CommunicatorParameters { port = ReadArgs() }); } // If it fails, we check if there are any external brains in the scene // If there are : Launch the communicator on the default port // If there arn't, there is no need for a communicator and it is set // to null catch { communicator = null; var externalBrain = brains.FirstOrDefault(b => b.brainType == BrainType.External); if (externalBrain != null) { communicator = new MLAgents.RPCCommunicator( new MLAgents.CommunicatorParameters { port = 5005 }); } } brainBatcher = new MLAgents.Batcher(communicator); // Initialize Brains and communicator (if present) foreach (var brain in brains) { brain.InitializeBrain(this, brainBatcher); } if (communicator != null) { isCommunicatorOn = true; var academyParameters = new MLAgents.CommunicatorObjects.UnityRLInitializationOutput(); academyParameters.Name = gameObject.name; academyParameters.Version = kApiVersion; foreach (var brain in brains) { var bp = brain.brainParameters; academyParameters.BrainParameters.Add( MLAgents.Batcher.BrainParametersConvertor( bp, brain.gameObject.name, (MLAgents.CommunicatorObjects.BrainTypeProto) brain.brainType)); } academyParameters.EnvironmentParameters = new MLAgents.CommunicatorObjects.EnvironmentParametersProto(); foreach (var key in resetParameters.Keys) { academyParameters.EnvironmentParameters.FloatParameters.Add( key, resetParameters[key] ); } var pythonParameters = brainBatcher.SendAcademyParameters(academyParameters); Random.InitState(pythonParameters.Seed); Application.logMessageReceived += HandleLog; logPath = Path.GetFullPath(".") + "/MLAgentsSDK.log"; logWriter = new StreamWriter(logPath, false); logWriter.WriteLine(System.DateTime.Now.ToString()); logWriter.WriteLine(" "); logWriter.Close(); } // If a communicator is enabled/provided, then we assume we are in // training mode. In the absence of a communicator, we assume we are // in inference mode. isInference = !isCommunicatorOn; BrainDecideAction += () => { }; AgentSetStatus += (m, d, i) => { }; AgentResetIfDone += () => { }; AgentSendState += () => { }; AgentAct += () => { }; AgentForceReset += () => { }; // Configure the environment using the configurations provided by // the developer in the Editor. SetIsInference(!brainBatcher.GetIsTraining()); ConfigureEnvironment(); } private void UpdateResetParameters() { var newResetParameters = brainBatcher.GetEnvironmentParameters(); if (newResetParameters != null) { foreach (var kv in newResetParameters.FloatParameters) { resetParameters[kv.Key] = kv.Value; } } } void HandleLog(string logString, string stackTrace, LogType type) { logWriter = new StreamWriter(logPath, true); logWriter.WriteLine(type.ToString()); logWriter.WriteLine(logString); logWriter.WriteLine(stackTrace); logWriter.Close(); } /// /// Configures the environment settings depending on the training/inference /// mode and the corresponding parameters passed in the Editor. /// void ConfigureEnvironment() { if (isInference) { ConfigureEnvironmentHelper(inferenceConfiguration); Monitor.SetActive(true); } else { ConfigureEnvironmentHelper(trainingConfiguration); Monitor.SetActive(false); } } /// /// Helper method for initializing the environment based on the provided /// configuration. /// /// /// Environment configuration (specified in the Editor). /// static void ConfigureEnvironmentHelper(EnvironmentConfiguration config) { Screen.SetResolution(config.width, config.height, false); QualitySettings.SetQualityLevel(config.qualityLevel, true); Time.timeScale = config.timeScale; Time.captureFramerate = 60; Application.targetFrameRate = config.targetFrameRate; } /// /// Initializes the academy and environment. Called during the waking-up /// phase of the environment before any of the scene objects/agents have /// been initialized. /// public virtual void InitializeAcademy() { } /// /// Specifies the academy behavior at every step of the environment. /// public virtual void AcademyStep() { } /// /// Specifies the academy behavior when being reset (i.e. at the completion /// of a global episode). /// public virtual void AcademyReset() { } /// /// Returns the flag. /// /// /// true, if current mode is inference, false if training. /// public bool GetIsInference() { return isInference; } /// /// Sets the flag to the provided value. If /// the new flag differs from the current flag value, this signals that /// the environment configuration needs to be updated. /// /// /// Environment mode, if true then inference, otherwise training. /// public void SetIsInference(bool isInference) { if (this.isInference != isInference) { this.isInference = isInference; // This signals to the academy that at the next environment step // the engine configurations need updating to the respective mode // (i.e. training vs inference) configuraiton. modeSwitched = true; } } /// /// Returns the current episode counter. /// /// /// Current episode number. /// public int GetEpisodeCount() { return episodeCount; } /// /// Returns the current step counter (within the current epside). /// /// /// Current episode number. /// public int GetStepCount() { return stepCount; } /// /// Sets the done flag to true. /// public void Done() { done = true; } /// /// Returns whether or not the academy is done. /// /// /// true, if academy is done, false otherwise. /// public bool IsDone() { return done; } /// /// Returns whether or not the communicator is on. /// /// /// true, if communicator is on, false otherwise. /// public bool IsCommunicatorOn() { return isCommunicatorOn; } /// /// Forces the full reset. The done flags are not affected. Is either /// called the first reset at inference and every external reset /// at training. /// void ForcedFullReset() { EnvironmentReset(); AgentForceReset(); firstAcademyReset = true; } /// /// Performs a single environment update to the Academy, Brain and Agent /// objects within the environment. /// void EnvironmentStep() { if (modeSwitched) { ConfigureEnvironment(); modeSwitched = false; } if ((isCommunicatorOn) && (lastCommunicatorMessageNumber != brainBatcher.GetNumberMessageReceived())) { lastCommunicatorMessageNumber = brainBatcher.GetNumberMessageReceived(); if (brainBatcher.GetCommand() == MLAgents.CommunicatorObjects.CommandProto.Reset) { UpdateResetParameters(); SetIsInference(!brainBatcher.GetIsTraining()); ForcedFullReset(); } if (brainBatcher.GetCommand() == MLAgents.CommunicatorObjects.CommandProto.Quit) { #if UNITY_EDITOR EditorApplication.isPlaying = false; #endif Application.Quit(); return; } } else if (!firstAcademyReset) { UpdateResetParameters(); ForcedFullReset(); } if ((stepCount >= maxSteps) && maxSteps > 0) { maxStepReached = true; Done(); } AgentSetStatus(maxStepReached, done, stepCount); brainBatcher.RegisterAcademyDoneFlag(done); if (done) { EnvironmentReset(); } AgentResetIfDone(); AgentSendState(); BrainDecideAction(); AcademyStep(); AgentAct(); stepCount += 1; } /// /// Resets the environment, including the Academy. /// void EnvironmentReset() { stepCount = 0; episodeCount++; done = false; maxStepReached = false; AcademyReset(); } /// /// Monobehavior function that dictates each environment step. /// void FixedUpdate() { EnvironmentStep(); } /// /// Helper method that retrieves the Brain objects that are currently /// specified as children of the Academy within the Editor. /// /// Academy. /// /// List of brains currently attached to academy. /// static List GetBrains(GameObject academy) { List brains = new List(); var transform = academy.transform; for (var i = 0; i < transform.childCount; i++) { var child = transform.GetChild(i); var brain = child.GetComponent(); if (brain != null && child.gameObject.activeSelf) { brains.Add(brain); } } return brains; } } }