simple TicTacToe example

3 年前 · f5bf6e08
--- a/Project/Assets/ML-Agents/Examples/TicTacToe.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe.meta
+fileFormatVersion: 2
+guid: 8cd2b21690cdb42118493d209626abec
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/config/ppo/TicTacToe.yaml
+++ b/config/ppo/TicTacToe.yaml
+behaviors:
+  TicTacToe:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: constant
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
+    reward_signals:
+      extrinsic:
+        gamma: 0.99
+        strength: 1.0
+    keep_checkpoints: 5
+    max_steps: 50000
+    time_horizon: 16
+    summary_freq: 1000
+    threaded: false
+    self_play:
+      save_steps: 5000
+      team_change: 20000
+      swap_steps: 200
+      window: 10
+      play_against_latest_model_ratio: 0.5
+      initial_elo: 1200.0
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes.meta
+fileFormatVersion: 2
+guid: 7f3247b825a0a4531afd458547cedcbf
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes/TicTacToe.unity
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes/TicTacToe.unity
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!29 &1
+OcclusionCullingSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  m_OcclusionBakeSettings:
+    smallestOccluder: 5
+    smallestHole: 0.25
+    backfaceThreshold: 100
+  m_SceneGUID: 00000000000000000000000000000000
+  m_OcclusionCullingData: {fileID: 0}
+--- !u!104 &2
+RenderSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 9
+  m_Fog: 0
+  m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1}
+  m_FogMode: 3
+  m_FogDensity: 0.01
+  m_LinearFogStart: 0
+  m_LinearFogEnd: 300
+  m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1}
+  m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1}
+  m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1}
+  m_AmbientIntensity: 1
+  m_AmbientMode: 0
+  m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1}
+  m_SkyboxMaterial: {fileID: 10304, guid: 0000000000000000f000000000000000, type: 0}
+  m_HaloStrength: 0.5
+  m_FlareStrength: 1
+  m_FlareFadeSpeed: 3
+  m_HaloTexture: {fileID: 0}
+  m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0}
+  m_DefaultReflectionMode: 0
+  m_DefaultReflectionResolution: 128
+  m_ReflectionBounces: 1
+  m_ReflectionIntensity: 1
+  m_CustomReflection: {fileID: 0}
+  m_Sun: {fileID: 0}
+  m_IndirectSpecularColor: {r: 0.44657898, g: 0.49641287, b: 0.5748173, a: 1}
+  m_UseRadianceAmbientProbe: 0
+--- !u!157 &3
+LightmapSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 11
+  m_GIWorkflowMode: 0
+  m_GISettings:
+    serializedVersion: 2
+    m_BounceScale: 1
+    m_IndirectOutputScale: 1
+    m_AlbedoBoost: 1
+    m_EnvironmentLightingMode: 0
+    m_EnableBakedLightmaps: 1
+    m_EnableRealtimeLightmaps: 1
+  m_LightmapEditorSettings:
+    serializedVersion: 10
+    m_Resolution: 2
+    m_BakeResolution: 40
+    m_AtlasSize: 1024
+    m_AO: 0
+    m_AOMaxDistance: 1
+    m_CompAOExponent: 1
+    m_CompAOExponentDirect: 0
+    m_Padding: 2
+    m_LightmapParameters: {fileID: 0}
+    m_LightmapsBakeMode: 1
+    m_TextureCompression: 1
+    m_FinalGather: 0
+    m_FinalGatherFiltering: 1
+    m_FinalGatherRayCount: 256
+    m_ReflectionCompression: 2
+    m_MixedBakeMode: 2
+    m_BakeBackend: 1
+    m_PVRSampling: 1
+    m_PVRDirectSampleCount: 32
+    m_PVRSampleCount: 500
+    m_PVRBounces: 2
+    m_PVRFilterTypeDirect: 0
+    m_PVRFilterTypeIndirect: 0
+    m_PVRFilterTypeAO: 0
+    m_PVRFilteringMode: 1
+    m_PVRCulling: 1
+    m_PVRFilteringGaussRadiusDirect: 1
+    m_PVRFilteringGaussRadiusIndirect: 5
+    m_PVRFilteringGaussRadiusAO: 2
+    m_PVRFilteringAtrousPositionSigmaDirect: 0.5
+    m_PVRFilteringAtrousPositionSigmaIndirect: 2
+    m_PVRFilteringAtrousPositionSigmaAO: 1
+    m_ShowResolutionOverlay: 1
+  m_LightingDataAsset: {fileID: 0}
+  m_UseShadowmask: 1
+--- !u!196 &4
+NavMeshSettings:
+  serializedVersion: 2
+  m_ObjectHideFlags: 0
+  m_BuildSettings:
+    serializedVersion: 2
+    agentTypeID: 0
+    agentRadius: 0.5
+    agentHeight: 2
+    agentSlope: 45
+    agentClimb: 0.4
+    ledgeDropHeight: 0
+    maxJumpAcrossDistance: 0
+    minRegionArea: 2
+    manualCellSize: 0
+    cellSize: 0.16666667
+    manualTileSize: 0
+    tileSize: 256
+    accuratePlacement: 0
+    debug:
+      m_Flags: 0
+  m_NavMeshData: {fileID: 0}
+--- !u!1 &105600135
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 105600137}
+  - component: {fileID: 105600136}
+  m_Layer: 0
+  m_Name: Directional Light
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!108 &105600136
+Light:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 105600135}
+  m_Enabled: 1
+  serializedVersion: 8
+  m_Type: 1
+  m_Color: {r: 1, g: 0.95686275, b: 0.8392157, a: 1}
+  m_Intensity: 1
+  m_Range: 10
+  m_SpotAngle: 30
+  m_CookieSize: 10
+  m_Shadows:
+    m_Type: 2
+    m_Resolution: -1
+    m_CustomResolution: -1
+    m_Strength: 1
+    m_Bias: 0.05
+    m_NormalBias: 0.4
+    m_NearPlane: 0.2
+  m_Cookie: {fileID: 0}
+  m_DrawHalo: 0
+  m_Flare: {fileID: 0}
+  m_RenderMode: 0
+  m_CullingMask:
+    serializedVersion: 2
+    m_Bits: 4294967295
+  m_Lightmapping: 4
+  m_LightShadowCasterMode: 0
+  m_AreaSize: {x: 1, y: 1}
+  m_BounceIntensity: 1
+  m_ColorTemperature: 6570
+  m_UseColorTemperature: 0
+  m_ShadowRadius: 0
+  m_ShadowAngle: 0
+--- !u!4 &105600137
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 105600135}
+  m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261}
+  m_LocalPosition: {x: 0, y: 3, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 1
+  m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0}
+--- !u!1 &887871232
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 887871233}
+  m_Layer: 0
+  m_Name: TicTacToeManager
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &887871233
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 887871232}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children:
+  - {fileID: 1871104840}
+  m_Father: {fileID: 0}
+  m_RootOrder: 2
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &1871104839
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 1871104840}
+  - component: {fileID: 1871104842}
+  - component: {fileID: 1871104841}
+  m_Layer: 0
+  m_Name: PlayerX
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &1871104840
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1871104839}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 887871233}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &1871104841
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1871104839}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 70e67251099c241d394f5b5ae4dd1c83, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  agentParameters:
+    maxStep: 0
+  hasUpgradedFromAgentParameters: 1
+  MaxStep: 0
+  playerType: 0
+--- !u!114 &1871104842
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1871104839}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 5d1c4e0b1822b495aa52bc52839ecb30, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_BrainParameters:
+    VectorObservationSize: 27
+    NumStackedVectorObservations: 1
+    m_ActionSpec:
+      m_NumContinuousActions: 0
+      BranchSizes: 09000000
+    VectorActionSize: 09000000
+    VectorActionDescriptions: []
+    VectorActionSpaceType: 0
+    hasUpgradedBrainParametersWithActionSpec: 1
+  m_Model: {fileID: 0}
+  m_InferenceDevice: 2
+  m_BehaviorType: 0
+  m_BehaviorName: My Behavior
+  TeamId: 0
+  m_UseChildSensors: 1
+  m_UseChildActuators: 1
+  m_ObservableAttributeHandling: 0
+--- !u!1 &2116523437
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 2116523440}
+  - component: {fileID: 2116523439}
+  - component: {fileID: 2116523438}
+  m_Layer: 0
+  m_Name: Main Camera
+  m_TagString: MainCamera
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!81 &2116523438
+AudioListener:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 2116523437}
+  m_Enabled: 1
+--- !u!20 &2116523439
+Camera:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 2116523437}
+  m_Enabled: 1
+  serializedVersion: 2
+  m_ClearFlags: 1
+  m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0}
+  m_projectionMatrixMode: 1
+  m_SensorSize: {x: 36, y: 24}
+  m_LensShift: {x: 0, y: 0}
+  m_GateFitMode: 2
+  m_FocalLength: 50
+  m_NormalizedViewPortRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+  near clip plane: 0.3
+  far clip plane: 1000
+  field of view: 60
+  orthographic: 0
+  orthographic size: 5
+  m_Depth: -1
+  m_CullingMask:
+    serializedVersion: 2
+    m_Bits: 4294967295
+  m_RenderingPath: -1
+  m_TargetTexture: {fileID: 0}
+  m_TargetDisplay: 0
+  m_TargetEye: 3
+  m_HDR: 1
+  m_AllowMSAA: 1
+  m_AllowDynamicResolution: 0
+  m_ForceIntoRT: 0
+  m_OcclusionCulling: 1
+  m_StereoConvergence: 10
+  m_StereoSeparation: 0.022
+--- !u!4 &2116523440
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 2116523437}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 1, z: -10}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes/TicTacToe.unity.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scenes/TicTacToe.unity.meta
+fileFormatVersion: 2
+guid: 2999a69e73697429ab58c3bd498cc601
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts.meta
+fileFormatVersion: 2
+guid: 73cc3f2c4d51540e38e50663c51c7954
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeAgent.cs
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeAgent.cs
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public enum PlayerType
+{
+    PlayerX,
+    PlayerO
+}
+
+public class TicTacToeAgent : Agent
+{
+    public PlayerType playerType;
+
+    private TicTacToeGame m_Game;
+    private System.Random m_Random;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        m_Game = GetComponentInParent<TicTacToeGame>();
+        m_Random = new System.Random(GetInstanceID());
+    }
+
+    public override void CollectObservations(VectorSensor sensor)
+    {
+        var rows = m_Game.Board.GetLength(0);
+        var cols = m_Game.Board.GetLength(1);
+
+        for (int r = 0; r < rows; r++)
+        {
+            for (int c = 0; c < cols; c++)
+            {
+                // One-hot encoding of the board state *depending on the agent's player type*
+                // Set the 0th element if the board is empty
+                // Set the 1st element if the board is my type
+                // Set the 2nd element if the board is my opponent's type
+                BoardStatus boardVal = m_Game.Board[r, c];
+                bool isMine = (boardVal == BoardStatus.FilledX && playerType == PlayerType.PlayerX) ||
+                              (boardVal == BoardStatus.FilledO && playerType == PlayerType.PlayerO);
+                int oneHotIndex = (boardVal == BoardStatus.Empty) ? 0 : (isMine) ? 1 : 2;
+                sensor.AddOneHotObservation(oneHotIndex, 3);
+            }
+        }
+    }
+
+    public override void WriteDiscreteActionMask(IDiscreteActionMask actionMask)
+    {
+        var rows = m_Game.Board.GetLength(0);
+        var cols = m_Game.Board.GetLength(1);
+        int i = 0;
+
+        for (int r = 0; r < rows; r++)
+        {
+            for (int c = 0; c < cols; c++)
+            {
+                BoardStatus boardVal = m_Game.Board[r, c];
+                // Disallow moves on non-empty squares
+                if (boardVal != BoardStatus.Empty)
+                {
+                    actionMask.WriteMask(0, new[] { i });
+                }
+                i++;
+            }
+        }
+    }
+
+    public override void OnActionReceived(ActionBuffers actions)
+    {
+        var rows = m_Game.Board.GetLength(0);
+        var index = actions.DiscreteActions[0];
+        var r = index / rows;
+        var c = index % rows;
+
+        // The trainer can occasionally (about 1 in a million steps) select an invalid move.
+        // So make sure to check the square is empty before setting it.
+        if (m_Game.Board[r, c] == BoardStatus.Empty)
+        {
+            m_Game.Board[r, c] = (playerType == PlayerType.PlayerX)
+                ? BoardStatus.FilledX
+                : BoardStatus.FilledO;
+        }
+    }
+
+    public override void Heuristic(in ActionBuffers actionsOut)
+    {
+        // Pick a random empty square.
+        var validMoves = new List<int>();
+        var rows = m_Game.Board.GetLength(0);
+        var cols = m_Game.Board.GetLength(1);
+        int i = 0;
+
+        for (int r = 0; r < rows; r++)
+        {
+            for (int c = 0; c < cols; c++)
+            {
+                if (m_Game.Board[r, c] == BoardStatus.Empty)
+                {
+                    validMoves.Add(i);
+                }
+
+                i++;
+            }
+        }
+
+        var choice = validMoves[m_Random.Next(validMoves.Count)];
+        var discreteActions = actionsOut.DiscreteActions;
+        discreteActions[0] = choice;
+    }
+}
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeAgent.cs.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeAgent.cs.meta
+fileFormatVersion: 2
+guid: 70e67251099c241d394f5b5ae4dd1c83
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeGame.cs
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeGame.cs
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using Unity.MLAgents;
+using UnityEngine;
+
+public enum BoardStatus
+{
+    Empty,
+    FilledX,
+    FilledO
+}
+
+public enum WinState
+{
+    NoWinnerYet,
+    WinnerX,
+    WinnerO,
+    Draw
+}
+
+public class TicTacToeGame : MonoBehaviour
+{
+    public BoardStatus[,] Board = new BoardStatus[3, 3];
+    private PlayerType m_CurrentPlayer = PlayerType.PlayerX;
+    private TicTacToeAgent m_AgentX;
+    private TicTacToeAgent m_AgentO;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        Academy.Instance.AutomaticSteppingEnabled = false;
+        var agents = GetComponentsInChildren<TicTacToeAgent>();
+        foreach (var agent in agents)
+        {
+            if (agent.playerType == PlayerType.PlayerX)
+            {
+                m_AgentX = agent;
+            }
+            else if (agent.playerType == PlayerType.PlayerO)
+            {
+                m_AgentO = agent;
+            }
+            else
+            {
+                throw new UnityAgentsException("Unknown player type.");
+            }
+        }
+
+        Debug.Assert(m_AgentX != null);
+        Debug.Assert(m_AgentO != null);
+    }
+
+    void InitBoard()
+    {
+        var rows = Board.GetLength(0);
+        var cols = Board.GetLength(1);
+
+        for (int r = 0; r < rows; r++)
+        {
+            for (int c = 0; c < cols; c++)
+            {
+                Board[r, c] = BoardStatus.Empty;
+            }
+        }
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        Agent currentAgent = (m_CurrentPlayer == PlayerType.PlayerX) ? m_AgentX : m_AgentO;
+        currentAgent.RequestDecision();
+        Academy.Instance.EnvironmentStep();
+
+        var winState = CheckForWin();
+        if (winState == WinState.NoWinnerYet)
+        {
+            // Swap
+            m_CurrentPlayer = m_CurrentPlayer == PlayerType.PlayerX ? PlayerType.PlayerO : PlayerType.PlayerX;
+        }
+        else
+        {
+            //Debug.Log(this.ToString());
+            float rewardX = 0.0f;
+            float rewardO = 0.0f;
+
+            if (winState == WinState.WinnerX)
+            {
+                rewardX = 1.0f;
+                rewardO = -1.0f;
+                //Debug.Log("X wins!");
+            }
+            else if (winState == WinState.WinnerO)
+            {
+                rewardX = -1.0f;
+                rewardO = 1.0f;
+                //Debug.Log("O wins!");
+            }
+            else
+            {
+                //Debug.Log("Nobody wins :(");
+            }
+            m_AgentX.AddReward(rewardX);
+            m_AgentX.EndEpisode();
+
+            m_AgentO.AddReward(rewardO);
+            m_AgentO.EndEpisode();
+
+            InitBoard();
+        }
+    }
+
+    public WinState CheckForWin()
+    {
+        // Assume 3 rows and columns for now, not interested in generalizing.
+
+        // Check rows
+        for (var r = 0; r < 3; r++)
+        {
+            if (Board[r, 0] == BoardStatus.Empty)
+            {
+                continue;
+            }
+
+            if (Board[r, 1] == Board[r, 0] && Board[r, 2] == Board[r, 0])
+            {
+                // all the same in the row
+                return (Board[r, 0] == BoardStatus.FilledX) ? WinState.WinnerX : WinState.WinnerO;
+            }
+        }
+
+        for (var c = 0; c < 3; c++)
+        {
+            if (Board[0, c] == BoardStatus.Empty)
+            {
+                continue;
+            }
+
+            if (Board[1, c] == Board[0, c] && Board[2, c] == Board[0, c])
+            {
+                // all the same in the column
+                return (Board[0, c] == BoardStatus.FilledX) ? WinState.WinnerX : WinState.WinnerO;
+            }
+        }
+
+        // Check diagonals
+        {
+            if (Board[0, 0] == Board[1, 1] && Board[0, 0] == Board[2, 2] && Board[0, 0] != BoardStatus.Empty)
+            {
+                return (Board[0, 0] == BoardStatus.FilledX) ? WinState.WinnerX : WinState.WinnerO;
+            }
+
+            if (Board[0, 2] == Board[1, 1] && Board[0, 2] == Board[2, 0] && Board[0, 2] != BoardStatus.Empty)
+            {
+                return (Board[0, 2] == BoardStatus.FilledX) ? WinState.WinnerX : WinState.WinnerO;
+            }
+        }
+
+        // No winner, check if there are still moves or it's a draw
+        for (var r = 0; r < 3; r++)
+        {
+            for (var c = 0; c < 3; c++)
+            {
+                if (Board[r, c] == BoardStatus.Empty)
+                {
+                    return WinState.NoWinnerYet;
+                }
+            }
+        }
+
+        // No winner, no empty squares, do it's a draw
+        return WinState.Draw;
+    }
+
+    static string BoardStatusToString(BoardStatus s)
+    {
+        switch (s)
+        {
+            case BoardStatus.Empty:
+                return ".";
+            case BoardStatus.FilledX:
+                return "X";
+            case BoardStatus.FilledO:
+                return "O";
+            default:
+                throw new ArgumentOutOfRangeException(nameof(s), s, null);
+        }
+    }
+
+    public override string ToString()
+    {
+        var b00 = BoardStatusToString(Board[0, 0]);
+        var b01 = BoardStatusToString(Board[0, 1]);
+        var b02 = BoardStatusToString(Board[0, 2]);
+
+        var b10 = BoardStatusToString(Board[1, 0]);
+        var b11 = BoardStatusToString(Board[1, 1]);
+        var b12 = BoardStatusToString(Board[1, 2]);
+
+        var b20 = BoardStatusToString(Board[2, 0]);
+        var b21 = BoardStatusToString(Board[2, 1]);
+        var b22 = BoardStatusToString(Board[2, 2]);
+        return $"\n{b00}{b01}{b02}\n{b10}{b11}{b12}\n{b20}{b21}{b22}";
+    }
+}
--- a/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeGame.cs.meta
+++ b/Project/Assets/ML-Agents/Examples/TicTacToe/Scripts/TicTacToeGame.cs.meta
+fileFormatVersion: 2
+guid: 724fe1e451e454c798bbb93e3fa8c215
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: