[MLA-488] Fix observations in demonstration drawer (#3771)

* WIP observation shapes

* WIP observation shapes

* fix Observation shape serialization

* rename field, redo screenshot

* docstring

* changelog

Project/Assets/ML-Agents/Examples/3DBall/Demos/Expert3DBall.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/3DBall/Demos/Expert3DBall.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/3DBall/Demos/Expert3DBallHard.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/3DBall/Demos/Expert3DBallHard.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Basic/Demos/ExpertBasic.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Basic/Demos/ExpertBasic.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Bouncer/Demos/ExpertBouncer.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Bouncer/Demos/ExpertBouncer.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Crawler/Demos/ExpertCrawlerDyn.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Crawler/Demos/ExpertCrawlerDyn.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Crawler/Demos/ExpertCrawlerSta.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Crawler/Demos/ExpertCrawlerSta.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/FoodCollector/Demos/ExpertFood.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/FoodCollector/Demos/ExpertFood.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/GridWorld/Demos/ExpertGrid.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/GridWorld/Demos/ExpertGrid.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Hallway/Demos/ExpertHallway.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Hallway/Demos/ExpertHallway.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/PushBlock/Demos/ExpertPush.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/PushBlock/Demos/ExpertPush.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Pyramids/Demos/ExpertPyramid.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Pyramids/Demos/ExpertPyramid.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Reacher/Demos/ExpertReacher.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Reacher/Demos/ExpertReacher.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Tennis/Demos/ExpertTennis.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Tennis/Demos/ExpertTennis.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/Assets/ML-Agents/Examples/Walker/Demos/ExpertWalker.demo.meta

   fileIDToRecycleName:
     11400000: Assets/ML-Agents/Examples/Walker/Demos/ExpertWalker.demo
   externalObjects: {}
-  userData: ' (MLAgents.Demonstrations.Demonstration)'
+  userData: ' (MLAgents.Demonstrations.DemonstrationSummary)'
   assetBundleName: 
   assetBundleVariant: 
   script: {fileID: 11500000, guid: 7bd65ce151aaa4a41a45312543c56be1, type: 3}

Project/ProjectSettings/ProjectVersion.txt

-m_EditorVersion: 2018.4.18f1
+m_EditorVersion: 2018.4.17f1

com.unity.ml-agents/CHANGELOG.md

  - `StackingSensor` was changed from `internal` visibility to `public`
  - Updated Barracuda to 0.6.3-preview.
 
+ ### Bug Fixes
+ - Fixed a display bug when viewing Demonstration files in the inspector. The shapes of the observations in the file now display correctly. (#3771)
+
 ## [0.15.1-preview] - 2020-03-30
 ### Bug Fixes
  - Raise the wall in CrawlerStatic scene to prevent Agent from falling off. (#3650)

com.unity.ml-agents/Editor/DemonstrationDrawer.cs

+using System.Collections.Generic;
 using System.Text;
 using UnityEditor;
 using MLAgents.Demonstrations;
 namespace MLAgents.Editor
 {
     /// <summary>
-    /// Renders a custom UI for Demonstration Scriptable Object.
+    /// Renders a custom UI for DemonstrationSummary ScriptableObject.
-    [CustomEditor(typeof(Demonstration))]
+    [CustomEditor(typeof(DemonstrationSummary))]
+        SerializedProperty m_ObservationShapes;
+            m_ObservationShapes = serializedObject.FindProperty("observationSummaries");
         }
 
         /// <summary>
         {
             var nameProp = property.FindPropertyRelative("demonstrationName");
-            var expProp = property.FindPropertyRelative("numberExperiences");
-            var epiProp = property.FindPropertyRelative("numberEpisodes");
-            var rewProp = property.FindPropertyRelative("meanReward");
+            var experiencesProp = property.FindPropertyRelative("numberSteps");
+            var episodesProp = property.FindPropertyRelative("numberEpisodes");
+            var rewardsProp = property.FindPropertyRelative("meanReward");
-            var expLabel = expProp.displayName + ": " + expProp.intValue;
-            var epiLabel = epiProp.displayName + ": " + epiProp.intValue;
-            var rewLabel = rewProp.displayName + ": " + rewProp.floatValue;
+            var experiencesLabel = experiencesProp.displayName + ": " + experiencesProp.intValue;
+            var episodesLabel = episodesProp.displayName + ": " + episodesProp.intValue;
+            var rewardsLabel = rewardsProp.displayName + ": " + rewardsProp.floatValue;
-            EditorGUILayout.LabelField(expLabel);
-            EditorGUILayout.LabelField(epiLabel);
-            EditorGUILayout.LabelField(rewLabel);
+            EditorGUILayout.LabelField(experiencesLabel);
+            EditorGUILayout.LabelField(episodesLabel);
+            EditorGUILayout.LabelField(rewardsLabel);
-        /// Constructs label for action size array.
+        /// Constructs label for a serialized integer array.
-        static string BuildActionArrayLabel(SerializedProperty actionSizeProperty)
+        static string BuildIntArrayLabel(SerializedProperty actionSizeProperty)
         {
             var actionSize = actionSizeProperty.arraySize;
             var actionLabel = new StringBuilder("[ ");
         }
 
         /// <summary>
-        /// Renders Inspector UI for Brain Parameters of Demonstration.
+        /// Renders Inspector UI for BrainParameters of a DemonstrationSummary.
+        /// Only the Action size and type are used from the BrainParameters.
-        void MakeBrainParametersProperty(SerializedProperty property)
+        void MakeActionsProperty(SerializedProperty property)
-            var vecObsSizeProp = property.FindPropertyRelative("vectorObservationSize");
-            var numStackedProp = property.FindPropertyRelative("numStackedVectorObservations");
-            var vecObsSizeLabel = vecObsSizeProp.displayName + ": " + vecObsSizeProp.intValue;
-            var numStackedLabel = numStackedProp.displayName + ": " + numStackedProp.intValue;
-                actSizeProperty.displayName + ": " + BuildActionArrayLabel(actSizeProperty);
+                actSizeProperty.displayName + ": " + BuildIntArrayLabel(actSizeProperty);
-            EditorGUILayout.LabelField(vecObsSizeLabel);
-            EditorGUILayout.LabelField(numStackedLabel);
+        /// <summary>
+        /// Render the observation shapes of a DemonstrationSummary.
+        /// </summary>
+        /// <param name="obsSummariesProperty"></param>
+        void MakeObservationsProperty(SerializedProperty obsSummariesProperty)
+        {
+            var shapesLabels = new List<string>();
+            var numObservations = obsSummariesProperty.arraySize;
+            for (var i = 0; i < numObservations; i++)
+            {
+                var summary = obsSummariesProperty.GetArrayElementAtIndex(i);
+                var shapeProperty = summary.FindPropertyRelative("shape");
+                shapesLabels.Add(BuildIntArrayLabel(shapeProperty));
+            }
+
+            var shapeLabel = $"Shapes: {string.Join(",  ", shapesLabels)}";
+            EditorGUILayout.LabelField(shapeLabel);
+
+        }
+
+
+            EditorGUI.indentLevel++;
-            EditorGUILayout.LabelField("Brain Parameters", EditorStyles.boldLabel);
-            MakeBrainParametersProperty(m_BrainParameters);
+            EditorGUI.indentLevel--;
+
+            EditorGUILayout.LabelField("Observations", EditorStyles.boldLabel);
+            EditorGUI.indentLevel++;
+            MakeObservationsProperty(m_ObservationShapes);
+            EditorGUI.indentLevel--;
+
+            EditorGUILayout.LabelField("Actions", EditorStyles.boldLabel);
+            EditorGUI.indentLevel++;
+            MakeActionsProperty(m_BrainParameters);
+            EditorGUI.indentLevel--;
+
             serializedObject.ApplyModifiedProperties();
         }
     }

com.unity.ml-agents/Editor/DemonstrationImporter.cs

 using System;
+using System.Collections.Generic;
 using System.IO;
 using MLAgents.CommunicatorObjects;
 using UnityEditor;
 
             try
             {
-                // Read first two proto objects containing metadata and brain parameters.
+                // Read first three proto objects containing metadata, brain parameters, and observations.
                 Stream reader = File.OpenRead(ctx.assetPath);
 
                 var metaDataProto = DemonstrationMetaProto.Parser.ParseDelimitedFrom(reader);
                 var brainParamsProto = BrainParametersProto.Parser.ParseDelimitedFrom(reader);
                 var brainParameters = brainParamsProto.ToBrainParameters();
 
+                // Read the first AgentInfoActionPair so that we can get the observation sizes.
+                List<ObservationSummary> observationSummaries;
+                try
+                {
+                    var agentInfoActionPairProto = AgentInfoActionPairProto.Parser.ParseDelimitedFrom(reader);
+                    observationSummaries = agentInfoActionPairProto.GetObservationSummaries();
+                }
+                catch
+                {
+                    // Just in case there weren't any AgentInfoActionPair or they couldn't be read.
+                    observationSummaries = new List<ObservationSummary>();
+                }
+
-                var demonstration = ScriptableObject.CreateInstance<Demonstration>();
-                demonstration.Initialize(brainParameters, metaData);
-                userData = demonstration.ToString();
+                var demonstrationSummary = ScriptableObject.CreateInstance<DemonstrationSummary>();
+                demonstrationSummary.Initialize(brainParameters, metaData, observationSummaries);
+                userData = demonstrationSummary.ToString();
-                ctx.AddObjectToAsset(ctx.assetPath, demonstration, texture);
-                ctx.SetMainObject(demonstration);
+                ctx.AddObjectToAsset(ctx.assetPath, demonstrationSummary, texture);
+                ctx.SetMainObject(demonstrationSummary);
             }
             catch
             {

com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs

 {
     internal static class GrpcExtensions
     {
+        #region AgentInfo
         /// <summary>
         /// Converts a AgentInfo to a protobuf generated AgentInfoActionPairProto
         /// </summary>
         }
 
         /// <summary>
+        /// Get summaries for the observations in the AgentInfo part of the AgentInfoActionPairProto.
+        /// </summary>
+        /// <param name="infoActionPair"></param>
+        /// <returns></returns>
+        public static List<ObservationSummary> GetObservationSummaries(this AgentInfoActionPairProto infoActionPair)
+        {
+            List<ObservationSummary> summariesOut = new List<ObservationSummary>();
+            var agentInfo = infoActionPair.AgentInfo;
+            foreach (var obs in agentInfo.Observations)
+            {
+                var summary = new ObservationSummary();
+                summary.shape = obs.Shape.ToArray();
+                summariesOut.Add(summary);
+            }
+
+            return summariesOut;
+        }
+
+
+        #endregion
+
+        #region BrainParameters
+        /// <summary>
         /// Converts a Brain into to a Protobuf BrainInfoProto so it can be sent
         /// </summary>
         /// <returns>The BrainInfoProto generated.</returns>
         }
 
         /// <summary>
+        /// Convert a BrainParametersProto to a BrainParameters struct.
+        /// </summary>
+        /// <param name="bpp">An instance of a brain parameters protobuf object.</param>
+        /// <returns>A BrainParameters struct.</returns>
+        public static BrainParameters ToBrainParameters(this BrainParametersProto bpp)
+        {
+            var bp = new BrainParameters
+            {
+                vectorActionSize = bpp.VectorActionSize.ToArray(),
+                vectorActionDescriptions = bpp.VectorActionDescriptions.ToArray(),
+                vectorActionSpaceType = (SpaceType)bpp.VectorActionSpaceType
+            };
+            return bp;
+        }
+
+        #endregion
+
+        #region DemonstrationMetaData
+        /// <summary>
         /// Convert metadata object to proto object.
         /// </summary>
         public static DemonstrationMetaProto ToProto(this DemonstrationMetaData dm)
                 ApiVersion = DemonstrationMetaData.ApiVersion,
                 MeanReward = dm.meanReward,
-                NumberSteps = dm.numberExperiences,
+                NumberSteps = dm.numberSteps,
                 NumberEpisodes = dm.numberEpisodes,
                 DemonstrationName = dm.demonstrationName
             };
             var dm = new DemonstrationMetaData
             {
                 numberEpisodes = demoProto.NumberEpisodes,
-                numberExperiences = demoProto.NumberSteps,
+                numberSteps = demoProto.NumberSteps,
                 meanReward = demoProto.MeanReward,
                 demonstrationName = demoProto.DemonstrationName
             };
             }
             return dm;
         }
-
-        /// <summary>
-        /// Convert a BrainParametersProto to a BrainParameters struct.
-        /// </summary>
-        /// <param name="bpp">An instance of a brain parameters protobuf object.</param>
-        /// <returns>A BrainParameters struct.</returns>
-        public static BrainParameters ToBrainParameters(this BrainParametersProto bpp)
-        {
-            var bp = new BrainParameters
-            {
-                vectorActionSize = bpp.VectorActionSize.ToArray(),
-                vectorActionDescriptions = bpp.VectorActionDescriptions.ToArray(),
-                vectorActionSpaceType = (SpaceType)bpp.VectorActionSpaceType
-            };
-            return bp;
-        }
+        #endregion
 
         public static UnityRLInitParameters ToUnityRLInitParameters(this UnityRLInitializationInputProto inputProto)
         {
             };
         }
 
+        #region AgentAction
         public static AgentAction ToAgentAction(this AgentActionProto aap)
         {
             return new AgentAction
             }
             return agentActions;
         }
+        #endregion
+        #region Observations
         public static ObservationProto ToProto(this Observation obs)
         {
             ObservationProto obsProto = null;
             observationProto.Shape.AddRange(shape);
             return observationProto;
         }
+        #endregion
     }
 }

com.unity.ml-agents/Runtime/Demonstrations/DemonstrationWriter.cs

             }
 
             // Increment meta-data counters.
-            m_MetaData.numberExperiences++;
+            m_MetaData.numberSteps++;
             m_CumulativeReward += info.reward;
             if (info.done)
             {

docs/Training-Imitation-Learning.md

 
 <p align="center">
   <img src="images/demo_component.png"
-       alt="BC Teacher Helper"
+       alt="Demonstration Recorder"
        width="375" border="10" />
 </p>
 
 
 <p align="center">
   <img src="images/demo_inspector.png"
-       alt="BC Teacher Helper"
+       alt="Demonstration Inspector"
        width="375" border="10" />
 </p>
 
         gail:
             demo_path: <path_to_your_demo_file>
             ...
-```
+```

com.unity.ml-agents/Runtime/Demonstrations/DemonstrationMetaData.cs

+using System;
+using UnityEngine;
+using MLAgents.Policies;
+using UnityEngine.Serialization;
+
+namespace MLAgents.Demonstrations
+{
+    /// <summary>
+    /// Demonstration meta-data.
+    /// Kept in a struct for easy serialization and deserialization.
+    /// </summary>
+    [Serializable]
+    internal class DemonstrationMetaData
+    {
+        [FormerlySerializedAs("numberExperiences")]
+        public int numberSteps;
+        public int numberEpisodes;
+        public float meanReward;
+        public string demonstrationName;
+        public const int ApiVersion = 1;
+    }
+}

com.unity.ml-agents/Runtime/Demonstrations/DemonstrationMetaData.cs.meta

+fileFormatVersion: 2
+guid: af5f3b4258a2d4ead90e733f30cfaa7a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.ml-agents/Runtime/Demonstrations/DemonstrationSummary.cs

+using System;
+using System.Collections.Generic;
+using UnityEngine;
+using MLAgents.Policies;
+
+namespace MLAgents.Demonstrations
+{
+    /// <summary>
+    /// Summary of a loaded Demonstration file. Only used for display in the Inspector.
+    /// </summary>
+    [Serializable]
+    internal class DemonstrationSummary : ScriptableObject
+    {
+        public DemonstrationMetaData metaData;
+        public BrainParameters brainParameters;
+        public List<ObservationSummary> observationSummaries;
+
+        public void Initialize(BrainParameters brainParams,
+            DemonstrationMetaData demonstrationMetaData, List<ObservationSummary> obsSummaries)
+        {
+            brainParameters = brainParams;
+            metaData = demonstrationMetaData;
+            observationSummaries = obsSummaries;
+        }
+    }
+
+
+    /// <summary>
+    /// Summary of a loaded Observation. Currently only contains the shape of the Observation.
+    /// </summary>
+    /// <remarks>This is necessary because serialization doesn't support nested containers or arrays.</remarks>
+    [Serializable]
+    internal struct ObservationSummary
+    {
+        public int[] shape;
+    }
+}

com.unity.ml-agents/Runtime/Demonstrations/Demonstration.cs

-using System;
-using UnityEngine;
-using MLAgents.Policies;
-
-namespace MLAgents.Demonstrations
-{
-    /// <summary>
-    /// Demonstration Object. Contains meta-data regarding demonstration.
-    /// Used for imitation learning, or other forms of learning from data.
-    /// </summary>
-    [Serializable]
-    internal class Demonstration : ScriptableObject
-    {
-        public DemonstrationMetaData metaData;
-        public BrainParameters brainParameters;
-
-        public void Initialize(BrainParameters brainParams,
-            DemonstrationMetaData demonstrationMetaData)
-        {
-            brainParameters = brainParams;
-            metaData = demonstrationMetaData;
-        }
-    }
-
-    /// <summary>
-    /// Demonstration meta-data.
-    /// Kept in a struct for easy serialization and deserialization.
-    /// </summary>
-    [Serializable]
-    internal class DemonstrationMetaData
-    {
-        public int numberExperiences;
-        public int numberEpisodes;
-        public float meanReward;
-        public string demonstrationName;
-        public const int ApiVersion = 1;
-    }
-}