Recording pose for a random animation frame & performance improvements

com.unity.perception/Runtime/GroundTruth/Labelers/KeyPointLabeler.cs

             /// </summary>
             public string template_guid;
             /// <summary>
+            /// Pose ground truth for the current set of keypoints
+            /// </summary>
+            public string pose = "unset";
+            /// <summary>
             /// Array of all of the keypoints
             /// </summary>
             public KeyPoint[] keypoints;
                     keyPoints[idx].y = loc.y;
                     keyPoints[idx].state = 1;
                 }
+
+                // check if the model has a pose state component, if so we will add that to our model, this value
+                // is purposely not cached, because it could be added during runtime
+                var poseState = labeledEntity.GetComponent<PoseStateGroundTruthInfo>();
+                cachedData.keyPoints.pose = poseState != null ? poseState.poseState : "unset";
 
                 m_KeyPointEntries.Add(cachedData.keyPoints);
             }

com.unity.perception/Runtime/Randomization/Parameters/CategoricalParameter.cs

         public sealed override Type sampleType => typeof(T);
 
         /// <summary>
+        /// Returns the number of stored categories
+        /// </summary>
+        /// <returns>The number of stored categories</returns>
+        public int GetCategoryCount() => m_Categories.Count;
+
+        /// <summary>
         /// Returns the category stored at the specified index
         /// </summary>
         /// <param name="index">The index of the category to lookup</param>

com.unity.perception/Runtime/Randomization/Parameters/ParameterTypes/CategorialParameters/AnimationClipParameter.cs

 using System;
+using UnityEngine.Perception.GroundTruth;
 
 namespace UnityEngine.Experimental.Perception.Randomization.Parameters
 {
     [Serializable]
-    public class AnimationClipParameter : CategoricalParameter<AnimationClip> { }
+    public class AnimationClipParameter : CategoricalParameter<AnimationPoseLabel> { }
 }

com.unity.perception/Runtime/Randomization/Randomizers/RandomizerExamples/Randomizers/AnimationRandomizer.cs

 using UnityEngine.Experimental.Perception.Randomization.Parameters;
 using UnityEngine.Experimental.Perception.Randomization.Randomizers.SampleRandomizers.Tags;
 using UnityEngine.Experimental.Perception.Randomization.Samplers;
+using UnityEngine.Perception.GroundTruth;
-    /// Chooses a random of frame of a random clip for a gameobject
+    /// Chooses a random of frame of a random clip for a game object
     /// </summary>
     [Serializable]
     [AddRandomizerMenu("Perception/Animation Randomizer")]
-        Dictionary<GameObject, (PlayableGraph, Animator)> m_GraphMap;
+
+        class CachedData
+        {
+            public PlayableGraph graph;
+            public AnimationPlayableOutput output;
+            public Dictionary<string, AnimationClipPlayable> playables = new Dictionary<string, AnimationClipPlayable>();
+            public PoseStateGroundTruthInfo poseState;
+        }
+        Dictionary<GameObject, CachedData> m_CacheMap;
-            m_GraphMap = new Dictionary<GameObject, (PlayableGraph, Animator)>();
+            m_CacheMap = new Dictionary<GameObject, CachedData>();
-        (PlayableGraph, Animator) GetGraph(GameObject gameObject)
+        CachedData InitializeCacheData(AnimationRandomizerTag tag)
-            if (!m_GraphMap.ContainsKey(gameObject))
+            var cachedData = new CachedData { graph = PlayableGraph.Create() };
+            cachedData.graph.SetTimeUpdateMode(DirectorUpdateMode.GameTime);
+            var animator = tag.gameObject.GetComponent<Animator>();
+            cachedData.output = AnimationPlayableOutput.Create(cachedData.graph, "Animation", animator);
+
+            for (var i = 0; i < tag.animationClips.GetCategoryCount(); i++)
+            {
+                var clip = tag.animationClips.GetCategory(i);
+                var playable = AnimationClipPlayable.Create(cachedData.graph, clip.animationClip);
+                cachedData.playables[clip.animationClip.name] = playable;
+            }
+
+            cachedData.poseState = tag.gameObject.GetComponent<PoseStateGroundTruthInfo>();
+            if (cachedData.poseState == null)
-                var graph = PlayableGraph.Create();
-                graph.SetTimeUpdateMode(DirectorUpdateMode.GameTime);
-                var animator = gameObject.GetComponent<Animator>();
-                m_GraphMap[gameObject] = (graph, animator);
+                cachedData.poseState = tag.gameObject.AddComponent<PoseStateGroundTruthInfo>();
-            return m_GraphMap[gameObject];
+            return cachedData;
+        }
+
+        CachedData GetGraph(AnimationRandomizerTag tag)
+        {
+            if (!m_CacheMap.ContainsKey(tag.gameObject))
+            {
+                m_CacheMap[tag.gameObject] = InitializeCacheData(tag);
+
+            }
+            return m_CacheMap[tag.gameObject];
         }
 
         /// <inheritdoc/>
             foreach (var taggedObject in taggedObjects)
             {
-                var (graph, animator) = GetGraph(taggedObject.gameObject);
-
+                var cached = GetGraph(tag);
-                CategoricalParameter<AnimationClip> param = new AnimationClipParameter();
-                param.SetOptions(clips);
-                var clip = param.Sample();
+                var animationPoseLabel = clips.Sample();
+                var clip = animationPoseLabel.animationClip;
-                var output = AnimationPlayableOutput.Create(graph, "Animation", animator);
-                var playable = AnimationClipPlayable.Create(graph, clip);
-                output.SetSourcePlayable(playable);
-                var l = clip.length;
-                var t = m_FloatParameter.Sample();
-                playable.SetTime(t * l);
+                var playable = cached.playables[clip.name];
+                cached.output.SetSourcePlayable(cached.playables[clip.name]);
+                var t = m_FloatParameter.Sample() * clip.length;
+
+                playable.SetTime(t);
-                graph.Play();
+
+                cached.graph.Play();
+
+                cached.poseState.poseState = animationPoseLabel.GetPoseAtTime(t);
+            }
+        }
+
+        protected override void OnScenarioComplete()
+        {
+            foreach (var cache in m_CacheMap.Values)
+            {
+                foreach (var p in cache.playables.Values)
+                    cache.graph.DestroyPlayable(p);
+
+                cache.graph.DestroyOutput(cache.output);
+                cache.graph.Destroy();
             }
         }
     }

com.unity.perception/Runtime/Randomization/Randomizers/RandomizerExamples/Tags/AnimationRandomizerTag.cs

+using UnityEngine.Experimental.Perception.Randomization.Parameters;
+using UnityEngine.Perception.GroundTruth;
+
 namespace UnityEngine.Experimental.Perception.Randomization.Randomizers.SampleRandomizers.Tags
 {
     /// <summary>
         /// <summary>
         /// A list of animation clips from which to choose
         /// </summary>
-        public AnimationClip[] animationClips;
+        public AnimationClipParameter animationClips;
+
+
     }
 }

com.unity.perception/Runtime/GroundTruth/Labelers/AnimationPoseLabel.cs

+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    /// <summary>
+    /// Record that maps a pose to a timestamp
+    /// </summary>
+    [Serializable]
+    public class PoseTimestampRecord
+    {
+        /// <summary>
+        /// The duration to use for this timestamp in seconds
+        /// </summary>
+        public float duration;
+        /// <summary>
+        /// The label to use for any captures inside of this time period
+        /// </summary>
+        public string poseLabel;
+    }
+
+    /// <summary>
+    /// The animation pose label is a mapping that file that maps a time range in an animation clip to a ground truth
+    /// pose. The timestamp record is defined by a pose label and a duration. The timestamp records are order dependent
+    /// and build on the previous entries. This means that if the first record has a duration of 5, then it will be the label
+    /// for all points in the clip from 0 (the beginning) to the five second mark. The next record will then go from the end
+    /// of the previous clip to its duration. If there is time left over in the flip, the final entry will be used.
+    /// </summary>
+    [CreateAssetMenu(fileName = "AnimationPoseTimestamp", menuName = "Perception/Animation Pose Timestamps")]
+    public class AnimationPoseLabel : ScriptableObject
+    {
+        /// <summary>
+        /// The animation clip used for all of the timestamps
+        /// </summary>
+        public AnimationClip animationClip;
+        /// <summary>
+        /// The list of timestamps, order dependent
+        /// </summary>
+        public List<PoseTimestampRecord> timestamps;
+
+        /// <summary>
+        /// Retrieves the pose for the clip at the current time.
+        /// </summary>
+        /// <param name="time">The time in question</param>
+        /// <returns>The pose for the passed in time</returns>
+        public string GetPoseAtTime(float time)
+        {
+            var totalTime = 0f;
+
+            // The amount of timestamps are expected to be small, so at the current time
+            // no performance improvements are deemed necesssary
+            foreach (var t in timestamps)
+            {
+                totalTime += t.duration;
+                if (time < totalTime) return t.poseLabel;
+            }
+
+            return timestamps.Last().poseLabel;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/AnimationPoseLabel.cs.meta

+fileFormatVersion: 2
+guid: 4c69656f5dd14516a3a18e42b3b43a4e
+timeCreated: 1611270313

com.unity.perception/Runtime/GroundTruth/Labelers/PoseStateGroundTruthInfo.cs

+using UnityEngine;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    /// <summary>
+    /// Pose state ground truth information that can be applied to a gameobject. The typical use case
+    /// is that this will be added to a labeled gameobject by a randomizer.
+    /// </summary>
+    public class PoseStateGroundTruthInfo : MonoBehaviour
+    {
+        /// <summary>
+        /// The pose state
+        /// </summary>
+        public string poseState;
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/PoseStateGroundTruthInfo.cs.meta

+fileFormatVersion: 2
+guid: 509316760ae346d1b4cac25f103fa5a7
+timeCreated: 1611613163