Commit of working keypoint labeler

com.unity.perception/Runtime/GroundTruth/Labelers/CocoKeypointTemplate.asset

+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 37a7d6f1a40c45a2981a6291f0d03337, type: 3}
+  m_Name: CocoKeypointTemplate
+  m_EditorClassIdentifier: 
+  templateName: Coco
+  jointTexture: {fileID: 2800000, guid: e381cbaaf29614168bafc8f7ec5dbfe9, type: 3}
+  skeletonTexture: {fileID: 2800000, guid: e381cbaaf29614168bafc8f7ec5dbfe9, type: 3}
+  keyPoints:
+  - label: nose
+    associateToRig: 0
+    rigLabel: 0
+    color: {r: 1, g: 0, b: 0, a: 1}
+  - label: neck
+    associateToRig: 1
+    rigLabel: 9
+    color: {r: 1, g: 0.3397382, b: 0, a: 1}
+  - label: right_shoulder
+    associateToRig: 1
+    rigLabel: 14
+    color: {r: 1, g: 0.5694697, b: 0, a: 1}
+  - label: right_elbow
+    associateToRig: 1
+    rigLabel: 16
+    color: {r: 1, g: 0.8258381, b: 0, a: 1}
+  - label: right_wrist
+    associateToRig: 1
+    rigLabel: 18
+    color: {r: 0.6454141, g: 1, b: 0, a: 1}
+  - label: left_shoulder
+    associateToRig: 1
+    rigLabel: 13
+    color: {r: 0.33125544, g: 1, b: 0, a: 1}
+  - label: left_elbow
+    associateToRig: 1
+    rigLabel: 15
+    color: {r: 0.04907012, g: 1, b: 0, a: 1}
+  - label: left_wrist
+    associateToRig: 1
+    rigLabel: 17
+    color: {r: 0, g: 1, b: 0.16702724, a: 1}
+  - label: right_hip
+    associateToRig: 1
+    rigLabel: 2
+    color: {r: 0, g: 1, b: 0.36656523, a: 1}
+  - label: right_knee
+    associateToRig: 1
+    rigLabel: 4
+    color: {r: 0, g: 1, b: 0.58708096, a: 1}
+  - label: right_ankle
+    associateToRig: 1
+    rigLabel: 6
+    color: {r: 0, g: 1, b: 0.7695224, a: 1}
+  - label: left_hip
+    associateToRig: 1
+    rigLabel: 1
+    color: {r: 0, g: 1, b: 1, a: 1}
+  - label: left_knee
+    associateToRig: 1
+    rigLabel: 3
+    color: {r: 0, g: 0.63836884, b: 1, a: 1}
+  - label: left_ankle
+    associateToRig: 1
+    rigLabel: 5
+    color: {r: 0, g: 0.29786587, b: 1, a: 1}
+  - label: right_eye
+    associateToRig: 1
+    rigLabel: 22
+    color: {r: 0.45002556, g: 0, b: 1, a: 1}
+  - label: left_eye
+    associateToRig: 1
+    rigLabel: 21
+    color: {r: 0.9471822, g: 0, b: 1, a: 1}
+  - label: right_ear
+    associateToRig: 0
+    rigLabel: 22
+    color: {r: 1, g: 0, b: 0.6039734, a: 1}
+  - label: left_ear
+    associateToRig: 0
+    rigLabel: 21
+    color: {r: 1, g: 0, b: 0.11927748, a: 1}
+  skeleton:
+  - joint1: 0
+    joint2: 1
+    color: {r: 0.014684939, g: 0.05894964, b: 0.6226415, a: 1}
+  - joint1: 1
+    joint2: 2
+    color: {r: 0.5283019, g: 0, b: 0.074745394, a: 1}
+  - joint1: 2
+    joint2: 3
+    color: {r: 0.7830189, g: 0.32108742, b: 0.07756319, a: 1}
+  - joint1: 3
+    joint2: 4
+    color: {r: 0.9622642, g: 0.85543716, b: 0, a: 1}
+  - joint1: 1
+    joint2: 5
+    color: {r: 0.7019608, g: 0.20392157, b: 0.11461401, a: 1}
+  - joint1: 5
+    joint2: 6
+    color: {r: 0.3374826, g: 0.9056604, b: 0.26059094, a: 1}
+  - joint1: 6
+    joint2: 7
+    color: {r: 0.04214221, g: 0.4811321, b: 0.03404236, a: 1}
+  - joint1: 1
+    joint2: 8
+    color: {r: 0, g: 0.764151, b: 0.22962166, a: 1}
+  - joint1: 8
+    joint2: 9
+    color: {r: 0, g: 1, b: 0.3301921, a: 1}
+  - joint1: 9
+    joint2: 10
+    color: {r: 0, g: 0.9433962, b: 0.71313965, a: 1}
+  - joint1: 1
+    joint2: 11
+    color: {r: 0, g: 1, b: 1, a: 1}
+  - joint1: 11
+    joint2: 12
+    color: {r: 0, g: 0.38122815, b: 0.9433962, a: 1}
+  - joint1: 12
+    joint2: 13
+    color: {r: 0.20773989, g: 0, b: 0.7169812, a: 1}
+  - joint1: 0
+    joint2: 14
+    color: {r: 1, g: 0, b: 0.88550186, a: 1}
+  - joint1: 0
+    joint2: 15
+    color: {r: 1, g: 0, b: 0.81438303, a: 1}
+  - joint1: 16
+    joint2: 14
+    color: {r: 0.5743165, g: 0, b: 1, a: 1}
+  - joint1: 17
+    joint2: 15
+    color: {r: 0.8962264, g: 0, b: 0.12766689, a: 1}

com.unity.perception/Runtime/GroundTruth/Labelers/CocoKeypointTemplate.asset.meta

+fileFormatVersion: 2
+guid: a29b79d8ce98945a0855b1addec08d86
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs

+using System;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    public class JointLabel : MonoBehaviour
+    {
+        [Serializable]
+        public class TemplateData
+        {
+            public KeyPointTemplate template;
+            public string label;
+        };
+        public List<TemplateData> templateInformation;
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/JointLabel.cs.meta

+fileFormatVersion: 2
+guid: 8cf4fa374b134b1680755f8280ae8e7d
+timeCreated: 1610577744

com.unity.perception/Runtime/GroundTruth/Labelers/KeyPointLabeler.cs

+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Unity.Collections;
+using Unity.Entities;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    [Serializable]
+    public sealed class KeyPointLabeler : CameraLabeler
+    {
+        public KeyPointTemplate activeTemplate;
+
+        public override string description
+        {
+            get => "Produces keypoint annotations for all visible labeled objects that have a humanoid animation avatar component.";
+            protected set { }
+        }
+
+        ///<inheritdoc/>
+        protected override bool supportsVisualization => true;
+
+        // ReSharper disable MemberCanBePrivate.Global
+        public string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
+        public IdLabelConfig idLabelConfig;
+        // ReSharper restore MemberCanBePrivate.Global
+
+        AnnotationDefinition m_AnnotationDefinition;
+        EntityQuery m_EntityQuery;
+        Texture2D m_MissingTexture;
+
+        protected override void Setup()
+        {
+            if (idLabelConfig == null)
+                throw new InvalidOperationException("KeyPointLabeler's idLabelConfig field must be assigned");
+
+            m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", new []{TemplateToJson(activeTemplate)},
+                "pixel coordinates of keypoints in a model, along with skeletal connectivity data", id: new Guid(annotationId));
+
+            m_EntityQuery = World.DefaultGameObjectInjectionWorld.EntityManager.CreateEntityQuery(typeof(Labeling), typeof(GroundTruthInfo));
+
+            m_KeyPointEntries = new List<KeyPointEntry>();
+
+            m_MissingTexture = new Texture2D(1, 1);
+
+            m_KnownStatus = new Dictionary<uint, CachedData>();
+        }
+
+        protected override void OnBeginRendering()
+        {
+            var reporter = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
+
+            var entities = m_EntityQuery.ToEntityArray(Allocator.TempJob);
+            var entityManager = World.DefaultGameObjectInjectionWorld.EntityManager;
+
+            m_KeyPointEntries.Clear();
+
+            foreach (var entity in entities)
+            {
+                ProcessEntity(entityManager.GetComponentObject<Labeling>(entity));
+            }
+
+            entities.Dispose();
+
+            reporter.ReportValues(m_KeyPointEntries);
+        }
+
+        // ReSharper disable InconsistentNaming
+        // ReSharper disable NotAccessedField.Global
+        [Serializable]
+        public struct KeyPointEntry
+        {
+
+
+            public int label_id;
+            public uint instance_id;
+            public string template_guid;
+            public KeyPoint[] keypoints;
+        }
+
+        [Serializable]
+        public struct KeyPoint
+        {
+            public int index;
+            public float x;
+            public float y;
+            public int state;
+        }
+        // ReSharper restore InconsistentNaming
+        // ReSharper restore NotAccessedField.Global
+
+        Vector3 ConvertToScreenSpace(Vector3 worldLocation)
+        {
+            var pt = perceptionCamera.attachedCamera.WorldToScreenPoint(worldLocation);
+            pt.y = Screen.height - pt.y;
+            return pt;
+        }
+
+        List<KeyPointEntry> m_KeyPointEntries;
+
+        struct CachedData
+        {
+            public bool status;
+            public Animator animator;
+            public KeyPointEntry keyPoints;
+            public List<(JointLabel, int)> overrides;
+        }
+
+        Dictionary<uint, CachedData> m_KnownStatus;
+
+        bool TryToGetTemplateIndexForJoint(KeyPointTemplate template, JointLabel joint, out int index)
+        {
+            index = -1;
+
+            foreach (var jointTemplate in joint.templateInformation.Where(jointTemplate => jointTemplate.template == template))
+            {
+                for (var i = 0; i < template.keyPoints.Length; i++)
+                {
+                    if (template.keyPoints[i].label == jointTemplate.label)
+                    {
+                        index = i;
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        bool DoesTemplateContainJoint(JointLabel jointLabel)
+        {
+            foreach (var template in jointLabel.templateInformation)
+            {
+                if (template.template == activeTemplate)
+                {
+                    if (activeTemplate.keyPoints.Any(i => i.label == template.label))
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        void ProcessEntity(Labeling labeledEntity)
+        {
+            if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId))
+            {
+                var cached = new CachedData()
+                {
+                    status = false,
+                    animator = null,
+                    keyPoints = new KeyPointEntry()
+                };
+
+                if (idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
+                {
+                    var entityGameObject = labeledEntity.gameObject;
+                    var animator = entityGameObject.transform.GetComponentInChildren<Animator>();
+                    if (animator != null)
+                    {
+                        var avatar = animator.avatar;
+                        if (avatar.isValid && avatar.isHuman)
+                        {
+                            cached.animator = animator;
+                            cached.keyPoints.instance_id = labeledEntity.instanceId;
+                            cached.keyPoints.label_id = labelEntry.id;
+                            cached.keyPoints.template_guid = activeTemplate.templateID.ToString();
+                            cached.keyPoints.keypoints = new KeyPoint[activeTemplate.keyPoints.Length];
+
+                            for (var i = 0; i < cached.keyPoints.keypoints.Length; i++)
+                            {
+                                cached.keyPoints.keypoints[i].index = i;
+                                cached.keyPoints.keypoints[i].state = 0;
+                            }
+
+                            cached.overrides = new List<(JointLabel, int)>();
+                            cached.status = true;
+
+                            foreach (var joint in entityGameObject.transform.GetComponentsInChildren<JointLabel>())
+                            {
+                                if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx))
+                                {
+                                    cached.overrides.Add((joint, idx));
+                                }
+                            }
+                        }
+                    }
+                }
+
+                m_KnownStatus[labeledEntity.instanceId] = cached;
+            }
+
+            var cachedData = m_KnownStatus[labeledEntity.instanceId];
+
+            if (cachedData.status)
+            {
+                var animator = cachedData.animator;
+                var keyPoints = cachedData.keyPoints.keypoints;
+
+                // Go through all of the rig keypoints and get their location
+                for (var i = 0; i < activeTemplate.keyPoints.Length; i++)
+                {
+                    var pt = activeTemplate.keyPoints[i];
+                    if (pt.associateToRig)
+                    {
+                        var loc = ConvertToScreenSpace(animator.GetBoneTransform(pt.rigLabel).position);
+                        keyPoints[i].index = i;
+                        keyPoints[i].x = loc.x;
+                        keyPoints[i].y = loc.y;
+                        keyPoints[i].state = 1;
+                    }
+                }
+
+                // Go through all of the additional or override points defined by joint labels and get
+                // their locations
+                foreach (var (joint, idx) in cachedData.overrides)
+                {
+                    var loc = ConvertToScreenSpace(joint.transform.position);
+                    keyPoints[idx].index = idx;
+                    keyPoints[idx].x = loc.x;
+                    keyPoints[idx].y = loc.y;
+                    keyPoints[idx].state = 1;
+                }
+
+                m_KeyPointEntries.Add(cachedData.keyPoints);
+            }
+        }
+
+        Rect ToBoxRect(float x, float y, float halfSize = 3.0f)
+        {
+            return new Rect(x - halfSize, y - halfSize, halfSize * 2, halfSize * 2);
+        }
+
+        void DrawPoint(float x, float y, Color color, Texture2D texture)
+        {
+            var oldColor = GUI.color;
+            GUI.color = color;
+            GUI.DrawTexture(ToBoxRect(x, y, 4), texture);
+            GUI.color = oldColor;
+        }
+
+        float Magnitude(float p1X, float p1Y, float p2X, float p2Y)
+        {
+            var x = p2X - p1X;
+            var y = p2Y - p1Y;
+            return Mathf.Sqrt(x * x + y * y);
+        }
+
+        void DrawLine (float p1X, float p1Y, float p2X, float p2Y, Color color, Texture texture)
+        {
+            var oldColor = GUI.color;
+
+            GUI.color = color;
+
+            var matrixBackup = GUI.matrix;
+            const float width = 8.0f;
+            var angle = Mathf.Atan2 (p2Y - p1Y, p2X - p1X) * 180f / Mathf.PI;
+
+            var length = Magnitude(p1X, p1Y, p2X, p2Y);
+
+            GUIUtility.RotateAroundPivot (angle, new Vector2(p1X, p1Y));
+            const float halfWidth = width * 0.5f;
+            GUI.DrawTexture (new Rect (p1X - halfWidth, p1Y - halfWidth, length, width), texture);
+
+            GUI.matrix = matrixBackup;
+            GUI.color = oldColor;
+        }
+
+        protected override void OnVisualize()
+        {
+            var jointTexture = activeTemplate.jointTexture;
+            if (jointTexture == null) jointTexture = m_MissingTexture;
+
+            var skeletonTexture = activeTemplate.skeletonTexture;
+            if (skeletonTexture == null) skeletonTexture = m_MissingTexture;
+
+            foreach (var entry in m_KeyPointEntries)
+            {
+                foreach (var bone in activeTemplate.skeleton)
+                {
+                    var joint1 = entry.keypoints[bone.joint1];
+                    var joint2 = entry.keypoints[bone.joint2];
+
+                    if (joint1.state != 0 && joint2.state != 0)
+                    {
+                        DrawLine(joint1.x, joint1.y, joint2.x, joint2.y, bone.color, skeletonTexture);
+                    }
+                }
+
+                foreach (var keypoint in entry.keypoints)
+                {
+                    if (keypoint.state != 0)
+                        DrawPoint(keypoint.x, keypoint.y, activeTemplate.keyPoints[keypoint.index].color, jointTexture);
+                }
+            }
+        }
+
+        // ReSharper disable InconsistentNaming
+        // ReSharper disable NotAccessedField.Local
+        [Serializable]
+        struct JointJson
+        {
+            public string label;
+            public int index;
+        }
+
+        [Serializable]
+        struct SkeletonJson
+        {
+            public int joint1;
+            public int joint2;
+        }
+
+        [Serializable]
+        struct KeyPointJson
+        {
+            public string template_id;
+            public string template_name;
+            public JointJson[] key_points;
+            public SkeletonJson[] skeleton;
+        }
+        // ReSharper restore InconsistentNaming
+        // ReSharper restore NotAccessedField.Local
+
+        KeyPointJson TemplateToJson(KeyPointTemplate input)
+        {
+            var json = new KeyPointJson();
+            json.template_id = input.templateID.ToString();
+            json.template_name = input.templateName;
+            json.key_points = new JointJson[input.keyPoints.Length];
+            json.skeleton = new SkeletonJson[input.skeleton.Length];
+
+            for (var i = 0; i < input.keyPoints.Length; i++)
+            {
+                json.key_points[i] = new JointJson
+                {
+                    label = input.keyPoints[i].label,
+                    index = i
+                };
+            }
+
+            for (var i = 0; i < input.skeleton.Length; i++)
+            {
+                json.skeleton[i] = new SkeletonJson()
+                {
+                    joint1 = input.skeleton[i].joint1,
+                    joint2 = input.skeleton[i].joint2
+                };
+            }
+
+            return json;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/KeyPointLabeler.cs.meta

+fileFormatVersion: 2
+guid: 377d37b913b843b6985fa57c13cb732c
+timeCreated: 1610383503

com.unity.perception/Runtime/GroundTruth/Labelers/KeyPointTemplate.cs

+using System;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    [Serializable]
+    public class KeyPointDefinition
+    {
+        public string label;
+        public bool associateToRig = true;
+        public HumanBodyBones rigLabel = HumanBodyBones.Head;
+        public Color color;
+    }
+
+    [Serializable]
+    public class SkeletonDefinition
+    {
+        public int joint1;
+        public int joint2;
+        public Color color;
+    }
+
+    [CreateAssetMenu(fileName = "KeypointTemplate", menuName = "Perception/Keypoint Template", order = 2)]
+    public class KeyPointTemplate : ScriptableObject
+    {
+        public Guid templateID;
+        public string templateName;
+        public Texture2D jointTexture;
+        public Texture2D skeletonTexture;
+        public KeyPointDefinition[] keyPoints;
+        public SkeletonDefinition[] skeleton;
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/KeyPointTemplate.cs.meta

+fileFormatVersion: 2
+guid: 37a7d6f1a40c45a2981a6291f0d03337
+timeCreated: 1610633739

com.unity.perception/Runtime/GroundTruth/Resources/joint_circle.png.meta

+fileFormatVersion: 2
+guid: e381cbaaf29614168bafc8f7ec5dbfe9
+TextureImporter:
+  internalIDToNameTable: []
+  externalObjects: {}
+  serializedVersion: 11
+  mipmaps:
+    mipMapMode: 0
+    enableMipMap: 0
+    sRGBTexture: 1
+    linearTexture: 0
+    fadeOut: 0
+    borderMipMap: 0
+    mipMapsPreserveCoverage: 0
+    alphaTestReferenceValue: 0.5
+    mipMapFadeDistanceStart: 1
+    mipMapFadeDistanceEnd: 3
+  bumpmap:
+    convertToNormalMap: 0
+    externalNormalMap: 0
+    heightScale: 0.25
+    normalMapFilter: 0
+  isReadable: 0
+  streamingMipmaps: 0
+  streamingMipmapsPriority: 0
+  vTOnly: 0
+  grayScaleToAlpha: 0
+  generateCubemap: 6
+  cubemapConvolution: 0
+  seamlessCubemap: 0
+  textureFormat: 1
+  maxTextureSize: 2048
+  textureSettings:
+    serializedVersion: 2
+    filterMode: -1
+    aniso: -1
+    mipBias: -100
+    wrapU: 1
+    wrapV: 1
+    wrapW: -1
+  nPOTScale: 0
+  lightmap: 0
+  compressionQuality: 50
+  spriteMode: 1
+  spriteExtrude: 1
+  spriteMeshType: 1
+  alignment: 0
+  spritePivot: {x: 0.5, y: 0.5}
+  spritePixelsToUnits: 100
+  spriteBorder: {x: 0, y: 0, z: 0, w: 0}
+  spriteGenerateFallbackPhysicsShape: 1
+  alphaUsage: 1
+  alphaIsTransparency: 1
+  spriteTessellationDetail: -1
+  textureType: 8
+  textureShape: 1
+  singleChannelComponent: 0
+  flipbookRows: 1
+  flipbookColumns: 1
+  maxTextureSizeSet: 0
+  compressionQualitySet: 0
+  textureFormatSet: 0
+  ignorePngGamma: 0
+  applyGammaDecoding: 0
+  platformSettings:
+  - serializedVersion: 3
+    buildTarget: DefaultTexturePlatform
+    maxTextureSize: 2048
+    resizeAlgorithm: 0
+    textureFormat: -1
+    textureCompression: 1
+    compressionQuality: 50
+    crunchedCompression: 0
+    allowsAlphaSplitting: 0
+    overridden: 0
+    androidETC2FallbackOverride: 0
+    forceMaximumCompressionQuality_BC6H_BC7: 0
+  - serializedVersion: 3
+    buildTarget: Standalone
+    maxTextureSize: 2048
+    resizeAlgorithm: 0
+    textureFormat: -1
+    textureCompression: 1
+    compressionQuality: 50
+    crunchedCompression: 0
+    allowsAlphaSplitting: 0
+    overridden: 0
+    androidETC2FallbackOverride: 0
+    forceMaximumCompressionQuality_BC6H_BC7: 0
+  spriteSheet:
+    serializedVersion: 2
+    sprites: []
+    outline: []
+    physicsShape: []
+    bones: []
+    spriteID: 5e97eb03825dee720800000000000000
+    internalID: 0
+    vertices: []
+    indices: 
+    edges: []
+    weights: []
+    secondaryTextures: []
+  spritePackingTag: 
+  pSDRemoveMatte: 0
+  pSDShowRemoveMatteOption: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: