using System; using System.Collections.Generic; using System.Linq; using Unity.Collections; using Unity.Entities; using UnityEngine.Serialization; namespace UnityEngine.Perception.GroundTruth { /// /// Produces keypoint annotations for a humanoid model. This labeler supports generic /// . Template values are mapped to rigged /// . Custom joints can be /// created by applying to empty game objects at a body /// part's location. /// [Serializable] public sealed class KeyPointLabeler : CameraLabeler { /// /// The active keypoint template. Required to annotate keypoint data. /// public KeyPointTemplate activeTemplate; /// public override string description { get => "Produces keypoint annotations for all visible labeled objects that have a humanoid animation avatar component."; protected set { } } /// protected override bool supportsVisualization => true; // ReSharper disable MemberCanBePrivate.Global /// /// The GUID id to associate with the annotations produced by this labeler. /// public string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2"; /// /// The which associates objects with labels. /// public IdLabelConfig idLabelConfig; // ReSharper restore MemberCanBePrivate.Global AnnotationDefinition m_AnnotationDefinition; EntityQuery m_EntityQuery; Texture2D m_MissingTexture; /// /// Action that gets triggered when a new frame of key points are computed. /// public event Action> KeyPointsComputed; /// /// Creates a new key point labeler. This constructor creates a labeler that /// is not valid until a and /// are assigned. /// public KeyPointLabeler() { } /// /// Creates a new key point labeler. /// /// The Id label config for the labeler /// The active keypoint template public KeyPointLabeler(IdLabelConfig config, KeyPointTemplate template) { this.idLabelConfig = config; this.activeTemplate = template; } /// /// Array of animation pose labels which map animation clip times to ground truth pose labels. /// public List animationPoseConfigs; /// protected override void Setup() { if (idLabelConfig == null) throw new InvalidOperationException("KeyPointLabeler's idLabelConfig field must be assigned"); m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", new []{TemplateToJson(activeTemplate)}, "pixel coordinates of keypoints in a model, along with skeletal connectivity data", id: new Guid(annotationId)); m_EntityQuery = World.DefaultGameObjectInjectionWorld.EntityManager.CreateEntityQuery(typeof(Labeling), typeof(GroundTruthInfo)); m_KeyPointEntries = new List(); // Texture to use in case the template does not contain a texture for the joints or the skeletal connections m_MissingTexture = new Texture2D(1, 1); m_KnownStatus = new Dictionary(); } /// protected override void OnBeginRendering() { var reporter = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition); var entities = m_EntityQuery.ToEntityArray(Allocator.TempJob); var entityManager = World.DefaultGameObjectInjectionWorld.EntityManager; m_KeyPointEntries.Clear(); foreach (var entity in entities) { ProcessEntity(entityManager.GetComponentObject(entity)); } entities.Dispose(); KeyPointsComputed?.Invoke(m_KeyPointEntries); reporter.ReportValues(m_KeyPointEntries); } // ReSharper disable InconsistentNaming // ReSharper disable NotAccessedField.Global // ReSharper disable NotAccessedField.Local /// /// Record storing all of the keypoint data of a labeled gameobject. /// [Serializable] public class KeyPointEntry { /// /// The label id of the entity /// public int label_id; /// /// The instance id of the entity /// public uint instance_id; /// /// The template that the points are based on /// public string template_guid; /// /// Pose ground truth for the current set of keypoints /// public string pose = "unset"; /// /// Array of all of the keypoints /// public KeyPoint[] keypoints; } /// /// The values of a specific keypoint /// [Serializable] public class KeyPoint { /// /// The index of the keypoint in the template file /// public int index; /// /// The keypoint's x-coordinate pixel location /// public float x; /// /// The keypoint's y-coordinate pixel location /// public float y; /// /// The state of the point, /// 0 = not present, /// 1 = keypoint is present but not visible, /// 2 = keypoint is present and visible /// public int state; } // ReSharper restore InconsistentNaming // ReSharper restore NotAccessedField.Global // ReSharper restore NotAccessedField.Local // Converts a coordinate from world space into pixel space Vector3 ConvertToScreenSpace(Vector3 worldLocation) { var pt = perceptionCamera.attachedCamera.WorldToScreenPoint(worldLocation); pt.y = Screen.height - pt.y; return pt; } List m_KeyPointEntries; struct CachedData { public bool status; public Animator animator; public KeyPointEntry keyPoints; public List<(JointLabel, int)> overrides; } Dictionary m_KnownStatus; bool TryToGetTemplateIndexForJoint(KeyPointTemplate template, JointLabel joint, out int index) { index = -1; foreach (var jointTemplate in joint.templateInformation.Where(jointTemplate => jointTemplate.template == template)) { for (var i = 0; i < template.keyPoints.Length; i++) { if (template.keyPoints[i].label == jointTemplate.label) { index = i; return true; } } } return false; } bool DoesTemplateContainJoint(JointLabel jointLabel) { foreach (var template in jointLabel.templateInformation) { if (template.template == activeTemplate) { if (activeTemplate.keyPoints.Any(i => i.label == template.label)) { return true; } } } return false; } void ProcessEntity(Labeling labeledEntity) { // Cache out the data of a labeled game object the first time we see it, this will // save performance each frame. Also checks to see if a labeled game object can be annotated. if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId)) { var cached = new CachedData() { status = false, animator = null, keyPoints = new KeyPointEntry(), overrides = new List<(JointLabel, int)>() }; if (idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry)) { var entityGameObject = labeledEntity.gameObject; cached.keyPoints.instance_id = labeledEntity.instanceId; cached.keyPoints.label_id = labelEntry.id; cached.keyPoints.template_guid = activeTemplate.templateID.ToString(); cached.keyPoints.keypoints = new KeyPoint[activeTemplate.keyPoints.Length]; for (var i = 0; i < cached.keyPoints.keypoints.Length; i++) { cached.keyPoints.keypoints[i] = new KeyPoint { index = i, state = 0 }; } var animator = entityGameObject.transform.GetComponentInChildren(); if (animator != null) { cached.animator = animator; cached.status = true; } foreach (var joint in entityGameObject.transform.GetComponentsInChildren()) { if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx)) { cached.overrides.Add((joint, idx)); cached.status = true; } } } m_KnownStatus[labeledEntity.instanceId] = cached; } var cachedData = m_KnownStatus[labeledEntity.instanceId]; if (cachedData.status) { var animator = cachedData.animator; var keyPoints = cachedData.keyPoints.keypoints; // Go through all of the rig keypoints and get their location for (var i = 0; i < activeTemplate.keyPoints.Length; i++) { var pt = activeTemplate.keyPoints[i]; if (pt.associateToRig) { var bone = animator.GetBoneTransform(pt.rigLabel); if (bone != null) { var loc = ConvertToScreenSpace(bone.position); keyPoints[i].index = i; keyPoints[i].x = loc.x; keyPoints[i].y = loc.y; keyPoints[i].state = 2; } } } // Go through all of the additional or override points defined by joint labels and get // their locations foreach (var (joint, idx) in cachedData.overrides) { var loc = ConvertToScreenSpace(joint.transform.position); keyPoints[idx].index = idx; keyPoints[idx].x = loc.x; keyPoints[idx].y = loc.y; keyPoints[idx].state = 2; } cachedData.keyPoints.pose = "unset"; if (cachedData.animator != null) { cachedData.keyPoints.pose = GetPose(cachedData.animator); } m_KeyPointEntries.Add(cachedData.keyPoints); } } string GetPose(Animator animator) { var info = animator.GetCurrentAnimatorClipInfo(0); if (info != null && info.Length > 0) { var clip = info[0].clip; var timeOffset = animator.GetCurrentAnimatorStateInfo(0).normalizedTime; if (animationPoseConfigs != null) { foreach (var p in animationPoseConfigs) { if (p != null && p.animationClip == clip) { var time = timeOffset; var label = p.GetPoseAtTime(time); return label; } } } } return "unset"; } /// protected override void OnVisualize() { var jointTexture = activeTemplate.jointTexture; if (jointTexture == null) jointTexture = m_MissingTexture; var skeletonTexture = activeTemplate.skeletonTexture; if (skeletonTexture == null) skeletonTexture = m_MissingTexture; foreach (var entry in m_KeyPointEntries) { foreach (var bone in activeTemplate.skeleton) { var joint1 = entry.keypoints[bone.joint1]; var joint2 = entry.keypoints[bone.joint2]; if (joint1.state != 0 && joint2.state != 0) { VisualizationHelper.DrawLine(joint1.x, joint1.y, joint2.x, joint2.y, bone.color, 8, skeletonTexture); } } foreach (var keypoint in entry.keypoints) { if (keypoint.state != 0) VisualizationHelper.DrawPoint(keypoint.x, keypoint.y, activeTemplate.keyPoints[keypoint.index].color, 8, jointTexture); } } } // ReSharper disable InconsistentNaming // ReSharper disable NotAccessedField.Local [Serializable] struct JointJson { public string label; public int index; } [Serializable] struct SkeletonJson { public int joint1; public int joint2; } [Serializable] struct KeyPointJson { public string template_id; public string template_name; public JointJson[] key_points; public SkeletonJson[] skeleton; } // ReSharper restore InconsistentNaming // ReSharper restore NotAccessedField.Local KeyPointJson TemplateToJson(KeyPointTemplate input) { var json = new KeyPointJson(); json.template_id = input.templateID.ToString(); json.template_name = input.templateName; json.key_points = new JointJson[input.keyPoints.Length]; json.skeleton = new SkeletonJson[input.skeleton.Length]; for (var i = 0; i < input.keyPoints.Length; i++) { json.key_points[i] = new JointJson { label = input.keyPoints[i].label, index = i }; } for (var i = 0; i < input.skeleton.Length; i++) { json.skeleton[i] = new SkeletonJson() { joint1 = input.skeleton[i].joint1, joint2 = input.skeleton[i].joint2 }; } return json; } } }