using System; using System.Collections.Generic; using System.Linq; using Unity.Collections; using Unity.Profiling; using UnityEngine.Serialization; using Unity.Simulation; using UnityEngine.Perception.GroundTruth.DataModel; using UnityEngine.Perception.GroundTruth.Exporters.Solo; using UnityEngine.Rendering; namespace UnityEngine.Perception.GroundTruth { /// /// Produces 2d bounding box annotations for all visible objects each frame. /// [Serializable] public sealed class BoundingBox2DLabeler : CameraLabeler { public class BoundingBoxAnnotationDefinition : AnnotationDefinition { static readonly string k_Id = "bounding box"; static readonly string k_Description = "Bounding box for each labeled object visible to the sensor"; static readonly string k_AnnotationType = "bounding box"; public BoundingBoxAnnotationDefinition() : base(k_Id, k_Description, k_AnnotationType) { } public BoundingBoxAnnotationDefinition(IEnumerable spec) : base(k_Id, k_Description, k_AnnotationType) { this.spec = spec; } [Serializable] public struct DefinitionEntry : IMessageProducer { public DefinitionEntry(int id, string name) { labelId = id; labelName = name; } public int labelId; public string labelName; public void ToMessage(IMessageBuilder builder) { builder.AddInt("label_id", labelId); builder.AddString("label_name", labelName); } } public IEnumerable spec; public override void ToMessage(IMessageBuilder builder) { base.ToMessage(builder); foreach (var e in spec) { var nested = builder.AddNestedMessageToVector("spec"); e.ToMessage(nested); } } } BoundingBoxAnnotationDefinition m_AnnotationDefinition; /// /// Bounding boxes for all of the labeled objects in a capture /// [Serializable] public class BoundingBoxAnnotation : Annotation { public struct Entry { // The instance ID of the object public int instanceId; public int labelId; // The type of the object public string labelName; /// /// (xy) pixel location of the object's bounding box /// public Vector2 origin; /// /// (width/height) dimensions of the bounding box /// public Vector2 dimension; public void ToMessage(IMessageBuilder builder) { builder.AddInt("instance_id", instanceId); builder.AddInt("label_id", labelId); builder.AddString("label_name", labelName); builder.AddFloatVector("origin", new[] { origin.x, origin.y }); builder.AddFloatVector("dimension", new[] { dimension.x, dimension.y }); } } /// /// The bounding boxes recorded by the annotator /// public List boxes; public override void ToMessage(IMessageBuilder builder) { base.ToMessage(builder); foreach (var e in boxes) { var nested = builder.AddNestedMessageToVector("values"); e.ToMessage(nested); } } } /// public override string description { get => "Produces 2D bounding box annotations for all visible objects that bear a label defined in this labeler's associated label configuration."; protected set {} } static ProfilerMarker s_BoundingBoxCallback = new ProfilerMarker("OnBoundingBoxesReceived"); /// /// The GUID id to associate with the annotations produced by this labeler. /// public static string annotationId = "bounding box"; /// /// The which associates objects with labels. /// [FormerlySerializedAs("labelingConfiguration")] public IdLabelConfig idLabelConfig; Dictionary m_AsyncData; List m_ToVisualize; Vector2 m_OriginalScreenSize = Vector2.zero; Texture m_BoundingBoxTexture; Texture m_LabelTexture; GUIStyle m_Style; /// /// Creates a new BoundingBox2DLabeler. Be sure to assign before adding to a . /// public BoundingBox2DLabeler() { } /// /// Creates a new BoundingBox2DLabeler with the given . /// /// The label config for resolving the label for each object. public BoundingBox2DLabeler(IdLabelConfig labelConfig) { this.idLabelConfig = labelConfig; } /// protected override bool supportsVisualization => true; /// /// Event information for /// internal struct BoundingBoxesCalculatedEventArgs { /// /// The on which the data was derived. This may be multiple frames in the past. /// public int frameCount; /// /// Bounding boxes. /// public IEnumerable data; } /// /// Event which is called each frame a semantic segmentation image is read back from the GPU. /// internal event Action boundingBoxesCalculated; /// protected override void Setup() { if (idLabelConfig == null) throw new InvalidOperationException("BoundingBox2DLabeler's idLabelConfig field must be assigned"); m_AsyncData = new Dictionary(); var spec = idLabelConfig.GetAnnotationSpecification().Select(i => new BoundingBoxAnnotationDefinition.DefinitionEntry { labelId = i.label_id, labelName = i.label_name }); m_AnnotationDefinition = new BoundingBoxAnnotationDefinition(spec); DatasetCapture.Instance.RegisterAnnotationDefinition(m_AnnotationDefinition); #if false m_BoundingBoxAnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("bounding box", idLabelConfig.GetAnnotationSpecification(), "Bounding box for each labeled object visible to the sensor", id: new Guid(annotationId)); #endif perceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfosCalculated; visualizationEnabled = supportsVisualization; // Record the original screen size. The screen size can change during play, and the visual bounding // boxes need to be scaled appropriately m_OriginalScreenSize = new Vector2(Screen.width, Screen.height); m_BoundingBoxTexture = Resources.Load("outline_box"); m_LabelTexture = Resources.Load("solid_white"); m_Style = new GUIStyle(); m_Style.normal.textColor = Color.black; m_Style.fontSize = 16; m_Style.padding = new RectOffset(4, 4, 4, 4); m_Style.contentOffset = new Vector2(4, 0); m_Style.alignment = TextAnchor.MiddleLeft; } /// protected override void OnBeginRendering(ScriptableRenderContext scriptableRenderContext) { m_AsyncData[Time.frameCount] = (perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition), idLabelConfig.CreateLabelEntryMatchCache(Allocator.TempJob)); } void OnRenderedObjectInfosCalculated(int frameCount, NativeArray renderedObjectInfos) { if (!m_AsyncData.TryGetValue(frameCount, out var asyncData)) return; m_AsyncData.Remove(frameCount); using (s_BoundingBoxCallback.Auto()) { var bbValues = new List(); for (var i = 0; i < renderedObjectInfos.Length; i++) { var objectInfo = renderedObjectInfos[i]; if (!asyncData.labelEntryMatchCache.TryGetLabelEntryFromInstanceId(objectInfo.instanceId, out var labelEntry, out _)) continue; bbValues.Add(new BoundingBoxAnnotation.Entry { labelId = labelEntry.id, labelName = labelEntry.label, instanceId = (int)objectInfo.instanceId, origin = new Vector2(objectInfo.boundingBox.x, objectInfo.boundingBox.y), dimension = new Vector2(objectInfo.boundingBox.width, objectInfo.boundingBox.height) } ); } if (!CaptureOptions.useAsyncReadbackIfSupported && frameCount != Time.frameCount) Debug.LogWarning("Not on current frame: " + frameCount + "(" + Time.frameCount + ")"); m_ToVisualize = bbValues; boundingBoxesCalculated?.Invoke(new BoundingBoxesCalculatedEventArgs() { data = bbValues, frameCount = frameCount }); #if true var toReport = new BoundingBoxAnnotation { sensorId = perceptionCamera.ID, Id = m_AnnotationDefinition.id, annotationType = m_AnnotationDefinition.annotationType, description = m_AnnotationDefinition.description, boxes = bbValues }; asyncData.annotation.Report(toReport); asyncData.labelEntryMatchCache.Dispose(); #endif } } /// protected override void OnVisualize() { if (m_ToVisualize == null) return; GUI.depth = 5; // The player screen can be dynamically resized during play, need to // scale the bounding boxes appropriately from the original screen size var screenRatioWidth = Screen.width / m_OriginalScreenSize.x; var screenRatioHeight = Screen.height / m_OriginalScreenSize.y; foreach (var box in m_ToVisualize) { var x = box.origin.x * screenRatioWidth; var y = box.origin.y * screenRatioHeight; var boxRect = new Rect(x, y, box.dimension.x * screenRatioWidth, box.dimension.y * screenRatioHeight); var labelWidth = Math.Min(120, box.dimension.x * screenRatioWidth); var labelRect = new Rect(x, y - 17, labelWidth, 17); GUI.DrawTexture(boxRect, m_BoundingBoxTexture, ScaleMode.StretchToFill, true, 0, Color.yellow, 3, 0.25f); GUI.DrawTexture(labelRect, m_LabelTexture, ScaleMode.StretchToFill, true, 0, Color.yellow, 0, 0); GUI.Label(labelRect, box.labelName + "_" + box.instanceId, m_Style); } } } }