Initial 3D bounding boxes working with in perception

Initial cut before full testing (although a lot of unit testing has been done) and peer review (so a lot may change)

TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity

   - id: 1
   - id: 2
   - id: 3
+  - id: 4
   showVisualizations: 1
   references:
     version: 1
-        enabled: 1
+        enabled: 0
         objectCountMetricId: 51da3c27-369d-4929-aea6-d01614635ce2
         m_LabelConfig: {fileID: 11400000, guid: cedcacfb1d9beb34fbbb231166c472fe,
           type: 2}
       data:
-        enabled: 1
+        enabled: 0
         annotationId: f9f22e05-443f-4602-a422-ebe4ea9b55cb
         idLabelConfig: {fileID: 11400000, guid: cedcacfb1d9beb34fbbb231166c472fe,
           type: 2}
       data:
-        enabled: 1
+        enabled: 0
         objectInfoMetricId: 5ba92024-b3b7-41a7-9d3f-c03a6a8ddd01
         idLabelConfig: {fileID: 11400000, guid: cedcacfb1d9beb34fbbb231166c472fe,
           type: 2}
       data:
-        enabled: 1
+        enabled: 0
+    00000004:
+      type: {class: BoundingBox3DLabeler, ns: UnityEngine.Perception.GroundTruth,
+        asm: Unity.Perception.Runtime}
+      data:
+        enabled: 1
+        annotationId: 0bfbe00d-00fa-4555-88d1-471b58449f5c
+        idLabelConfig: {fileID: 11400000, guid: cedcacfb1d9beb34fbbb231166c472fe,
+          type: 2}
 --- !u!114 &963194231
 MonoBehaviour:
   m_ObjectHideFlags: 0

com.unity.perception/Runtime/GroundTruth/GroundTruthCrossPipelinePass.cs

             {
                 var labelSetupSystem = World.DefaultGameObjectInjectionWorld?.GetExistingSystem<GroundTruthLabelSetupSystem>();
                 labelSetupSystem?.Activate(this);
+
+
                 m_IsActivated = true;
             }
         }

com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Materials/SegmentationMaterial.mat

         m_Offset: {x: 0, y: 0}
     m_Floats:
     - _AlphaClip: 1
-    - _BackTransparency: 0
+    - _BackTransparency: 0.048913043
     - _Blend: 2
     - _BlendOp: 0
     - _BumpScale: 1
     - _QueueOffset: 0
     - _ReceiveShadows: 1
     - _SampleGI: 0
-    - _SegmentTransparency: 0.8
+    - _SegmentTransparency: 0.61818254
     - _Shininess: 0
     - _Smoothness: 0.5
     - _SmoothnessSource: 0

com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/VisualizationUI.prefab

   m_RenderMode: 0
   m_Camera: {fileID: 0}
   m_PlaneDistance: 100
-  m_PixelPerfect: 0
+  m_PixelPerfect: 1
   m_ReceivesEvents: 1
   m_OverrideSorting: 0
   m_OverridePixelPerfect: 0

com.unity.perception/Tests/Runtime/GroundTruthTests/TestHelper.cs

             return planeObject;
         }
 
+        public static GameObject CreateLabeledCube(float scale = 10, string label = "label", float x = 0, float y = 0, float z = 0, float roll = 0, float pitch = 0, float yaw = 0)
+        {
+            var cube = GameObject.CreatePrimitive(PrimitiveType.Cube);
+            cube.transform.SetPositionAndRotation(new Vector3(x, y, z), Quaternion.Euler(pitch, yaw, roll));
+            cube.transform.localScale = new Vector3(scale, scale, scale);
+            var labeling = cube.AddComponent<Labeling>();
+            labeling.labels.Add(label);
+            return cube;
+        }
+
 #if UNITY_EDITOR
         public static void LoadAndStartRenderDocCapture(out UnityEditor.EditorWindow gameView)
         {

com.unity.perception/Runtime/GroundTruth/IGroundTruthUpdater.cs

+using System.Collections.Generic;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    public interface IGroundTruthUpdater
+    {
+        void OnBeginUpdate(int count);
+        void OnUpdateEntity(Labeling labeling, GroundTruthInfo groundTruthInfo);
+
+        void OnEndUpdate();
+    }
+}

com.unity.perception/Runtime/GroundTruth/IGroundTruthUpdater.cs.meta

+fileFormatVersion: 2
+guid: c12f77198d5a412ba5b18034135d7846
+timeCreated: 1596720636

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBox3DLabeler.cs.meta

+fileFormatVersion: 2
+guid: 8c19a08d5a9b48ac86e37d001ed7463f
+timeCreated: 1596554354

com.unity.perception/Runtime/GroundTruth/Labelers/GroundTruthUpdateSystem.cs

+using System.Collections;
+using System.Collections.Generic;
+using Unity.Collections;
+using Unity.Entities;
+using Unity.Entities.UniversalDelegates;
+using UnityEngine;
+using UnityEngine.Perception.GroundTruth;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    public class GroundTruthUpdateSystem : ComponentSystem
+    {
+        List<IGroundTruthUpdater> m_ActiveUpdaters = new List<IGroundTruthUpdater>();
+
+        public void Activate(IGroundTruthUpdater updater)
+        {
+            m_ActiveUpdaters.Add(updater);
+        }
+
+        public void Deactivate(IGroundTruthUpdater updater)
+        {
+            m_ActiveUpdaters.Remove(updater);
+        }
+
+        EntityQueryBuilder m_QueryBuilder;
+        EntityQuery m_Query;
+
+        protected override void OnCreate()
+        {
+            //These are here to inform the system runner the queries we are interested in. Without these calls, OnUpdate() might not be called
+            GetEntityQuery(ComponentType.ReadOnly<Labeling>());
+
+            m_QueryBuilder = Entities.WithAllReadOnly<Labeling, GroundTruthInfo>();
+            m_Query = m_QueryBuilder.ToEntityQuery();
+        }
+
+        protected override void OnUpdate()
+        {
+            var count = m_Query.CalculateEntityCount();
+
+            foreach (var updater in m_ActiveUpdaters)
+            {
+                updater.OnBeginUpdate(count);
+                m_QueryBuilder.ForEach((Entity entity, Labeling labeling, ref GroundTruthInfo groundTruth) =>
+                {
+                    updater.OnUpdateEntity(labeling, groundTruth);
+                });
+                updater.OnEndUpdate();
+            }
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Labelers/GroundTruthUpdateSystem.cs.meta

+fileFormatVersion: 2
+guid: 2c6db511440054e7192d6fda3368019f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBox3DLabeler.cs

+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using Unity.Entities;
+using Unity.Profiling;
+using Unity.Simulation;
+using UnityEngine.Scripting;
+using UnityEngine.Serialization;
+using UnityEngine.UIElements;
+
+namespace UnityEngine.Perception.GroundTruth
+{
+    // ##### 3D bounding box
+    //
+    // A json file that stored collections of 3D bounding boxes.
+    // Each bounding box record maps a tuple of (instance, label) to translation, size and rotation that draws a 3D bounding box,
+    // as well as velocity and acceleration (optional) of the 3D bounding box.
+    // All location data is given with respect to the **sensor coordinate system**.
+    //
+    //
+    // bounding_box_3d {
+    //      label_id:     <int> -- Integer identifier of the label
+    //      label_name:   <str> -- String identifier of the label
+    //      instance_id:  <str> -- UUID of the instance.
+    //      translation:  <float, float, float> -- 3d bounding box's center location in meters as center_x, center_y, center_z with respect to global coordinate system.
+    //      size:         <float, float, float> -- 3d bounding box size in meters as width, length, height.
+    //      rotation:     <float, float, float, float> -- 3d bounding box orientation as quaternion: w, x, y, z.
+    //      velocity:     <float, float, float>  -- 3d bounding box velocity in meters per second as v_x, v_y, v_z.
+    //      acceleration: <float, float, float> [optional] -- 3d bounding box acceleration in meters per second^2 as a_x, a_y, a_z.
+    //  }
+
+    public sealed class BoundingBox3DLabeler : CameraLabeler, IGroundTruthUpdater
+    {
+        public enum OutputMode
+        {
+            Verbose,
+            Kitti
+        }
+
+        [SuppressMessage("ReSharper", "InconsistentNaming")]
+        public abstract class BoxData
+        {
+            public int label_id;
+            public string label_name;
+            public uint instance_id;
+        }
+
+        public class KittiData : BoxData
+        {
+            public float[] translation;
+            public float[] size;
+            public float yaw;
+        }
+
+        public class VerboseData : BoxData
+        {
+            public float[] translation;
+            public float[] size;
+            public float[] rotation;
+            public float[] velocity; // TODO
+            public float[] acceleration; // TODO
+        }
+
+        static ProfilerMarker s_BoundingBoxCallback = new ProfilerMarker("OnBoundingBoxes3DReceived");
+
+        public string annotationId = "0bfbe00d-00fa-4555-88d1-471b58449f5c";
+
+        Dictionary<int, AsyncAnnotation> m_AsyncAnnotations;
+        AnnotationDefinition m_AnnotationDefinition;
+        BoxData[] m_BoundingBoxValues;
+
+        public OutputMode mode = OutputMode.Kitti;
+
+        public IdLabelConfig idLabelConfig;
+
+        protected override bool supportsVisualization => false;
+
+        public event Action<int, BoxData> BoundingBoxComputed;
+
+        public BoundingBox3DLabeler() {}
+
+        public BoundingBox3DLabeler(IdLabelConfig labelConfig)
+        {
+            this.idLabelConfig = labelConfig;
+        }
+
+        protected override void Setup()
+        {
+            if (idLabelConfig == null)
+                throw new InvalidOperationException("BoundingBox2DLabeler's idLabelConfig field must be assigned");
+
+            var updater = World.DefaultGameObjectInjectionWorld?.GetOrCreateSystem<GroundTruthUpdateSystem>();
+            updater?.Activate(this);
+
+            m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
+
+            m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("bounding box 3D", idLabelConfig.GetAnnotationSpecification(),
+                "Bounding box for each labeled object visible to the sensor", id: new Guid(annotationId));
+        }
+
+        protected override void Cleanup()
+        {
+            var updater = World.DefaultGameObjectInjectionWorld?.GetExistingSystem<GroundTruthUpdateSystem>();
+            updater?.Deactivate(this);
+        }
+
+        int m_CurrentIndex;
+        int m_CurrentFrame;
+
+        public void OnBeginUpdate(int count)
+        {
+            if (m_BoundingBoxValues == null || count != m_BoundingBoxValues.Length)
+                m_BoundingBoxValues = new BoxData[count];
+
+            m_CurrentIndex = 0;
+            m_CurrentFrame = Time.frameCount;
+        }
+
+        static BoxData Convert(IdLabelEntry label, uint instanceId, Vector3 center, Vector3 extents, Quaternion rotation, OutputMode outputMode)
+        {
+            return outputMode == OutputMode.Kitti ?
+                ConvertToKitti(label, instanceId, center, extents, rotation) :
+                ConvertToVerboseData(label, instanceId, center, extents, rotation);
+        }
+
+        static BoxData ConvertToVerboseData(IdLabelEntry label, uint instanceId, Vector3 center, Vector3 extents, Quaternion rotation)
+        {
+            return new VerboseData
+            {
+                label_id = label.id,
+                label_name = label.label,
+                instance_id = instanceId,
+                translation = new float[] { center.x, center.y, center.z },
+                size = new float[] { extents.x, extents.y, extents.z },
+                rotation = new float[] { rotation.x, rotation.y, rotation.z, rotation.w },
+                velocity = null,
+                acceleration = null
+            };
+        }
+
+        static BoxData ConvertToKitti(IdLabelEntry label, uint instanceId, Vector3 center, Vector3 extents, Quaternion rotation)
+        {
+            return new KittiData
+            {
+                label_id = label.id,
+                label_name = label.label,
+                instance_id = instanceId,
+                translation = new float[] { center.x, center.y, center.z },
+                // unity stores bounds as half extents, kitti expects full values
+                size = new float[] { extents.x * 2, extents.y * 2, extents.z * 2 },
+                yaw = rotation.eulerAngles.y,
+            };
+        }
+
+        public void OnUpdateEntity(Labeling labeling, GroundTruthInfo groundTruthInfo)
+        {
+            using (s_BoundingBoxCallback.Auto())
+            {
+                var meshFilters = labeling.gameObject.GetComponentsInChildren<MeshFilter>();
+
+                if (meshFilters == null || meshFilters.Length == 0) return;
+
+                if (idLabelConfig.TryGetLabelEntryFromInstanceId(groundTruthInfo.instanceId, out var labelEntry))
+                {
+                    var camTrans = perceptionCamera.transform;
+                    var transform = labeling.transform;
+                    var bounds = new Bounds(Vector3.zero, Vector3.zero);
+                    var boundsUnset = true;
+
+                    foreach (var mesh in meshFilters)
+                    {
+                        var currentTransform = mesh.gameObject.transform;
+                        var targetTransform = labeling.transform;
+
+                        // Need to copy the bounds because we are going to move them, and do not want to change
+                        // the bounds of the mesh
+                        var meshBounds = mesh.mesh.bounds;
+                        var transformedBounds = new Bounds(meshBounds.center, meshBounds.size);
+
+                        var tmp = currentTransform.TransformPoint(bounds.center);
+                        var tmp2 = bounds.center + transform.position;
+                        var tmp3 = bounds.center + transform.localPosition;
+                        var tmp4 = tmp3 + labeling.transform.position;
+                        var tmp5 = tmp3 + labeling.transform.localPosition;
+#if false
+                        //transformedBounds.center += currentTransform.position;
+                        //transformedBounds.extents = Vector3.Scale(currentTransform.localScale, transformedBounds.extents);
+                        //transformedBounds.extents = currentTransform.rotation * transformedBounds.extents;
+                        transformedBounds.center = currentTransform.TransformPoint(transformedBounds.center);
+                        transformedBounds.extents = currentTransform.TransformVector(transformedBounds.extents);
+#else
+                        while (currentTransform != labeling.transform)
+                        {
+                            Debug.Log("P: " + currentTransform.position);
+
+                            transformedBounds.center += currentTransform.localPosition;
+                            //transformedBounds.extents = Vector3.Scale(currentTransform.localScale, transformedBounds.extents);
+                            //transformedBounds.extents = Quaternion.Inverse(currentTransform.rotation) * transformedBounds.extents;
+                            //transformedBounds.center = currentTransform.InverseTransformPoint(transformedBounds.center);
+                            //transformedBounds.extents = currentTransform.InverseTransformVector(transformedBounds.extents);
+                            //transformedBounds.center += currentTransform.Translate(transformedBounds.center);
+                            //transformedBounds.center = currentTransform.TransformPoint(transformedBounds.center);
+                            transformedBounds.extents = currentTransform.TransformVector(transformedBounds.extents);
+                            currentTransform = currentTransform.parent;
+                        }
+#endif
+                        if (boundsUnset)
+                        {
+                            bounds.center = transformedBounds.center;
+                            bounds.extents = transformedBounds.extents;
+                            boundsUnset = false;
+                        }
+                        else
+                            bounds.Encapsulate(transformedBounds);
+                    }
+
+
+
+                    // Need to transform our bounding box by the parent transform, but it should not be rotated, the bounding box should
+                    // always be in respect to local transform
+                    bounds.center = labeling.transform.TransformPoint(bounds.center);
+                    bounds.extents = Vector3.Scale(bounds.extents, labeling.transform.localScale);
+
+                    // Transform the points at the end...
+
+                    var center = bounds.center;
+                    var localRotation = Quaternion.Inverse(camTrans.rotation) * transform.rotation;
+                    var localCenter = camTrans.InverseTransformPoint(center);
+
+                    var converted = Convert(labelEntry, groundTruthInfo.instanceId, localCenter, bounds.extents, localRotation, mode);
+                    BoundingBoxComputed?.Invoke(m_CurrentFrame, converted);
+                    m_BoundingBoxValues[m_CurrentIndex++] = converted;
+                }
+            }
+        }
+
+        public void Working_OnUpdateEntity(Labeling labeling, GroundTruthInfo groundTruthInfo)
+        {
+            using (s_BoundingBoxCallback.Auto())
+            {
+                var meshFilters = labeling.gameObject.GetComponentsInChildren<MeshFilter>();
+
+                if (meshFilters == null || meshFilters.Length == 0) return;
+
+                if (idLabelConfig.TryGetLabelEntryFromInstanceId(groundTruthInfo.instanceId, out var labelEntry))
+                {
+                    var camTrans = perceptionCamera.transform;
+                    var transform = labeling.transform;
+                    var bounds = new Bounds(Vector3.zero, Vector3.zero);
+                    var boundsUnset = true;
+
+                    foreach (var mesh in meshFilters)
+                    {
+                        // Need to copy the bounds because we are going to move them, and do not want to change
+                        // the bounds of the mesh
+                        var meshBounds = mesh.mesh.bounds;
+                        var transformedBounds = new Bounds(meshBounds.center, meshBounds.size);
+
+                        // Only transform meshes that are not a part of the parent transform...
+                        if (mesh.gameObject != labeling.gameObject)
+                        {
+                            // Properly get the bounds of the mesh with respect to its parent.
+                            var meshTrans = mesh.gameObject.transform;
+                            transformedBounds.center += meshTrans.localPosition;
+                            transformedBounds.extents = Vector3.Scale(meshTrans.localScale, transformedBounds.extents);
+                            transformedBounds.extents = meshTrans.localRotation * transformedBounds.extents;
+                        }
+
+                        if (boundsUnset)
+                        {
+                            bounds.center = transformedBounds.center;
+                            bounds.extents = transformedBounds.extents;
+                            boundsUnset = false;
+                        }
+                        else
+                            bounds.Encapsulate(transformedBounds);
+                    }
+
+                    var center = transform.position + bounds.center;
+                    var localRotation = Quaternion.Inverse(transform.rotation) * camTrans.rotation;
+                    var localCenter = camTrans.InverseTransformPoint(center);
+                    var extents =  Vector3.Scale(transform.localScale,bounds.extents);
+
+                    var converted = Convert(labelEntry, groundTruthInfo.instanceId, localCenter, extents, localRotation, mode);
+                    BoundingBoxComputed?.Invoke(m_CurrentFrame, converted);
+                    m_BoundingBoxValues[m_CurrentIndex++] = converted;
+                }
+            }
+        }
+
+        public void OnEndUpdate()
+        {
+            perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition).ReportValues(m_BoundingBoxValues);
+        }
+    }
+}

com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox3dTests.cs

+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+using System.Xml.Schema;
+using NUnit.Framework;
+using Unity.Mathematics;
+using UnityEditor.VersionControl;
+using UnityEngine;
+using UnityEngine.Perception.GroundTruth;
+using UnityEngine.TestTools;
+
+namespace GroundTruthTests
+{
+    [TestFixture]
+    public class BoundingBox3dTests : GroundTruthTestBase
+    {
+        static string PrintBox(BoundingBox3DLabeler.BoxData box)
+        {
+            var sb = new StringBuilder();
+            sb.Append("label id: " + box.label_id + " ");
+            sb.Append("label_name: " + box.label_name + " ");
+            sb.Append("instance_id: " + box.instance_id + " ");
+
+            switch (box)
+            {
+                case BoundingBox3DLabeler.KittiData k1:
+                    sb.Append("translation: (" + k1.translation[0] + ", " + k1.translation[1] + ", " + k1.translation[2] + ") ");
+                    sb.Append("size: (" + k1.size[0] + ", " + k1.size[1] + ", " + k1.size[2] + ") ");
+                    sb.Append("yaw: " + k1.yaw);
+                    break;
+                case BoundingBox3DLabeler.VerboseData v:
+                {
+                    sb.Append("translation: (" + v.translation[0] + ", " + v.translation[1] + ", " + v.translation[2] + ") ");
+                    sb.Append("size: (" + v.size[0] + ", " + v.size[1] + ", " + v.size[2] + ") ");
+                    sb.Append("rotation: " + +v.rotation[0] + ", " + v.rotation[1] + ", " + v.rotation[2] + ", " + v.rotation[3] + ") ");
+                    if (v.velocity == null)
+                        sb.Append("velocity: null ");
+                    else
+                        sb.Append("velocity: (" + v.velocity[0] + ", " + v.velocity[1] + ", " + v.velocity[2] + ") ");
+                    if (v.acceleration == null)
+                        sb.Append("acceleration: null");
+                    else
+                        sb.Append("acceleration: (" + v.acceleration[0] + ", " + v.acceleration[1] + ", " + v.acceleration[2] + ")");
+                    break;
+                }
+            }
+
+            return sb.ToString();
+        }
+
+        const float k_Delta = 0.0001f;
+
+        [UnityTest]
+        public IEnumerator CameraOffset_PrduceProperTranslationTest()
+        {
+            var expected = new ExpectedResult[]
+            {
+                new ExpectedResult
+                {
+                    instanceId = 1,
+                    labelId = 1,
+                    labelName = "label",
+                    position = new Vector3(0, 0, 10),
+                    scale = new Vector3(5, 5, 5),
+                    rotation = Quaternion.identity
+                }
+            };
+            var target = TestHelper.CreateLabeledCube();
+            var cameraPosition = new Vector3(0, 0, -10);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expected);
+        }
+
+        [UnityTest]
+        public IEnumerator CameraOffsetAndRotated_PrduceProperTranslationTest()
+        {
+            var expected = new ExpectedResult[]
+            {
+                new ExpectedResult
+                {
+                    instanceId = 1,
+                    labelId = 1,
+                    labelName = "label",
+                    position = new Vector3(0, 0, Mathf.Sqrt(200)),
+                    scale = new Vector3(5, 5, 5),
+                    rotation = Quaternion.identity
+                }
+            };
+            var target = TestHelper.CreateLabeledCube(x: 10, y: 0, z: 10, yaw: 45);
+            var cameraPosition = new Vector3(0, 0, 0);
+            var cameraRotation = Quaternion.Euler(0, 45, 0);
+            return ExecuteTest(target, cameraPosition, cameraRotation, expected);
+        }
+
+        [UnityTest]
+        public IEnumerator SimpleMultiMesh_PrduceProperTranslationTest()
+        {
+            var expected = new ExpectedResult[]
+            {
+                new ExpectedResult
+                {
+                    instanceId = 1,
+                    labelId = 1,
+                    labelName = "label",
+                    position = new Vector3(0, 0, 10),
+                    scale = new Vector3(6.5f, 2.5f, 2.5f),
+                    rotation = Quaternion.identity
+                }
+            };
+            var target = CreateMultiMeshGameObject();
+            target.transform.position = Vector3.zero;
+            var cameraPosition = new Vector3(0, 0, -10);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expected);
+        }
+
+        [UnityTest]
+        public IEnumerator MultiInheritedMesh_PrduceProperTranslationTest()
+        {
+            var expected = new ExpectedResult[]
+            {
+                new ExpectedResult
+                {
+                    instanceId = 1,
+                    labelId = 2,
+                    labelName = "car",
+                    position = new Vector3(0, 0.525f, 20),
+                    scale = new Vector3(2f, 0.875f, 2.4f),
+                    rotation = Quaternion.identity
+                },
+            };
+
+            var target = CreateTestReallyBadCar(new Vector3(0, 0.35f, 20), Quaternion.identity);
+            target.transform.localPosition = new Vector3(0, 0, 20);
+            var cameraPosition = new Vector3(0, 0, 0);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expected);
+        }
+
+        [UnityTest]
+        public IEnumerator MultiInheritedMeshDifferentLabels_PrduceProperTranslationTest()
+        {
+            var wheelScale = new Vector3(0.35f, 1.0f, 0.35f);
+            var wheelRot = Quaternion.Euler(0, 0, 90);
+
+            var expected = new ExpectedResult[]
+            {
+                new ExpectedResult
+                {
+                    instanceId = 1,
+                    labelId = 2,
+                    labelName = "car",
+                    position = new Vector3(0, 1.05f, 20),
+                    scale = new Vector3(1f, 0.7f, 2.4f),
+                    rotation = Quaternion.identity
+                },
+                new ExpectedResult
+                {
+                    instanceId = 2,
+                    labelId = 3,
+                    labelName = "wheel",
+                    position = new Vector3(1, 0.35f, 18.6f),
+                    scale = wheelScale,
+                    rotation = wheelRot
+                },
+                new ExpectedResult
+                {
+                    instanceId = 3,
+                    labelId = 3,
+                    labelName = "wheel",
+                    position = new Vector3(1, 0.35f, 21.45f),
+                    scale = wheelScale,
+                    rotation = wheelRot
+                },
+                new ExpectedResult
+                {
+                    instanceId = 4,
+                    labelId = 3,
+                    labelName = "wheel",
+                    position = new Vector3(-1, 0.35f, 18.6f),
+                    scale = wheelScale,
+                    rotation = wheelRot
+                },
+                new ExpectedResult
+                {
+                    instanceId = 5,
+                    labelId = 3,
+                    labelName = "wheel",
+                    position = new Vector3(-1, 0.35f, 21.45f),
+                    scale = wheelScale,
+                    rotation = wheelRot
+                }
+            };
+
+            var target = CreateTestReallyBadCar(new Vector3(0, 0.35f, 20), Quaternion.identity, false);
+            //target.transform.localPosition = new Vector3(0, 0, 20);
+            var cameraPosition = new Vector3(0, 0, 0);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expected);
+        }
+
+        [UnityTest]
+        public IEnumerator SpinningBoxTest()
+        {
+            var pos = new Vector3(0, 0, 10);
+            var expectedPosition = new Vector3[]{pos, pos, pos, pos, pos};
+            var scale = new Vector3(15f, 15f, 15f);
+            var expectedScale = new Vector3[] { scale, scale, scale, scale, scale };
+
+            var rot = new Quaternion[5];
+
+            var spinner = Quaternion.Euler(0, 15, 0);
+            for (var i = 0; i < 5; i++)
+            {
+                rot[0] = Quaternion.Euler(0, 30, 0);
+                rot[1] = rot[0] * spinner;
+                rot[2] = rot[1] * spinner;
+                rot[3] = rot[2] * spinner;
+                rot[4] = rot[3] * spinner;
+            }
+
+            var target = CreateDynamicBox(spinning: true);
+            target.transform.localPosition = new Vector3(0, 0, 10);
+            var cameraPosition = new Vector3(0, 0, 0);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expectedPosition, expectedScale, rot, BoundingBox3DLabeler.OutputMode.Verbose);
+        }
+
+        [UnityTest]
+        public IEnumerator MovingboxTest()
+        {
+            var pos = new Vector3(-40, 0, 10);
+            var movement = new Vector3(10, 0, 0);
+            var expectedPosition = new Vector3[]
+            {
+                pos + movement * 2,
+                pos + movement * 3,
+                pos + movement * 4,
+                pos + movement * 5,
+                pos + movement * 6
+            };
+            var scale = new Vector3(15, 15, 15);
+            var expectedScale = new Vector3[] { scale, scale, scale, scale, scale };
+            var rot = Quaternion.identity;
+            var expectedRot = new Quaternion[] { rot, rot, rot, rot, rot };
+            var target = CreateDynamicBox(moving: true);
+            target.transform.localPosition = pos;
+            var cameraPosition = new Vector3(0, 0, 0);
+            var cameraRotation = Quaternion.identity;
+            return ExecuteTest(target, cameraPosition, cameraRotation, expectedPosition, expectedScale, expectedRot, BoundingBox3DLabeler.OutputMode.Verbose);
+        }
+
+        public struct ExpectedResult
+        {
+            public int labelId;
+            public string labelName;
+            public int instanceId;
+            public Vector3 position;
+            public Vector3 scale;
+            public Quaternion rotation;
+        }
+
+        private IEnumerator ExecuteTest(GameObject target, Vector3 cameraPos, Quaternion cameraRotation, ExpectedResult[] expectations)
+        {
+            var receivedResults = new List<(int, BoundingBox3DLabeler.BoxData)>();
+            var cameraObject = SetupCamera(SetupLabelConfig(), (frame, data) =>
+            {
+                receivedResults.Add((frame, data));
+            });
+
+            cameraObject.Item1.transform.position = cameraPos;
+            cameraObject.Item1.transform.rotation = cameraRotation;
+
+            AddTestObjectForCleanup(cameraObject.Item1);
+
+            var firstTime = true;
+
+            foreach (var mode in (BoundingBox3DLabeler.OutputMode[])Enum.GetValues(typeof(BoundingBox3DLabeler.OutputMode)))
+            {
+                cameraObject.Item1.SetActive(false);
+                receivedResults.Clear();
+                cameraObject.Item2.mode = mode;
+                cameraObject.Item1.SetActive(true);
+
+                if (firstTime)
+                {
+                    firstTime = false;
+                    yield return null;
+                }
+
+                yield return null;
+
+                Assert.AreEqual(expectations.Length, receivedResults.Count);
+
+                for (var i = 0; i < expectations.Length; i++)
+                {
+                    var b = receivedResults[i].Item2;
+
+                    Debug.Log(PrintBox(b));
+
+                    Assert.AreEqual(expectations[i].labelId, b.label_id);
+                    Assert.AreEqual(expectations[i].labelName, b.label_name);
+                    Assert.AreEqual(expectations[i].instanceId, b.instance_id);
+
+                    switch (mode)
+                    {
+                        case BoundingBox3DLabeler.OutputMode.Verbose:
+                            Assert.IsAssignableFrom<BoundingBox3DLabeler.VerboseData>(b);
+                            var v = b as BoundingBox3DLabeler.VerboseData;
+                            TestVerboseResults(v, expectations[i]);
+                            break;
+                        case BoundingBox3DLabeler.OutputMode.Kitti:
+                            Assert.IsAssignableFrom<BoundingBox3DLabeler.KittiData>(b);
+                            var k = b as BoundingBox3DLabeler.KittiData;
+                            TestKittiResults(k, expectations[i]);
+                            break;
+                        default:
+                            throw new ArgumentOutOfRangeException();
+                    }
+                }
+            }
+
+            DestroyTestObject(cameraObject.Item1);
+            UnityEngine.Object.DestroyImmediate(target);
+        }
+
+
+
+        private IEnumerator ExecuteTest(GameObject target, Vector3 cameraPos, Quaternion cameraRotation, Vector3[] expectedPosition, Vector3[] expectedScale, Quaternion[] expectedRotation, BoundingBox3DLabeler.OutputMode mode)
+        {
+            var receivedResults = new List<(int, BoundingBox3DLabeler.BoxData)>();
+            var cameraObject = SetupCamera(SetupLabelConfig(), (frame, data) =>
+            {
+                receivedResults.Add((frame, data));
+            });
+
+            cameraObject.Item1.transform.position = cameraPos;
+            cameraObject.Item1.transform.rotation = cameraRotation;
+
+            AddTestObjectForCleanup(cameraObject.Item1);
+
+            cameraObject.Item1.SetActive(false);
+            receivedResults.Clear();
+            cameraObject.Item2.mode = mode;
+            cameraObject.Item1.SetActive(true);
+
+            yield return null;
+
+            for (int i = 0; i < expectedPosition.Length; i++)
+            {
+                yield return null;
+
+                Assert.AreEqual(i + 1, receivedResults.Count);
+                var b = receivedResults[i].Item2;
+
+                Debug.Log(PrintBox(b));
+
+                Assert.AreEqual(1, b.label_id);
+                Assert.AreEqual("label", b.label_name);
+                Assert.AreEqual(1, b.instance_id);
+
+                // TODO fix this test up to use the expected results as an input
+                var expected = new ExpectedResult
+                {
+                    position = expectedPosition[i],
+                    rotation = expectedRotation[i],
+                    scale = expectedScale[i],
+                };
+
+                switch (mode)
+                {
+                    case BoundingBox3DLabeler.OutputMode.Verbose:
+                        Assert.IsAssignableFrom<BoundingBox3DLabeler.VerboseData>(b);
+                        var v = b as BoundingBox3DLabeler.VerboseData;
+                        TestVerboseResults(v, expected);
+                        break;
+                    case BoundingBox3DLabeler.OutputMode.Kitti:
+                        Assert.IsAssignableFrom<BoundingBox3DLabeler.KittiData>(b);
+                        var k = b as BoundingBox3DLabeler.KittiData;
+                        TestKittiResults(k, expected);
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+            }
+
+            DestroyTestObject(cameraObject.Item1);
+            UnityEngine.Object.DestroyImmediate(target);
+        }
+
+        static IdLabelConfig SetupLabelConfig()
+        {
+            var labelConfig = ScriptableObject.CreateInstance<IdLabelConfig>();
+            labelConfig.Init(new List<IdLabelEntry>()
+            {
+                new IdLabelEntry
+                {
+                    id = 1,
+                    label = "label"
+                },
+                new IdLabelEntry
+                {
+
+                    id = 2,
+                    label= "car"
+                },
+                new IdLabelEntry
+                {
+                    id = 3,
+                    label= "wheel"
+                },
+            });
+
+            return labelConfig;
+        }
+
+        static (GameObject, BoundingBox3DLabeler) SetupCamera(IdLabelConfig config, Action<int, BoundingBox3DLabeler.BoxData> computeListener)
+        {
+            var cameraObject = new GameObject();
+            cameraObject.SetActive(false);
+            var camera = cameraObject.AddComponent<Camera>();
+            camera.orthographic = true;
+            camera.orthographicSize = 1;
+
+            var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
+            perceptionCamera.captureRgbImages = false;
+            var bboxLabeler = new BoundingBox3DLabeler(config);
+            if (computeListener != null)
+                bboxLabeler.BoundingBoxComputed += computeListener;
+
+            perceptionCamera.AddLabeler(bboxLabeler);
+
+            return (cameraObject, bboxLabeler);
+        }
+
+        public void TestVerboseResults(BoundingBox3DLabeler.VerboseData data, ExpectedResult e)
+        {
+            Assert.IsNotNull(data);
+            Assert.AreEqual(e.position[0], data.translation[0], k_Delta);
+            Assert.AreEqual(e.position[1], data.translation[1], k_Delta);
+            Assert.AreEqual(e.position[2], data.translation[2], k_Delta);
+            Assert.AreEqual(e.scale[0], data.size[0], k_Delta);
+            Assert.AreEqual(e.scale[1], data.size[1], k_Delta);
+            Assert.AreEqual(e.scale[2], data.size[2], k_Delta);
+            Assert.AreEqual(e.rotation[0], data.rotation[0], k_Delta);
+            Assert.AreEqual(e.rotation[1], data.rotation[1], k_Delta);
+            Assert.AreEqual(e.rotation[2], data.rotation[2], k_Delta);
+            Assert.AreEqual(e.rotation[3], data.rotation[3], k_Delta);
+            Assert.IsNull(data.velocity);
+            Assert.IsNull(data.acceleration);
+        }
+
+        public void TestKittiResults(BoundingBox3DLabeler.KittiData data, ExpectedResult e)
+        {
+            var size = e.scale * 2;
+
+            Assert.IsNotNull(data);
+            Assert.AreEqual(e.position[0], data.translation[0], k_Delta);
+            Assert.AreEqual(e.position[1], data.translation[1], k_Delta);
+            Assert.AreEqual(e.position[2], data.translation[2], k_Delta);
+            Assert.AreEqual(size[0], data.size[0], k_Delta);
+            Assert.AreEqual(size[1], data.size[1], k_Delta);
+            Assert.AreEqual(size[2], data.size[2], k_Delta);
+            Assert.AreEqual(e.rotation.eulerAngles.y, data.yaw, k_Delta);
+        }
+
+        private static GameObject CreateTestReallyBadCar(Vector3 position, Quaternion rotation, bool underOneLabel = true)
+        {
+            var badCar = new GameObject("BadCar");
+            badCar.transform.position = position;
+            badCar.transform.rotation = rotation;
+            if (underOneLabel)
+            {
+                var labeling = badCar.AddComponent<Labeling>();
+                labeling.labels.Add("car");
+            }
+
+            var body = GameObject.CreatePrimitive(PrimitiveType.Cube);
+            body.name = "body";
+            body.transform.parent = badCar.transform;
+            body.transform.localPosition = new Vector3(0, 0.7f, 0);
+            body.transform.localScale = new Vector3(2f, 1.4f, 4.8f);
+            if (!underOneLabel)
+            {
+                var labeling = body.AddComponent<Labeling>();
+                labeling.labels.Add("car");
+            }
+
+            var wheel = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+            wheel.name = "wheel1";
+            wheel.transform.parent = badCar.transform;
+            wheel.transform.localPosition = new Vector3(1f, 0, -1.4f);
+            wheel.transform.localRotation = Quaternion.Euler(0, 0, 90);
+            wheel.transform.localScale = new Vector3(0.7f, 1, 0.7f);
+            if (!underOneLabel)
+            {
+                var labeling = wheel.AddComponent<Labeling>();
+                labeling.labels.Add("wheel");
+            }
+
+            wheel = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+            wheel.name = "wheel2";
+            wheel.transform.parent = badCar.transform;
+            wheel.transform.localPosition = new Vector3(1f, 0, 1.45f);
+            wheel.transform.localRotation = Quaternion.Euler(0, 0, 90);
+            wheel.transform.localScale = new Vector3(0.7f, 1, 0.7f);
+            if (!underOneLabel)
+            {
+                var labeling = wheel.AddComponent<Labeling>();
+                labeling.labels.Add("wheel");
+            }
+
+            wheel = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+            wheel.name = "wheel3";
+            wheel.transform.parent = badCar.transform;
+            wheel.transform.localPosition = new Vector3(-1f, 0, -1.4f);
+            wheel.transform.localRotation = Quaternion.Euler(0, 0, 90);
+            wheel.transform.localScale = new Vector3(0.7f, 1, 0.7f);
+            if (!underOneLabel)
+            {
+                var labeling = wheel.AddComponent<Labeling>();
+                labeling.labels.Add("wheel");
+            }
+
+            wheel = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+            wheel.name = "wheel4";
+            wheel.transform.parent = badCar.transform;
+            wheel.transform.localPosition = new Vector3(-1f, 0, 1.45f);
+            wheel.transform.localRotation = Quaternion.Euler(0, 0, 90);
+            wheel.transform.localScale = new Vector3(0.7f, 1, 0.7f);
+            if (!underOneLabel)
+            {
+                var labeling = wheel.AddComponent<Labeling>();
+                labeling.labels.Add("wheel");
+            }
+
+            return badCar;
+        }
+
+        private GameObject CreateMultiMeshGameObject()
+        {
+            var go = new GameObject();
+            var labeling = go.AddComponent<Labeling>();
+            labeling.labels.Add("label");
+
+            var left = GameObject.CreatePrimitive(PrimitiveType.Cube);
+            left.transform.parent = go.transform;
+            left.transform.localPosition = new Vector3(-4, 0, 0);
+            left.transform.localScale = new Vector3(5, 5, 5);
+
+            var right = GameObject.CreatePrimitive(PrimitiveType.Cube);
+            right.transform.parent = go.transform;
+            right.transform.localPosition = new Vector3(4, 0, 0);
+            right.transform.localScale = new Vector3(5, 5, 5);
+
+            var center = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+            center.transform.parent = go.transform;
+            center.transform.localPosition = Vector3.zero;
+            center.transform.localRotation = Quaternion.Euler(0, 0, 90);
+            center.transform.localScale = new Vector3(1, 3, 1);
+
+            return go;
+        }
+
+        class CubeSpinner : MonoBehaviour
+        {
+            public Quaternion rotationPerFrame = Quaternion.Euler(0, 15, 0);
+
+            void Update()
+            {
+                transform.localRotation *= rotationPerFrame;
+            }
+        }
+
+        class CubeMover : MonoBehaviour
+        {
+            public Vector3 distancePerFrame = new Vector3(10, 0 , 0);
+
+            void Update()
+            {
+                transform.localPosition += distancePerFrame;
+            }
+        }
+
+        private static GameObject CreateDynamicBox(bool spinning = false, bool moving = false, Vector3? translation = null, Quaternion? rotation = null)
+        {
+            var cube = GameObject.CreatePrimitive(PrimitiveType.Cube);
+            cube.name = "Cube";
+            var labeling = cube.AddComponent<Labeling>();
+            labeling.labels.Add("label");
+            cube.transform.position = new Vector3(0f, 0f, 10f);
+            cube.transform.localScale = new Vector3(30f, 30f, 30f);
+            if (spinning)
+            {
+                var spin = cube.AddComponent<CubeSpinner>();
+                spin.rotationPerFrame = rotation ?? spin.rotationPerFrame;
+            }
+            if (moving)
+            {
+                var move = cube.AddComponent<CubeMover>();
+                move.distancePerFrame = translation ?? move.distancePerFrame;
+            }
+            return cube;
+        }
+    }
+}

com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox3dTests.cs.meta

+fileFormatVersion: 2
+guid: d9782f8e97d04481d86de232fdb7dc62
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: