Merge branch 'coco_perception_output_hybrid' into h-i

com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs

 using System;
 using System.Collections.Generic;
+using System.Linq;
+using UnityEngine.Perception.GroundTruth.Exporters;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
 
 #if UNITY_EDITOR_WIN || UNITY_EDITOR_OSX
 using UnityEditor.Perception.Visualizer;
     {
         Dictionary<SerializedProperty, CameraLabelerDrawer> m_CameraLabelerDrawers = new Dictionary<SerializedProperty, CameraLabelerDrawer>();
         ReorderableList m_LabelersList;
+        string[] m_ExporterList;
+        string m_OutputMode = "Perception";
+        int m_OutputModeIndex = -1;
+
         public void OnEnable()
         {
             m_LabelersList = new ReorderableList(this.serializedObject, labelersProperty, true, true, true, true);
             m_LabelersList.drawElementCallback = DrawElement;
             m_LabelersList.onAddCallback += OnAdd;
             m_LabelersList.onRemoveCallback += OnRemove;
+
+            m_OutputMode = PlayerPrefs.GetString(SimulationState.outputFormatMode);
+            m_ExporterList = TypeCache.GetTypesDerivedFrom<IDatasetExporter>().Select(exporter => exporter.Name).ToArray();
+            if (m_ExporterList.Any())
+                m_OutputModeIndex = Array.IndexOf(m_ExporterList, m_OutputMode);
+
+            m_OutputMode = PlayerPrefs.GetString(SimulationState.outputFormatMode, "Perception");
         }
 
         float GetElementHeight(int index)
 
         const string k_FrametimeTitle = "Simulation Delta Time";
         const float k_DeltaTimeTooLarge = 200;
+
         public override void OnInspectorGUI()
         {
             using(new EditorGUI.DisabledScope(EditorApplication.isPlaying))
                 EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureRgbImages)),new GUIContent("Save Camera RGB Output to Disk", "For each captured frame, save an RGB image of the camera's output to disk."));
+
+                if (m_ExporterList.Any())
+                {
+                    if (m_OutputModeIndex < 0)
+                    {
+                        m_OutputMode = nameof(PerceptionExporter);
+                        Debug.LogWarning($"Could not find the output format \'{m_OutputMode}\' in the list of available exporters, setting the exporter to \'{m_OutputMode}\'");
+                        m_OutputModeIndex = Array.IndexOf(m_ExporterList, m_OutputMode);
+                    }
+                    var selected = EditorGUILayout.Popup(new GUIContent("Output Format", ""), m_OutputModeIndex, m_ExporterList);
+
+                    if (m_OutputModeIndex != selected)
+                    {
+                        m_OutputModeIndex = selected;
+                        m_OutputMode = m_ExporterList[selected];
+                        PlayerPrefs.SetString(SimulationState.outputFormatMode, m_OutputMode);
+                    }
+                }
+
                 EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureTriggerMode)),new GUIContent("Capture Trigger Mode", $"The method of triggering captures for this camera. In {nameof(CaptureTriggerMode.Scheduled)} mode, captures happen automatically based on a start frame and frame delta time. In {nameof(CaptureTriggerMode.Manual)} mode, captures should be triggered manually through calling the {nameof(perceptionCamera.RequestCapture)} method of {nameof(PerceptionCamera)}."));
 
                 GUILayout.Space(5);

com.unity.perception/Editor/Randomization/Editors/RunInUnitySimulationWindow.cs

 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
 using UnityEditor.UIElements;
 using UnityEngine;
+using UnityEngine.Perception.GroundTruth;
+using UnityEngine.Perception.GroundTruth.Exporters;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
 using UnityEngine.Perception.Randomization.Samplers;
 using UnityEngine.Perception.Randomization.Scenarios;
 using UnityEngine.SceneManagement;
         TextField m_BuildPathField;
         TextField m_SelectedBuildPathTextField;
         TextField m_BuildIdField;
+
+        string[] m_OutputFormats;
+        ToolbarMenu m_OutputFormatMenu;
+        int m_OutputFormatIndex;
 
         [MenuItem("Window/Run in Unity Simulation")]
         static void ShowWindow()
                 m_RandomSeedField.value = BitConverter.ToUInt32(bytes, 0);
             };
 
+            m_OutputFormats = TypeCache.GetTypesDerivedFrom<IDatasetExporter>().Select(exporter => exporter.Name).ToArray();
+            m_OutputFormatMenu = root.Q<ToolbarMenu>("output-format");
+            var i = 0;
+            foreach (var format in m_OutputFormats)
+            {
+                var index = i++;
+                m_OutputFormatMenu.menu.AppendAction(
+                    format,
+                    action =>
+                    {
+                        m_OutputFormatIndex = index;
+                        m_OutputFormatMenu.text = format;
+                    });
+            }
+
-            for (var i = 0; i < m_SysParamDefinitions.Length; i++)
+            for (i = 0; i < m_SysParamDefinitions.Length; i++)
             {
                 var index = i;
                 var param = m_SysParamDefinitions[i];
             m_ProjectIdLabel.text = $"Project ID: {CloudProjectSettings.projectId}";
             m_PrevExecutionIdLabel.text = $"Execution ID: {PlayerPrefs.GetString("SimWindow/prevExecutionId")}";
             m_PrevRandomSeedLabel.text = $"Random Seed: {PlayerPrefs.GetString("SimWindow/prevRandomSeed")}";
+
+            m_OutputFormatMenu.text = PlayerPrefs.GetString(SimulationState.outputFormatMode, nameof(PerceptionExporter));
         }
 
         static string IncrementRunName(string runName)
                 sysParamIndex = m_SysParamIndex,
                 scenarioConfig = (TextAsset)m_ScenarioConfigField.value,
                 currentOpenScenePath = SceneManager.GetSceneAt(0).path,
-                currentScenario = FindObjectOfType<ScenarioBase>()
+                currentScenario = FindObjectOfType<ScenarioBase>(),
+                outputFormat = m_OutputFormats[m_OutputFormatIndex]
             };
             var runGuid = Guid.NewGuid();
             PerceptionEditorAnalytics.ReportRunInUnitySimulationStarted(
             constants["totalIterations"] = m_RunParameters.totalIterations;
             constants["instanceCount"] = m_RunParameters.instanceCount;
             constants["randomSeed"] = m_RunParameters.randomSeed;
+            constants["outputFormat"] = m_RunParameters.outputFormat;
 
             var appParamName = $"{m_RunParameters.runName}";
             var appParamsString = JsonConvert.SerializeObject(configuration, Formatting.Indented);
             PlayerPrefs.SetInt("SimWindow/sysParamIndex", m_RunParameters.sysParamIndex);
             PlayerPrefs.SetString("SimWindow/scenarioConfig",
                 m_RunParameters.scenarioConfig != null ? m_RunParameters.scenarioConfigAssetPath : string.Empty);
+            PlayerPrefs.SetString(SimulationState.outputFormatMode, m_RunParameters.outputFormat);
 
             SetFieldsFromPlayerPreferences();
         }
             public TextAsset scenarioConfig;
             public string currentOpenScenePath;
             public ScenarioBase currentScenario;
+            public string outputFormat;
 
             public string scenarioConfigAssetPath => AssetDatabase.GetAssetPath(scenarioConfig);
         }

com.unity.perception/Editor/Randomization/Uxml/RunInUnitySimulationWindow.uxml

                 <editor:ToolbarMenu name="sys-param" class="unity-base-field__input" style="border-width: 1px;"/>
             </VisualElement>
 
+            <VisualElement class="unity-base-field" tooltip="TODO">
+                <Label text="Output Format" class="unity-base-field__label"/>
+                <editor:ToolbarMenu name="output-format" class="unity-base-field__input" style="border-width: 1px;"/>
+            </VisualElement>
+
             <Label text="Optional Configuration" class="sim-window__header-1" style="margin-top: 18px;"/>
             <editor:ObjectField name="scenario-config" label="Scenario JSON Config" allow-scene-object="false"
                 tooltip="Selects a scenario JSON configuration to load during the run.

com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs

             return ego;
         }
 
+
+
         /// <summary>
         /// Register a new sensor under the given ego.
         /// </summary>
         }
     }
 
-
     /// <summary>
     /// Capture trigger modes for sensors.
     /// </summary>
                 throw new ArgumentException("The given annotationDefinition is invalid", nameof(annotationDefinition));
 
             return DatasetCapture.SimulationState.ReportAnnotationAsync(annotationDefinition, this);
+        }
+
+        public string GetRgbCaptureFilename(string defaultFilename, params(string, object)[] additionalSensorValues)
+        {
+            return DatasetCapture.SimulationState.GetRgbCaptureFilename(defaultFilename, additionalSensorValues);
         }
 
         /// <summary>

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs

 
         [SuppressMessage("ReSharper", "InconsistentNaming")]
         [SuppressMessage("ReSharper", "NotAccessedField.Local")]
-        internal struct BoundingBoxValue
+        public struct BoundingBoxValue
+            public int frame;
             public string label_name;
             public uint instance_id;
             public float x;
         /// The GUID id to associate with the annotations produced by this labeler.
         /// </summary>
         public string annotationId = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
+
+        /// <summary>
+        /// The annotation ID that the COCO exporter will use to know whether an ID is coming from a bounding box 2 labeler.
+        /// </summary>
+        public static string annotationIdForCocoExport = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
+
         /// <summary>
         /// The <see cref="IdLabelConfig"/> which associates objects with labels.
         /// </summary>
                     m_BoundingBoxValues.Add(new BoundingBoxValue
                     {
                         label_id = labelEntry.id,
+                        frame = frameCount,
                         label_name = labelEntry.label,
                         instance_id = objectInfo.instanceId,
                         x = objectInfo.boundingBox.x,

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

         /// The GUID id to associate with the annotations produced by this labeler.
         /// </summary>
         public string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
+
+        /// <summary>
+        /// The annotation ID that the COCO exporter will use to know whether an ID is coming from a keypoint labeler.
+        /// </summary>
+        public static string annotationIdForCocoExport = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
+
         /// <summary>
         /// The <see cref="IdLabelConfig"/> which associates objects with labels.
         /// </summary>
             if (idLabelConfig == null)
                 throw new InvalidOperationException($"{nameof(KeypointLabeler)}'s idLabelConfig field must be assigned");
 
-            m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", new []{TemplateToJson(activeTemplate)},
+            m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", TemplateToJson(activeTemplate, idLabelConfig),
                 "pixel coordinates of keypoints in a model, along with skeletal connectivity data", id: new Guid(annotationId));
 
             // Texture to use in case the template does not contain a texture for the joints or the skeletal connections
             m_AsyncAnnotations[m_CurrentFrame] = (annotation, keypoints);
 
             foreach (var label in LabelManager.singleton.registeredLabels)
-                ProcessLabel(label);
+                ProcessLabel(m_CurrentFrame, label);
         }
 
         // ReSharper disable InconsistentNaming
             /// The label id of the entity
             /// </summary>
             public int label_id;
+
+            public int frame;
+
             /// <summary>
             /// The instance id of the entity
             /// </summary>
             return false;
         }
 
-        void ProcessLabel(Labeling labeledEntity)
+        void ProcessLabel(int frame, Labeling labeledEntity)
         {
             if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
                 return;
 
                 cached.keypoints.instance_id = labeledEntity.instanceId;
                 cached.keypoints.label_id = labelEntry.id;
+                cached.keypoints.frame = -1;
                 cached.keypoints.template_guid = activeTemplate.templateID;
 
                 cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];
             }
 
             var cachedData = m_KnownStatus[labeledEntity.instanceId];
+            cachedData.keypoints.frame = frame;
 
             if (cachedData.status)
             {
                 var cachedKeypointEntry = cachedData.keypoints;
                 var keypointEntry = new KeypointEntry()
                 {
+                    frame = cachedKeypointEntry.frame,
                     instance_id = cachedKeypointEntry.instance_id,
                     keypoints = cachedKeypointEntry.keypoints.ToArray(),
                     label_id = cachedKeypointEntry.label_id,
         // ReSharper disable InconsistentNaming
         // ReSharper disable NotAccessedField.Local
         [Serializable]
-        struct JointJson
+        public struct JointJson
         {
             public string label;
             public int index;
         [Serializable]
-        struct SkeletonJson
+        public struct SkeletonJson
         {
             public int joint1;
             public int joint2;
         [Serializable]
-        struct KeypointJson
+        public struct KeypointJson
+            public int label_id;
+            public string label_name;
             public string template_id;
             public string template_name;
             public JointJson[] key_points;
         // ReSharper restore NotAccessedField.Local
 
-        KeypointJson TemplateToJson(KeypointTemplate input)
+        KeypointJson[] TemplateToJson(KeypointTemplate input, IdLabelConfig labelConfig)
-            var json = new KeypointJson();
-            json.template_id = input.templateID;
-            json.template_name = input.templateName;
-            json.key_points = new JointJson[input.keypoints.Length];
-            json.skeleton = new SkeletonJson[input.skeleton.Length];
+            var jsons = new KeypointJson[labelConfig.labelEntries.Count];
+            var idx = 0;
-            for (var i = 0; i < input.keypoints.Length; i++)
+            foreach (var cfg in labelConfig.labelEntries)
-                json.key_points[i] = new JointJson
+                var json = new KeypointJson();
+                json.label_id = cfg.id;
+                json.label_name = cfg.label;
+                json.template_id = input.templateID;
+                json.template_name = input.templateName;
+                json.key_points = new JointJson[input.keypoints.Length];
+                json.skeleton = new SkeletonJson[input.skeleton.Length];
+
+                for (var i = 0; i < input.keypoints.Length; i++)
-                    label = input.keypoints[i].label,
-                    index = i,
-                    color = input.keypoints[i].color
-                };
-            }
+                    json.key_points[i] = new JointJson
+                    {
+                        label = input.keypoints[i].label,
+                        index = i,
+                        color = input.keypoints[i].color
+                    };
+                }
-            for (var i = 0; i < input.skeleton.Length; i++)
-            {
-                json.skeleton[i] = new SkeletonJson()
+                for (var i = 0; i < input.skeleton.Length; i++)
-                    joint1 = input.skeleton[i].joint1,
-                    joint2 = input.skeleton[i].joint2,
-                    color = input.skeleton[i].color
-                };
+                    json.skeleton[i] = new SkeletonJson()
+                    {
+                        joint1 = input.skeleton[i].joint1,
+                        joint2 = input.skeleton[i].joint2,
+                        color = input.skeleton[i].color
+                    };
+                }
+
+                jsons[idx++] = json;
-            return json;
+            return jsons;
-}
+}

com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs

         }
 
         [SuppressMessage("ReSharper", "InconsistentNaming")]
-        internal struct LabelEntrySpec
+        public struct LabelEntrySpec
         {
             /// <summary>
             /// The label id prepared for reporting in the annotation

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

 #endif
 #if URP_PRESENT
 using UnityEngine.Rendering.Universal;
+#pragma warning disable 649
 #endif
 
 namespace UnityEngine.Perception.GroundTruth
 
         void CaptureRgbData(Camera cam)
         {
+            // TODO - Steve - this could be a place where we override the capture of the RGB image to allow the
+            // active reporter to determine where we should save RGB images, or even write them out
+
+
+            var width = cam.pixelWidth;
+            var height = cam.pixelHeight;
+
+            var frameCount = Time.frameCount;
+            var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{frameCount}.png";
+            SetPersistentSensorData("camera_width", width);
+            SetPersistentSensorData("camera_height", height);
+            SetPersistentSensorData("full_path", captureFilename);
+            SetPersistentSensorData("frame", frameCount);
-            var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{Time.frameCount}.png";
+
-            var width = cam.pixelWidth;
-            var height = cam.pixelHeight;
 
             colorFunctor = r =>
             {

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

 using Unity.Collections;
 using Unity.Simulation;
 using UnityEngine;
+using UnityEngine.Perception.GroundTruth.Exporters;
+using UnityEngine.Perception.GroundTruth.Exporters.Coco;
+using UnityEngine.Perception.GroundTruth.Exporters.CocoHybrid;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew;
-    partial class SimulationState
+    public partial class SimulationState
     {
         HashSet<SensorHandle> m_ActiveSensors = new HashSet<SensorHandle>();
         Dictionary<SensorHandle, SensorData> m_Sensors = new Dictionary<SensorHandle, SensorData>();
+
+        IDatasetExporter _ActiveReporter = null;
 
         // Always use the property SequenceTimeMs instead
         int m_FrameCountLastUpdatedSequenceTime;
         float m_LastTimeScale;
         readonly string m_OutputDirectoryName;
         string m_OutputDirectoryPath;
+
+        public const string outputFormatMode = "outputFormatMode";
 
         public bool IsRunning { get; private set; }
 
 
         public SimulationState(string outputDirectory)
         {
+            _ActiveReporter = null;
+
             PlayerPrefs.SetString(defaultOutputBaseDirectory, Configuration.Instance.GetStorageBasePath());
             m_OutputDirectoryName = outputDirectory;
             var basePath = PlayerPrefs.GetString(userBaseDirectoryKey, string.Empty);
                     basePath = Configuration.localPersistentDataPath;
                 }
             }
+/*
+            //var activeReporterString = PlayerPrefs.GetString(activeReporterKey, defaultReporter);
+            var activeReporterString = "coco";
+            if (activeReporterString == "perceptionOutput")
+            {
+                m_ActiveReporter = new PerceptionExporter();
+            }
+            else
+            {
+                m_ActiveReporter = new CocoExporter();
+            }
+*/
+
+
+        }
+
+        IDatasetExporter GetActiveReporter()
+        {
+            if (_ActiveReporter != null) return _ActiveReporter;
+
+            var mode = PlayerPrefs.GetString(outputFormatMode, nameof(CocoExporter));
+#if false
+            // TODO figure out how to do this with just the class name and not have to have the switch
+
+            var exporter = Activator.CreateInstance(Type.GetType(mode) ?? typeof(PerceptionExporter));
+
+            if (exporter is IDatasetExporter casted)
+            {
+                m_ActiveReporter = casted;
+            }
+#else
+            Debug.Log($"SS - Sim State setting active reporter: {mode}");
+
+            switch (mode)
+            {
+                case nameof(PerceptionExporter):
+                    _ActiveReporter = new PerceptionExporter();
+                    break;
+                case nameof(CocoExporter):
+                    //_ActiveReporter = new CocoExporter();
+                    _ActiveReporter = new CocoHybridExporter();
+                    break;
+                case nameof(PerceptionNewExporter):
+                    _ActiveReporter = new PerceptionNewExporter();
+                    break;
+                default:
+                    _ActiveReporter = new PerceptionExporter();
+                    break;
+            }
+
+#endif
+            Debug.Log("Calling SS::OnSimulationBegin");
+            _ActiveReporter?.OnSimulationBegin(Manager.Instance.GetDirectoryFor(m_OutputDirectoryName));
+
+            return _ActiveReporter;
+        }
+
+        public string GetRgbCaptureFilename(string defaultFilename, params(string, object)[] additionalSensorValues)
+        {
+            var directory = GetActiveReporter()?.GetRgbCaptureFilename(additionalSensorValues);
+            return directory == string.Empty ? defaultFilename : directory;
         }
 
         /// <summary>
-        class PendingCapture
+        public class PendingCapture
         {
             public Guid Id;
             public SensorHandle SensorHandle;
             public EgoHandle egoHandle;
         }
 
-        struct AnnotationData
+        public struct AnnotationData
+            public IEnumerable<object> RawValues;
             public bool IsAssigned => Path != null || ValuesJson != null;
 
             public AnnotationData(AnnotationDefinition annotationDefinition, string path, JArray valuesJson)
 
             sensorData.lastCaptureFrameCount = Time.frameCount;
             m_Sensors[sensorHandle] = sensorData;
+
+            // SB - maybe this can all be moved to the other capture area
+            var width = -1;
+            var height = -1;
+            var fullPath = filename;
+            var frameCount = 0;
+
+            foreach (var i in additionalSensorValues)
+            {
+                switch (i.Item1)
+                {
+                    case "camera_width":
+                        width = (int)i.Item2;
+                        break;
+                    case "camera_height":
+                        height = (int)i.Item2;
+                        break;
+                    case "full_path":
+                        fullPath = (string)i.Item2;
+                        break;
+                    case "frame":
+                        frameCount = (int)i.Item2;
+                        break;
+                }
+            }
+
+            GetActiveReporter()?.OnCaptureReported(frameCount, width, height, filename);
         }
 
         static string GetFormatFromFilename(string filename)
             if (m_PendingMetrics.Count > 0)
                 Debug.LogError($"Simulation ended with pending metrics: {string.Join(", ", m_PendingMetrics.Select(c => $"id:{c.MetricId} step:{c.Step}"))}");
 
+            if (m_AdditionalInfoTypeData.Any())
+            {
+                List<IdLabelConfig.LabelEntrySpec> labels = new List<IdLabelConfig.LabelEntrySpec>();
+
+                foreach (var infoTypeData in m_AdditionalInfoTypeData)
+                {
+                    if (infoTypeData.specValues == null) continue;
+
+                    foreach (var spec in infoTypeData.specValues)
+                    {
+                        if (spec is IdLabelConfig.LabelEntrySpec entrySpec)
+                        {
+                            labels.Add(entrySpec);
+                        }
+                    }
+
+                    Debug.Log($"adt: {infoTypeData}");
+
+                }
+            }
+
+
+            Debug.Log($"Calling SS::OnSimulationEnd");
+            GetActiveReporter()?.OnSimulationEnd();
+
 
         public AnnotationDefinition RegisterAnnotationDefinition<TSpec>(string name, TSpec[] specValues, string description, string format, Guid id)
         {
             RegisterAdditionalInfoType(name, specValues, description, format, id, AdditionalInfoKind.Annotation);
+
+            GetActiveReporter()?.OnAnnotationRegistered(id, specValues); // <- Not sure about this one either
 
             return new AnnotationDefinition(id);
         }
                 }
             }
 
-            ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray);
+            var values2 = new List<object>();
+            foreach (var v in values)
+            {
+                values2.Add(v);
+            }
+            ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray, values:values2);
-        void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray)
+        void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray, IEnumerable<object> values = null)
         {
             if (!asyncAnnotation.IsPending)
                 throw new InvalidOperationException("AsyncAnnotation has already been reported and cannot be reported again.");
 
             annotationData.Path = filename;
             annotationData.ValuesJson = jArray;
+            annotationData.RawValues = values;
 
             annotationTuple.Item2 = annotationData;
             pendingCapture.Annotations[annotationIndex] = annotationTuple;

com.unity.perception/Runtime/GroundTruth/SimulationState_Json.cs

 
 namespace UnityEngine.Perception.GroundTruth
 {
-    partial class SimulationState
+    public partial class SimulationState
     {
         const Formatting k_Formatting = Formatting.Indented;
 
             m_WriteToDiskSampler.Begin();
 
             var path = Path.Combine(OutputDirectory, filename);
+            Debug.Log($"ss - sensors.json - {path}");
             File.WriteAllText(path, contents);
             Manager.Instance.ConsumerFileProduced(path);
             m_WriteToDiskSampler.End();
 
             void Write(List<PendingCapture> pendingCaptures, SimulationState simulationState, int captureFileIndex)
             {
+                GetActiveReporter()?.ProcessPendingCaptures(pendingCaptures, simulationState);
+#if false
                 simulationState.m_SerializeCapturesAsyncSampler.Begin();
 
                 //lazily allocate for fast zero-write frames
                 simulationState.WriteJObjectToFile(capturesJObject,
                     $"captures_{captureFileIndex:000}.json");
                 simulationState.m_SerializeCapturesAsyncSampler.End();
+#endif
             }
 
             if (flush)

com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenario.cs

 using System;
 using Unity.Simulation;
+using UnityEngine.Perception.GroundTruth;
 
 namespace UnityEngine.Perception.Randomization.Scenarios
 {
             {
                 var filePath = new Uri(Configuration.Instance.SimulationConfig.app_param_uri).LocalPath;
                 LoadConfigurationFromFile(filePath);
+                PlayerPrefs.SetString(SimulationState.outputFormatMode, constants.outputFormat);
             }
             else
             {

com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenarioConstants.cs

         /// </summary>
         [Tooltip("The Unity Simulation instance index of the currently executing worker.")]
         public int instanceIndex;
+
+        public string outputFormat;
     }
 }

com.unity.perception/Runtime/GroundTruth/Exporters.meta

+fileFormatVersion: 2
+guid: 5353392e887128948bc94b5e0dd9ff73
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/Coco.meta

+fileFormatVersion: 2
+guid: b543148039c075741a0d54a25d56f573
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat.meta

+fileFormatVersion: 2
+guid: a40dd8c5379f64748a0942b0475badc6
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs.meta

+fileFormatVersion: 2
+guid: 28e95774bd5a26d47973d1213bcc4c26
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs.meta

+fileFormatVersion: 2
+guid: c38f43837b9cf984f92853858398e0b4
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs

+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Unity.Simulation;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat
+{
+    public class PerceptionExporter : IDatasetExporter
+    {
+        const Formatting k_Formatting = Formatting.Indented;
+        string outputDirectory = string.Empty;
+        int captureFileIndex = 0;
+
+        public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
+        {
+            return string.Empty;
+        }
+
+        public void OnSimulationBegin(string directoryName)
+        {
+            Debug.Log($"SS - Perception - OnSimBegin");
+            outputDirectory = directoryName;
+        }
+
+        public void OnSimulationEnd()
+        {
+            Debug.Log($"SS - Perception - OnSimEnd");
+            // do nothing :-)
+        }
+
+        public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
+        {
+            // do nothing :-)
+        }
+
+        public static void WriteJObjectToFile(JObject jObject, string directory, string filename)
+        {
+            var stringWriter = new StringWriter(new StringBuilder(256), CultureInfo.InvariantCulture);
+            using (var jsonTextWriter = new JsonTextWriter(stringWriter))
+            {
+                jsonTextWriter.Formatting = k_Formatting;
+                jObject.WriteTo(jsonTextWriter);
+            }
+
+            var contents = stringWriter.ToString();
+
+            var path = Path.Combine(directory, filename);
+            File.WriteAllText(path, contents);
+
+            // TODO what to do about this...
+            Manager.Instance.ConsumerFileProduced(path);
+        }
+
+        public Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
+        {
+            //lazily allocate for fast zero-write frames
+            var capturesJArray = new JArray();
+
+            foreach (var pendingCapture in pendingCaptures)
+                capturesJArray.Add(JObjectFromPendingCapture(pendingCapture));
+
+            var capturesJObject = new JObject();
+            capturesJObject.Add("version", DatasetCapture.SchemaVersion);
+            capturesJObject.Add("captures", capturesJArray);
+
+            Debug.Log("SS - perception - writing");
+
+            WriteJObjectToFile(capturesJObject, outputDirectory, $"captures_{captureFileIndex:000}.json");
+
+            // TODO what to do about this...
+            return null;
+        }
+
+        public Task OnCaptureReported(int frame, int width, int height, string filename)
+        {
+            // do nothing :-)
+            return null;
+        }
+
+        public static JToken JObjectFromPendingCapture(SimulationState.PendingCapture pendingCapture)
+        {
+            var sensorJObject = new JObject();//new SensorCaptureJson
+            sensorJObject["sensor_id"] = pendingCapture.SensorHandle.Id.ToString();
+            sensorJObject["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
+            sensorJObject["modality"] = pendingCapture.SensorData.modality;
+            sensorJObject["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.position);
+            sensorJObject["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.rotation);
+
+            if (pendingCapture.AdditionalSensorValues != null)
+            {
+                foreach (var(name, value) in pendingCapture.AdditionalSensorValues)
+                    sensorJObject.Add(name, DatasetJsonUtility.ToJToken(value));
+            }
+
+            var egoCaptureJson = new JObject();
+            egoCaptureJson["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
+            egoCaptureJson["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.position);
+            egoCaptureJson["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.rotation);
+            egoCaptureJson["velocity"] = pendingCapture.SensorSpatialData.EgoVelocity.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoVelocity.Value) : null;
+            egoCaptureJson["acceleration"] = pendingCapture.SensorSpatialData.EgoAcceleration.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoAcceleration.Value) : null;
+
+            var capture = new JObject();
+            capture["id"] = pendingCapture.Id.ToString();
+            capture["sequence_id"] = pendingCapture.SequenceId.ToString();
+            capture["step"] = pendingCapture.Step;
+            capture["timestamp"] = pendingCapture.Timestamp;
+            capture["sensor"] = sensorJObject;
+            capture["ego"] = egoCaptureJson;
+            capture["filename"] = pendingCapture.Path;
+            capture["format"] = GetFormatFromFilename(pendingCapture.Path);
+
+            if (pendingCapture.Annotations.Any())
+                capture["annotations"] = new JArray(pendingCapture.Annotations.Select(JObjectFromAnnotation).ToArray());
+
+            return capture;
+        }
+
+        public static JObject JObjectFromAnnotation((Annotation, SimulationState.AnnotationData) annotationInfo)
+        {
+            var annotationJObject = new JObject();
+            annotationJObject["id"] = annotationInfo.Item1.Id.ToString();
+            annotationJObject["annotation_definition"] = annotationInfo.Item2.AnnotationDefinition.Id.ToString();
+            if (annotationInfo.Item2.Path != null)
+                annotationJObject["filename"] = annotationInfo.Item2.Path;
+
+            if (annotationInfo.Item2.ValuesJson != null)
+                annotationJObject["values"] = annotationInfo.Item2.ValuesJson;
+
+            return annotationJObject;
+        }
+
+        static string GetFormatFromFilename(string filename)
+        {
+            var ext = Path.GetExtension(filename);
+            if (ext == null)
+                return null;
+
+            if (ext.StartsWith("."))
+                ext = ext.Substring(1);
+
+            return ext.ToUpperInvariant();
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew.meta

+fileFormatVersion: 2
+guid: 8b80f219b9e258043aeae665de9f8d90
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs

+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Unity.Simulation;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew
+{
+    public static class AnnotationHandler
+    {
+        [Serializable]
+        struct BoundingBox2dRecord
+        {
+            public uint instanceId;
+            public int frame;
+            public int labelId;
+            public string labelName;
+            public float x;
+            public float y;
+            public float width;
+            public float height;
+            public string annotationId;
+            public string annotationDefinition;
+
+            public static BoundingBox2dRecord FromBoundingBoxValue(Guid annotationId, Guid annotationDefinition, BoundingBox2DLabeler.BoundingBoxValue bbox)
+            {
+                return new BoundingBox2dRecord
+                {
+                    instanceId = bbox.instance_id,
+                    frame = bbox.frame,
+                    labelId = bbox.label_id,
+                    labelName = bbox.label_name,
+                    x = bbox.x,
+                    y = bbox.y,
+                    width = bbox.width,
+                    height = bbox.height,
+                    annotationId = annotationId.ToString(),
+                    annotationDefinition = annotationDefinition.ToString()
+                };
+            }
+
+            public string ToJson()
+            {
+                return JsonUtility.ToJson(this, true);
+            }
+        }
+
+        public static async Task WriteOutJson(string path, string filename, string json)
+        {
+            if (true)
+            {
+                json = JToken.Parse(json).ToString(Formatting.Indented);
+            }
+
+            var writePath = Path.Combine(path, filename);
+            var file = File.CreateText(writePath);
+
+            await file.WriteAsync(json);
+            file.Close();
+
+            Manager.Instance.ConsumerFileProduced(writePath);
+        }
+
+        static async Task HandleBoundingBoxAnnotation(string path, Annotation annotation, AnnotationDefinition def, BoundingBox2DLabeler.BoundingBoxValue bbox)
+        {
+
+
+            var id = annotation.Id;
+            var defId = def.Id;
+            var converted = BoundingBox2dRecord.FromBoundingBoxValue(id, defId, bbox);
+            var filename = $"frame_{converted.frame}_id_{converted.instanceId}_bounding_box_2d.json";
+            var writePath = Path.Combine(path, filename);
+            var file = File.CreateText(writePath);
+            await file.WriteAsync(converted.ToJson());
+            file.Close();
+            Manager.Instance.ConsumerFileProduced(writePath);
+        }
+
+
+        public static async Task HandleAnnotation(string path, Annotation annotation, AnnotationDefinition def, object annotatedData)
+        {
+            switch (annotatedData)
+            {
+                case BoundingBox2DLabeler.BoundingBoxValue bbox:
+                    await HandleBoundingBoxAnnotation(path, annotation, def, bbox);
+                    break;
+            }
+
+
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs.meta

+fileFormatVersion: 2
+guid: 9c350f412031d374ebe3027d0e8283de
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Unity.Simulation;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew
+{
+    public class PerceptionNewExporter : IDatasetExporter
+    {
+        public bool prettyPrint = true;
+
+        string m_DirectoryName = string.Empty;
+
+        int m_UnknownFrameCount = 0;
+
+        public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
+        {
+            var frameArray = additionalSensorValues.Where(p => p.Item1 == "frame").Select(p => p.Item2);
+            var frame = frameArray.Any() ? (int)frameArray.First() : m_UnknownFrameCount++;
+
+            return Path.Combine(m_DirectoryName, $"rgb_{frame}.png");
+        }
+
+        public void OnSimulationBegin(string directoryName)
+        {
+            Debug.Log($"SS - New Perception - OnSimBegin");
+            m_Metadata = new Metadata
+            {
+                version = "0.0.1",
+                image_width = 0,
+                image_height = 0,
+                dataset_size = 0
+            };
+
+            m_DirectoryName = directoryName + Path.DirectorySeparatorChar + Guid.NewGuid() + Path.DirectorySeparatorChar;
+            if (!Directory.Exists(m_DirectoryName))
+                Directory.CreateDirectory(m_DirectoryName);
+        }
+
+        [Serializable]
+        struct Metadata
+        {
+            public string version;
+            public int image_width;
+            public int image_height;
+            public int dataset_size;
+        }
+
+        Metadata m_Metadata;
+
+        public void OnSimulationEnd()
+        {
+            Debug.Log($"SS - New Perception - OnSimEnd");
+
+            var writePath = Path.Combine(m_DirectoryName, "metadata.json");
+            var file = File.CreateText(writePath);
+
+            Debug.Log("SS - New Perception - writing");
+
+            file.Write(JsonUtility.ToJson(m_Metadata, true));
+            file.Close();
+
+            Manager.Instance.ConsumerFileProduced(writePath);
+
+            Task.WhenAll(m_PendingTasks);
+        }
+
+        public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
+        {
+            // Right now, do nothing :-)
+        }
+
+        static bool GetFilenameForAnnotation(object rawData, out string filename)
+        {
+            filename = string.Empty;
+            if (rawData is BoundingBox2DLabeler.BoundingBoxValue bbox)
+            {
+                var frame = bbox.frame;
+                filename = $"frame_{frame}_bounding_bocx_2d.json";
+                return true;
+            }
+
+            return false;
+        }
+
+        // TODO - handle the 1000's of file writes we will be doing in a more intelligent fashion. Perhaps create a bg thread
+        // that reads json records off of a queue and writes them out
+
+        List<Task> m_PendingTasks = new List<Task>();
+
+        public Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
+        {
+            foreach (var cap in pendingCaptures)
+            {
+                foreach (var (annotation, annotationData) in cap.Annotations)
+                {
+                    // Create a file for the annotation
+
+#if false
+                    if (annotationData.RawValues.Any())
+                    {
+                        var first = annotationData.RawValues.First();
+
+                        if (GetFilenameForAnnotation(first, frame, out var filename))
+                        {
+                            var json = new StringBuilder("{");
+                            json.Append(annotationData.ValuesJson);
+                            json.Append("}");
+                            m_PendingTasks.Add(AnnotationHandler.WriteOutJson(m_DirectoryName, filename, json.ToString()));
+#if false
+                            // Need to revisit this and handle this in a performant way
+                            var jObject = PerceptionExporter.JObjectFromAnnotation((annotation, annotationData));
+                            PerceptionExporter.WriteJObjectToFile(jObject, m_DirectoryName, filename);
+#endif
+                        }
+                    }
+#endif
+                    foreach (var rawValue in annotationData.RawValues)
+                    {
+
+                        if (GetFilenameForAnnotation(rawValue, out var filename))
+                        {
+#if true
+                            var json = new StringBuilder();
+                            json.Append(annotationData.ValuesJson);
+                            m_PendingTasks.Add(AnnotationHandler.WriteOutJson(m_DirectoryName, filename, json.ToString()));
+#else
+                            // Need to revisit this and handle this in a performant way
+                            var jObject = PerceptionExporter.JObjectFromAnnotation((annotation, annotationData));
+                            PerceptionExporter.WriteJObjectToFile(jObject, m_DirectoryName, filename);
+#endif
+                        }
+                    }
+                }
+            }
+            return Task.CompletedTask;
+        }
+
+        public Task OnCaptureReported(int frame, int width, int height, string filename)
+        {
+            m_Metadata.dataset_size++;
+            m_Metadata.image_height = height;
+            m_Metadata.image_width = width;
+            return null;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs.meta

+fileFormatVersion: 2
+guid: 50287c95e34d8354d935c14caf487750
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs

+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters
+{
+    public interface IDatasetExporter
+    {
+        string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues);
+
+        void OnSimulationBegin(string directoryName);
+        void OnSimulationEnd();
+
+        void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values);
+
+        Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState);
+
+        Task OnCaptureReported(int frame, int width, int height, string filename);
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid.meta

+fileFormatVersion: 2
+guid: 71de1979b5988b34d8d7e41110c57c56
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs

+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using UnityEngine.Perception.GroundTruth.Exporters.Coco;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.CocoHybrid
+{
+    public class CocoHybridExporter : IDatasetExporter
+    {
+        CocoExporter m_Coco = new CocoExporter();
+        PerceptionExporter m_Perception = new PerceptionExporter();
+
+        public string GetRgbCaptureFilename(params (string, object)[] additionalSensorValues)
+        {
+            return m_Coco.GetRgbCaptureFilename(additionalSensorValues) + m_Perception.GetRgbCaptureFilename(additionalSensorValues);
+        }
+
+        public void OnSimulationBegin(string directoryName)
+        {
+            m_Coco.OnSimulationBegin(directoryName + "_coco");
+            m_Perception.OnSimulationBegin(directoryName);
+        }
+
+        public void OnSimulationEnd()
+        {
+            m_Coco.OnSimulationEnd();
+            m_Perception.OnSimulationEnd();
+        }
+
+        public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
+        {
+            m_Coco.OnAnnotationRegistered(annotationId, values);
+            m_Perception.OnAnnotationRegistered(annotationId, values);
+        }
+
+        public async Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
+        {
+            var cocoTask = m_Coco.ProcessPendingCaptures(pendingCaptures, simState);
+            var perceptionTask = m_Perception.ProcessPendingCaptures(pendingCaptures, simState);
+            await cocoTask;
+            await perceptionTask;
+        }
+
+        public async Task OnCaptureReported(int frame, int width, int height, string filename)
+        {
+            var cocoTask = m_Coco.OnCaptureReported(frame, width, height, filename);
+            var perceptionTask = m_Perception.OnCaptureReported(frame, width, height, filename);
+            await cocoTask;
+            await perceptionTask;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs.meta

+fileFormatVersion: 2
+guid: 75965f62c369c564b89229b8cbb10d93
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs.meta

+fileFormatVersion: 2
+guid: 0f587bc3e200ea542bd7725e7f64978c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs.meta

+fileFormatVersion: 2
+guid: d25254cc4c0ad734cb76497c70a40088
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs.meta

+fileFormatVersion: 2
+guid: a715e9572e6cc314098dcdf401bb9273
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs

+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
+{
+    public static class CocoTypes
+    {
+        public class CocoData
+        {
+            public Info info;
+            public Image[] images;
+            public ObjectDetectionAnnotation[] annotations;
+            public ObjectDetectionCategory[] categories;
+            public License[] licenses;
+        }
+
+        public class Info
+        {
+            public int year;
+            public string version;
+            public string description;
+            public string contributor;
+            public string url;
+            public string date_created;
+        }
+
+        [Serializable]
+        public class License
+        {
+            public int id;
+            public string name;
+            public string url;
+        }
+
+        [Serializable]
+        public class Licenses
+        {
+            public License[] licenses;
+        }
+
+        public class Image
+        {
+            public int id;
+            public int width;
+            public int height;
+            public string file_name;
+            public int license;
+            public string flickr_url;
+            public string coco_url;
+            public string date_captured;
+        }
+
+        [Serializable]
+        public class ObjectDetectionAnnotation
+        {
+            [JsonProperty(Order = -2)]
+            public int id;
+            [JsonProperty(Order = -2)]
+            public int image_id;
+            [JsonProperty(Order = -2)]
+            public int category_id;
+            [JsonProperty(Order = -2)]
+            public float[] segmentation;
+            [JsonProperty(Order = -2)]
+            public float area;
+            [JsonProperty(Order = -2)]
+            public float[] bbox;
+            [JsonProperty(Order = -2)]
+            public int iscrowd;
+
+            public static ObjectDetectionAnnotation FromBoundingBoxValue(BoundingBox2DLabeler.BoundingBoxValue bbox)
+            {
+                return new ObjectDetectionAnnotation
+                {
+                    id = (int)bbox.instance_id,
+                    image_id = bbox.frame,
+                    category_id = bbox.label_id,
+                    segmentation = new float[]{},
+                    area = bbox.width * bbox.height,
+                    bbox = new []{bbox.x, bbox.y, bbox.width, bbox.height},
+                    iscrowd = 0
+                };
+            }
+        }
+
+        [Serializable]
+        public class ObjectDetectionCategory
+        {
+            [JsonProperty(Order = -2)]
+            public int id;
+            [JsonProperty(Order = -2)]
+            public string name = string.Empty;
+            [JsonProperty(Order = -2)]
+            public string supercategory = string.Empty;
+        }
+
+        [Serializable]
+        public class ObjectDetectionCategories
+        {
+            public List<ObjectDetectionCategory> categories;
+        }
+
+        public class KeypointAnnotation : ObjectDetectionAnnotation
+        {
+            public int num_keypoints;
+            public float[] keypoints;
+
+
+            public void CopyObjectDetectionData(ObjectDetectionAnnotation objDetection)
+            {
+                if (objDetection.id == this.id)
+                {
+                    image_id = objDetection.image_id;
+                    area = objDetection.area;
+                    bbox = objDetection.bbox;
+                    iscrowd = objDetection.iscrowd;
+                }
+            }
+
+            public static KeypointAnnotation FromKeypointValue(KeypointLabeler.KeypointEntry keypoint)
+            {
+                var outKeypoint = new KeypointAnnotation()
+                {
+                    id = (int)keypoint.instance_id,
+                    image_id = keypoint.frame,
+                    category_id = keypoint.label_id,
+                    segmentation = new float[]{},
+                    area = 0,
+                    bbox = new []{0f},
+                    iscrowd = 0,
+                    num_keypoints = keypoint.keypoints.Length,
+                    keypoints = new float[keypoint.keypoints.Length * 3]
+                };
+
+                var i = 0;
+                foreach (var k in keypoint.keypoints)
+                {
+                    outKeypoint.keypoints[i++] = k.x;
+                    outKeypoint.keypoints[i++] = k.y;
+                    outKeypoint.keypoints[i++] = k.state;
+                }
+
+                return outKeypoint;
+            }
+        }
+
+        [Serializable]
+        public class KeypointCategory : ObjectDetectionCategory
+        {
+            public string[] keypoints;
+            public int[][] skeleton;
+        }
+
+
+
+        [Serializable]
+        public class KeypointCategories
+        {
+            public KeypointCategory[] categories;
+        }
+    }
+
+
+}

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs

+using System;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
+{
+    public static class AnnotationHandler
+    {
+        public static object HandleAnnotation(AsyncAnnotation asyncAnnotation, object annotation)
+        {
+            switch (annotation)
+            {
+                case BoundingBox2DLabeler.BoundingBoxValue bbox:
+                    return CocoTypes.ObjectDetectionAnnotation.FromBoundingBoxValue(bbox);
+                case KeypointLabeler.KeypointEntry keypoint:
+                    return CocoTypes.KeypointAnnotation.FromKeypointValue(keypoint);
+            }
+
+            return null;
+        }
+
+        public static CocoTypes.KeypointCategory ToKeypointCategory(KeypointLabeler.KeypointJson keypointJson)
+        {
+            var keypoints = new string[keypointJson.key_points.Length];
+            var skeleton = new int[keypointJson.skeleton.Length][];
+
+            foreach (var kp in keypointJson.key_points)
+            {
+                keypoints[kp.index] = kp.label;
+            }
+
+            var i = 0;
+            foreach (var bone in keypointJson.skeleton)
+            {
+                var joints = new int[]
+                {
+                    bone.joint1,
+                    bone.joint2
+                };
+                skeleton[i++] = joints;
+            }
+
+            return new CocoTypes.KeypointCategory
+            {
+                id = keypointJson.label_id,
+                name = keypointJson.label_name,
+                supercategory = keypointJson.label_name,
+                keypoints = keypoints,
+                skeleton = skeleton
+            };
+        }
+
+
+        public static object HandleCameraCapture(int id, int width, int height, string filename)
+        {
+            var image = new CocoTypes.Image()
+            {
+                id = id,
+                width = width,
+                height = height,
+                file_name = filename,
+                license = 0,
+                flickr_url = "",
+                coco_url = "",
+                date_captured =  DateTime.Today.ToString("D")
+            };
+
+            return image;
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Unity.Simulation;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
+{
+    public class CocoExporter : IDatasetExporter
+    {
+        bool m_PrettyPrint = true;
+
+        bool m_ReportingObjectDetection;
+        bool m_ReportingKeypoints;
+
+        bool m_Initialized;
+        string m_DirectoryName = string.Empty;
+
+        string m_RgbCaptureFilename;
+        StreamWriter m_RgbCaptureStream;
+
+        string m_ObjectDetectionFilename;
+        StreamWriter m_ObjectDetectionStream;
+        Task m_ObjectDetectionWritingTask;
+
+        string m_KeypointFilename;
+        StreamWriter m_KeypointDetectionStream;
+        Task m_KeypointDetectionWritingTask;
+
+        CocoTypes.ObjectDetectionCategories m_ObjectDetectionCategories;
+        string m_ObjectDetectionCategoryFilename;
+        StreamWriter m_ObjectDetectionCategoryStream;
+        Task m_ObjectDetectionCategoryWritingTask;
+
+        CocoTypes.KeypointCategories m_KeypointCategories;
+        string m_KeypointCategoryFilename;
+        StreamWriter m_KeypointCategoryStream;
+        Task m_KeypointCategoryWritingTask;
+
+        Guid m_SessionGuid;
+
+        public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
+        {
+            return string.Empty;
+        }
+
+        public void OnSimulationBegin(string directoryName)
+        {
+            Debug.Log($"SS - COCO - OnSimBegin");
+            m_DirectoryName = directoryName;
+            m_DataCaptured = false;
+        }
+
+        async Task AwaitAllWrites()
+        {
+            Debug.Log("SS - coco - writing");
+
+            WriteOutCategories();
+
+            if (m_ObjectDetectionWritingTask != null)
+            {
+                await m_ObjectDetectionWritingTask;
+                await m_ObjectDetectionStream.WriteAsync("]");
+            }
+
+            if (m_KeypointDetectionWritingTask != null)
+            {
+                await m_KeypointDetectionWritingTask;
+                await m_KeypointDetectionStream.WriteAsync("]");
+            }
+
+            if (m_RgbCaptureWritingTask != null)
+            {
+                await m_RgbCaptureWritingTask;
+                await m_RgbCaptureStream.WriteAsync("]");
+            }
+
+            if (m_ObjectDetectionCategoryWritingTask != null)
+            {
+                await m_ObjectDetectionCategoryWritingTask;
+            }
+
+            if (m_KeypointCategoryWritingTask != null)
+            {
+                await m_KeypointCategoryWritingTask;
+            }
+
+            m_RgbCaptureStream?.Close();
+            m_ObjectDetectionStream?.Close();
+            m_ObjectDetectionCategoryStream?.Close();
+            m_KeypointDetectionStream?.Close();
+            m_KeypointCategoryStream?.Close();
+        }
+
+
+        public async void OnSimulationEnd()
+        {
+            Debug.Log($"SS - COCO - OnSimEnd");
+            if (!m_DataCaptured) return;
+
+            await AwaitAllWrites();
+
+            if (m_ReportingObjectDetection)
+            {
+                await WriteObjectDetectionFile();
+            }
+
+            if (m_ReportingKeypoints)
+            {
+                await WriteKeypointFile();
+            }
+
+            File.Delete(m_RgbCaptureFilename);
+
+            m_Initialized = false;
+
+        }
+
+        void InitializeCaptureFiles()
+        {
+            if (m_Initialized) return;
+
+            if (!Directory.Exists(m_DirectoryName))
+                Directory.CreateDirectory(m_DirectoryName);
+
+            m_SessionGuid = Guid.NewGuid();
+
+            //var prefix = m_DirectoryName + Path.DirectorySeparatorChar + m_SessionGuid;
+
+
+            m_RgbCaptureFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_captures.json");
+            m_RgbCaptureStream = File.CreateText(m_RgbCaptureFilename);
+
+            m_ObjectDetectionFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_box_annotations.json");
+            if (m_ReportingObjectDetection)
+            {
+                m_ObjectDetectionStream = File.CreateText(m_ObjectDetectionFilename);
+                m_ObjectDetectionCategoryFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_obj_detection_categories.json");
+            }
+
+            m_KeypointFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_keypoint_annotations.json");
+            if (m_ReportingKeypoints)
+            {
+                m_KeypointDetectionStream = File.CreateText(m_KeypointFilename);
+                m_KeypointCategoryFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_keypoint_categories.json");
+            }
+
+            m_Initialized = true;
+        }
+
+        static void AggregateFile(string filename, StringBuilder aggregated, bool skipFirstCharacter = false)
+        {
+            var sr = new StreamReader(filename);
+
+            var length = (int)sr.BaseStream.Length;
+            var start = 0;
+
+            if (length == 0) return;
+
+            var buffer = new char[length];
+            sr.Read(buffer, start, length);
+
+            if (skipFirstCharacter)
+            {
+                length--;
+                start++;
+            }
+
+            aggregated.Append(buffer, start, length);
+
+            sr.Dispose();
+        }
+
+        async Task WriteObjectDetectionFile()
+        {
+            var stringBuilder = new StringBuilder();
+
+            stringBuilder.Append("{");
+
+            // Create the json header
+            CreateHeaderInfo(stringBuilder);
+            stringBuilder.Append(",");
+
+            CreateLicenseInfo(stringBuilder);
+
+            // Read in the contents of the captures
+            stringBuilder.Append(",\"images\":");
+            AggregateFile(m_RgbCaptureFilename, stringBuilder);
+
+            // Read in the contents of the object detection
+            stringBuilder.Append(",\"annotations\":");
+            AggregateFile(m_ObjectDetectionFilename, stringBuilder);
+            stringBuilder.Append(",");
+
+            // Read in the contents of the object detection categories
+            AggregateFile(m_ObjectDetectionCategoryFilename, stringBuilder, true);
+
+            var json = stringBuilder.ToString();
+
+            if (m_PrettyPrint)
+            {
+                json = JToken.Parse(json).ToString(Formatting.Indented);
+            }
+
+            Debug.Log($"SS - COCO - writing to path: {m_DirectoryName}, file: coco_object_detection_annotations.json");
+
+            // Write out the files
+            var filename = Path.Combine(m_DirectoryName, "coco_object_detection_annotations.json");
+
+            Debug.Log($"SS - COCO - file: {filename}");
+
+
+            var cocoStream = File.CreateText(filename);
+            await cocoStream.WriteAsync(json);
+            cocoStream.Close();
+
+            Manager.Instance.ConsumerFileProduced(filename);
+
+            File.Delete(m_ObjectDetectionFilename);
+            File.Delete(m_ObjectDetectionCategoryFilename);
+        }
+
+        async Task WriteKeypointFile()
+        {
+            var stringBuilder = new StringBuilder();
+
+            stringBuilder.Append("{");
+
+            // Create the json header
+            CreateHeaderInfo(stringBuilder);
+            stringBuilder.Append(",");
+
+            CreateLicenseInfo(stringBuilder);
+
+            // Read in the contents of the captures
+            stringBuilder.Append(",\"images\":");
+            AggregateFile(m_RgbCaptureFilename, stringBuilder);
+
+            // Read in the contents of the object detection
+            stringBuilder.Append(",\"annotations\":");
+            AggregateFile(m_KeypointFilename, stringBuilder);
+            stringBuilder.Append(",");
+
+            // Read in the contents of the object detection categories
+            AggregateFile(m_KeypointCategoryFilename, stringBuilder, true);
+
+            var json = stringBuilder.ToString();
+
+            if (m_PrettyPrint)
+            {
+                json = JToken.Parse(json).ToString(Formatting.Indented);
+            }
+
+            // Write out the files
+            var filename = Path.Combine(m_DirectoryName, "coco_keypoint_annotations.json");
+            var cocoStream = File.CreateText(filename);
+            await cocoStream.WriteAsync(json);
+            cocoStream.Dispose();
+
+            Manager.Instance.ConsumerFileProduced(filename);
+
+            File.Delete(m_KeypointFilename);
+            File.Delete(m_KeypointCategoryFilename);
+        }
+
+        bool m_DataCaptured;
+
+        void WriteOutCategories()
+        {
+            // 3 cases
+            //   1) Just have object detection
+            //   2) Just have keypoint detection
+            //   3) Have both, which includes writing out object detection in object detection coco output
+            //      and merging the object detection & keypoints and writing out that data in the keypoint
+            //      coco output data
+            if (m_ReportingObjectDetection)
+            {
+                m_ObjectDetectionCategoryStream = File.CreateText(m_ObjectDetectionCategoryFilename);
+                var json = JsonUtility.ToJson(m_ObjectDetectionCategories);
+                m_ObjectDetectionCategoryWritingTask = m_ObjectDetectionCategoryStream.WriteAsync(json);
+            }
+
+            if (m_ReportingKeypoints)
+            {
+                // TODO - revisit this, but right now we are going to add the keypoint info to all of the categories,
+                // we can get away with this because we only support one set of keypoint definitions per simulation
+                // currently.
+
+                if (m_KeypointCategories.categories.Length < 1) return;
+
+                m_KeypointCategoryStream = File.CreateText(m_KeypointCategoryFilename);
+
+                var merged = m_KeypointCategories;
+
+                if (m_ReportingObjectDetection)
+                {
+                    merged = new CocoTypes.KeypointCategories
+                    {
+                        categories = new CocoTypes.KeypointCategory[m_ObjectDetectionCategories.categories.Count]
+                    };
+
+                    var i = 0;
+                    foreach (var odc in m_ObjectDetectionCategories.categories)
+                    {
+                        merged.categories[i++] = MergeCategories(odc, m_KeypointCategories.categories[0]);
+                    }
+                }
+
+#if false
+                var json = JsonUtility.ToJson(merged);
+#else
+                var builder = new StringBuilder();
+                var sw = new StringWriter(builder);
+
+                var serializer = new JsonSerializer();
+
+                using (var writer = new JsonTextWriter(sw))
+                {
+                    serializer.Serialize(writer, merged);
+                }
+
+                var json = builder.ToString();
+#endif
+                m_KeypointCategoryWritingTask = m_KeypointCategoryStream.WriteAsync(json);
+            }
+        }
+
+        public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
+        {
+            if (annotationId.ToString() == BoundingBox2DLabeler.annotationIdForCocoExport)
+            {
+                m_ReportingObjectDetection = true;
+                m_ObjectDetectionCategories = new CocoTypes.ObjectDetectionCategories
+                {
+                    categories = new List<CocoTypes.ObjectDetectionCategory>()
+                };
+
+                foreach (var value in values)
+                {
+                    if (value is IdLabelConfig.LabelEntrySpec spec)
+                    {
+                        var rec = new CocoTypes.ObjectDetectionCategory
+                        {
+                            id = spec.label_id,
+                            name = spec.label_name,
+                            supercategory = spec.label_name
+                        };
+
+                        m_ObjectDetectionCategories.categories.Add(rec);
+                    }
+                }
+            }
+
+            if (annotationId.ToString() == KeypointLabeler.annotationIdForCocoExport)
+            {
+                m_ReportingKeypoints = true;
+                var categories = new CocoTypes.KeypointCategory[values.Length];
+                var i = 0;
+
+                foreach (var value in values)
+                {
+                    if (value is KeypointLabeler.KeypointJson keypointJson)
+                    {
+                        categories[i++] = AnnotationHandler.ToKeypointCategory(keypointJson);
+                    }
+                }
+
+                m_KeypointCategories = new CocoTypes.KeypointCategories
+                {
+                    categories = categories
+                };
+            }
+        }
+
+
+        static string versionEntry = "0.0.1";
+        static string descriptionEntry = "Description of dataset";
+        static string contributorEntry = "Anonymous";
+        static string urlEntry = "Not Set";
+
+        static void CreateHeaderInfo(StringBuilder stringBuilder)
+        {
+            stringBuilder.Append("\"info\":");
+
+            var dateTime = DateTime.Today;
+            var info = new CocoTypes.Info
+            {
+                year = int.Parse(dateTime.ToString("yyyy")),
+                version = versionEntry,
+                description = descriptionEntry,
+                contributor = contributorEntry,
+                url = urlEntry,
+                date_created = DateTime.Today.ToString("D")
+            };
+            stringBuilder.Append(JsonUtility.ToJson(info));
+        }
+
+        static void CreateLicenseInfo(StringBuilder stringBuilder)
+        {
+            var licenses = new CocoTypes.Licenses
+            {
+                licenses = new[]
+                {
+                    new CocoTypes.License
+                    {
+                        id = 0,
+                        name = "No License",
+                        url = "Not Set"
+                    }
+                }
+            };
+
+            var tmpJson = JsonUtility.ToJson(licenses);
+
+            // Remove the start and end '{' from the licenses json
+            stringBuilder.Append(tmpJson.Substring(1, tmpJson.Length - 2));
+        }
+
+        bool m_FirstBoxAnnotation = true;
+        bool m_FirstKeypointAnnotation = true;
+
+        bool m_FirstCapture = true;
+        Task m_RgbCaptureWritingTask;
+
+        static CocoTypes.KeypointCategory MergeCategories(CocoTypes.ObjectDetectionCategory od, CocoTypes.KeypointCategory kp)
+        {
+            return new CocoTypes.KeypointCategory
+            {
+                id = od.id,
+                name = od.name,
+                supercategory = od.supercategory,
+                keypoints = kp.keypoints,
+                skeleton = kp.skeleton
+            };
+        }
+
+        public async Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
+        {
+            var boxJson = string.Empty;
+            var keypointJson = string.Empty;
+
+            foreach (var cap in pendingCaptures)
+            {
+                var boxes = new Dictionary<int, CocoTypes.ObjectDetectionAnnotation>();
+
+                foreach (var annotation in cap.Annotations)
+                {
+                    var tmp = ProcessBoundingBoxAnnotations(annotation.Item2.RawValues);
+
+                    foreach (var box in tmp.Values)
+                    {
+                        boxes[box.id] = box;
+
+                        if (m_FirstBoxAnnotation)
+                        {
+                            boxJson = "[";
+                            m_FirstBoxAnnotation = false;
+                        }
+                        else
+                            boxJson += ",";
+
+                        boxJson += JsonUtility.ToJson(box);
+                    }
+
+                }
+
+                foreach (var annotation in cap.Annotations)
+                {
+                    var keypoints = ProcessKeypointAnnotations(annotation.Item2.RawValues, boxes);
+
+                    foreach (var kp in keypoints.Values)
+                    {
+                        if (m_FirstKeypointAnnotation)
+                        {
+                            keypointJson = "[";
+                            m_FirstKeypointAnnotation = false;
+                        }
+                        else
+                            keypointJson += ",";
+
+                        var builder = new StringBuilder();
+                        var stringWriter = new StringWriter(builder);
+
+                        var serializer = new JsonSerializer();
+
+                        using (var writer = new JsonTextWriter(stringWriter))
+                        {
+                            serializer.Serialize(writer, kp);
+                        }
+
+                        keypointJson += builder.ToString();
+
+                    }
+                }
+            }
+
+            if (m_ObjectDetectionWritingTask != null)
+                await m_ObjectDetectionWritingTask;
+
+            if (boxJson != string.Empty)
+            {
+                InitializeCaptureFiles();
+                m_ObjectDetectionWritingTask = m_ObjectDetectionStream.WriteAsync(boxJson);
+            }
+
+            if (m_KeypointDetectionWritingTask != null)
+                await m_KeypointDetectionWritingTask;
+
+            if (keypointJson != string.Empty)
+            {
+                InitializeCaptureFiles();
+                m_KeypointDetectionWritingTask = m_KeypointDetectionStream.WriteAsync(keypointJson);
+            }
+        }
+
+        static Dictionary<int, CocoTypes.ObjectDetectionAnnotation> ProcessBoundingBoxAnnotations(IEnumerable<object> annotations)
+        {
+            var map = new Dictionary<int, CocoTypes.ObjectDetectionAnnotation>();
+            foreach (var annotation in annotations)
+            {
+                if (annotation is BoundingBox2DLabeler.BoundingBoxValue bbox)
+                {
+                    var coco = CocoTypes.ObjectDetectionAnnotation.FromBoundingBoxValue(bbox);
+                    map[coco.id] = coco;
+                }
+            }
+
+            return map;
+        }
+
+        static Dictionary<int, CocoTypes.KeypointAnnotation> ProcessKeypointAnnotations(IEnumerable<object> annotations, Dictionary<int, CocoTypes.ObjectDetectionAnnotation> boundingBoxes)
+        {
+            var map = new Dictionary<int, CocoTypes.KeypointAnnotation>();
+
+            foreach (var annotation in annotations)
+            {
+                if (annotation is KeypointLabeler.KeypointEntry keypoint)
+                {
+                    var coco = CocoTypes.KeypointAnnotation.FromKeypointValue(keypoint);
+                    if (boundingBoxes.ContainsKey(coco.id))
+                    {
+                        coco.CopyObjectDetectionData(boundingBoxes[coco.id]);
+                    }
+
+                    map[coco.id] = coco;
+                }
+            }
+
+            return map;
+        }
+
+        public async Task OnCaptureReported(int frame, int width, int height, string filename)
+        {
+            InitializeCaptureFiles();
+
+            var json = string.Empty;
+
+            var converted = AnnotationHandler.HandleCameraCapture(frame, width, height, filename);
+            if (m_FirstCapture)
+            {
+                json = "[";
+                m_FirstCapture = false;
+            }
+            else
+                json += ",";
+
+            json += JsonUtility.ToJson(converted);
+
+            if (m_RgbCaptureWritingTask != null)
+                await m_RgbCaptureWritingTask;
+
+            if (json != string.Empty)
+                m_RgbCaptureWritingTask = m_RgbCaptureStream.WriteAsync(json);
+
+            m_DataCaptured = true;
+        }
+    }
+}