perception output and UI working

com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs

 
         PerceptionCamera perceptionCamera => ((PerceptionCamera)this.target);
 
+        OutputMode m_OutputMode = OutputMode.Perception;
+
         public void OnEnable()
         {
             m_LabelersList = new ReorderableList(this.serializedObject, labelersProperty, true, true, true, true);
             m_LabelersList.drawElementCallback = DrawElement;
             m_LabelersList.onAddCallback += OnAdd;
             m_LabelersList.onRemoveCallback += OnRemove;
+
+            m_OutputMode = (OutputMode)PlayerPrefs.GetInt(SimulationState.outputFormatMode, 0);
         }
 
         float GetElementHeight(int index)
 
         const string k_FrametimeTitle = "Simulation Delta Time";
         const float k_DeltaTimeTooLarge = 200;
+
         public override void OnInspectorGUI()
         {
             using(new EditorGUI.DisabledScope(EditorApplication.isPlaying))
                 EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureRgbImages)),new GUIContent("Save Camera RGB Output to Disk", "For each captured frame, save an RGB image of the camera's output to disk."));
+
+                var mode = (OutputMode)EditorGUILayout.EnumPopup("Output Format", m_OutputMode);
+
+                if (mode != m_OutputMode)
+                {
+                    m_OutputMode = mode;
+                    PlayerPrefs.SetInt(SimulationState.outputFormatMode, (int)m_OutputMode);
+                }
+
                 EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureTriggerMode)),new GUIContent("Capture Trigger Mode", $"The method of triggering captures for this camera. In {nameof(CaptureTriggerMode.Scheduled)} mode, captures happen automatically based on a start frame and frame delta time. In {nameof(CaptureTriggerMode.Manual)} mode, captures should be triggered manually through calling the {nameof(perceptionCamera.RequestCapture)} method of {nameof(PerceptionCamera)}."));
 
                 GUILayout.Space(5);

com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs

         }
     }
 
+    public enum OutputMode
+    {
+        Perception,
+        COCO
+    }
 
     /// <summary>
     /// Capture trigger modes for sensors.

com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs

             }
         }
 
+        public static string versionEntry = "0.0.1";
+        public static string descriptionEntry = "Description of dataset";
+        public static string contributorEntry = "Anonymous";
+        public static string urlEntry = "Not Set";
+
+        public static CocoTypes.License[] licenses = new []
+        {
+            new CocoTypes.License
+            {
+                id = 0,
+                name = "No License",
+                url = "Not Set"
+            }
+        };
+
         static void CreateHeaderInfo(StringBuilder stringBuilder)
         {
             stringBuilder.Append("\"info\":");
             {
                 year = int.Parse(dateTime.ToString("yyyy")),
-                version = "0.0.1",
-                description = "temp output of coco data",
-                contributor = "Tyler Durden",
-                url = "https://ytmnd.com",
+                version = versionEntry,
+                description = descriptionEntry,
+                contributor = contributorEntry,
+                url = urlEntry,
                 date_created = DateTime.Today.ToString("D")
             };
             stringBuilder.Append(JsonUtility.ToJson(info));
         {
-            var licenses = new CocoTypes.Licenses()
-            {
-                licenses = new[]
-                {
-                    new CocoTypes.License
-                    {
-                        id = 0,
-                        name = "Unity License",
-                        url = "https://unity3d.com"
-                    }
-                }
-            };
-
             var tmpJson = JsonUtility.ToJson(licenses);
 
             // Remove the start and end '{' from the licenses json

com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs

 
         [SuppressMessage("ReSharper", "InconsistentNaming")]
         [SuppressMessage("ReSharper", "NotAccessedField.Local")]
-        internal struct BoundingBoxValue
+        public struct BoundingBoxValue
+            public int frame;
             public string label_name;
             public uint instance_id;
             public float x;
         /// <summary>
         /// The GUID id to associate with the annotations produced by this labeler.
         /// </summary>
-        public string annotationId = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
+        public static string annotationId = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
         /// <summary>
         /// The <see cref="IdLabelConfig"/> which associates objects with labels.
         /// </summary>
                     m_BoundingBoxValues.Add(new BoundingBoxValue
                     {
                         label_id = labelEntry.id,
+                        frame = frameCount,
                         label_name = labelEntry.label,
                         instance_id = objectInfo.instanceId,
                         x = objectInfo.boundingBox.x,

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

         /// <summary>
         /// The GUID id to associate with the annotations produced by this labeler.
         /// </summary>
-        public string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
+        public static string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
         /// <summary>
         /// The <see cref="IdLabelConfig"/> which associates objects with labels.
         /// </summary>
         // ReSharper disable InconsistentNaming
         // ReSharper disable NotAccessedField.Local
         [Serializable]
-        struct JointJson
+        public struct JointJson
         {
             public string label;
             public int index;
         [Serializable]
-        struct SkeletonJson
+        public struct SkeletonJson
         {
             public int joint1;
             public int joint2;
         [Serializable]
-        struct KeypointJson
+        public struct KeypointJson
         {
             public string template_id;
             public string template_name;

com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs

         }
 
         [SuppressMessage("ReSharper", "InconsistentNaming")]
-        internal struct LabelEntrySpec
+        public struct LabelEntrySpec
         {
             /// <summary>
             /// The label id prepared for reporting in the annotation

com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs

 #endif
 #if URP_PRESENT
 using UnityEngine.Rendering.Universal;
+#pragma warning disable 649
 #endif
 
 namespace UnityEngine.Perception.GroundTruth
         /// The method of triggering captures for this camera.
         /// </summary>
         public CaptureTriggerMode captureTriggerMode = CaptureTriggerMode.Scheduled;
+
+        public OutputMode outputMode = OutputMode.Perception;
 
         /// <summary>
         /// Have this unscheduled (manual capture) camera affect simulation timings (similar to a scheduled camera) by
 
             Profiler.BeginSample("CaptureDataFromLastFrame");
 
+            var width = cam.pixelWidth;
+            var height = cam.pixelHeight;
+
+            var frameCount = Time.frameCount;
+            var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{frameCount}.png";
+
+            SetPersistentSensorData("camera_width", width);
+            SetPersistentSensorData("camera_height", height);
+            SetPersistentSensorData("full_path", captureFilename);
+            SetPersistentSensorData("frame", frameCount);
-            var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{Time.frameCount}.png";
-            var dxRootPath = $"{rgbDirectory}/{k_RgbFilePrefix}{Time.frameCount}.png";
+            var dxRootPath = $"{rgbDirectory}/{k_RgbFilePrefix}{frameCount}.png";
-            var width = cam.pixelWidth;
-            var height = cam.pixelHeight;
 
             colorFunctor = r =>
             {

com.unity.perception/Runtime/GroundTruth/SimulationState.cs

 using Unity.Collections;
 using Unity.Simulation;
 using UnityEngine;
+using UnityEngine.Perception.GroundTruth.Exporters;
+using UnityEngine.Perception.GroundTruth.Exporters.Coco;
+using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
-    partial class SimulationState
+    public partial class SimulationState
     {
         HashSet<SensorHandle> m_ActiveSensors = new HashSet<SensorHandle>();
         Dictionary<SensorHandle, SensorData> m_Sensors = new Dictionary<SensorHandle, SensorData>();
+
+        IDatasetReporter m_ActiveReporter = new PerceptionExporter();
 
         // Always use the property SequenceTimeMs instead
         int m_FrameCountLastUpdatedSequenceTime;
         float m_LastTimeScale;
         readonly string m_OutputDirectoryName;
         string m_OutputDirectoryPath;
+
+        public const string outputFormatMode = "outputFormatMode";
 
         public bool IsRunning { get; private set; }
 
 
         public SimulationState(string outputDirectory)
         {
+            var mode = (OutputMode)PlayerPrefs.GetInt(outputFormatMode, 0);
+
+            m_ActiveReporter = mode switch
+            {
+                OutputMode.Perception => new PerceptionExporter(),
+                OutputMode.COCO => new CocoExporter(),
+                _ => m_ActiveReporter
+            };
+
             PlayerPrefs.SetString(defaultOutputBaseDirectory, Configuration.Instance.GetStorageBasePath());
             m_OutputDirectoryName = outputDirectory;
             var basePath = PlayerPrefs.GetString(userBaseDirectoryKey, string.Empty);
                     basePath = Configuration.localPersistentDataPath;
                 }
             }
+/*
+            //var activeReporterString = PlayerPrefs.GetString(activeReporterKey, defaultReporter);
+            var activeReporterString = "coco";
+            if (activeReporterString == "perceptionOutput")
+            {
+                m_ActiveReporter = new PerceptionExporter();
+            }
+            else
+            {
+                m_ActiveReporter = new CocoExporter();
+            }
+*/
+
+            m_ActiveReporter?.OnSimulationBegin(Manager.Instance.GetDirectoryFor(m_OutputDirectoryName));
         }
 
         /// <summary>
-        class PendingCapture
+        public class PendingCapture
         {
             public Guid Id;
             public SensorHandle SensorHandle;
             public EgoHandle egoHandle;
         }
 
-        struct AnnotationData
+        public struct AnnotationData
+            public IEnumerable<object> RawValues;
             public bool IsAssigned => Path != null || ValuesJson != null;
 
             public AnnotationData(AnnotationDefinition annotationDefinition, string path, JArray valuesJson)
 
             sensorData.lastCaptureFrameCount = Time.frameCount;
             m_Sensors[sensorHandle] = sensorData;
+
+            // SB - maybe this can all be moved to the other capture area
+            var width = -1;
+            var height = -1;
+            var fullPath = filename;
+            var frameCount = 0;
+
+            foreach (var i in additionalSensorValues)
+            {
+                switch (i.Item1)
+                {
+                    case "camera_width":
+                        width = (int)i.Item2;
+                        break;
+                    case "camera_height":
+                        height = (int)i.Item2;
+                        break;
+                    case "full_path":
+                        fullPath = (string)i.Item2;
+                        break;
+                    case "frame":
+                        frameCount = (int)i.Item2;
+                        break;
+                }
+            }
+
+            m_ActiveReporter.OnCaptureReported(frameCount, width, height, filename);
         }
 
         static string GetFormatFromFilename(string filename)
             if (m_PendingMetrics.Count > 0)
                 Debug.LogError($"Simulation ended with pending metrics: {string.Join(", ", m_PendingMetrics.Select(c => $"id:{c.MetricId} step:{c.Step}"))}");
 
+            if (m_AdditionalInfoTypeData.Any())
+            {
+                List<IdLabelConfig.LabelEntrySpec> labels = new List<IdLabelConfig.LabelEntrySpec>();
+
+                foreach (var infoTypeData in m_AdditionalInfoTypeData)
+                {
+                    if (infoTypeData.specValues == null) continue;
+
+                    foreach (var spec in infoTypeData.specValues)
+                    {
+                        if (spec is IdLabelConfig.LabelEntrySpec entrySpec)
+                        {
+                            labels.Add(entrySpec);
+                        }
+                    }
+
+                    Debug.Log($"adt: {infoTypeData}");
+
+                }
+            }
+
+
+            m_ActiveReporter?.OnSimulationEnd();
         }
 
         public AnnotationDefinition RegisterAnnotationDefinition<TSpec>(string name, TSpec[] specValues, string description, string format, Guid id)
 
             RegisterAdditionalInfoType(name, specValues, description, format, id, AdditionalInfoKind.Annotation);
+
+            m_ActiveReporter.OnAnnotationRegistered(id, specValues); // <- Not sure about this one either
 
             return new AnnotationDefinition(id);
         }
                 }
             }
 
-            ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray);
+            var values2 = new List<object>();
+            foreach (var v in values)
+            {
+                values2.Add(v);
+            }
+            ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray, values:values2);
-        void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray)
+        void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray, IEnumerable<object> values = null)
         {
             if (!asyncAnnotation.IsPending)
                 throw new InvalidOperationException("AsyncAnnotation has already been reported and cannot be reported again.");
 
             annotationData.Path = filename;
             annotationData.ValuesJson = jArray;
+            annotationData.RawValues = values;
 
             annotationTuple.Item2 = annotationData;
             pendingCapture.Annotations[annotationIndex] = annotationTuple;

com.unity.perception/Runtime/GroundTruth/SimulationState_Json.cs

 
 namespace UnityEngine.Perception.GroundTruth
 {
-    partial class SimulationState
+    public partial class SimulationState
     {
         const Formatting k_Formatting = Formatting.Indented;
 
 
             void Write(List<PendingCapture> pendingCaptures, SimulationState simulationState, int captureFileIndex)
             {
+                m_ActiveReporter.ProcessPendingCaptures(pendingCaptures, simulationState);
+#if false
                 simulationState.m_SerializeCapturesAsyncSampler.Begin();
 
                 //lazily allocate for fast zero-write frames
                 simulationState.WriteJObjectToFile(capturesJObject,
                     $"captures_{captureFileIndex:000}.json");
                 simulationState.m_SerializeCapturesAsyncSampler.End();
+#endif
             }
 
             if (flush)

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat.meta

+fileFormatVersion: 2
+guid: a40dd8c5379f64748a0942b0475badc6
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs

+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Unity.Simulation;
+
+namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat
+{
+    public class PerceptionExporter : IDatasetReporter
+    {
+        const Formatting k_Formatting = Formatting.Indented;
+        string outputDirectory = string.Empty;
+        int captureFileIndex = 0;
+
+        public void OnSimulationBegin(string directoryName)
+        {
+            outputDirectory = directoryName;
+        }
+
+        public void OnSimulationEnd()
+        {
+            // do nothing :-)
+        }
+
+        public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
+        {
+            // do nothing :-)
+        }
+
+        void WriteJObjectToFile(JObject jObject, string filename)
+        {
+            var stringWriter = new StringWriter(new StringBuilder(256), CultureInfo.InvariantCulture);
+            using (var jsonTextWriter = new JsonTextWriter(stringWriter))
+            {
+                jsonTextWriter.Formatting = k_Formatting;
+                jObject.WriteTo(jsonTextWriter);
+            }
+
+            var contents = stringWriter.ToString();
+
+            var path = Path.Combine(outputDirectory, filename);
+            File.WriteAllText(path, contents);
+
+            // TODO what to do about this...
+            Manager.Instance.ConsumerFileProduced(path);
+        }
+
+        public Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
+        {
+            //lazily allocate for fast zero-write frames
+            var capturesJArray = new JArray();
+
+            foreach (var pendingCapture in pendingCaptures)
+                capturesJArray.Add(JObjectFromPendingCapture(pendingCapture));
+
+            var capturesJObject = new JObject();
+            capturesJObject.Add("version", DatasetCapture.SchemaVersion);
+            capturesJObject.Add("captures", capturesJArray);
+
+            WriteJObjectToFile(capturesJObject, $"captures_{captureFileIndex:000}.json");
+
+            // TODO what to do about this...
+            return null;
+        }
+
+        public Task OnCaptureReported(int frame, int width, int height, string filename)
+        {
+            // do nothing :-)
+            return null;
+        }
+
+        static JToken JObjectFromPendingCapture(SimulationState.PendingCapture pendingCapture)
+        {
+            var sensorJObject = new JObject();//new SensorCaptureJson
+            sensorJObject["sensor_id"] = pendingCapture.SensorHandle.Id.ToString();
+            sensorJObject["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
+            sensorJObject["modality"] = pendingCapture.SensorData.modality;
+            sensorJObject["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.position);
+            sensorJObject["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.rotation);
+
+            if (pendingCapture.AdditionalSensorValues != null)
+            {
+                foreach (var(name, value) in pendingCapture.AdditionalSensorValues)
+                    sensorJObject.Add(name, DatasetJsonUtility.ToJToken(value));
+            }
+
+            var egoCaptureJson = new JObject();
+            egoCaptureJson["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
+            egoCaptureJson["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.position);
+            egoCaptureJson["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.rotation);
+            egoCaptureJson["velocity"] = pendingCapture.SensorSpatialData.EgoVelocity.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoVelocity.Value) : null;
+            egoCaptureJson["acceleration"] = pendingCapture.SensorSpatialData.EgoAcceleration.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoAcceleration.Value) : null;
+
+            var capture = new JObject();
+            capture["id"] = pendingCapture.Id.ToString();
+            capture["sequence_id"] = pendingCapture.SequenceId.ToString();
+            capture["step"] = pendingCapture.Step;
+            capture["timestamp"] = pendingCapture.Timestamp;
+            capture["sensor"] = sensorJObject;
+            capture["ego"] = egoCaptureJson;
+            capture["filename"] = pendingCapture.Path;
+            capture["format"] = GetFormatFromFilename(pendingCapture.Path);
+
+            if (pendingCapture.Annotations.Any())
+                capture["annotations"] = new JArray(pendingCapture.Annotations.Select(JObjectFromAnnotation).ToArray());
+
+            return capture;
+        }
+
+        static JObject JObjectFromAnnotation((Annotation, SimulationState.AnnotationData) annotationInfo)
+        {
+            var annotationJObject = new JObject();
+            annotationJObject["id"] = annotationInfo.Item1.Id.ToString();
+            annotationJObject["annotation_definition"] = annotationInfo.Item2.AnnotationDefinition.Id.ToString();
+            if (annotationInfo.Item2.Path != null)
+                annotationJObject["filename"] = annotationInfo.Item2.Path;
+
+            if (annotationInfo.Item2.ValuesJson != null)
+                annotationJObject["values"] = annotationInfo.Item2.ValuesJson;
+
+            return annotationJObject;
+        }
+
+        static string GetFormatFromFilename(string filename)
+        {
+            var ext = Path.GetExtension(filename);
+            if (ext == null)
+                return null;
+
+            if (ext.StartsWith("."))
+                ext = ext.Substring(1);
+
+            return ext.ToUpperInvariant();
+        }
+    }
+}

com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs.meta

+fileFormatVersion: 2
+guid: c38f43837b9cf984f92853858398e0b4
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: