浏览代码

Merge branch 'coco_perception_output_hybrid' into h-i

/h-i
Mohsen Kamalzadeh 3 年前
当前提交
123d60c1
共有 38 个文件被更改,包括 1712 次插入42 次删除
  1. 34
      com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs
  2. 33
      com.unity.perception/Editor/Randomization/Editors/RunInUnitySimulationWindow.cs
  3. 5
      com.unity.perception/Editor/Randomization/Uxml/RunInUnitySimulationWindow.uxml
  4. 8
      com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs
  5. 10
      com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs
  6. 80
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  7. 2
      com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs
  8. 18
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs
  9. 143
      com.unity.perception/Runtime/GroundTruth/SimulationState.cs
  10. 6
      com.unity.perception/Runtime/GroundTruth/SimulationState_Json.cs
  11. 2
      com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenario.cs
  12. 2
      com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenarioConstants.cs
  13. 8
      com.unity.perception/Runtime/GroundTruth/Exporters.meta
  14. 8
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco.meta
  15. 8
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat.meta
  16. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs.meta
  17. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs.meta
  18. 150
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs
  19. 8
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew.meta
  20. 94
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs
  21. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs.meta
  22. 151
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs
  23. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs.meta
  24. 20
      com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs
  25. 8
      com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid.meta
  26. 53
      com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs
  27. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs.meta
  28. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs.meta
  29. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs.meta
  30. 11
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs.meta
  31. 165
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs
  32. 69
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs
  33. 581
      com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs

34
com.unity.perception/Editor/GroundTruth/PerceptionCameraEditor.cs


using System;
using System.Collections.Generic;
using System.Linq;
using UnityEngine.Perception.GroundTruth.Exporters;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
#if UNITY_EDITOR_WIN || UNITY_EDITOR_OSX
using UnityEditor.Perception.Visualizer;

{
Dictionary<SerializedProperty, CameraLabelerDrawer> m_CameraLabelerDrawers = new Dictionary<SerializedProperty, CameraLabelerDrawer>();
ReorderableList m_LabelersList;
string[] m_ExporterList;
string m_OutputMode = "Perception";
int m_OutputModeIndex = -1;
public void OnEnable()
{
m_LabelersList = new ReorderableList(this.serializedObject, labelersProperty, true, true, true, true);

m_LabelersList.drawElementCallback = DrawElement;
m_LabelersList.onAddCallback += OnAdd;
m_LabelersList.onRemoveCallback += OnRemove;
m_OutputMode = PlayerPrefs.GetString(SimulationState.outputFormatMode);
m_ExporterList = TypeCache.GetTypesDerivedFrom<IDatasetExporter>().Select(exporter => exporter.Name).ToArray();
if (m_ExporterList.Any())
m_OutputModeIndex = Array.IndexOf(m_ExporterList, m_OutputMode);
m_OutputMode = PlayerPrefs.GetString(SimulationState.outputFormatMode, "Perception");
}
float GetElementHeight(int index)

const string k_FrametimeTitle = "Simulation Delta Time";
const float k_DeltaTimeTooLarge = 200;
public override void OnInspectorGUI()
{
using(new EditorGUI.DisabledScope(EditorApplication.isPlaying))

EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureRgbImages)),new GUIContent("Save Camera RGB Output to Disk", "For each captured frame, save an RGB image of the camera's output to disk."));
if (m_ExporterList.Any())
{
if (m_OutputModeIndex < 0)
{
m_OutputMode = nameof(PerceptionExporter);
Debug.LogWarning($"Could not find the output format \'{m_OutputMode}\' in the list of available exporters, setting the exporter to \'{m_OutputMode}\'");
m_OutputModeIndex = Array.IndexOf(m_ExporterList, m_OutputMode);
}
var selected = EditorGUILayout.Popup(new GUIContent("Output Format", ""), m_OutputModeIndex, m_ExporterList);
if (m_OutputModeIndex != selected)
{
m_OutputModeIndex = selected;
m_OutputMode = m_ExporterList[selected];
PlayerPrefs.SetString(SimulationState.outputFormatMode, m_OutputMode);
}
}
EditorGUILayout.PropertyField(serializedObject.FindProperty(nameof(perceptionCamera.captureTriggerMode)),new GUIContent("Capture Trigger Mode", $"The method of triggering captures for this camera. In {nameof(CaptureTriggerMode.Scheduled)} mode, captures happen automatically based on a start frame and frame delta time. In {nameof(CaptureTriggerMode.Manual)} mode, captures should be triggered manually through calling the {nameof(perceptionCamera.RequestCapture)} method of {nameof(PerceptionCamera)}."));
GUILayout.Space(5);

33
com.unity.perception/Editor/Randomization/Editors/RunInUnitySimulationWindow.cs


using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;

using UnityEditor.UIElements;
using UnityEngine;
using UnityEngine.Perception.GroundTruth;
using UnityEngine.Perception.GroundTruth.Exporters;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
using UnityEngine.Perception.Randomization.Samplers;
using UnityEngine.Perception.Randomization.Scenarios;
using UnityEngine.SceneManagement;

TextField m_BuildPathField;
TextField m_SelectedBuildPathTextField;
TextField m_BuildIdField;
string[] m_OutputFormats;
ToolbarMenu m_OutputFormatMenu;
int m_OutputFormatIndex;
[MenuItem("Window/Run in Unity Simulation")]
static void ShowWindow()

m_RandomSeedField.value = BitConverter.ToUInt32(bytes, 0);
};
m_OutputFormats = TypeCache.GetTypesDerivedFrom<IDatasetExporter>().Select(exporter => exporter.Name).ToArray();
m_OutputFormatMenu = root.Q<ToolbarMenu>("output-format");
var i = 0;
foreach (var format in m_OutputFormats)
{
var index = i++;
m_OutputFormatMenu.menu.AppendAction(
format,
action =>
{
m_OutputFormatIndex = index;
m_OutputFormatMenu.text = format;
});
}
for (var i = 0; i < m_SysParamDefinitions.Length; i++)
for (i = 0; i < m_SysParamDefinitions.Length; i++)
{
var index = i;
var param = m_SysParamDefinitions[i];

m_ProjectIdLabel.text = $"Project ID: {CloudProjectSettings.projectId}";
m_PrevExecutionIdLabel.text = $"Execution ID: {PlayerPrefs.GetString("SimWindow/prevExecutionId")}";
m_PrevRandomSeedLabel.text = $"Random Seed: {PlayerPrefs.GetString("SimWindow/prevRandomSeed")}";
m_OutputFormatMenu.text = PlayerPrefs.GetString(SimulationState.outputFormatMode, nameof(PerceptionExporter));
}
static string IncrementRunName(string runName)

sysParamIndex = m_SysParamIndex,
scenarioConfig = (TextAsset)m_ScenarioConfigField.value,
currentOpenScenePath = SceneManager.GetSceneAt(0).path,
currentScenario = FindObjectOfType<ScenarioBase>()
currentScenario = FindObjectOfType<ScenarioBase>(),
outputFormat = m_OutputFormats[m_OutputFormatIndex]
};
var runGuid = Guid.NewGuid();
PerceptionEditorAnalytics.ReportRunInUnitySimulationStarted(

constants["totalIterations"] = m_RunParameters.totalIterations;
constants["instanceCount"] = m_RunParameters.instanceCount;
constants["randomSeed"] = m_RunParameters.randomSeed;
constants["outputFormat"] = m_RunParameters.outputFormat;
var appParamName = $"{m_RunParameters.runName}";
var appParamsString = JsonConvert.SerializeObject(configuration, Formatting.Indented);

PlayerPrefs.SetInt("SimWindow/sysParamIndex", m_RunParameters.sysParamIndex);
PlayerPrefs.SetString("SimWindow/scenarioConfig",
m_RunParameters.scenarioConfig != null ? m_RunParameters.scenarioConfigAssetPath : string.Empty);
PlayerPrefs.SetString(SimulationState.outputFormatMode, m_RunParameters.outputFormat);
SetFieldsFromPlayerPreferences();
}

public TextAsset scenarioConfig;
public string currentOpenScenePath;
public ScenarioBase currentScenario;
public string outputFormat;
public string scenarioConfigAssetPath => AssetDatabase.GetAssetPath(scenarioConfig);
}

5
com.unity.perception/Editor/Randomization/Uxml/RunInUnitySimulationWindow.uxml


<editor:ToolbarMenu name="sys-param" class="unity-base-field__input" style="border-width: 1px;"/>
</VisualElement>
<VisualElement class="unity-base-field" tooltip="TODO">
<Label text="Output Format" class="unity-base-field__label"/>
<editor:ToolbarMenu name="output-format" class="unity-base-field__input" style="border-width: 1px;"/>
</VisualElement>
<Label text="Optional Configuration" class="sim-window__header-1" style="margin-top: 18px;"/>
<editor:ObjectField name="scenario-config" label="Scenario JSON Config" allow-scene-object="false"
tooltip="Selects a scenario JSON configuration to load during the run.

8
com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs


return ego;
}
/// <summary>
/// Register a new sensor under the given ego.
/// </summary>

}
}
/// <summary>
/// Capture trigger modes for sensors.
/// </summary>

throw new ArgumentException("The given annotationDefinition is invalid", nameof(annotationDefinition));
return DatasetCapture.SimulationState.ReportAnnotationAsync(annotationDefinition, this);
}
public string GetRgbCaptureFilename(string defaultFilename, params(string, object)[] additionalSensorValues)
{
return DatasetCapture.SimulationState.GetRgbCaptureFilename(defaultFilename, additionalSensorValues);
}
/// <summary>

10
com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs


[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]
internal struct BoundingBoxValue
public struct BoundingBoxValue
public int frame;
public string label_name;
public uint instance_id;
public float x;

/// The GUID id to associate with the annotations produced by this labeler.
/// </summary>
public string annotationId = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
/// <summary>
/// The annotation ID that the COCO exporter will use to know whether an ID is coming from a bounding box 2 labeler.
/// </summary>
public static string annotationIdForCocoExport = "f9f22e05-443f-4602-a422-ebe4ea9b55cb";
/// <summary>
/// The <see cref="IdLabelConfig"/> which associates objects with labels.
/// </summary>

m_BoundingBoxValues.Add(new BoundingBoxValue
{
label_id = labelEntry.id,
frame = frameCount,
label_name = labelEntry.label,
instance_id = objectInfo.instanceId,
x = objectInfo.boundingBox.x,

80
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs


/// The GUID id to associate with the annotations produced by this labeler.
/// </summary>
public string annotationId = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
/// <summary>
/// The annotation ID that the COCO exporter will use to know whether an ID is coming from a keypoint labeler.
/// </summary>
public static string annotationIdForCocoExport = "8b3ef246-daa7-4dd5-a0e8-a943f6e7f8c2";
/// <summary>
/// The <see cref="IdLabelConfig"/> which associates objects with labels.
/// </summary>

if (idLabelConfig == null)
throw new InvalidOperationException($"{nameof(KeypointLabeler)}'s idLabelConfig field must be assigned");
m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", new []{TemplateToJson(activeTemplate)},
m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition("keypoints", TemplateToJson(activeTemplate, idLabelConfig),
"pixel coordinates of keypoints in a model, along with skeletal connectivity data", id: new Guid(annotationId));
// Texture to use in case the template does not contain a texture for the joints or the skeletal connections

m_AsyncAnnotations[m_CurrentFrame] = (annotation, keypoints);
foreach (var label in LabelManager.singleton.registeredLabels)
ProcessLabel(label);
ProcessLabel(m_CurrentFrame, label);
}
// ReSharper disable InconsistentNaming

/// The label id of the entity
/// </summary>
public int label_id;
public int frame;
/// <summary>
/// The instance id of the entity
/// </summary>

return false;
}
void ProcessLabel(Labeling labeledEntity)
void ProcessLabel(int frame, Labeling labeledEntity)
{
if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
return;

cached.keypoints.instance_id = labeledEntity.instanceId;
cached.keypoints.label_id = labelEntry.id;
cached.keypoints.frame = -1;
cached.keypoints.template_guid = activeTemplate.templateID;
cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];

}
var cachedData = m_KnownStatus[labeledEntity.instanceId];
cachedData.keypoints.frame = frame;
if (cachedData.status)
{

var cachedKeypointEntry = cachedData.keypoints;
var keypointEntry = new KeypointEntry()
{
frame = cachedKeypointEntry.frame,
instance_id = cachedKeypointEntry.instance_id,
keypoints = cachedKeypointEntry.keypoints.ToArray(),
label_id = cachedKeypointEntry.label_id,

// ReSharper disable InconsistentNaming
// ReSharper disable NotAccessedField.Local
[Serializable]
struct JointJson
public struct JointJson
{
public string label;
public int index;

[Serializable]
struct SkeletonJson
public struct SkeletonJson
{
public int joint1;
public int joint2;

[Serializable]
struct KeypointJson
public struct KeypointJson
public int label_id;
public string label_name;
public string template_id;
public string template_name;
public JointJson[] key_points;

// ReSharper restore NotAccessedField.Local
KeypointJson TemplateToJson(KeypointTemplate input)
KeypointJson[] TemplateToJson(KeypointTemplate input, IdLabelConfig labelConfig)
var json = new KeypointJson();
json.template_id = input.templateID;
json.template_name = input.templateName;
json.key_points = new JointJson[input.keypoints.Length];
json.skeleton = new SkeletonJson[input.skeleton.Length];
var jsons = new KeypointJson[labelConfig.labelEntries.Count];
var idx = 0;
for (var i = 0; i < input.keypoints.Length; i++)
foreach (var cfg in labelConfig.labelEntries)
json.key_points[i] = new JointJson
var json = new KeypointJson();
json.label_id = cfg.id;
json.label_name = cfg.label;
json.template_id = input.templateID;
json.template_name = input.templateName;
json.key_points = new JointJson[input.keypoints.Length];
json.skeleton = new SkeletonJson[input.skeleton.Length];
for (var i = 0; i < input.keypoints.Length; i++)
label = input.keypoints[i].label,
index = i,
color = input.keypoints[i].color
};
}
json.key_points[i] = new JointJson
{
label = input.keypoints[i].label,
index = i,
color = input.keypoints[i].color
};
}
for (var i = 0; i < input.skeleton.Length; i++)
{
json.skeleton[i] = new SkeletonJson()
for (var i = 0; i < input.skeleton.Length; i++)
joint1 = input.skeleton[i].joint1,
joint2 = input.skeleton[i].joint2,
color = input.skeleton[i].color
};
json.skeleton[i] = new SkeletonJson()
{
joint1 = input.skeleton[i].joint1,
joint2 = input.skeleton[i].joint2,
color = input.skeleton[i].color
};
}
jsons[idx++] = json;
return json;
return jsons;
}
}

2
com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs


}
[SuppressMessage("ReSharper", "InconsistentNaming")]
internal struct LabelEntrySpec
public struct LabelEntrySpec
{
/// <summary>
/// The label id prepared for reporting in the annotation

18
com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs


#endif
#if URP_PRESENT
using UnityEngine.Rendering.Universal;
#pragma warning disable 649
#endif
namespace UnityEngine.Perception.GroundTruth

void CaptureRgbData(Camera cam)
{
// TODO - Steve - this could be a place where we override the capture of the RGB image to allow the
// active reporter to determine where we should save RGB images, or even write them out
var width = cam.pixelWidth;
var height = cam.pixelHeight;
var frameCount = Time.frameCount;
var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{frameCount}.png";
SetPersistentSensorData("camera_width", width);
SetPersistentSensorData("camera_height", height);
SetPersistentSensorData("full_path", captureFilename);
SetPersistentSensorData("frame", frameCount);
var captureFilename = $"{Manager.Instance.GetDirectoryFor(rgbDirectory)}/{k_RgbFilePrefix}{Time.frameCount}.png";
var width = cam.pixelWidth;
var height = cam.pixelHeight;
colorFunctor = r =>
{

143
com.unity.perception/Runtime/GroundTruth/SimulationState.cs


using Unity.Collections;
using Unity.Simulation;
using UnityEngine;
using UnityEngine.Perception.GroundTruth.Exporters;
using UnityEngine.Perception.GroundTruth.Exporters.Coco;
using UnityEngine.Perception.GroundTruth.Exporters.CocoHybrid;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew;
partial class SimulationState
public partial class SimulationState
{
HashSet<SensorHandle> m_ActiveSensors = new HashSet<SensorHandle>();
Dictionary<SensorHandle, SensorData> m_Sensors = new Dictionary<SensorHandle, SensorData>();

IDatasetExporter _ActiveReporter = null;
// Always use the property SequenceTimeMs instead
int m_FrameCountLastUpdatedSequenceTime;

float m_LastTimeScale;
readonly string m_OutputDirectoryName;
string m_OutputDirectoryPath;
public const string outputFormatMode = "outputFormatMode";
public bool IsRunning { get; private set; }

public SimulationState(string outputDirectory)
{
_ActiveReporter = null;
PlayerPrefs.SetString(defaultOutputBaseDirectory, Configuration.Instance.GetStorageBasePath());
m_OutputDirectoryName = outputDirectory;
var basePath = PlayerPrefs.GetString(userBaseDirectoryKey, string.Empty);

basePath = Configuration.localPersistentDataPath;
}
}
/*
//var activeReporterString = PlayerPrefs.GetString(activeReporterKey, defaultReporter);
var activeReporterString = "coco";
if (activeReporterString == "perceptionOutput")
{
m_ActiveReporter = new PerceptionExporter();
}
else
{
m_ActiveReporter = new CocoExporter();
}
*/
}
IDatasetExporter GetActiveReporter()
{
if (_ActiveReporter != null) return _ActiveReporter;
var mode = PlayerPrefs.GetString(outputFormatMode, nameof(CocoExporter));
#if false
// TODO figure out how to do this with just the class name and not have to have the switch
var exporter = Activator.CreateInstance(Type.GetType(mode) ?? typeof(PerceptionExporter));
if (exporter is IDatasetExporter casted)
{
m_ActiveReporter = casted;
}
#else
Debug.Log($"SS - Sim State setting active reporter: {mode}");
switch (mode)
{
case nameof(PerceptionExporter):
_ActiveReporter = new PerceptionExporter();
break;
case nameof(CocoExporter):
//_ActiveReporter = new CocoExporter();
_ActiveReporter = new CocoHybridExporter();
break;
case nameof(PerceptionNewExporter):
_ActiveReporter = new PerceptionNewExporter();
break;
default:
_ActiveReporter = new PerceptionExporter();
break;
}
#endif
Debug.Log("Calling SS::OnSimulationBegin");
_ActiveReporter?.OnSimulationBegin(Manager.Instance.GetDirectoryFor(m_OutputDirectoryName));
return _ActiveReporter;
}
public string GetRgbCaptureFilename(string defaultFilename, params(string, object)[] additionalSensorValues)
{
var directory = GetActiveReporter()?.GetRgbCaptureFilename(additionalSensorValues);
return directory == string.Empty ? defaultFilename : directory;
}
/// <summary>

class PendingCapture
public class PendingCapture
{
public Guid Id;
public SensorHandle SensorHandle;

public EgoHandle egoHandle;
}
struct AnnotationData
public struct AnnotationData
public IEnumerable<object> RawValues;
public bool IsAssigned => Path != null || ValuesJson != null;
public AnnotationData(AnnotationDefinition annotationDefinition, string path, JArray valuesJson)

sensorData.lastCaptureFrameCount = Time.frameCount;
m_Sensors[sensorHandle] = sensorData;
// SB - maybe this can all be moved to the other capture area
var width = -1;
var height = -1;
var fullPath = filename;
var frameCount = 0;
foreach (var i in additionalSensorValues)
{
switch (i.Item1)
{
case "camera_width":
width = (int)i.Item2;
break;
case "camera_height":
height = (int)i.Item2;
break;
case "full_path":
fullPath = (string)i.Item2;
break;
case "frame":
frameCount = (int)i.Item2;
break;
}
}
GetActiveReporter()?.OnCaptureReported(frameCount, width, height, filename);
}
static string GetFormatFromFilename(string filename)

if (m_PendingMetrics.Count > 0)
Debug.LogError($"Simulation ended with pending metrics: {string.Join(", ", m_PendingMetrics.Select(c => $"id:{c.MetricId} step:{c.Step}"))}");
if (m_AdditionalInfoTypeData.Any())
{
List<IdLabelConfig.LabelEntrySpec> labels = new List<IdLabelConfig.LabelEntrySpec>();
foreach (var infoTypeData in m_AdditionalInfoTypeData)
{
if (infoTypeData.specValues == null) continue;
foreach (var spec in infoTypeData.specValues)
{
if (spec is IdLabelConfig.LabelEntrySpec entrySpec)
{
labels.Add(entrySpec);
}
}
Debug.Log($"adt: {infoTypeData}");
}
}
Debug.Log($"Calling SS::OnSimulationEnd");
GetActiveReporter()?.OnSimulationEnd();
public AnnotationDefinition RegisterAnnotationDefinition<TSpec>(string name, TSpec[] specValues, string description, string format, Guid id)
{

RegisterAdditionalInfoType(name, specValues, description, format, id, AdditionalInfoKind.Annotation);
GetActiveReporter()?.OnAnnotationRegistered(id, specValues); // <- Not sure about this one either
return new AnnotationDefinition(id);
}

}
}
ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray);
var values2 = new List<object>();
foreach (var v in values)
{
values2.Add(v);
}
ReportAsyncAnnotationResult(asyncAnnotation, filename, jArray, values:values2);
void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray)
void ReportAsyncAnnotationResult(AsyncAnnotation asyncAnnotation, string filename, JArray jArray, IEnumerable<object> values = null)
{
if (!asyncAnnotation.IsPending)
throw new InvalidOperationException("AsyncAnnotation has already been reported and cannot be reported again.");

annotationData.Path = filename;
annotationData.ValuesJson = jArray;
annotationData.RawValues = values;
annotationTuple.Item2 = annotationData;
pendingCapture.Annotations[annotationIndex] = annotationTuple;

6
com.unity.perception/Runtime/GroundTruth/SimulationState_Json.cs


namespace UnityEngine.Perception.GroundTruth
{
partial class SimulationState
public partial class SimulationState
{
const Formatting k_Formatting = Formatting.Indented;

m_WriteToDiskSampler.Begin();
var path = Path.Combine(OutputDirectory, filename);
Debug.Log($"ss - sensors.json - {path}");
File.WriteAllText(path, contents);
Manager.Instance.ConsumerFileProduced(path);
m_WriteToDiskSampler.End();

void Write(List<PendingCapture> pendingCaptures, SimulationState simulationState, int captureFileIndex)
{
GetActiveReporter()?.ProcessPendingCaptures(pendingCaptures, simulationState);
#if false
simulationState.m_SerializeCapturesAsyncSampler.Begin();
//lazily allocate for fast zero-write frames

simulationState.WriteJObjectToFile(capturesJObject,
$"captures_{captureFileIndex:000}.json");
simulationState.m_SerializeCapturesAsyncSampler.End();
#endif
}
if (flush)

2
com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenario.cs


using System;
using Unity.Simulation;
using UnityEngine.Perception.GroundTruth;
namespace UnityEngine.Perception.Randomization.Scenarios
{

{
var filePath = new Uri(Configuration.Instance.SimulationConfig.app_param_uri).LocalPath;
LoadConfigurationFromFile(filePath);
PlayerPrefs.SetString(SimulationState.outputFormatMode, constants.outputFormat);
}
else
{

2
com.unity.perception/Runtime/Randomization/Scenarios/UnitySimulationScenarioConstants.cs


/// </summary>
[Tooltip("The Unity Simulation instance index of the currently executing worker.")]
public int instanceIndex;
public string outputFormat;
}
}

8
com.unity.perception/Runtime/GroundTruth/Exporters.meta


fileFormatVersion: 2
guid: 5353392e887128948bc94b5e0dd9ff73
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
com.unity.perception/Runtime/GroundTruth/Exporters/Coco.meta


fileFormatVersion: 2
guid: b543148039c075741a0d54a25d56f573
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat.meta


fileFormatVersion: 2
guid: a40dd8c5379f64748a0942b0475badc6
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

11
com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs.meta


fileFormatVersion: 2
guid: 28e95774bd5a26d47973d1213bcc4c26
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs.meta


fileFormatVersion: 2
guid: c38f43837b9cf984f92853858398e0b4
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

150
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionFormat/PerceptionExporter.cs


using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Unity.Simulation;
namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat
{
public class PerceptionExporter : IDatasetExporter
{
const Formatting k_Formatting = Formatting.Indented;
string outputDirectory = string.Empty;
int captureFileIndex = 0;
public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
{
return string.Empty;
}
public void OnSimulationBegin(string directoryName)
{
Debug.Log($"SS - Perception - OnSimBegin");
outputDirectory = directoryName;
}
public void OnSimulationEnd()
{
Debug.Log($"SS - Perception - OnSimEnd");
// do nothing :-)
}
public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
{
// do nothing :-)
}
public static void WriteJObjectToFile(JObject jObject, string directory, string filename)
{
var stringWriter = new StringWriter(new StringBuilder(256), CultureInfo.InvariantCulture);
using (var jsonTextWriter = new JsonTextWriter(stringWriter))
{
jsonTextWriter.Formatting = k_Formatting;
jObject.WriteTo(jsonTextWriter);
}
var contents = stringWriter.ToString();
var path = Path.Combine(directory, filename);
File.WriteAllText(path, contents);
// TODO what to do about this...
Manager.Instance.ConsumerFileProduced(path);
}
public Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
{
//lazily allocate for fast zero-write frames
var capturesJArray = new JArray();
foreach (var pendingCapture in pendingCaptures)
capturesJArray.Add(JObjectFromPendingCapture(pendingCapture));
var capturesJObject = new JObject();
capturesJObject.Add("version", DatasetCapture.SchemaVersion);
capturesJObject.Add("captures", capturesJArray);
Debug.Log("SS - perception - writing");
WriteJObjectToFile(capturesJObject, outputDirectory, $"captures_{captureFileIndex:000}.json");
// TODO what to do about this...
return null;
}
public Task OnCaptureReported(int frame, int width, int height, string filename)
{
// do nothing :-)
return null;
}
public static JToken JObjectFromPendingCapture(SimulationState.PendingCapture pendingCapture)
{
var sensorJObject = new JObject();//new SensorCaptureJson
sensorJObject["sensor_id"] = pendingCapture.SensorHandle.Id.ToString();
sensorJObject["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
sensorJObject["modality"] = pendingCapture.SensorData.modality;
sensorJObject["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.position);
sensorJObject["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.SensorPose.rotation);
if (pendingCapture.AdditionalSensorValues != null)
{
foreach (var(name, value) in pendingCapture.AdditionalSensorValues)
sensorJObject.Add(name, DatasetJsonUtility.ToJToken(value));
}
var egoCaptureJson = new JObject();
egoCaptureJson["ego_id"] = pendingCapture.SensorData.egoHandle.Id.ToString();
egoCaptureJson["translation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.position);
egoCaptureJson["rotation"] = DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoPose.rotation);
egoCaptureJson["velocity"] = pendingCapture.SensorSpatialData.EgoVelocity.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoVelocity.Value) : null;
egoCaptureJson["acceleration"] = pendingCapture.SensorSpatialData.EgoAcceleration.HasValue ? DatasetJsonUtility.ToJToken(pendingCapture.SensorSpatialData.EgoAcceleration.Value) : null;
var capture = new JObject();
capture["id"] = pendingCapture.Id.ToString();
capture["sequence_id"] = pendingCapture.SequenceId.ToString();
capture["step"] = pendingCapture.Step;
capture["timestamp"] = pendingCapture.Timestamp;
capture["sensor"] = sensorJObject;
capture["ego"] = egoCaptureJson;
capture["filename"] = pendingCapture.Path;
capture["format"] = GetFormatFromFilename(pendingCapture.Path);
if (pendingCapture.Annotations.Any())
capture["annotations"] = new JArray(pendingCapture.Annotations.Select(JObjectFromAnnotation).ToArray());
return capture;
}
public static JObject JObjectFromAnnotation((Annotation, SimulationState.AnnotationData) annotationInfo)
{
var annotationJObject = new JObject();
annotationJObject["id"] = annotationInfo.Item1.Id.ToString();
annotationJObject["annotation_definition"] = annotationInfo.Item2.AnnotationDefinition.Id.ToString();
if (annotationInfo.Item2.Path != null)
annotationJObject["filename"] = annotationInfo.Item2.Path;
if (annotationInfo.Item2.ValuesJson != null)
annotationJObject["values"] = annotationInfo.Item2.ValuesJson;
return annotationJObject;
}
static string GetFormatFromFilename(string filename)
{
var ext = Path.GetExtension(filename);
if (ext == null)
return null;
if (ext.StartsWith("."))
ext = ext.Substring(1);
return ext.ToUpperInvariant();
}
}
}

8
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew.meta


fileFormatVersion: 2
guid: 8b80f219b9e258043aeae665de9f8d90
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

94
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs


using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Unity.Simulation;
namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew
{
public static class AnnotationHandler
{
[Serializable]
struct BoundingBox2dRecord
{
public uint instanceId;
public int frame;
public int labelId;
public string labelName;
public float x;
public float y;
public float width;
public float height;
public string annotationId;
public string annotationDefinition;
public static BoundingBox2dRecord FromBoundingBoxValue(Guid annotationId, Guid annotationDefinition, BoundingBox2DLabeler.BoundingBoxValue bbox)
{
return new BoundingBox2dRecord
{
instanceId = bbox.instance_id,
frame = bbox.frame,
labelId = bbox.label_id,
labelName = bbox.label_name,
x = bbox.x,
y = bbox.y,
width = bbox.width,
height = bbox.height,
annotationId = annotationId.ToString(),
annotationDefinition = annotationDefinition.ToString()
};
}
public string ToJson()
{
return JsonUtility.ToJson(this, true);
}
}
public static async Task WriteOutJson(string path, string filename, string json)
{
if (true)
{
json = JToken.Parse(json).ToString(Formatting.Indented);
}
var writePath = Path.Combine(path, filename);
var file = File.CreateText(writePath);
await file.WriteAsync(json);
file.Close();
Manager.Instance.ConsumerFileProduced(writePath);
}
static async Task HandleBoundingBoxAnnotation(string path, Annotation annotation, AnnotationDefinition def, BoundingBox2DLabeler.BoundingBoxValue bbox)
{
var id = annotation.Id;
var defId = def.Id;
var converted = BoundingBox2dRecord.FromBoundingBoxValue(id, defId, bbox);
var filename = $"frame_{converted.frame}_id_{converted.instanceId}_bounding_box_2d.json";
var writePath = Path.Combine(path, filename);
var file = File.CreateText(writePath);
await file.WriteAsync(converted.ToJson());
file.Close();
Manager.Instance.ConsumerFileProduced(writePath);
}
public static async Task HandleAnnotation(string path, Annotation annotation, AnnotationDefinition def, object annotatedData)
{
switch (annotatedData)
{
case BoundingBox2DLabeler.BoundingBoxValue bbox:
await HandleBoundingBoxAnnotation(path, annotation, def, bbox);
break;
}
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/AnnotationHandler.cs.meta


fileFormatVersion: 2
guid: 9c350f412031d374ebe3027d0e8283de
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

151
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs


using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Unity.Simulation;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
namespace UnityEngine.Perception.GroundTruth.Exporters.PerceptionNew
{
public class PerceptionNewExporter : IDatasetExporter
{
public bool prettyPrint = true;
string m_DirectoryName = string.Empty;
int m_UnknownFrameCount = 0;
public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
{
var frameArray = additionalSensorValues.Where(p => p.Item1 == "frame").Select(p => p.Item2);
var frame = frameArray.Any() ? (int)frameArray.First() : m_UnknownFrameCount++;
return Path.Combine(m_DirectoryName, $"rgb_{frame}.png");
}
public void OnSimulationBegin(string directoryName)
{
Debug.Log($"SS - New Perception - OnSimBegin");
m_Metadata = new Metadata
{
version = "0.0.1",
image_width = 0,
image_height = 0,
dataset_size = 0
};
m_DirectoryName = directoryName + Path.DirectorySeparatorChar + Guid.NewGuid() + Path.DirectorySeparatorChar;
if (!Directory.Exists(m_DirectoryName))
Directory.CreateDirectory(m_DirectoryName);
}
[Serializable]
struct Metadata
{
public string version;
public int image_width;
public int image_height;
public int dataset_size;
}
Metadata m_Metadata;
public void OnSimulationEnd()
{
Debug.Log($"SS - New Perception - OnSimEnd");
var writePath = Path.Combine(m_DirectoryName, "metadata.json");
var file = File.CreateText(writePath);
Debug.Log("SS - New Perception - writing");
file.Write(JsonUtility.ToJson(m_Metadata, true));
file.Close();
Manager.Instance.ConsumerFileProduced(writePath);
Task.WhenAll(m_PendingTasks);
}
public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
{
// Right now, do nothing :-)
}
static bool GetFilenameForAnnotation(object rawData, out string filename)
{
filename = string.Empty;
if (rawData is BoundingBox2DLabeler.BoundingBoxValue bbox)
{
var frame = bbox.frame;
filename = $"frame_{frame}_bounding_bocx_2d.json";
return true;
}
return false;
}
// TODO - handle the 1000's of file writes we will be doing in a more intelligent fashion. Perhaps create a bg thread
// that reads json records off of a queue and writes them out
List<Task> m_PendingTasks = new List<Task>();
public Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
{
foreach (var cap in pendingCaptures)
{
foreach (var (annotation, annotationData) in cap.Annotations)
{
// Create a file for the annotation
#if false
if (annotationData.RawValues.Any())
{
var first = annotationData.RawValues.First();
if (GetFilenameForAnnotation(first, frame, out var filename))
{
var json = new StringBuilder("{");
json.Append(annotationData.ValuesJson);
json.Append("}");
m_PendingTasks.Add(AnnotationHandler.WriteOutJson(m_DirectoryName, filename, json.ToString()));
#if false
// Need to revisit this and handle this in a performant way
var jObject = PerceptionExporter.JObjectFromAnnotation((annotation, annotationData));
PerceptionExporter.WriteJObjectToFile(jObject, m_DirectoryName, filename);
#endif
}
}
#endif
foreach (var rawValue in annotationData.RawValues)
{
if (GetFilenameForAnnotation(rawValue, out var filename))
{
#if true
var json = new StringBuilder();
json.Append(annotationData.ValuesJson);
m_PendingTasks.Add(AnnotationHandler.WriteOutJson(m_DirectoryName, filename, json.ToString()));
#else
// Need to revisit this and handle this in a performant way
var jObject = PerceptionExporter.JObjectFromAnnotation((annotation, annotationData));
PerceptionExporter.WriteJObjectToFile(jObject, m_DirectoryName, filename);
#endif
}
}
}
}
return Task.CompletedTask;
}
public Task OnCaptureReported(int frame, int width, int height, string filename)
{
m_Metadata.dataset_size++;
m_Metadata.image_height = height;
m_Metadata.image_width = width;
return null;
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Exporters/PerceptionNew/PerceptionNewExporter.cs.meta


fileFormatVersion: 2
guid: 50287c95e34d8354d935c14caf487750
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

20
com.unity.perception/Runtime/GroundTruth/Exporters/IDatasetExporter.cs


using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace UnityEngine.Perception.GroundTruth.Exporters
{
public interface IDatasetExporter
{
string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues);
void OnSimulationBegin(string directoryName);
void OnSimulationEnd();
void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values);
Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState);
Task OnCaptureReported(int frame, int width, int height, string filename);
}
}

8
com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid.meta


fileFormatVersion: 2
guid: 71de1979b5988b34d8d7e41110c57c56
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

53
com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs


using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine.Perception.GroundTruth.Exporters.Coco;
using UnityEngine.Perception.GroundTruth.Exporters.PerceptionFormat;
namespace UnityEngine.Perception.GroundTruth.Exporters.CocoHybrid
{
public class CocoHybridExporter : IDatasetExporter
{
CocoExporter m_Coco = new CocoExporter();
PerceptionExporter m_Perception = new PerceptionExporter();
public string GetRgbCaptureFilename(params (string, object)[] additionalSensorValues)
{
return m_Coco.GetRgbCaptureFilename(additionalSensorValues) + m_Perception.GetRgbCaptureFilename(additionalSensorValues);
}
public void OnSimulationBegin(string directoryName)
{
m_Coco.OnSimulationBegin(directoryName + "_coco");
m_Perception.OnSimulationBegin(directoryName);
}
public void OnSimulationEnd()
{
m_Coco.OnSimulationEnd();
m_Perception.OnSimulationEnd();
}
public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
{
m_Coco.OnAnnotationRegistered(annotationId, values);
m_Perception.OnAnnotationRegistered(annotationId, values);
}
public async Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
{
var cocoTask = m_Coco.ProcessPendingCaptures(pendingCaptures, simState);
var perceptionTask = m_Perception.ProcessPendingCaptures(pendingCaptures, simState);
await cocoTask;
await perceptionTask;
}
public async Task OnCaptureReported(int frame, int width, int height, string filename)
{
var cocoTask = m_Coco.OnCaptureReported(frame, width, height, filename);
var perceptionTask = m_Perception.OnCaptureReported(frame, width, height, filename);
await cocoTask;
await perceptionTask;
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Exporters/CocoHybrid/CocoHybridExporter.cs.meta


fileFormatVersion: 2
guid: 75965f62c369c564b89229b8cbb10d93
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs.meta


fileFormatVersion: 2
guid: 0f587bc3e200ea542bd7725e7f64978c
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs.meta


fileFormatVersion: 2
guid: d25254cc4c0ad734cb76497c70a40088
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs.meta


fileFormatVersion: 2
guid: a715e9572e6cc314098dcdf401bb9273
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

165
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoTypes.cs


using System;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
{
public static class CocoTypes
{
public class CocoData
{
public Info info;
public Image[] images;
public ObjectDetectionAnnotation[] annotations;
public ObjectDetectionCategory[] categories;
public License[] licenses;
}
public class Info
{
public int year;
public string version;
public string description;
public string contributor;
public string url;
public string date_created;
}
[Serializable]
public class License
{
public int id;
public string name;
public string url;
}
[Serializable]
public class Licenses
{
public License[] licenses;
}
public class Image
{
public int id;
public int width;
public int height;
public string file_name;
public int license;
public string flickr_url;
public string coco_url;
public string date_captured;
}
[Serializable]
public class ObjectDetectionAnnotation
{
[JsonProperty(Order = -2)]
public int id;
[JsonProperty(Order = -2)]
public int image_id;
[JsonProperty(Order = -2)]
public int category_id;
[JsonProperty(Order = -2)]
public float[] segmentation;
[JsonProperty(Order = -2)]
public float area;
[JsonProperty(Order = -2)]
public float[] bbox;
[JsonProperty(Order = -2)]
public int iscrowd;
public static ObjectDetectionAnnotation FromBoundingBoxValue(BoundingBox2DLabeler.BoundingBoxValue bbox)
{
return new ObjectDetectionAnnotation
{
id = (int)bbox.instance_id,
image_id = bbox.frame,
category_id = bbox.label_id,
segmentation = new float[]{},
area = bbox.width * bbox.height,
bbox = new []{bbox.x, bbox.y, bbox.width, bbox.height},
iscrowd = 0
};
}
}
[Serializable]
public class ObjectDetectionCategory
{
[JsonProperty(Order = -2)]
public int id;
[JsonProperty(Order = -2)]
public string name = string.Empty;
[JsonProperty(Order = -2)]
public string supercategory = string.Empty;
}
[Serializable]
public class ObjectDetectionCategories
{
public List<ObjectDetectionCategory> categories;
}
public class KeypointAnnotation : ObjectDetectionAnnotation
{
public int num_keypoints;
public float[] keypoints;
public void CopyObjectDetectionData(ObjectDetectionAnnotation objDetection)
{
if (objDetection.id == this.id)
{
image_id = objDetection.image_id;
area = objDetection.area;
bbox = objDetection.bbox;
iscrowd = objDetection.iscrowd;
}
}
public static KeypointAnnotation FromKeypointValue(KeypointLabeler.KeypointEntry keypoint)
{
var outKeypoint = new KeypointAnnotation()
{
id = (int)keypoint.instance_id,
image_id = keypoint.frame,
category_id = keypoint.label_id,
segmentation = new float[]{},
area = 0,
bbox = new []{0f},
iscrowd = 0,
num_keypoints = keypoint.keypoints.Length,
keypoints = new float[keypoint.keypoints.Length * 3]
};
var i = 0;
foreach (var k in keypoint.keypoints)
{
outKeypoint.keypoints[i++] = k.x;
outKeypoint.keypoints[i++] = k.y;
outKeypoint.keypoints[i++] = k.state;
}
return outKeypoint;
}
}
[Serializable]
public class KeypointCategory : ObjectDetectionCategory
{
public string[] keypoints;
public int[][] skeleton;
}
[Serializable]
public class KeypointCategories
{
public KeypointCategory[] categories;
}
}
}

69
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/AnnotationHandler.cs


using System;
namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
{
public static class AnnotationHandler
{
public static object HandleAnnotation(AsyncAnnotation asyncAnnotation, object annotation)
{
switch (annotation)
{
case BoundingBox2DLabeler.BoundingBoxValue bbox:
return CocoTypes.ObjectDetectionAnnotation.FromBoundingBoxValue(bbox);
case KeypointLabeler.KeypointEntry keypoint:
return CocoTypes.KeypointAnnotation.FromKeypointValue(keypoint);
}
return null;
}
public static CocoTypes.KeypointCategory ToKeypointCategory(KeypointLabeler.KeypointJson keypointJson)
{
var keypoints = new string[keypointJson.key_points.Length];
var skeleton = new int[keypointJson.skeleton.Length][];
foreach (var kp in keypointJson.key_points)
{
keypoints[kp.index] = kp.label;
}
var i = 0;
foreach (var bone in keypointJson.skeleton)
{
var joints = new int[]
{
bone.joint1,
bone.joint2
};
skeleton[i++] = joints;
}
return new CocoTypes.KeypointCategory
{
id = keypointJson.label_id,
name = keypointJson.label_name,
supercategory = keypointJson.label_name,
keypoints = keypoints,
skeleton = skeleton
};
}
public static object HandleCameraCapture(int id, int width, int height, string filename)
{
var image = new CocoTypes.Image()
{
id = id,
width = width,
height = height,
file_name = filename,
license = 0,
flickr_url = "",
coco_url = "",
date_captured = DateTime.Today.ToString("D")
};
return image;
}
}
}

581
com.unity.perception/Runtime/GroundTruth/Exporters/Coco/CocoExporter.cs


using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Unity.Simulation;
namespace UnityEngine.Perception.GroundTruth.Exporters.Coco
{
public class CocoExporter : IDatasetExporter
{
bool m_PrettyPrint = true;
bool m_ReportingObjectDetection;
bool m_ReportingKeypoints;
bool m_Initialized;
string m_DirectoryName = string.Empty;
string m_RgbCaptureFilename;
StreamWriter m_RgbCaptureStream;
string m_ObjectDetectionFilename;
StreamWriter m_ObjectDetectionStream;
Task m_ObjectDetectionWritingTask;
string m_KeypointFilename;
StreamWriter m_KeypointDetectionStream;
Task m_KeypointDetectionWritingTask;
CocoTypes.ObjectDetectionCategories m_ObjectDetectionCategories;
string m_ObjectDetectionCategoryFilename;
StreamWriter m_ObjectDetectionCategoryStream;
Task m_ObjectDetectionCategoryWritingTask;
CocoTypes.KeypointCategories m_KeypointCategories;
string m_KeypointCategoryFilename;
StreamWriter m_KeypointCategoryStream;
Task m_KeypointCategoryWritingTask;
Guid m_SessionGuid;
public string GetRgbCaptureFilename(params(string, object)[] additionalSensorValues)
{
return string.Empty;
}
public void OnSimulationBegin(string directoryName)
{
Debug.Log($"SS - COCO - OnSimBegin");
m_DirectoryName = directoryName;
m_DataCaptured = false;
}
async Task AwaitAllWrites()
{
Debug.Log("SS - coco - writing");
WriteOutCategories();
if (m_ObjectDetectionWritingTask != null)
{
await m_ObjectDetectionWritingTask;
await m_ObjectDetectionStream.WriteAsync("]");
}
if (m_KeypointDetectionWritingTask != null)
{
await m_KeypointDetectionWritingTask;
await m_KeypointDetectionStream.WriteAsync("]");
}
if (m_RgbCaptureWritingTask != null)
{
await m_RgbCaptureWritingTask;
await m_RgbCaptureStream.WriteAsync("]");
}
if (m_ObjectDetectionCategoryWritingTask != null)
{
await m_ObjectDetectionCategoryWritingTask;
}
if (m_KeypointCategoryWritingTask != null)
{
await m_KeypointCategoryWritingTask;
}
m_RgbCaptureStream?.Close();
m_ObjectDetectionStream?.Close();
m_ObjectDetectionCategoryStream?.Close();
m_KeypointDetectionStream?.Close();
m_KeypointCategoryStream?.Close();
}
public async void OnSimulationEnd()
{
Debug.Log($"SS - COCO - OnSimEnd");
if (!m_DataCaptured) return;
await AwaitAllWrites();
if (m_ReportingObjectDetection)
{
await WriteObjectDetectionFile();
}
if (m_ReportingKeypoints)
{
await WriteKeypointFile();
}
File.Delete(m_RgbCaptureFilename);
m_Initialized = false;
}
void InitializeCaptureFiles()
{
if (m_Initialized) return;
if (!Directory.Exists(m_DirectoryName))
Directory.CreateDirectory(m_DirectoryName);
m_SessionGuid = Guid.NewGuid();
//var prefix = m_DirectoryName + Path.DirectorySeparatorChar + m_SessionGuid;
m_RgbCaptureFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_captures.json");
m_RgbCaptureStream = File.CreateText(m_RgbCaptureFilename);
m_ObjectDetectionFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_box_annotations.json");
if (m_ReportingObjectDetection)
{
m_ObjectDetectionStream = File.CreateText(m_ObjectDetectionFilename);
m_ObjectDetectionCategoryFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_obj_detection_categories.json");
}
m_KeypointFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_keypoint_annotations.json");
if (m_ReportingKeypoints)
{
m_KeypointDetectionStream = File.CreateText(m_KeypointFilename);
m_KeypointCategoryFilename = Path.Combine(m_DirectoryName, m_SessionGuid + "_coco_keypoint_categories.json");
}
m_Initialized = true;
}
static void AggregateFile(string filename, StringBuilder aggregated, bool skipFirstCharacter = false)
{
var sr = new StreamReader(filename);
var length = (int)sr.BaseStream.Length;
var start = 0;
if (length == 0) return;
var buffer = new char[length];
sr.Read(buffer, start, length);
if (skipFirstCharacter)
{
length--;
start++;
}
aggregated.Append(buffer, start, length);
sr.Dispose();
}
async Task WriteObjectDetectionFile()
{
var stringBuilder = new StringBuilder();
stringBuilder.Append("{");
// Create the json header
CreateHeaderInfo(stringBuilder);
stringBuilder.Append(",");
CreateLicenseInfo(stringBuilder);
// Read in the contents of the captures
stringBuilder.Append(",\"images\":");
AggregateFile(m_RgbCaptureFilename, stringBuilder);
// Read in the contents of the object detection
stringBuilder.Append(",\"annotations\":");
AggregateFile(m_ObjectDetectionFilename, stringBuilder);
stringBuilder.Append(",");
// Read in the contents of the object detection categories
AggregateFile(m_ObjectDetectionCategoryFilename, stringBuilder, true);
var json = stringBuilder.ToString();
if (m_PrettyPrint)
{
json = JToken.Parse(json).ToString(Formatting.Indented);
}
Debug.Log($"SS - COCO - writing to path: {m_DirectoryName}, file: coco_object_detection_annotations.json");
// Write out the files
var filename = Path.Combine(m_DirectoryName, "coco_object_detection_annotations.json");
Debug.Log($"SS - COCO - file: {filename}");
var cocoStream = File.CreateText(filename);
await cocoStream.WriteAsync(json);
cocoStream.Close();
Manager.Instance.ConsumerFileProduced(filename);
File.Delete(m_ObjectDetectionFilename);
File.Delete(m_ObjectDetectionCategoryFilename);
}
async Task WriteKeypointFile()
{
var stringBuilder = new StringBuilder();
stringBuilder.Append("{");
// Create the json header
CreateHeaderInfo(stringBuilder);
stringBuilder.Append(",");
CreateLicenseInfo(stringBuilder);
// Read in the contents of the captures
stringBuilder.Append(",\"images\":");
AggregateFile(m_RgbCaptureFilename, stringBuilder);
// Read in the contents of the object detection
stringBuilder.Append(",\"annotations\":");
AggregateFile(m_KeypointFilename, stringBuilder);
stringBuilder.Append(",");
// Read in the contents of the object detection categories
AggregateFile(m_KeypointCategoryFilename, stringBuilder, true);
var json = stringBuilder.ToString();
if (m_PrettyPrint)
{
json = JToken.Parse(json).ToString(Formatting.Indented);
}
// Write out the files
var filename = Path.Combine(m_DirectoryName, "coco_keypoint_annotations.json");
var cocoStream = File.CreateText(filename);
await cocoStream.WriteAsync(json);
cocoStream.Dispose();
Manager.Instance.ConsumerFileProduced(filename);
File.Delete(m_KeypointFilename);
File.Delete(m_KeypointCategoryFilename);
}
bool m_DataCaptured;
void WriteOutCategories()
{
// 3 cases
// 1) Just have object detection
// 2) Just have keypoint detection
// 3) Have both, which includes writing out object detection in object detection coco output
// and merging the object detection & keypoints and writing out that data in the keypoint
// coco output data
if (m_ReportingObjectDetection)
{
m_ObjectDetectionCategoryStream = File.CreateText(m_ObjectDetectionCategoryFilename);
var json = JsonUtility.ToJson(m_ObjectDetectionCategories);
m_ObjectDetectionCategoryWritingTask = m_ObjectDetectionCategoryStream.WriteAsync(json);
}
if (m_ReportingKeypoints)
{
// TODO - revisit this, but right now we are going to add the keypoint info to all of the categories,
// we can get away with this because we only support one set of keypoint definitions per simulation
// currently.
if (m_KeypointCategories.categories.Length < 1) return;
m_KeypointCategoryStream = File.CreateText(m_KeypointCategoryFilename);
var merged = m_KeypointCategories;
if (m_ReportingObjectDetection)
{
merged = new CocoTypes.KeypointCategories
{
categories = new CocoTypes.KeypointCategory[m_ObjectDetectionCategories.categories.Count]
};
var i = 0;
foreach (var odc in m_ObjectDetectionCategories.categories)
{
merged.categories[i++] = MergeCategories(odc, m_KeypointCategories.categories[0]);
}
}
#if false
var json = JsonUtility.ToJson(merged);
#else
var builder = new StringBuilder();
var sw = new StringWriter(builder);
var serializer = new JsonSerializer();
using (var writer = new JsonTextWriter(sw))
{
serializer.Serialize(writer, merged);
}
var json = builder.ToString();
#endif
m_KeypointCategoryWritingTask = m_KeypointCategoryStream.WriteAsync(json);
}
}
public void OnAnnotationRegistered<TSpec>(Guid annotationId, TSpec[] values)
{
if (annotationId.ToString() == BoundingBox2DLabeler.annotationIdForCocoExport)
{
m_ReportingObjectDetection = true;
m_ObjectDetectionCategories = new CocoTypes.ObjectDetectionCategories
{
categories = new List<CocoTypes.ObjectDetectionCategory>()
};
foreach (var value in values)
{
if (value is IdLabelConfig.LabelEntrySpec spec)
{
var rec = new CocoTypes.ObjectDetectionCategory
{
id = spec.label_id,
name = spec.label_name,
supercategory = spec.label_name
};
m_ObjectDetectionCategories.categories.Add(rec);
}
}
}
if (annotationId.ToString() == KeypointLabeler.annotationIdForCocoExport)
{
m_ReportingKeypoints = true;
var categories = new CocoTypes.KeypointCategory[values.Length];
var i = 0;
foreach (var value in values)
{
if (value is KeypointLabeler.KeypointJson keypointJson)
{
categories[i++] = AnnotationHandler.ToKeypointCategory(keypointJson);
}
}
m_KeypointCategories = new CocoTypes.KeypointCategories
{
categories = categories
};
}
}
static string versionEntry = "0.0.1";
static string descriptionEntry = "Description of dataset";
static string contributorEntry = "Anonymous";
static string urlEntry = "Not Set";
static void CreateHeaderInfo(StringBuilder stringBuilder)
{
stringBuilder.Append("\"info\":");
var dateTime = DateTime.Today;
var info = new CocoTypes.Info
{
year = int.Parse(dateTime.ToString("yyyy")),
version = versionEntry,
description = descriptionEntry,
contributor = contributorEntry,
url = urlEntry,
date_created = DateTime.Today.ToString("D")
};
stringBuilder.Append(JsonUtility.ToJson(info));
}
static void CreateLicenseInfo(StringBuilder stringBuilder)
{
var licenses = new CocoTypes.Licenses
{
licenses = new[]
{
new CocoTypes.License
{
id = 0,
name = "No License",
url = "Not Set"
}
}
};
var tmpJson = JsonUtility.ToJson(licenses);
// Remove the start and end '{' from the licenses json
stringBuilder.Append(tmpJson.Substring(1, tmpJson.Length - 2));
}
bool m_FirstBoxAnnotation = true;
bool m_FirstKeypointAnnotation = true;
bool m_FirstCapture = true;
Task m_RgbCaptureWritingTask;
static CocoTypes.KeypointCategory MergeCategories(CocoTypes.ObjectDetectionCategory od, CocoTypes.KeypointCategory kp)
{
return new CocoTypes.KeypointCategory
{
id = od.id,
name = od.name,
supercategory = od.supercategory,
keypoints = kp.keypoints,
skeleton = kp.skeleton
};
}
public async Task ProcessPendingCaptures(List<SimulationState.PendingCapture> pendingCaptures, SimulationState simState)
{
var boxJson = string.Empty;
var keypointJson = string.Empty;
foreach (var cap in pendingCaptures)
{
var boxes = new Dictionary<int, CocoTypes.ObjectDetectionAnnotation>();
foreach (var annotation in cap.Annotations)
{
var tmp = ProcessBoundingBoxAnnotations(annotation.Item2.RawValues);
foreach (var box in tmp.Values)
{
boxes[box.id] = box;
if (m_FirstBoxAnnotation)
{
boxJson = "[";
m_FirstBoxAnnotation = false;
}
else
boxJson += ",";
boxJson += JsonUtility.ToJson(box);
}
}
foreach (var annotation in cap.Annotations)
{
var keypoints = ProcessKeypointAnnotations(annotation.Item2.RawValues, boxes);
foreach (var kp in keypoints.Values)
{
if (m_FirstKeypointAnnotation)
{
keypointJson = "[";
m_FirstKeypointAnnotation = false;
}
else
keypointJson += ",";
var builder = new StringBuilder();
var stringWriter = new StringWriter(builder);
var serializer = new JsonSerializer();
using (var writer = new JsonTextWriter(stringWriter))
{
serializer.Serialize(writer, kp);
}
keypointJson += builder.ToString();
}
}
}
if (m_ObjectDetectionWritingTask != null)
await m_ObjectDetectionWritingTask;
if (boxJson != string.Empty)
{
InitializeCaptureFiles();
m_ObjectDetectionWritingTask = m_ObjectDetectionStream.WriteAsync(boxJson);
}
if (m_KeypointDetectionWritingTask != null)
await m_KeypointDetectionWritingTask;
if (keypointJson != string.Empty)
{
InitializeCaptureFiles();
m_KeypointDetectionWritingTask = m_KeypointDetectionStream.WriteAsync(keypointJson);
}
}
static Dictionary<int, CocoTypes.ObjectDetectionAnnotation> ProcessBoundingBoxAnnotations(IEnumerable<object> annotations)
{
var map = new Dictionary<int, CocoTypes.ObjectDetectionAnnotation>();
foreach (var annotation in annotations)
{
if (annotation is BoundingBox2DLabeler.BoundingBoxValue bbox)
{
var coco = CocoTypes.ObjectDetectionAnnotation.FromBoundingBoxValue(bbox);
map[coco.id] = coco;
}
}
return map;
}
static Dictionary<int, CocoTypes.KeypointAnnotation> ProcessKeypointAnnotations(IEnumerable<object> annotations, Dictionary<int, CocoTypes.ObjectDetectionAnnotation> boundingBoxes)
{
var map = new Dictionary<int, CocoTypes.KeypointAnnotation>();
foreach (var annotation in annotations)
{
if (annotation is KeypointLabeler.KeypointEntry keypoint)
{
var coco = CocoTypes.KeypointAnnotation.FromKeypointValue(keypoint);
if (boundingBoxes.ContainsKey(coco.id))
{
coco.CopyObjectDetectionData(boundingBoxes[coco.id]);
}
map[coco.id] = coco;
}
}
return map;
}
public async Task OnCaptureReported(int frame, int width, int height, string filename)
{
InitializeCaptureFiles();
var json = string.Empty;
var converted = AnnotationHandler.HandleCameraCapture(frame, width, height, filename);
if (m_FirstCapture)
{
json = "[";
m_FirstCapture = false;
}
else
json += ",";
json += JsonUtility.ToJson(converted);
if (m_RgbCaptureWritingTask != null)
await m_RgbCaptureWritingTask;
if (json != string.Empty)
m_RgbCaptureWritingTask = m_RgbCaptureStream.WriteAsync(json);
m_DataCaptured = true;
}
}
}
正在加载...
取消
保存