浏览代码

Major refactoring. Moving RenderedObjectInfo and instance segmentation back into PerceptionCamera. Moving instance id -> labelentry cache into LabelingConfiguration. Properly implementing the labelers.

Still need to implement the hookups in PerceptionCamera to call into the labelers at runtime and write more tests.
/labeler_mock_mb
Jon Hogins 4 年前
当前提交
88c3dc1a
共有 30 个文件被更改,包括 631 次插入643 次删除
  1. 119
      TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity
  2. 6
      TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset
  3. 36
      com.unity.perception/Editor/GroundTruth/BoundingBoxLabelerEditor.cs
  4. 5
      com.unity.perception/Runtime/GroundTruth/GroundTruthRendererFeature.cs
  5. 23
      com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs
  6. 153
      com.unity.perception/Runtime/GroundTruth/Labelers/RenderedObjectInfoLabeler.cs
  7. 49
      com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs
  8. 45
      com.unity.perception/Runtime/GroundTruth/Labeling/LabelingConfiguration.cs
  9. 30
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs
  10. 9
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs
  11. 13
      com.unity.perception/Runtime/GroundTruth/RenderedObjectInfo.cs
  12. 77
      com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs
  13. 12
      com.unity.perception/Runtime/GroundTruth/SemanticSegmentationCrossPipelinePass.cs
  14. 5
      com.unity.perception/Tests/Editor/PerceptionCameraEditorTests.cs
  15. 29
      com.unity.perception/Tests/Runtime/GroundTruthTests/PerceptionCameraIntegrationTests.cs
  16. 50
      com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs
  17. 22
      com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs
  18. 26
      com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs
  19. 3
      com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs.meta
  20. 112
      com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs
  21. 3
      com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs.meta
  22. 54
      com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs
  23. 3
      com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs.meta
  24. 162
      com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs
  25. 11
      com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs.meta
  26. 25
      com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs
  27. 192
      com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox2DTests.cs
  28. 0
      /com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs.meta
  29. 0
      /com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs.meta
  30. 0
      /com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs

119
TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity


m_TrainingDataDestination: TrainingData
m_LightProbeSampleCountMultiplier: 4
m_LightingDataAsset: {fileID: 0}
m_UseShadowmask: 0
m_UseShadowmask: 1
--- !u!196 &4
NavMeshSettings:
serializedVersion: 2

- component: {fileID: 963194229}
- component: {fileID: 963194227}
- component: {fileID: 963194231}
- component: {fileID: 963194236}
- component: {fileID: 963194235}
- component: {fileID: 963194234}
- component: {fileID: 963194233}
- component: {fileID: 963194232}
m_Layer: 0
m_Name: Main Camera
m_TagString: MainCamera

period: 0.0166
startTime: 0
captureRgbImages: 1
labelers:
- id: 0
- id: 1
- id: 2
- id: 3
references:
version: 1
00000000:
type: {class: RenderedObjectInfoLabeler, ns: UnityEngine.Perception.GroundTruth,
asm: Unity.Perception.Runtime}
data:
enabled: 1
foldout: 0
objectInfoMetricId: 5BA92024-B3B7-41A7-9D3F-C03A6A8DDD01
labelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
type: 2}
00000001:
type: {class: SemanticSegmentationLabeler, ns: UnityEngine.Perception.GroundTruth,
asm: Unity.Perception.Runtime}
data:
enabled: 1
foldout: 0
annotationId: 12F94D8D-5425-4DEB-9B21-5E53AD957D66
labelConfig: {fileID: 11400000, guid: c140c5aa05dd09e4fadaa26de31b1f39, type: 2}
00000002:
type: {class: BoundingBox2DLabeler, ns: UnityEngine.Perception.GroundTruth,
asm: Unity.Perception.Runtime}
data:
enabled: 1
foldout: 0
annotationId: F9F22E05-443F-4602-A422-EBE4EA9B55CB
labelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
type: 2}
00000003:
type: {class: ObjectCountLabeler, ns: UnityEngine.Perception.GroundTruth, asm: Unity.Perception.Runtime}
data:
enabled: 1
foldout: 0
objectCountMetricId: 51DA3C27-369D-4929-AEA6-D01614635CE2
m_LabelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
type: 2}
--- !u!114 &963194231
MonoBehaviour:
m_ObjectHideFlags: 0

m_EditorClassIdentifier:
light: {fileID: 705507993}
target: {fileID: 1640252278}
--- !u!114 &963194232
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 60775aefebb84bf49aff98525a6f7b55, type: 3}
m_Name:
m_EditorClassIdentifier:
totalFrames: 1000
--- !u!114 &963194233
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 6c08fb5eff78425db5567cf66f9cb3fc, type: 3}
m_Name:
m_EditorClassIdentifier:
annotationId: 12F94D8D-5425-4DEB-9B21-5E53AD957D66
labelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
type: 2}
--- !u!114 &963194234
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: d0127208de324c00a881af84cdd63da3, type: 3}
m_Name:
m_EditorClassIdentifier:
annotationId: F9F22E05-443F-4602-A422-EBE4EA9B55CB
--- !u!114 &963194235
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: c2fcc6ce8ec34a44ae0f1679f25b6fb4, type: 3}
m_Name:
m_EditorClassIdentifier:
produceObjectInfoMetrics: 1
objectInfoMetricId: 5BA92024-B3B7-41A7-9D3F-C03A6A8DDD01
produceObjectCountMetrics: 1
objectCountMetricId: 51DA3C27-369D-4929-AEA6-D01614635CE2
labelingConfiguration: {fileID: 11400000, guid: e74234fe725079e4aa7ecd74797ceb79,
type: 2}
--- !u!114 &963194236
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 43085b9d882c415b90fb9ebe4d765f26, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &1640252278
GameObject:
m_ObjectHideFlags: 0

6
TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset


m_Name: SemanticSegmentationLabelingConfiguration
m_EditorClassIdentifier: Unity.Perception.Runtime:UnityEngine.Perception.GroundTruth:SemanticSegmentationLabelConfig
LabelEntries:
- label: box
- label: Box
- label: cube
- label: Crate
- label: crate
- label: Cube
pixelValue: 30000
AutoAssignIds: 1
StartingLabelId: 1

36
com.unity.perception/Editor/GroundTruth/BoundingBoxLabelerEditor.cs


namespace UnityEditor.Perception.GroundTruth
{
[CustomEditor(typeof(BoundingBoxLabeler))]
public class BoundingBoxLabelerEditor : Editor
{
public override void OnInspectorGUI()
{
// base.OnInspectorGUI();
// return;
EditorGUILayout.PropertyField(this.serializedObject.FindProperty(nameof(BoundingBoxLabeler.annotationId)));
var serializedProperty = this.serializedObject.FindProperty(nameof(BoundingBoxLabeler.labelingConfiguration));
if (serializedProperty.objectReferenceValue != null)
{
EditorGUILayout.Separator();
var editor = Editor.CreateEditor(serializedProperty.objectReferenceValue);
editor.OnInspectorGUI();
}
}
}
// [CustomEditor(typeof(BoundingBoxLabeler))]
// public class BoundingBoxLabelerEditor : Editor
// {
// public override void OnInspectorGUI()
// {
// // base.OnInspectorGUI();
// // return;
// EditorGUILayout.PropertyField(this.serializedObject.FindProperty(nameof(BoundingBoxLabeler.annotationId)));
// var serializedProperty = this.serializedObject.FindProperty(nameof(BoundingBoxLabeler.labelingConfiguration));
//
// if (serializedProperty.objectReferenceValue != null)
// {
// EditorGUILayout.Separator();
// var editor = Editor.CreateEditor(serializedProperty.objectReferenceValue);
// editor.OnInspectorGUI();
// }
// }
// }
}

5
com.unity.perception/Runtime/GroundTruth/GroundTruthRendererFeature.cs


#if URP_PRESENT
using System;
using System.Collections.Generic;
using UnityEditor;
using UnityEngine;
using UnityEngine.Rendering;

{
SemanticSegmentationCrossPipelinePass m_SemanticSegmentationCrossPipelinePass;
public SemanticSegmentationUrpPass(Camera camera, RenderTexture targetTexture, LabelingConfiguration labelingConfiguration)
public SemanticSegmentationUrpPass(Camera camera, RenderTexture targetTexture, SemanticSegmentationLabelConfig labelConfig)
m_SemanticSegmentationCrossPipelinePass = new SemanticSegmentationCrossPipelinePass(camera, labelingConfiguration);
m_SemanticSegmentationCrossPipelinePass = new SemanticSegmentationCrossPipelinePass(camera, labelConfig);
ConfigureTarget(targetTexture, targetTexture.depthBuffer);
m_SemanticSegmentationCrossPipelinePass.Setup();
}

23
com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs


Dictionary<int, AsyncAnnotation> m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
AnnotationDefinition m_BoundingBoxAnnotationDefinition;
BoundingBoxValue[] m_BoundingBoxValues;
RenderedObjectInfoLabeler m_RenderedObjectInfoLabeler;
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]

public string label_name;
public int instance_id;
public uint instance_id;
}
public BoundingBox2DLabeler()
{
}
public BoundingBox2DLabeler(LabelingConfiguration labelConfig)
{
this.labelingConfiguration = labelConfig;
m_RenderedObjectInfoLabeler = (RenderedObjectInfoLabeler)PerceptionCamera.labelers.First(l => l is RenderedObjectInfoLabeler && ((RenderedObjectInfoLabeler)l).labelingConfiguration == labelingConfiguration);
if (m_RenderedObjectInfoLabeler == null)
{
PerceptionCamera.labelers
}
m_BoundingBoxAnnotationDefinition = SimulationManager.RegisterAnnotationDefinition("bounding box", m_RenderedObjectInfoLabeler.labelingConfiguration.GetAnnotationSpecification(),
m_BoundingBoxAnnotationDefinition = SimulationManager.RegisterAnnotationDefinition("bounding box", labelingConfiguration.GetAnnotationSpecification(),
m_RenderedObjectInfoLabeler.renderedObjectInfosCalculated += OnRenderedObjectInfosCalculated;
PerceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfosCalculated;
}
public override void OnBeginRendering()

for (var i = 0; i < renderedObjectInfos.Length; i++)
{
var objectInfo = renderedObjectInfos[i];
if (!m_RenderedObjectInfoLabeler.TryGetLabelEntryFromInstanceId(objectInfo.instanceId, out var labelEntry))
if (!labelingConfiguration.TryGetLabelEntryFromInstanceId(objectInfo.instanceId, out var labelEntry))
continue;
m_BoundingBoxValues[i] = new BoundingBoxValue

153
com.unity.perception/Runtime/GroundTruth/Labelers/RenderedObjectInfoLabeler.cs


using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using Unity.Entities;
using UnityEngine.Serialization;
namespace UnityEngine.Perception.GroundTruth
{

public bool produceObjectInfoMetrics = true;
public bool produceObjectCountMetrics = true;
/// <summary>
/// The ID to use for object count annotations in the resulting dataset
/// </summary>
public string objectCountMetricId = "51DA3C27-369D-4929-AEA6-D01614635CE2";
static ProfilerMarker s_RenderedObjectInfosCalculatedEvent = new ProfilerMarker("renderedObjectInfosCalculated event");
static ProfilerMarker s_ClassCountCallback = new ProfilerMarker("OnClassLabelsReceived");
[SuppressMessage("ReSharper", "InconsistentNaming")]
struct ObjectCountSpec
{
[UsedImplicitly]
public int label_id;
[UsedImplicitly]
public string label_name;
}
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]
struct ClassCountValue
{
public int label_id;
public string label_name;
public uint count;
}
// ReSharper disable InconsistentNaming
struct RenderedObjectInfoValue
{

public int instance_id;
public uint instance_id;
[UsedImplicitly]
public int visible_pixels;
}

ClassCountValue[] m_ClassCountValues;
Dictionary<int, AsyncMetric> m_ObjectCountAsyncMetrics = new Dictionary<int, AsyncMetric>();
MetricDefinition m_ObjectCountMetricDefinition;
internal event Action<NativeSlice<uint>, IReadOnlyList<LabelEntry>, int> classCountsReceived;
public string boundingBoxAnnotationId = "F9F22E05-443F-4602-A422-EBE4EA9B55CB";
Dictionary<int, AsyncAnnotation> m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
AnnotationDefinition m_BoundingBoxAnnotationDefinition;
BoundingBoxValue[] m_BoundingBoxValues;
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]
struct BoundingBoxValue
public RenderedObjectInfoLabeler()
{
}
public RenderedObjectInfoLabeler(LabelingConfiguration labelingConfiguration)
public int label_id;
public string label_name;
public int instance_id;
public float x;
public float y;
public float width;
public float height;
this.labelingConfiguration = labelingConfiguration;
public void Setup()
public override void Setup()
{
if (labelingConfiguration == null)
{

}
PerceptionCamera.InstanceSegmentationImageReadback += (frameCount, data, tex) =>
PerceptionCamera.RenderedObjectInfosCalculated += (frameCount, objectInfo) =>
using (s_RenderedObjectInfosCalculatedEvent.Auto())
renderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
if (produceObjectCountMetrics)
ProduceObjectCountMetric(classCounts, labelingConfiguration.LabelEntries, frameCount);
if (produceObjectInfoMetrics)
ProduceRenderedObjectInfoMetric(renderedObjectInfos, frameCount);
ProduceRenderedObjectInfoMetric(objectInfo, frameCount);
if (produceObjectCountMetrics)
if (m_RenderedObjectInfoMetricDefinition.Equals(default))
if (m_ObjectCountMetricDefinition.Equals(default(MetricDefinition)))
{
m_ObjectCountMetricDefinition = SimulationManager.RegisterMetricDefinition("object count", CreateLabelingMetricSpecs(),
"Counts of objects for each label in the sensor's view", id: new Guid(objectCountMetricId));
}
m_ObjectCountAsyncMetrics[Time.frameCount] = m_PerceptionCamera.SensorHandle.ReportMetricAsync(m_ObjectCountMetricDefinition);
}
if (produceObjectInfoMetrics)
{
if (m_RenderedObjectInfoMetricDefinition.Equals(default(MetricDefinition)))
{
m_RenderedObjectInfoMetricDefinition = SimulationManager.RegisterMetricDefinition(
"rendered object info",
CreateLabelingMetricSpecs(),
"Information about each labeled object visible to the sensor",
id: new Guid(objectInfoMetricId));
}
m_ObjectInfoAsyncMetrics[Time.frameCount] = m_PerceptionCamera.SensorHandle.ReportMetricAsync(m_RenderedObjectInfoMetricDefinition);
m_RenderedObjectInfoMetricDefinition = SimulationManager.RegisterMetricDefinition(
"rendered object info",
labelingConfiguration.GetAnnotationSpecification(),
"Information about each labeled object visible to the sensor",
id: new Guid(objectInfoMetricId));
}
ObjectCountSpec[] CreateLabelingMetricSpecs()
{
var labelingMetricSpec = labelingConfiguration.LabelEntries.Select((l) => new ObjectCountSpec()
{
label_id = l.id,
label_name = l.label,
}).ToArray();
return labelingMetricSpec;
}
void ProduceObjectCountMetric(NativeSlice<uint> counts, IReadOnlyList<LabelEntry> entries, int frameCount)
{
using (s_ClassCountCallback.Auto())
{
classCountsReceived?.Invoke(counts, entries, frameCount);
if (!m_ObjectCountAsyncMetrics.TryGetValue(frameCount, out var classCountAsyncMetric))
return;
m_ObjectCountAsyncMetrics.Remove(frameCount);
if (m_ClassCountValues == null || m_ClassCountValues.Length != entries.Count)
m_ClassCountValues = new ClassCountValue[entries.Count];
for (var i = 0; i < entries.Count; i++)
{
m_ClassCountValues[i] = new ClassCountValue()
{
label_id = entries[i].id,
label_name = entries[i].label,
count = counts[i]
};
}
classCountAsyncMetric.ReportValues(m_ClassCountValues);
}
m_ObjectInfoAsyncMetrics[Time.frameCount] = PerceptionCamera.SensorHandle.ReportMetricAsync(m_RenderedObjectInfoMetricDefinition);
}
void ProduceRenderedObjectInfoMetric(NativeArray<RenderedObjectInfo> renderedObjectInfos, int frameCount)

}
}
public bool TryGetLabelEntryFromInstanceId(int instanceId, out LabelEntry labelEntry)
{
return m_RenderedObjectInfoGenerator.TryGetLabelEntryFromInstanceId(instanceId, out labelEntry);
}
void OnDisable()
bool TryGetLabelEntryFromInstanceId(uint instanceId, out LabelEntry labelEntry)
if (m_RenderedObjectInfoGenerator != null)
{
World.DefaultGameObjectInjectionWorld?.GetExistingSystem<GroundTruthLabelSetupSystem>()?.Deactivate(m_RenderedObjectInfoGenerator);
m_RenderedObjectInfoGenerator?.Dispose();
m_RenderedObjectInfoGenerator = null;
}
return labelingConfiguration.TryGetLabelEntryFromInstanceId(instanceId, out labelEntry);
}
}
}

49
com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs


/// </summary>
public SemanticSegmentationLabelConfig labelConfig;
public event Action<int,NativeArray<uint>,RenderTexture> SemanticSegmentationImageReadback;
RenderTextureReader<short> m_SemanticSegmentationTextureReader;
RenderTextureReader<uint> m_SemanticSegmentationTextureReader;
#if HDRP_PRESENT
SemanticSegmentationPass m_SemanticSegmentationPass;

PerceptionCamera m_PerceptionCamera;
public SemanticSegmentationLabeler()
{
}
public SemanticSegmentationLabeler(SemanticSegmentationLabelConfig labelConfig)
{
this.labelConfig = labelConfig;
}
[UsedImplicitly]
public int label_id;
[UsedImplicitly]
public string label_name;
[UsedImplicitly]

struct AsyncSemanticSegmentationWrite
{
public short[] dataArray;
public NativeArray<uint> data;
void Start()
public override void Setup()
var myCamera = GetComponent<Camera>();
var myCamera = PerceptionCamera.GetComponent<Camera>();
if (labelingConfiguration == null)
if (labelConfig == null)
{
Debug.LogError("LabelingConfiguration must be set if producing ground truth data");
this.enabled = false;

semanticSegmentationTexture.name = "Labeling";
m_PerceptionCamera = GetComponent<PerceptionCamera>();
#if HDRP_PRESENT
var customPassVolume = this.GetComponent<CustomPassVolume>() ?? gameObject.AddComponent<CustomPassVolume>();

customPassVolume.customPasses.Add(m_SemanticSegmentationPass);
#endif
#if URP_PRESENT
m_PerceptionCamera.AddScriptableRenderPass(new SemanticSegmentationUrpPass(myCamera, semanticSegmentationTexture, labelingConfiguration));
PerceptionCamera.AddScriptableRenderPass(new SemanticSegmentationUrpPass(myCamera, semanticSegmentationTexture, labelConfig));
var specs = labelingConfiguration.LabelEntries.Select((l) => new SemanticSegmentationSpec()
var specs = labelConfig.LabelEntries.Select((l) => new SemanticSegmentationSpec()
label_id = l.id,
pixel_value = l.value
pixel_value = l.pixelValue
m_SemanticSegmentationTextureReader = new RenderTextureReader<short>(semanticSegmentationTexture, myCamera,
m_SemanticSegmentationTextureReader = new RenderTextureReader<uint>(semanticSegmentationTexture, myCamera,
m_PerceptionCamera.BeginRendering += ReportAsyncAnnotations;
void OnSemanticSegmentationImageRead(int frameCount, NativeArray<short> data)
void OnSemanticSegmentationImageRead(int frameCount, NativeArray<uint> data)
{
var dxLocalPath = Path.Combine(k_SemanticSegmentationDirectory, k_SegmentationFilePrefix) + frameCount + ".png";
var path = Path.Combine(Manager.Instance.GetDirectoryFor(k_SemanticSegmentationDirectory), k_SegmentationFilePrefix) + frameCount + ".png";

annotation.ReportFile(dxLocalPath);
var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncSemanticSegmentationWrite>>();
SemanticSegmentationImageReadback?.Invoke(frameCount, data, semanticSegmentationTexture);
dataArray = data.ToArray(),
data = new NativeArray<uint>(data, Allocator.TempJob),
width = semanticSegmentationTexture.width,
height = semanticSegmentationTexture.height,
path = path

Profiler.EndSample();
Profiler.BeginSample("Encode");
var pngBytes = ImageConversion.EncodeArrayToPNG(r.data.dataArray, GraphicsFormat.R8G8B8A8_UNorm, (uint)r.data.width, (uint)r.data.height);
var pngBytes = ImageConversion.EncodeArrayToPNG(r.data.data.ToArray(), GraphicsFormat.R8G8B8A8_UNorm, (uint)r.data.width, (uint)r.data.height);
r.data.data.Dispose();
void ReportAsyncAnnotations()
public override void OnBeginRendering()
m_AsyncAnnotations[Time.frameCount] = m_PerceptionCamera.SensorHandle.ReportAnnotationAsync(m_SemanticSegmentationAnnotationDefinition);
m_AsyncAnnotations[Time.frameCount] = PerceptionCamera.SensorHandle.ReportAnnotationAsync(m_SemanticSegmentationAnnotationDefinition);
void OnSimulationEnding()
public override void OnSimulationEnding()
{
m_SemanticSegmentationTextureReader?.WaitForAllImages();
m_SemanticSegmentationTextureReader?.Dispose();

45
com.unity.perception/Runtime/GroundTruth/Labeling/LabelingConfiguration.cs


using System.Diagnostics.CodeAnalysis;
using System.Linq;
using JetBrains.Annotations;
using Unity.Collections;
using UnityEngine.Serialization;
namespace UnityEngine.Perception.GroundTruth

[CreateAssetMenu(fileName = "LabelingConfiguration", menuName = "Perception/Labeling Configuration", order = 1)]
public class LabelingConfiguration : ScriptableObject
{
[FormerlySerializedAs("LabelingConfigurations")]
[FormerlySerializedAs("LabelEntries")]
[SerializeField]

/// Whether the inspector will start label ids at zero or one when <see cref="AutoAssignIds"/> is enabled.
/// </summary>
public StartingLabelId StartingLabelId = StartingLabelId.One;
LabelEntryMatchCache m_LabelEntryMatchCache;
/// <summary>
/// A sequence of <see cref="LabelEntry"/> which defines the labels relevant for this configuration and their values.

labelEntryIndex = -1;
labelEntry = default;
return false;
}
/// <summary>
/// Initialize the list of LabelEntries on this LabelingConfiguration. Should only be called immediately after instantiation.
/// </summary>
/// <param name="labelEntries">The LabelEntry values to associate with this LabelingConfiguration</param>
/// <exception cref="InvalidOperationException">Thrown if <see cref="TryGetLabelEntryFromInstanceId"/> has ever been called on this object.</exception>
public void Init(IEnumerable<LabelEntry> labelEntries)
{
if (m_LabelEntryMatchCache != null)
{
throw new InvalidOperationException("LabelingConfiguration.Init() may not be called after TryGetLabelEntryFromInstanceId has been called for the first time.");
}
this.m_LabelEntries = new List<LabelEntry>(labelEntries);
}
/// <summary>
/// Attempts to find the label id for the given instance id.
/// </summary>
/// <param name="instanceId">The instanceId of the object for which the labelId should be found</param>
/// <param name="labelEntry">The LabelEntry associated with the object. default if not found</param>
/// <returns>True if a labelId is found for the given instanceId.</returns>
public bool TryGetLabelEntryFromInstanceId(uint instanceId, out LabelEntry labelEntry)
{
return TryGetLabelEntryFromInstanceId(instanceId, out labelEntry, out var _);
}
/// <summary>
/// Attempts to find the label id for the given instance id.
/// </summary>
/// <param name="instanceId">The instanceId of the object for which the labelId should be found</param>
/// <param name="labelEntry">The LabelEntry associated with the object. default if not found</param>
/// <param name="index">The index of the found LabelEntry in <see cref="LabelEntries"/>. -1 if not found</param>
/// <returns>True if a labelId is found for the given instanceId.</returns>
public bool TryGetLabelEntryFromInstanceId(uint instanceId, out LabelEntry labelEntry, out int index)
{
if (m_LabelEntryMatchCache == null)
m_LabelEntryMatchCache = new LabelEntryMatchCache(this);
return m_LabelEntryMatchCache.TryGetLabelEntryFromInstanceId(instanceId, out labelEntry, out index);
}
[SuppressMessage("ReSharper", "InconsistentNaming")]

30
com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs


/// The <see cref="SensorHandle"/> associated with this camera. Use this to report additional annotations and metrics at runtime.
/// </summary>
public SensorHandle SensorHandle { get; private set; }
struct AsyncCaptureInfo
{
public int FrameCount;
public AsyncAnnotation SegmentationAsyncAnnotation;
public AsyncMetric ClassCountAsyncMetric;
public AsyncMetric RenderedObjectInfoAsyncMetric;
public AsyncAnnotation BoundingBoxAsyncMetric;
}
List<AsyncCaptureInfo> m_AsyncCaptureInfos = new List<AsyncCaptureInfo>();
static ProfilerMarker s_WriteFrame = new ProfilerMarker("Write Frame (PerceptionCamera)");
static ProfilerMarker s_FlipY = new ProfilerMarker("Flip Y (PerceptionCamera)");

SensorHandle = default;
CleanUpInstanceSegmentation();
}
}
[Serializable]
public abstract class CameraLabeler
{
public bool enabled;
public bool foldout;
protected PerceptionCamera PerceptionCamera { get; private set; }
protected SensorHandle SensorHandle { get; private set; }
public abstract void Setup();
public virtual void OnUpdate() {}
public virtual void OnBeginRendering() {}
internal void Init(PerceptionCamera perceptionCamera)
{
PerceptionCamera = perceptionCamera;
SensorHandle = perceptionCamera.SensorHandle;
}
}
}

9
com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs


void SetupInstanceSegmentation()
{
var camera = PerceptionCamera.GetComponent<Camera>();
var camera = GetComponent<Camera>();
var width = camera.pixelWidth;
var height = camera.pixelHeight;
m_InstanceSegmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));

World.DefaultGameObjectInjectionWorld.GetExistingSystem<GroundTruthLabelSetupSystem>().Activate(m_RenderedObjectInfoGenerator);
#if HDRP_PRESENT
var customPassVolume = this.GetComponent<CustomPassVolume>() ?? gameObject.AddComponent<CustomPassVolume>();

m_InstanceSegmentationReader = new RenderTextureReader<uint>(m_InstanceSegmentationTexture, camera, (frameCount, data, tex) =>
{
InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
if (RenderedObjectInfosCalculated != null)
{
m_RenderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var renderedObjectInfos, Allocator.Temp);
RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
renderedObjectInfos.Dispose();
}
});
}

13
com.unity.perception/Runtime/GroundTruth/RenderedObjectInfo.cs


/// <summary>
/// The instanceId of the rendered object.
/// </summary>
public int instanceId;
/// <summary>
/// The labelId of the object resolved by a <see cref="LabelingConfiguration"/>
/// </summary>
public int labelId;
public uint instanceId;
/// <summary>
/// The bounding box of the object in pixel coordinates.
/// </summary>

/// <inheritdoc />
public override string ToString()
{
return $"{nameof(instanceId)}: {instanceId}, {nameof(labelId)}: {labelId}, {nameof(boundingBox)}: {boundingBox}, {nameof(pixelCount)}: {pixelCount}";
return $"{nameof(instanceId)}: {instanceId}, {nameof(boundingBox)}: {boundingBox}, {nameof(pixelCount)}: {pixelCount}";
return instanceId == other.instanceId && labelId == other.labelId && boundingBox.Equals(other.boundingBox) && pixelCount == other.pixelCount;
return instanceId == other.instanceId && boundingBox.Equals(other.boundingBox) && pixelCount == other.pixelCount;
}
/// <inheritdoc />

{
unchecked
{
var hashCode = instanceId;
hashCode = (hashCode * 397) ^ labelId;
var hashCode = (int)instanceId;
hashCode = (hashCode * 397) ^ boundingBox.GetHashCode();
hashCode = (hashCode * 397) ^ pixelCount;
return hashCode;

77
com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs


using Unity.Jobs;
using Unity.Mathematics;
using Unity.Profiling;
using UnityEngine.UIElements;
namespace UnityEngine.Perception.GroundTruth
{

public class RenderedObjectInfoGenerator : IGroundTruthGenerator, IDisposable
public class RenderedObjectInfoGenerator
const int k_StartingObjectCount = 1 << 8;
public int instanceId;
public uint instanceId;
public int row;
public int left;
public int right;

for (var row = 0; row < rows; row++)
{
var rowSlice = new NativeSlice<uint>(segmentationImageData, width * row, width);
instanceId = -1,
instanceId = 0,
row = row + rowStart
};
for (var i = 0; i < rowSlice.Length; i++)

currentBB = new Object1DSpan
{
instanceId = (int)value,
instanceId = value,
left = i,
row = row + rowStart
};

}
}
NativeList<int> m_InstanceIdToLabelEntryIndexLookup;
LabelingConfiguration m_LabelingConfiguration;
// ReSharper disable once InvalidXmlDocComment
/// <summary>
/// Create a new CpuRenderedObjectInfoPass with the given LabelingConfiguration.
/// </summary>
/// <param name="labelingConfiguration">The LabelingConfiguration to use to determine labelId. Should match the
/// one used by the <seealso cref="InstanceSegmentationUrpPass"/> generating the input image. See <see cref="Compute"/></param>
public RenderedObjectInfoGenerator(LabelingConfiguration labelingConfiguration)
{
m_LabelingConfiguration = labelingConfiguration;
m_InstanceIdToLabelEntryIndexLookup = new NativeList<int>(k_StartingObjectCount, Allocator.Persistent);
}
/// <inheritdoc/>
public void SetupMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
if (m_LabelingConfiguration.TryGetMatchingConfigurationEntry(labeling, out _, out var index))
{
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
{
m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
}
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = index;
}
}
// ReSharper disable once InvalidXmlDocComment
/// <summary>

/// <param name="stride">Stride of the image data. Should be equal to the width of the image.</param>
/// <param name="boundingBoxOrigin">Whether bounding boxes should be top-left or bottom-right-based.</param>
/// <param name="renderedObjectInfos">When this method returns, filled with RenderedObjectInfo entries for each object visible in the frame.</param>
/// <param name="perLabelEntryObjectCount">When the method returns, filled with a NativeArray with the count of objects for each entry in <see cref="LabelingConfiguration.LabelEntries"/> in the LabelingConfiguration passed into the constructor.</param>
public void Compute(NativeArray<uint> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, out NativeArray<uint> perLabelEntryObjectCount, Allocator allocator)
public void Compute(NativeArray<uint> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
{
const int jobCount = 24;
var height = instanceSegmentationRawData.Length / stride;

JobHandle.CompleteAll(handles);
}
perLabelEntryObjectCount = new NativeArray<uint>(m_LabelingConfiguration.LabelEntries.Count, allocator);
var boundingBoxMap = new NativeHashMap<int, RenderedObjectInfo>(100, Allocator.Temp);
//perLabelEntryObjectCount = new NativeArray<uint>(m_LabelingConfiguration.LabelEntries.Count, allocator);
var boundingBoxMap = new NativeHashMap<uint, RenderedObjectInfo>(100, Allocator.Temp);
using (s_LabelMerge.Auto())
{
foreach (var boundingBoxList in jobBoundingBoxLists)

for (var i = 0; i < keyValueArrays.Keys.Length; i++)
{
var instanceId = keyValueArrays.Keys[i];
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
continue;
var labelIndex = m_InstanceIdToLabelEntryIndexLookup[instanceId];
var labelId = m_LabelingConfiguration.LabelEntries[labelIndex].id;
perLabelEntryObjectCount[labelIndex]++;
var renderedObjectInfo = keyValueArrays.Values[i];
var boundingBox = renderedObjectInfo.boundingBox;
if (boundingBoxOrigin == BoundingBoxOrigin.TopLeft)

renderedObjectInfos[i] = new RenderedObjectInfo
{
instanceId = instanceId,
labelId = labelId,
boundingBox = boundingBox,
pixelCount = renderedObjectInfo.pixelCount
};

}
handles.Dispose();
}
/// <summary>
/// Attempts to find the label id for the given instance id using the LabelingConfiguration passed into the constructor.
/// </summary>
/// <param name="instanceId">The instanceId of the object for which the labelId should be found</param>
/// <param name="labelEntry">The LabelEntry associated with the object. default if not found</param>
/// <returns>True if a labelId is found for the given instanceId.</returns>
public bool TryGetLabelEntryFromInstanceId(int instanceId, out LabelEntry labelEntry)
{
labelEntry = default;
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
return false;
var labelEntryIndex = m_InstanceIdToLabelEntryIndexLookup[instanceId];
labelEntry = m_LabelingConfiguration.LabelEntries[labelEntryIndex];
return true;
}
/// <inheritdoc />
public void Dispose()
{
m_InstanceIdToLabelEntryIndexLookup.Dispose();
}
}
}

12
com.unity.perception/Runtime/GroundTruth/SemanticSegmentationCrossPipelinePass.cs


const string k_ShaderName = "Perception/SemanticSegmentation";
static readonly int k_LabelingId = Shader.PropertyToID("LabelingId");
LabelingConfiguration m_LabelingConfiguration;
SemanticSegmentationLabelConfig m_LabelConfig;
//Serialize the shader so that the shader asset is included in player builds when the SemanticSegmentationPass is used.
//Currently commented out and shaders moved to Resources folder due to serialization crashes when it is enabled.

Material m_OverrideMaterial;
public SemanticSegmentationCrossPipelinePass(Camera targetCamera, LabelingConfiguration labelingConfiguration) : base(targetCamera)
public SemanticSegmentationCrossPipelinePass(Camera targetCamera, SemanticSegmentationLabelConfig labelConfig) : base(targetCamera)
this.m_LabelingConfiguration = labelingConfiguration;
this.m_LabelConfig = labelConfig;
}
public override void Setup()

public override void SetupMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
var entry = new LabelEntry();
foreach (var l in m_LabelingConfiguration.LabelEntries)
var entry = new SemanticSegmentationLabelEntry();
foreach (var l in m_LabelConfig.LabelEntries)
{
if (labeling.labels.Contains(l.label))
{

}
//Set the labeling ID so that it can be accessed in ClassSemanticSegmentationPass.shader
mpb.SetInt(k_LabelingId, entry.value);
mpb.SetInt(k_LabelingId, entry.pixelValue);
}
}
}

5
com.unity.perception/Tests/Editor/PerceptionCameraEditorTests.cs


var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
perceptionCamera.captureRgbImages = true;
cameraObject.AddComponent<BoundingBoxLabeler>();
var renderedObjectInfoLabeler = cameraObject.AddComponent<RenderedObjectInfoLabeler>();
renderedObjectInfoLabeler.labelingConfiguration = labelingConfiguration;
perceptionCamera.labelers.Add(new BoundingBox2DLabeler(labelingConfiguration));
perceptionCamera.labelers.Add(new RenderedObjectInfoLabeler(labelingConfiguration));
cameraObject.SetActive(true);
}

29
com.unity.perception/Tests/Runtime/GroundTruthTests/PerceptionCameraIntegrationTests.cs


""height"": {Screen.height / 2:F1}
}}";
var labelingConfiguration = CreateLabelingConfiguration();
SetupCamera(labelingConfiguration, pc =>
SetupCamera(pc =>
pc.gameObject.AddComponent<BoundingBoxLabeler>();
pc.labelers.Add(new BoundingBox2DLabeler(labelingConfiguration));
});
var plane = TestHelper.CreateLabeledPlane();

[UnityTest]
public IEnumerator EnableSemanticSegmentation_GeneratesCorrectDataset()
{
var labelingConfiguration = CreateLabelingConfiguration();
SetupCamera(labelingConfiguration, pc =>
SetupCamera(pc =>
var semanticSegmentationLabeler = pc.gameObject.AddComponent<SemanticSegmentationLabeler>();
semanticSegmentationLabeler.labelingConfiguration = labelingConfiguration;
pc.labelers.Add(new SemanticSegmentationLabeler(CreateSemanticSegmentationLabelConfig()));
});
string expectedImageFilename = $"segmentation_{Time.frameCount}.png";

var label = "label";
var labelingConfiguration = ScriptableObject.CreateInstance<LabelingConfiguration>();
labelingConfiguration.LabelEntries = new List<LabelEntry>
labelingConfiguration.Init(new List<LabelEntry>
{
new LabelEntry
{

}
});
return labelingConfiguration;
}
static SemanticSegmentationLabelConfig CreateSemanticSegmentationLabelConfig()
{
var label = "label";
var labelingConfiguration = ScriptableObject.CreateInstance<SemanticSegmentationLabelConfig>();
labelingConfiguration.LabelEntries = new List<SemanticSegmentationLabelEntry>
{
new SemanticSegmentationLabelEntry()
{
label = label,
pixelValue = 500
}
GameObject SetupCamera(LabelingConfiguration labelingConfiguration, Action<PerceptionCamera> initPerceptionCamera)
GameObject SetupCamera(Action<PerceptionCamera> initPerceptionCamera)
{
var cameraObject = new GameObject();
cameraObject.SetActive(false);

50
com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs


[UnityPlatform(exclude = new[] {RuntimePlatform.LinuxEditor, RuntimePlatform.LinuxPlayer})]
public class SegmentationPassTests : GroundTruthTestBase
{
const int k_SemanticPixelValue = 4;
public enum SegmentationKind
{
Instance,
Semantic
}
public IEnumerator SegmentationPassTestsWithEnumeratorPasses([Values(false, true)] bool useSkinnedMeshRenderer)
public IEnumerator SegmentationPassTestsWithEnumeratorPasses(
[Values(false, true)] bool useSkinnedMeshRenderer,
[Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind)
int expectedPixelValue = segmentationKind == SegmentationKind.Instance ? 1 : k_SemanticPixelValue;
int? frameStart = null;
Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived = (frameCount, data, tex) =>
{

timesSegmentationImageReceived++;
CollectionAssert.AreEqual(Enumerable.Repeat(1, data.Length), data);
CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data);
var cameraObject = SetupCamera(onSegmentationImageReceived);
var cameraObject = SetupCamera(segmentationKind, onSegmentationImageReceived);
//
// // Arbitrary wait for 5 frames for shaders to load. Workaround for issue with Shader.WarmupAllShaders()
// for (int i=0 ; i<5 ; ++i)

}
[UnityTest]
public IEnumerator SegmentationPassProducesCorrectValuesEachFrame()
public IEnumerator SegmentationPassProducesCorrectValuesEachFrame(
[Values(SegmentationKind.Instance, SegmentationKind.Semantic)] SegmentationKind segmentationKind)
{
int timesSegmentationImageReceived = 0;
Dictionary<int, int> expectedLabelAtFrame = null;

}
};
var cameraObject = SetupCamera(onSegmentationImageReceived);
var cameraObject = SetupCamera(segmentationKind, onSegmentationImageReceived);
int expectedPixelValue = segmentationKind == SegmentationKind.Instance ? 1 : k_SemanticPixelValue;
{Time.frameCount , 1},
{Time.frameCount + 1, 1},
{Time.frameCount + 2, 1}
{Time.frameCount , expectedPixelValue},
{Time.frameCount + 1, expectedPixelValue},
{Time.frameCount + 2, expectedPixelValue}
};
GameObject planeObject;

Assert.AreEqual(3, timesSegmentationImageReceived);
}
GameObject SetupCamera(Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
GameObject SetupCamera(SegmentationKind segmentationKind, Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
{
var cameraObject = new GameObject();
cameraObject.SetActive(false);

var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
perceptionCamera.captureRgbImages = false;
var instanceSegmentationLabeler = cameraObject.AddComponent<InstanceSegmentationLabeler>();
instanceSegmentationLabeler.InstanceSegmentationImageReadback += onSegmentationImageReceived;
if (segmentationKind == SegmentationKind.Instance)
perceptionCamera.InstanceSegmentationImageReadback += onSegmentationImageReceived;
else
{
var labelConfig = new SemanticSegmentationLabelConfig();
labelConfig.LabelEntries = new List<SemanticSegmentationLabelEntry>()
{
new SemanticSegmentationLabelEntry()
{
label = "label",
pixelValue = k_SemanticPixelValue
}
};
var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
semanticSegmentationLabeler.SemanticSegmentationImageReadback += onSegmentationImageReceived;
perceptionCamera.labelers.Add(semanticSegmentationLabeler);
}
AddTestObjectForCleanup(cameraObject);
cameraObject.SetActive(true);

22
com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs


//Graphics issues with OpenGL Linux Editor. https://jira.unity3d.com/browse/AISV-422
[UnityPlatform(exclude = new[] {RuntimePlatform.LinuxEditor, RuntimePlatform.LinuxPlayer})]
[TestFixture]
class ObjectCountTests : GroundTruthTestBase
class ObjectCountLabelerTests : GroundTruthTestBase
[Test]
public void NullLabelingConfiguration_ProducesInvalidOperationException()
{
Assert.Throws<InvalidOperationException>(() => new ObjectCountLabeler(null).Setup());
}
[UnityTest]
public IEnumerator ProducesCorrectValuesWithChangingObjects()
{

labelingConfiguration.LabelEntries = new List<LabelEntry>
labelingConfiguration.Init(new List<LabelEntry>
{
new LabelEntry
{

}
};
});
var cameraObject = SetupCamera(labelingConfiguration, (counts, labels, frameCount) =>
var cameraObject = SetupCamera(labelingConfiguration, (frameCount, counts, labels) =>
{
receivedResults.Add((counts.ToArray(), labels.ToArray(), frameCount));
});

}
static GameObject SetupCamera(LabelingConfiguration labelingConfiguration,
Action<NativeSlice<uint>, IReadOnlyList<LabelEntry>, int> onClassCountsReceived)
Action<int, NativeSlice<uint>, IReadOnlyList<LabelEntry>> onClassCountsReceived)
{
var cameraObject = new GameObject();
cameraObject.SetActive(false);

var perceptionCamera = cameraObject.AddComponent<PerceptionCamera>();
perceptionCamera.captureRgbImages = false;
cameraObject.AddComponent<InstanceSegmentationLabeler>();
var renderedObjectInfoLabeler = cameraObject.AddComponent<RenderedObjectInfoLabeler>();
renderedObjectInfoLabeler.labelingConfiguration = labelingConfiguration;
renderedObjectInfoLabeler.classCountsReceived += onClassCountsReceived;
var classCountLabeler = new ObjectCountLabeler(labelingConfiguration);
classCountLabeler.ObjectCountsComputed += onClassCountsReceived;
perceptionCamera.labelers.Add(classCountLabeler);
cameraObject.SetActive(true);
return cameraObject;

26
com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs


using System;
namespace UnityEngine.Perception.GroundTruth
{
[Serializable]
public abstract class CameraLabeler
{
public bool enabled;
public bool foldout;
protected PerceptionCamera PerceptionCamera { get; private set; }
protected SensorHandle SensorHandle { get; private set; }
public abstract void Setup();
public virtual void OnUpdate() {}
public virtual void OnBeginRendering() {}
public virtual void OnSimulationEnding() {}
internal void Init(PerceptionCamera perceptionCamera)
{
PerceptionCamera = perceptionCamera;
SensorHandle = perceptionCamera.SensorHandle;
}
}
}

3
com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs.meta


fileFormatVersion: 2
guid: 1f21ae5ed3374b7288ad9bd4ad367c59
timeCreated: 1593188098

112
com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs


using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Unity.Collections;
using Unity.Profiling;
namespace UnityEngine.Perception.GroundTruth
{
[Serializable]
public class ObjectCountLabeler : CameraLabeler
{
/// <summary>
/// The ID to use for object count annotations in the resulting dataset
/// </summary>
public string objectCountMetricId = "51DA3C27-369D-4929-AEA6-D01614635CE2";
public LabelingConfiguration labelingConfiguration => m_LabelingConfiguration;
/// <summary>
/// Fired when the object counts are computed for a frame.
/// </summary>
public event Action<int, NativeSlice<uint>,IReadOnlyList<LabelEntry>> ObjectCountsComputed;
[SerializeField]
LabelingConfiguration m_LabelingConfiguration;
static ProfilerMarker s_ClassCountCallback = new ProfilerMarker("OnClassLabelsReceived");
ClassCountValue[] m_ClassCountValues;
Dictionary<int, AsyncMetric> m_ObjectCountAsyncMetrics = new Dictionary<int, AsyncMetric>();
MetricDefinition m_ObjectCountMetricDefinition;
public ObjectCountLabeler()
{
}
public ObjectCountLabeler(LabelingConfiguration labelingConfiguration)
{
this.m_LabelingConfiguration = labelingConfiguration;
}
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]
struct ClassCountValue
{
public int label_id;
public string label_name;
public uint count;
}
public override void Setup()
{
PerceptionCamera.RenderedObjectInfosCalculated += (frameCount, objectInfo) =>
{
NativeArray<uint> objectCounts = ComputeObjectCounts(objectInfo);
ObjectCountsComputed?.Invoke(frameCount, objectCounts, labelingConfiguration.LabelEntries);
ProduceObjectCountMetric(objectCounts, m_LabelingConfiguration.LabelEntries, frameCount);
};
}
public override void OnBeginRendering()
{
if (m_ObjectCountMetricDefinition.Equals(default))
{
m_ObjectCountMetricDefinition = SimulationManager.RegisterMetricDefinition("object count", m_LabelingConfiguration.GetAnnotationSpecification(),
"Counts of objects for each label in the sensor's view", id: new Guid(objectCountMetricId));
}
m_ObjectCountAsyncMetrics[Time.frameCount] = PerceptionCamera.SensorHandle.ReportMetricAsync(m_ObjectCountMetricDefinition);
}
NativeArray<uint> ComputeObjectCounts(NativeArray<RenderedObjectInfo> objectInfo)
{
var objectCounts = new NativeArray<uint>(m_LabelingConfiguration.LabelEntries.Count, Allocator.Temp);
foreach (var info in objectInfo)
{
if (!m_LabelingConfiguration.TryGetLabelEntryFromInstanceId(info.instanceId, out _, out var labelIndex))
continue;
objectCounts[labelIndex]++;
}
return objectCounts;
}
void ProduceObjectCountMetric(NativeSlice<uint> counts, IReadOnlyList<LabelEntry> entries, int frameCount)
{
using (s_ClassCountCallback.Auto())
{
if (!m_ObjectCountAsyncMetrics.TryGetValue(frameCount, out var classCountAsyncMetric))
return;
m_ObjectCountAsyncMetrics.Remove(frameCount);
if (m_ClassCountValues == null || m_ClassCountValues.Length != entries.Count)
m_ClassCountValues = new ClassCountValue[entries.Count];
for (var i = 0; i < entries.Count; i++)
{
m_ClassCountValues[i] = new ClassCountValue()
{
label_id = entries[i].id,
label_name = entries[i].label,
count = counts[i]
};
}
classCountAsyncMetric.ReportValues(m_ClassCountValues);
}
}
}
}

3
com.unity.perception/Runtime/GroundTruth/Labelers/ObjectCountLabeler.cs.meta


fileFormatVersion: 2
guid: 715672f8c1f94106b6e84431e53c6096
timeCreated: 1593183448

54
com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs


using System;
using Unity.Collections;
using Unity.Entities;
using UnityEngine.UIElements;
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Cache of instance id -> label entry index for a LabelConfig. This is not well optimized and is the source of a known memory leak for apps that create new instances frequently
/// </summary>
class LabelEntryMatchCache : IGroundTruthGenerator, IDisposable
{
const int k_StartingObjectCount = 1 << 8;
NativeList<ushort> m_InstanceIdToLabelEntryIndexLookup;
LabelingConfiguration m_LabelingConfiguration;
public LabelEntryMatchCache(LabelingConfiguration labelingConfiguration)
{
m_LabelingConfiguration = labelingConfiguration;
World.DefaultGameObjectInjectionWorld.GetOrCreateSystem<GroundTruthLabelSetupSystem>().Activate(this);
m_InstanceIdToLabelEntryIndexLookup = new NativeList<ushort>(k_StartingObjectCount, Allocator.Persistent);
}
public bool TryGetLabelEntryFromInstanceId(uint instanceId, out LabelEntry labelEntry, out int index)
{
labelEntry = default;
index = -1;
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
return false;
index = m_InstanceIdToLabelEntryIndexLookup[(int)instanceId];
labelEntry = m_LabelingConfiguration.LabelEntries[index];
return true;
}
void IGroundTruthGenerator.SetupMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
if (m_LabelingConfiguration.TryGetMatchingConfigurationEntry(labeling, out _, out var index))
{
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
{
m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
}
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = (ushort)index;
}
}
public void Dispose()
{
World.DefaultGameObjectInjectionWorld.GetOrCreateSystem<GroundTruthLabelSetupSystem>().Deactivate(this);
m_InstanceIdToLabelEntryIndexLookup.Dispose();
}
}
}

3
com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs.meta


fileFormatVersion: 2
guid: e736a5078e0a44d69bdc643b3157f670
timeCreated: 1593182152

162
com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs


using System;
using System.Collections;
using System.Collections.Generic;
using NUnit.Framework;
using Unity.Collections;
using Unity.Entities;
using UnityEngine;
using UnityEngine.Perception.GroundTruth;
using UnityEngine.TestTools;
namespace GroundTruthTests
{
[TestFixture]
public class RenderedObjectInfoTests : GroundTruthTestBase
{
public class ProducesCorrectObjectInfoData
{
public RenderedObjectInfo[] renderedObjectInfosExpected;
public uint[] data;
public BoundingBoxOrigin boundingBoxOrigin;
public int stride;
public string name;
public ProducesCorrectObjectInfoData(uint[] data, RenderedObjectInfo[] renderedObjectInfosExpected, int stride, BoundingBoxOrigin boundingBoxOrigin, string name)
{
this.data = data;
this.renderedObjectInfosExpected = renderedObjectInfosExpected;
this.stride = stride;
this.name = name;
this.boundingBoxOrigin = boundingBoxOrigin;
}
public override string ToString()
{
return name;
}
}
public static IEnumerable ProducesCorrectBoundingBoxesTestCases()
{
yield return new ProducesCorrectObjectInfoData(
new uint[]
{
1, 1,
1, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 2, 2),
instanceId = 1,
pixelCount = 4
}
},
2,
BoundingBoxOrigin.BottomLeft,
"SimpleBox");
yield return new ProducesCorrectObjectInfoData(
new uint[]
{
1, 0, 2,
1, 0, 0
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 1, 2),
instanceId = 1,
pixelCount = 2
},
new RenderedObjectInfo()
{
boundingBox = new Rect(2, 0, 1, 1),
instanceId = 2,
pixelCount = 1
}
},
3,
BoundingBoxOrigin.BottomLeft,
"WithGaps");
yield return new ProducesCorrectObjectInfoData(
new uint[]
{
1, 2, 1,
1, 2, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 3, 2),
instanceId = 1,
pixelCount = 4
},
new RenderedObjectInfo()
{
boundingBox = new Rect(1, 0, 1, 2),
instanceId = 2,
pixelCount = 2
}
},
3,
BoundingBoxOrigin.BottomLeft,
"Interleaved");
yield return new ProducesCorrectObjectInfoData(
new uint[]
{
0, 0,
0, 0,
0, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(1, 0, 1, 1),
instanceId = 1,
pixelCount = 1
},
},
2,
BoundingBoxOrigin.TopLeft,
"TopLeft");
}
[UnityTest]
public IEnumerator ProducesCorrectBoundingBoxes([ValueSource(nameof(ProducesCorrectBoundingBoxesTestCases))] ProducesCorrectObjectInfoData producesCorrectObjectInfoData)
{
var label = "label";
var label2 = "label2";
var labelingConfiguration = ScriptableObject.CreateInstance<LabelingConfiguration>();
labelingConfiguration.Init(new List<LabelEntry>
{
new LabelEntry
{
id = 1,
label = label,
value = 500
},
new LabelEntry
{
id = 2,
label = label2,
value = 500
}
});
var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
//Put a plane in front of the camera
AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(.1f, label));
AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(.1f, label2));
yield return null;
var dataNativeArray = new NativeArray<uint>(producesCorrectObjectInfoData.data, Allocator.Persistent);
renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, out var boundingBoxes, Allocator.Temp);
CollectionAssert.AreEqual(producesCorrectObjectInfoData.renderedObjectInfosExpected, boundingBoxes.ToArray());
dataNativeArray.Dispose();
boundingBoxes.Dispose();
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs.meta


fileFormatVersion: 2
guid: 43085b9d882c415b90fb9ebe4d765f26
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

25
com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs


using System;
using Unity.Collections;
using UnityEngine.Experimental.Rendering;
#if HDRP_PRESENT
using UnityEngine.Rendering.HighDefinition;
#endif
namespace UnityEngine.Perception.GroundTruth {
// public class InstanceSegmentationLabeler : CameraLabeler
// {
// //Uncomment when we support saving instance segmentation labels
// //public bool saveImages = false;
// //public string annotationId = "E657461D-B950-42E1-8141-BEC9B4810241";
//
// public override void Setup()
// {
// PerceptionCamera.InstanceSegmentationImageReadback += OnInstanceSegmentationImageReadback;
// }
//
// void OnInstanceSegmentationImageReadback(int frameCount, NativeArray<uint> imageData, RenderTexture renderTexture)
// {
// throw new NotImplementedException();
// }
// }
}

192
com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox2DTests.cs


using System;
using System.Collections;
using System.Collections.Generic;
using NUnit.Framework;
using Unity.Collections;
using Unity.Entities;
using UnityEngine;
using UnityEngine.Perception.GroundTruth;
using UnityEngine.TestTools;
namespace GroundTruthTests
{
[TestFixture]
public class BoundingBox2DTests : GroundTruthTestBase
{
public class ProducesCorrectBoundingBoxesData
{
public uint[] classCountsExpected;
public RenderedObjectInfo[] boundingBoxesExpected;
public uint[] data;
public BoundingBoxOrigin boundingBoxOrigin;
public int stride;
public string name;
public ProducesCorrectBoundingBoxesData(uint[] data, RenderedObjectInfo[] boundingBoxesExpected, uint[] classCountsExpected, int stride, BoundingBoxOrigin boundingBoxOrigin, string name)
{
this.data = data;
this.boundingBoxesExpected = boundingBoxesExpected;
this.classCountsExpected = classCountsExpected;
this.stride = stride;
this.name = name;
this.boundingBoxOrigin = boundingBoxOrigin;
}
public override string ToString()
{
return name;
}
}
public static IEnumerable ProducesCorrectBoundingBoxesTestCases()
{
yield return new ProducesCorrectBoundingBoxesData(
new uint[]
{
1, 1,
1, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 2, 2),
instanceId = 1,
labelId = 1,
pixelCount = 4
}
}, new uint[]
{
1,
0
},
2,
BoundingBoxOrigin.BottomLeft,
"SimpleBox");
yield return new ProducesCorrectBoundingBoxesData(
new uint[]
{
1, 0, 2,
1, 0, 0
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 1, 2),
instanceId = 1,
labelId = 1,
pixelCount = 2
},
new RenderedObjectInfo()
{
boundingBox = new Rect(2, 0, 1, 1),
instanceId = 2,
labelId = 2,
pixelCount = 1
}
}, new uint[]
{
1,
1
},
3,
BoundingBoxOrigin.BottomLeft,
"WithGaps");
yield return new ProducesCorrectBoundingBoxesData(
new uint[]
{
1, 2, 1,
1, 2, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(0, 0, 3, 2),
instanceId = 1,
labelId = 1,
pixelCount = 4
},
new RenderedObjectInfo()
{
boundingBox = new Rect(1, 0, 1, 2),
instanceId = 2,
labelId = 2,
pixelCount = 2
}
}, new uint[]
{
1,
1
},
3,
BoundingBoxOrigin.BottomLeft,
"Interleaved");
yield return new ProducesCorrectBoundingBoxesData(
new uint[]
{
0, 0,
0, 0,
0, 1
}, new[]
{
new RenderedObjectInfo()
{
boundingBox = new Rect(1, 0, 1, 1),
instanceId = 1,
labelId = 1,
pixelCount = 1
},
}, new uint[]
{
1,
0
},
2,
BoundingBoxOrigin.TopLeft,
"TopLeft");
}
[UnityTest]
public IEnumerator ProducesCorrectBoundingBoxes([ValueSource(nameof(ProducesCorrectBoundingBoxesTestCases))] ProducesCorrectBoundingBoxesData producesCorrectBoundingBoxesData)
{
var label = "label";
var label2 = "label2";
var labelingConfiguration = ScriptableObject.CreateInstance<LabelingConfiguration>();
labelingConfiguration.LabelEntries = new List<LabelEntry>
{
new LabelEntry
{
id = 1,
label = label,
value = 500
},
new LabelEntry
{
id = 2,
label = label2,
value = 500
}
};
var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator(labelingConfiguration);
var groundTruthLabelSetupSystem = World.DefaultGameObjectInjectionWorld.GetExistingSystem<GroundTruthLabelSetupSystem>();
groundTruthLabelSetupSystem.Activate(renderedObjectInfoGenerator);
//Put a plane in front of the camera
AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(.1f, label));
AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(.1f, label2));
yield return null;
var dataNativeArray = new NativeArray<uint>(producesCorrectBoundingBoxesData.data, Allocator.Persistent);
renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectBoundingBoxesData.stride, producesCorrectBoundingBoxesData.boundingBoxOrigin, out var boundingBoxes, out var classCounts, Allocator.Temp);
CollectionAssert.AreEqual(producesCorrectBoundingBoxesData.boundingBoxesExpected, boundingBoxes.ToArray());
CollectionAssert.AreEqual(producesCorrectBoundingBoxesData.classCountsExpected, classCounts.ToArray());
dataNativeArray.Dispose();
boundingBoxes.Dispose();
classCounts.Dispose();
groundTruthLabelSetupSystem.Deactivate(renderedObjectInfoGenerator);
renderedObjectInfoGenerator.Dispose();
}
}
}

/com.unity.perception/Tests/Runtime/GroundTruthTests/BoundingBox2DTests.cs.meta → /com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs.meta

/com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountTests.cs.meta → /com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs.meta

/com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountTests.cs → /com.unity.perception/Tests/Runtime/GroundTruthTests/ObjectCountLabelerTests.cs

正在加载...
取消
保存