浏览代码

Instance segmentation (#112)

* Copy to Color32 in labeler

* Finishing touches on instance segmentation labeler

* Moved away from static initialization

* Fixes for some unit tests

* Update packages for newer version of burst

* Fixed some documentation

* Updated changelog

* Updated documentation

* Fixes for PR comments

* Fix to support labelers coming on and off line

* Updates for PR comments
/main
GitHub 4 年前
当前提交
d1e6558f
共有 32 个文件被更改,包括 1507 次插入192 次删除
  1. 5
      com.unity.perception/CHANGELOG.md
  2. 5
      com.unity.perception/Documentation~/PerceptionCamera.md
  3. 18
      com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs
  4. 14
      com.unity.perception/Runtime/GroundTruth/InstanceSegmentationCrossPipelinePass.cs
  5. 8
      com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs
  6. 7
      com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs
  7. 8
      com.unity.perception/Runtime/GroundTruth/Labelers/RenderedObjectInfoLabeler.cs
  8. 103
      com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs
  9. 1
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Materials/SegmentationMaterial.mat
  10. 11
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs
  11. 7
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs
  12. 6
      com.unity.perception/Runtime/GroundTruth/RenderedObjectInfo.cs
  13. 48
      com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs
  14. 11
      com.unity.perception/Runtime/GroundTruth/Resources/InstanceSegmentation.shader
  15. 71
      com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs
  16. 42
      com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs
  17. 8
      com.unity.perception/Tests/Runtime/GroundTruthTests/VisualizationTests.cs
  18. 1
      com.unity.perception/package.json
  19. 202
      com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs
  20. 3
      com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs.meta
  21. 186
      com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs
  22. 11
      com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs.meta
  23. 258
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/OverlayPanel.cs
  24. 3
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/OverlayPanel.cs.meta
  25. 104
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down.png
  26. 106
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down.png.meta
  27. 99
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down2.png
  28. 106
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down2.png.meta
  29. 5
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/instance_back.png
  30. 104
      com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/instance_back.png.meta
  31. 135
      com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceIdToColorMappingTests.cs
  32. 3
      com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceIdToColorMappingTests.cs.meta

5
com.unity.perception/CHANGELOG.md


### Added
Added support for labeling Terrain objects. Trees and details are not labeled but will occlude other objects.
Added instance segmentation labeler
Added support for full screen visual overlays and overlay manager
Updated perception to use burst 1.3.9
Changed InstanceSegmentationImageReadback event to provide a NativeArray\<Color32\> instead of NativeArray\<uint\>
Uniform and Normal samplers now serialize their random seeds
### Deprecated

5
com.unity.perception/Documentation~/PerceptionCamera.md


The SemanticSegmentationLabeler generates a 2D RGB image with the attached Camera. Unity draws objects in the color you associate with the label in the SemanticSegmentationLabelingConfiguration. If Unity can't find a label for an object, it draws it in black.
### InstanceSegmentationLabeler
The instance segmentation labeler generates a 2D RGB image with the attached camera. Unity draws each instance of a labeled
object with a unique color.
### BoundingBox2DLabeler
![Example bounding box visualization from SynthDet generated by the `SynthDet_Statistics` Jupyter notebook](images/bounding_boxes.png)
<br/>_Example bounding box visualization from SynthDet generated by the `SynthDet_Statistics` Jupyter notebook_

18
com.unity.perception/Runtime/GroundTruth/DatasetCapture.cs


}
/// <summary>
/// Report file-based and value-based data for this annotation.
/// </summary>
/// <param name="path">The path to the file containing the annotation data.</param>
/// <param name="values">The annotation data.</param>
/// <typeparam name="T">The type of the data.</typeparam>
/// <exception cref="ArgumentNullException">Thrown if path or values is null</exception>
public void ReportFileAndValues<T>(string path, IEnumerable<T> values)
{
if (path == null)
throw new ArgumentNullException(nameof(path));
if (values == null)
throw new ArgumentNullException(nameof(values));
m_SimulationState.ReportAsyncAnnotationResult(this, path, values);
}
/// <summary>
/// Report a value-based data for this annotation.
/// </summary>
/// <param name="values">The annotation data.</param>

14
com.unity.perception/Runtime/GroundTruth/InstanceSegmentationCrossPipelinePass.cs


Shader m_SegmentationShader;
Material m_OverrideMaterial;
int m_NextObjectIndex;
Dictionary<uint, uint> m_Ids;
/// <summary>
/// Create a new <see cref="InstanceSegmentationCrossPipelinePass"/> referencing the given

{
using (s_ExecuteMarker.Auto())
{
cmd.ClearRenderTarget(true, true, Color.clear);
cmd.ClearRenderTarget(true, true, Color.black);
var result = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, layerMask);
DrawRendererList(renderContext, cmd, RendererList.Create(result));

public override void SetupMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
mpb.SetInt(k_SegmentationIdProperty, (int)instanceId);
var found = InstanceIdToColorMapping.TryGetColorFromInstanceId(instanceId, out var color);
if (!found)
{
Debug.LogError($"Could not get a unique color for {instanceId}");
}
mpb.SetVector(k_SegmentationIdProperty, (Color)color);
#if PERCEPTION_DEBUG
Debug.Log($"Assigning id. Frame {Time.frameCount} id {id}");
#endif

8
com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs


// The player screen can be dynamically resized during play, need to
// scale the bounding boxes appropriately from the original screen size
float screenRatioWidth = Screen.width / m_OriginalScreenSize.x;
float screenRatioHeight = Screen.height / m_OriginalScreenSize.y;
var screenRatioWidth = Screen.width / m_OriginalScreenSize.x;
var screenRatioHeight = Screen.height / m_OriginalScreenSize.y;
float x = box.x * screenRatioWidth;
float y = box.y * screenRatioHeight;
var x = box.x * screenRatioWidth;
var y = box.y * screenRatioHeight;
var boxRect = new Rect(x, y, box.width * screenRatioWidth, box.height * screenRatioHeight);
var labelWidth = Math.Min(120, box.width * screenRatioWidth);

7
com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs


public HUDPanel hudPanel => perceptionCamera != null ? perceptionCamera.hudPanel : null;
/// <summary>
/// The overlay panel. Used to control which full screen image visual is displayed.
/// </summary>
public OverlayPanel overlayPanel => perceptionCamera != null ? perceptionCamera.overlayPanel : null;
/// <summary>
/// The <see cref="PerceptionCamera"/> that contains this labeler.
/// </summary>
protected PerceptionCamera perceptionCamera { get; private set; }

internal void VisualizeUI()
{
if (supportsVisualization)
if (supportsVisualization && !(this is IOverlayPanelProvider))
{
GUILayout.Label(GetType().Name);
GUILayout.BeginHorizontal();

8
com.unity.perception/Runtime/GroundTruth/Labelers/RenderedObjectInfoLabeler.cs


[UsedImplicitly]
public uint instance_id;
[UsedImplicitly]
public Color32 instance_color;
[UsedImplicitly]
}
// ReSharper restore InconsistentNaming

// Clear out all of the old entries...
hudPanel.RemoveEntries(this);
}
for (var i = 0; i < renderedObjectInfos.Length; i++)
{
var objectInfo = renderedObjectInfos[i];

{
label_id = labelEntry.id,
instance_id = objectInfo.instanceId,
visible_pixels = objectInfo.pixelCount
visible_pixels = objectInfo.pixelCount,
instance_color = objectInfo.instanceColor
};
if (visualize)

103
com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs


/// Only one SemanticSegmentationLabeler can render at once across all cameras.
/// </summary>
[Serializable]
public sealed class SemanticSegmentationLabeler : CameraLabeler
public sealed class SemanticSegmentationLabeler : CameraLabeler, IOverlayPanelProvider
{
///<inheritdoc/>
public override string description

/// </summary>
public RenderTexture targetTexture => m_TargetTextureOverride;
/// <inheritdoc cref="IOverlayPanelProvider"/>
public Texture overlayImage=> targetTexture;
/// <inheritdoc cref="IOverlayPanelProvider"/>
public string label => "SemanticSegmentation";
[Tooltip("(Optional) The RenderTexture on which semantic segmentation images will be drawn. Will be reformatted on startup.")]
[SerializeField]
RenderTexture m_TargetTextureOverride;

#endif
Dictionary<int, AsyncAnnotation> m_AsyncAnnotations;
private float segmentTransparency = 0.8f;
private float backgroundTransparency = 0.0f;
/// <summary>
/// Creates a new SemanticSegmentationLabeler. Be sure to assign <see cref="labelConfig"/> before adding to a <see cref="PerceptionCamera"/>.

public string path;
}
int camWidth = 0;
int camHeight = 0;
private GameObject segCanvas;
private GameObject segVisual = null;
private RawImage segImage = null;
GUIStyle labelStyle = null;
GUIStyle sliderStyle = null;
/// <inheritdoc/>
protected override bool supportsVisualization => true;

var myCamera = perceptionCamera.GetComponent<Camera>();
camWidth = myCamera.pixelWidth;
camHeight = myCamera.pixelHeight;
var camWidth = myCamera.pixelWidth;
var camHeight = myCamera.pixelHeight;
if (labelConfig == null)
{

visualizationEnabled = supportsVisualization;
}
private void SetupVisualizationElements()
{
segmentTransparency = 0.8f;
backgroundTransparency = 0.0f;
segVisual = GameObject.Instantiate(Resources.Load<GameObject>("SegmentTexture"));
segImage = segVisual.GetComponent<RawImage>();
segImage.material.SetFloat("_SegmentTransparency", segmentTransparency);
segImage.material.SetFloat("_BackTransparency", backgroundTransparency);
segImage.texture = targetTexture;
var rt = segVisual.transform as RectTransform;
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, camWidth);
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, camHeight);
if (segCanvas == null)
{
segCanvas = new GameObject(perceptionCamera.gameObject.name + "_segmentation_canvas");
segCanvas.AddComponent<RectTransform>();
var canvas = segCanvas.AddComponent<Canvas>();
canvas.renderMode = RenderMode.ScreenSpaceOverlay;
segCanvas.AddComponent<CanvasScaler>();
segVisual.transform.SetParent(segCanvas.transform, false);
}
labelStyle = new GUIStyle(GUI.skin.label) {padding = {left = 10}};
sliderStyle = new GUIStyle(GUI.skin.horizontalSlider) {margin = {left = 12}};
}
void OnSemanticSegmentationImageRead(int frameCount, NativeArray<Color32> data)
{
if (!m_AsyncAnnotations.TryGetValue(frameCount, out var annotation))

m_SemanticSegmentationTextureReader?.Dispose();
m_SemanticSegmentationTextureReader = null;
Object.Destroy(segCanvas);
segCanvas = null;
}
/// <inheritdoc/>
override protected void OnVisualizerEnabledChanged(bool enabled)
{
if (segVisual != null)
segVisual.SetActive(enabled);
}
/// <inheritdoc/>
protected override void OnVisualizeAdditionalUI()
{
if (segImage == null)
{
SetupVisualizationElements();
}
var rt = segVisual.transform as RectTransform;
if (rt != null && camHeight != Screen.height)
{
camHeight = Screen.height;
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, camHeight);
}
if (rt != null && camWidth != Screen.width)
{
camWidth = Screen.width;
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, Screen.width);
}
GUILayout.Space(4);
GUILayout.Label("Object Alpha:", labelStyle);
segmentTransparency = GUILayout.HorizontalSlider(segmentTransparency, 0.0f, 1.0f, sliderStyle, GUI.skin.horizontalSliderThumb);
GUILayout.Space(4);
GUILayout.Label("Background Alpha:", labelStyle);
backgroundTransparency = GUILayout.HorizontalSlider(backgroundTransparency, 0.0f, 1.0f, sliderStyle, GUI.skin.horizontalSliderThumb);
GUI.skin.label.padding.left = 0;
if (!GUI.changed) return;
segImage.material.SetFloat("_SegmentTransparency", segmentTransparency);
segImage.material.SetFloat("_BackTransparency", backgroundTransparency);
}
}
}

1
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Materials/SegmentationMaterial.mat


- _ReplaceColor: {r: 1, g: 1, b: 1, a: 0}
- _SoftParticleFadeParams: {r: 0, g: 0, b: 0, a: 0}
- _SpecColor: {r: 0.19999996, g: 0.19999996, b: 0.19999996, a: 1}
m_BuildTextureStacks: []
--- !u!114 &3089368426582508830
MonoBehaviour:
m_ObjectHideFlags: 11

11
com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs


}
internal HUDPanel hudPanel = null;
internal OverlayPanel overlayPanel = null;
void SetupVisualizationCamera(Camera cam)
{

s_VisualizedPerceptionCamera = this;
hudPanel = gameObject.AddComponent<HUDPanel>();
overlayPanel = gameObject.AddComponent<OverlayPanel>();
overlayPanel.perceptionCamera = this;
#endif
}

GUI.skin.label.padding = new RectOffset(0, 0, 1, 1);
GUI.skin.label.margin = new RectOffset(0, 0, 1, 1);
GUI.skin.label.wordWrap = true;
GUI.skin.label.alignment = TextAnchor.MiddleLeft;
GUI.skin.box.padding = new RectOffset(5, 5, 5, 5);
GUI.skin.toggle.margin = new RectOffset(0, 0, 0, 0);
GUI.skin.horizontalSlider.margin = new RectOffset(0, 0, 0, 0);

foreach (var labeler in m_Labelers.Where(labeler => labeler.isInitialized))
{
labeler.VisualizeUI();
GUILayout.Space(4);
// This needs to happen here so that the overlay panel controls
// are placed in the controls panel
overlayPanel.OnDrawGUI(x, 10, panelWidth, height);
}
void OnValidate()

7
com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs


using System;
using System.Collections.Generic;
using Unity.Collections;
using UnityEngine;
using UnityEngine.Experimental.Rendering;

/// Invoked when instance segmentation images are read back from the graphics system. The first parameter is the
/// Time.frameCount at which the objects were rendered. May be invoked many frames after the objects were rendered.
/// </summary>
public event Action<int, NativeArray<uint>, RenderTexture> InstanceSegmentationImageReadback;
public event Action<int, NativeArray<Color32>, RenderTexture> InstanceSegmentationImageReadback;
/// <summary>
/// Invoked when RenderedObjectInfos are calculated. The first parameter is the Time.frameCount at which the

RenderedObjectInfoGenerator m_RenderedObjectInfoGenerator;
RenderTexture m_InstanceSegmentationTexture;
RenderTextureReader<uint> m_InstanceSegmentationReader;
RenderTextureReader<Color32> m_InstanceSegmentationReader;
internal bool m_fLensDistortionEnabled = false;

m_fLensDistortionEnabled = true;
#endif
m_InstanceSegmentationReader = new RenderTextureReader<uint>(m_InstanceSegmentationTexture, myCamera, (frameCount, data, tex) =>
m_InstanceSegmentationReader = new RenderTextureReader<Color32>(m_InstanceSegmentationTexture, myCamera, (frameCount, data, tex) =>
{
InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
if (RenderedObjectInfosCalculated != null)

6
com.unity.perception/Runtime/GroundTruth/RenderedObjectInfo.cs


/// The number of pixels in the image matching this instance.
/// </summary>
public int pixelCount;
/// <summary>
/// The unique RGBA color for the instance.
/// </summary>
public Color32 instanceColor;
return $"{nameof(instanceId)}: {instanceId}, {nameof(boundingBox)}: {boundingBox}, {nameof(pixelCount)}: {pixelCount}";
return $"{nameof(instanceId)}: {instanceId}, {nameof(boundingBox)}: {boundingBox}, {nameof(pixelCount)}: {pixelCount}, {nameof(instanceColor)}: {instanceColor}";
}
/// <inheritdoc />

48
com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs


struct ComputeHistogramPerRowJob : IJob
{
[ReadOnly]
public NativeSlice<uint> segmentationImageData;
public NativeSlice<Color32> segmentationImageData;
public int width;
public int rows;
public int rowStart;

{
for (var row = 0; row < rows; row++)
{
var rowSlice = new NativeSlice<uint>(segmentationImageData, width * row, width);
var rowSlice = new NativeSlice<Color32>(segmentationImageData, width * row, width);
var currentBB = new Object1DSpan
{

for (var i = 0; i < rowSlice.Length; i++)
{
var value = rowSlice[i];
var packed = InstanceIdToColorMapping.GetPackedColorFromColor(rowSlice[i]);
// pixel color black (0,0,0,255) is reserved for no hit, so set it to id 0
var id = packed == 255 ? 0 : packed;
if (value != currentBB.instanceId)
if (id != currentBB.instanceId)
{
if (currentBB.instanceId > 0)
{

currentBB = new Object1DSpan
{
instanceId = value,
instanceId = id,
left = i,
row = row + rowStart
};

/// <param name="boundingBoxOrigin">Whether bounding boxes should be top-left or bottom-right-based.</param>
/// <param name="renderedObjectInfos">When this method returns, filled with RenderedObjectInfo entries for each object visible in the frame.</param>
/// <param name="allocator">The allocator to use for allocating renderedObjectInfos and perLabelEntryObjectCount.</param>
public void Compute(NativeArray<uint> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
public void Compute(NativeArray<Color32> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
{
const int jobCount = 24;
var height = instanceSegmentationRawData.Length / stride;

handles[jobIndex] = new ComputeHistogramPerRowJob
{
segmentationImageData = new NativeSlice<uint>(instanceSegmentationRawData, row * stride, stride * rowsThisJob),
segmentationImageData = new NativeSlice<Color32>(instanceSegmentationRawData, row * stride, stride * rowsThisJob),
width = stride,
rowStart = row,
rows = rowsThisJob,

renderedObjectInfos = new NativeArray<RenderedObjectInfo>(keyValueArrays.Keys.Length, allocator);
for (var i = 0; i < keyValueArrays.Keys.Length; i++)
{
var instanceId = keyValueArrays.Keys[i];
var renderedObjectInfo = keyValueArrays.Values[i];
var boundingBox = renderedObjectInfo.boundingBox;
if (boundingBoxOrigin == BoundingBoxOrigin.TopLeft)
var color = InstanceIdToColorMapping.GetColorFromPackedColor(keyValueArrays.Keys[i]);
if (InstanceIdToColorMapping.TryGetInstanceIdFromColor(color, out var instanceId))
var y = height - boundingBox.yMax;
boundingBox = new Rect(boundingBox.x, y, boundingBox.width, boundingBox.height);
var renderedObjectInfo = keyValueArrays.Values[i];
var boundingBox = renderedObjectInfo.boundingBox;
if (boundingBoxOrigin == BoundingBoxOrigin.TopLeft)
{
var y = height - boundingBox.yMax;
boundingBox = new Rect(boundingBox.x, y, boundingBox.width, boundingBox.height);
}
renderedObjectInfos[i] = new RenderedObjectInfo
{
instanceId = instanceId,
boundingBox = boundingBox,
pixelCount = renderedObjectInfo.pixelCount,
instanceColor = color
};
renderedObjectInfos[i] = new RenderedObjectInfo
else
instanceId = instanceId,
boundingBox = boundingBox,
pixelCount = renderedObjectInfo.pixelCount
};
Debug.LogError($"Could not generate instance ID for object, ID exceeded maximum ID");
}
}
keyValueArrays.Dispose();
}

11
com.unity.perception/Runtime/GroundTruth/Resources/InstanceSegmentation.shader


{
Properties
{
[PerObjectData] _SegmentationId("Segmentation ID", int) = 0
[PerObjectData] _SegmentationId("Segmentation ID", vector) = (0,0,0,1)
}
SubShader
{

float4 vertex : SV_POSITION;
};
uint _SegmentationId;
float4 _SegmentationId;
v2f vert (appdata v)
{

fixed4 frag (v2f i) : SV_Target
{
return fixed4(
UnpackUIntToFloat((uint)_SegmentationId, 0, 8),
UnpackUIntToFloat((uint)_SegmentationId, 8, 8),
UnpackUIntToFloat((uint)_SegmentationId, 16, 8),
UnpackUIntToFloat((uint)_SegmentationId, 24, 8)
);
return _SegmentationId;
}
ENDCG
}

71
com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs


public class ProducesCorrectObjectInfoData
{
public RenderedObjectInfo[] renderedObjectInfosExpected;
public uint[] data;
public Color32[] data;
public ProducesCorrectObjectInfoData(uint[] data, RenderedObjectInfo[] renderedObjectInfosExpected, int stride, BoundingBoxOrigin boundingBoxOrigin, string name)
public ProducesCorrectObjectInfoData(Color32[] data, RenderedObjectInfo[] renderedObjectInfosExpected, int stride, BoundingBoxOrigin boundingBoxOrigin, string name)
{
this.data = data;
this.renderedObjectInfosExpected = renderedObjectInfosExpected;

}
public static IEnumerable ProducesCorrectBoundingBoxesTestCases()
{
InstanceIdToColorMapping.TryGetColorFromInstanceId(1, out var color1);
InstanceIdToColorMapping.TryGetColorFromInstanceId(2, out var color2);
var empty = Color.black;
new uint[]
new Color32[]
1, 1,
1, 1
color1, color1,
color1, color1
}, new[]
{
new RenderedObjectInfo()

pixelCount = 4
pixelCount = 4,
instanceColor = color1
}
},
2,

new uint[]
new Color32[]
1, 0, 2,
1, 0, 0
color1, empty, color2,
color1, empty, empty
boundingBox = new Rect(0, 0, 1, 2),
instanceId = 1,
pixelCount = 2
boundingBox = new Rect(2, 0, 1, 1),
instanceId = 2,
pixelCount = 1,
instanceColor = color2
boundingBox = new Rect(2, 0, 1, 1),
instanceId = 2,
pixelCount = 1
boundingBox = new Rect(0, 0, 1, 2),
instanceId = 1,
pixelCount = 2,
instanceColor = color1
}
},
3,

new uint[]
new Color32[]
1, 2, 1,
1, 2, 1
color1, color2, color1,
color1, color2, color1
boundingBox = new Rect(0, 0, 3, 2),
instanceId = 1,
pixelCount = 4
boundingBox = new Rect(1, 0, 1, 2),
instanceId = 2,
pixelCount = 2,
instanceColor = color2
boundingBox = new Rect(1, 0, 1, 2),
instanceId = 2,
pixelCount = 2
boundingBox = new Rect(0, 0, 3, 2),
instanceId = 1,
pixelCount = 4,
instanceColor = color1
}
},
3,

new uint[]
new Color32[]
0, 0,
0, 0,
0, 1
empty, empty,
empty, empty,
empty, color1
}, new[]
{
new RenderedObjectInfo()

pixelCount = 1
pixelCount = 1,
instanceColor = color1
},
},
2,

AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(.1f, label2));
yield return null;
var dataNativeArray = new NativeArray<uint>(producesCorrectObjectInfoData.data, Allocator.Persistent);
var dataNativeArray = new NativeArray<Color32>(producesCorrectObjectInfoData.data, Allocator.Persistent);
renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, out var boundingBoxes, Allocator.Temp);

42
com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs


{
public RenderTexture source;
public Camera cameraSource;
RenderTextureReader<uint> m_Reader;
RenderTextureReader<Color32> m_Reader;
public event Action<int, NativeArray<uint>> SegmentationImageReceived;
public event Action<int, NativeArray<Color32>> SegmentationImageReceived;
m_Reader = new RenderTextureReader<uint>(source, cameraSource, ImageReadCallback);
m_Reader = new RenderTextureReader<Color32>(source, cameraSource, ImageReadCallback);
void ImageReadCallback(int frameCount, NativeArray<uint> data, RenderTexture renderTexture)
void ImageReadCallback(int frameCount, NativeArray<Color32> data, RenderTexture renderTexture)
{
if (SegmentationImageReceived != null)
SegmentationImageReceived(frameCount, data);

switch (segmentationKind)
{
case SegmentationKind.Instance:
expectedPixelValue = 1;
//expectedPixelValue = new Color32(0, 74, 255, 255);
expectedPixelValue = new Color32(255,0,0, 255);
cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived);
break;
case SegmentationKind.Semantic:

{
GameObject cameraObject = null;
PerceptionCamera perceptionCamera;
bool fLensDistortionEnabled = false;
bool fDone = false;
int frames = 0;
var dataBBox = new uint[]
var fLensDistortionEnabled = false;
var fDone = false;
var frames = 0;
#if false
var dataBBox = new Color32[]
1, 1,
1, 1
Color.blue, Color.blue,
Color.blue, Color.blue
#endif
Rect boundingBoxWithoutLensDistortion = new Rect();
Rect boundingBoxWithLensDistortion = new Rect();
var boundingBoxWithoutLensDistortion = new Rect();
var boundingBoxWithLensDistortion = new Rect();
void OnSegmentationImageReceived(int frameCount, NativeArray<uint> data, RenderTexture tex)
void OnSegmentationImageReceived(int frameCount, NativeArray<Color32> data, RenderTexture tex)
{
frames++;

else
{
var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp);
boundingBoxWithLensDistortion = boundingBoxes[0].boundingBox;

}
var cameraObject = segmentationKind == SegmentationKind.Instance ?
SetupCameraInstanceSegmentation(OnSegmentationImageReceived<uint>) :
SetupCameraInstanceSegmentation(OnSegmentationImageReceived<Color32>) :
object expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) 1 : k_SemanticPixelValue;
//object expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) new Color32(0, 74, 255, 255) : k_SemanticPixelValue;
object expectedPixelValue = segmentationKind == SegmentationKind.Instance ? (object) new Color32(255, 0, 0, 255) : k_SemanticPixelValue;
expectedLabelAtFrame = new Dictionary<int, object>
{
{Time.frameCount , expectedPixelValue},

Assert.AreEqual(3, timesSegmentationImageReceived);
}
GameObject SetupCameraInstanceSegmentation(Action<int, NativeArray<uint>, RenderTexture> onSegmentationImageReceived)
GameObject SetupCameraInstanceSegmentation(Action<int, NativeArray<Color32>, RenderTexture> onSegmentationImageReceived)
{
var cameraObject = SetupCamera(out var perceptionCamera, false);
perceptionCamera.InstanceSegmentationImageReadback += onSegmentationImageReceived;

8
com.unity.perception/Tests/Runtime/GroundTruthTests/VisualizationTests.cs


// Need to wait to make sure a visualization call is made so that the canvas will be constructed
yield return null;
Assert.IsNotNull(GameObject.Find(nameof(VisualizedCamera_SetsUpCanvas) + "_segmentation_canvas"));
Assert.IsNotNull(GameObject.Find("overlay_canvas"));
}
[Test]
public void TwoCamerasVisualizing_CausesWarningAndDisablesVisualization()

AddTestObjectForCleanup(object1);
//wait a frame to make sure visualize is called once
yield return null;
Assert.IsNotNull(GameObject.Find(nameof(DestroyCamera_RemovesVisualization) + "_segmentation_canvas"));
Assert.IsNotNull(GameObject.Find("overlay_canvas"));
Assert.IsNull(GameObject.Find(nameof(DestroyCamera_RemovesVisualization) + "_segmentation_canvas"));
Assert.IsNull(GameObject.Find("overlay_segmentation_canvas"));
}
[UnityTest]
public IEnumerator DestroyAndRecreateCamera_ProperlyVisualizes()

//wait a frame to make sure visualize is called once
yield return null;
Assert.IsNotNull(GameObject.Find(nameof(DestroyAndRecreateCamera_ProperlyVisualizes) + "2_segmentation_canvas"));
Assert.IsNotNull("overlay_canvas");
}
[UnityTest]

1
com.unity.perception/package.json


"dependencies": {
"com.unity.nuget.newtonsoft-json": "1.1.2",
"com.unity.render-pipelines.core": "7.1.6",
"com.unity.burst": "1.3.9",
"com.unity.entities": "0.8.0-preview.8",
"com.unity.simulation.client": "0.0.10-preview.9",
"com.unity.simulation.capture": "0.0.10-preview.13",

202
com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs


using System;
using System.Collections.Generic;
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Static class to procedurally generate a unique color for an instance ID. This algorithm
/// is deterministic, and will always return the same color for a ID, and the same ID for a color. ID 0 is reserved to
/// be an invalid ID and is mapped to color black (0,0,0,255). Invalid IDs always map to black, and black always maps to ID 0.
/// In order to try to create visually contrasting colors for IDs, there are a subset of IDs reserved (1-65)
/// to be generated by applying the golden ration to find the next color in the HSL spectrum. All of these
/// colors, and only theses colors, will be in the alpha channel 255. After the first 65 IDs, the color will be
/// determined by iterating through all available RGB values in the alpha channels from 264 - 1. Alpha channel 0 is marked as invalid.
/// This service will support over 4 billion unique IDs => colors [(256^4) - (256*2) + 64]
/// </summary>
public static class InstanceIdToColorMapping
{
// ReSharper disable once MemberCanBePrivate.Global
/// <summary>
/// The max ID supported by this class.
/// </summary>
public const uint maxId = uint.MaxValue - ((256 * 256 * 256) * 2) + k_HslCount;
static Dictionary<uint, uint> s_IdToColorCache;
static Dictionary<uint, uint> s_ColorToIdCache;
const uint k_HslCount = 64;
const uint k_ColorsPerAlpha = 256 * 256 * 256;
const uint k_InvalidPackedColor = 255; // packed uint for color (0, 0, 0, 255);
static readonly Color32 k_InvalidColor = new Color(0, 0, 0, 255);
static readonly float k_GoldenRatio = (1 + Mathf.Sqrt(5)) / 2;
const int k_HuesInEachValue = 30;
static void InitializeMaps()
{
s_IdToColorCache = new Dictionary<uint, uint>();
s_ColorToIdCache = new Dictionary<uint, uint>();
s_IdToColorCache[0] = k_InvalidPackedColor;
s_IdToColorCache[k_InvalidPackedColor] = 0;
for (uint i = 1; i <= k_HslCount; i++)
{
var color = GenerateHSLValueForId(i);
s_IdToColorCache[i] = color;
s_ColorToIdCache[color] = i;
}
}
static uint GenerateHSLValueForId(uint count)
{
count -= 1;
var ratio = count * k_GoldenRatio;
var hue = ratio - Mathf.Floor(ratio);
count /= k_HuesInEachValue;
ratio = count * k_GoldenRatio;
var value = 1 - (ratio - Mathf.Floor(ratio));
var color = (Color32)Color.HSVToRGB(hue, 1f, value);
color.a = 255;
return GetPackedColorFromColor(color);
}
static uint GetColorForId(uint id)
{
if (id > maxId || id == 0 || id == k_InvalidPackedColor) return k_InvalidPackedColor;
if (id <= k_HslCount)
{
if (s_IdToColorCache == null) InitializeMaps();
return s_IdToColorCache[id];
}
var altered_id = id - k_HslCount;
var rgb = altered_id % k_ColorsPerAlpha;
var alpha= 254 - (altered_id / k_ColorsPerAlpha);
return rgb << 8 | alpha;
}
static bool TryGetIdForColor(uint color, out uint id)
{
if (color == 0 || color == k_InvalidPackedColor)
{
id = 0;
return true;
}
var alpha = color & 0xff;
if (alpha == 255)
{
if (s_ColorToIdCache == null) InitializeMaps();
return s_ColorToIdCache.TryGetValue(color, out id);
}
else
{
var rgb = color >> 8;
id = k_HslCount + rgb + (256 * 256 * 256) * (254 - alpha);
return true;
}
}
static uint GetIdForColor(uint color)
{
if (!TryGetIdForColor(color, out var id))
{
throw new InvalidOperationException($"Passed in color: {color} was not one of the reserved colors for alpha channel 255");
}
return id;
}
/// <summary>
/// Packs a color32 (RGBA - 1 byte per channel) into a 32bit unsigned integer.
/// </summary>
/// <param name="color">The RGBA color.</param>
/// <returns>The packed unsigned int 32 of the color.</returns>
public static uint GetPackedColorFromColor(Color32 color)
{
var tmp = (uint) ((color.r << 24) | (color.g << 16) | (color.b << 8) | (color.a << 0));
return tmp;
}
/// <summary>
/// Converts a packed color (or unsigned 32bit representation of a color) into an RGBA color.
/// </summary>
/// <param name="color">The packed color</param>
/// <returns>The RGBA color</returns>
public static Color32 GetColorFromPackedColor(uint color)
{
return new Color32((byte)(color >> 24), (byte)(color >> 16), (byte)(color >> 8), (byte)color);
}
/// <summary>
/// Retrieve the color that is mapped to the passed in ID. If the ID is 0 or 255 false will be returned, and
/// color will be set to black.
/// </summary>
/// <param name="id">The ID of interest.</param>
/// <param name="color">Will be set to the color associated with the passed in ID.</param>
/// <returns>Returns true if the ID was mapped to a non-black color, otherwise returns false</returns>
public static bool TryGetColorFromInstanceId(uint id, out Color32 color)
{
color = k_InvalidColor;
if (id > maxId) return false;
var packed = GetColorForId(id);
if (packed == k_InvalidPackedColor) return false;
color = GetColorFromPackedColor(packed);
return true;
}
/// <summary>
/// Retrieve the color that is mapped to the passed in ID. If the ID is 0 or 255 the returned color will be black.
/// </summary>
/// <param name="id">The ID of interest.</param>
/// <returns>The color associated with the passed in ID, or black if no associated color exists.</returns>
/// <exception cref="IndexOutOfRangeException">Thrown if the passed in ID is greater than the largest supported ID <see cref="maxId"/></exception>
public static Color32 GetColorFromInstanceId(uint id)
{
if (id > maxId)
throw new IndexOutOfRangeException($"Passed in index: {id} is greater than max ID: {maxId}");
TryGetColorFromInstanceId(id, out var color);
return color;
}
/// <summary>
/// Retrieve the ID associated with the passed in color. If the passed in color is black or cannot be mapped to an ID
/// this service will return false, and the out id will be set to 0.
/// </summary>
/// <param name="color">The color to map to an ID.</param>
/// <param name="id">This value will be updated with the ID for the passed in color.</param>
/// <returns>This service will return true if an ID is properly mapped to a color, otherwise it will return false.</returns>
public static bool TryGetInstanceIdFromColor(Color32 color, out uint id)
{
var packed = GetPackedColorFromColor(color);
if (!TryGetIdForColor(packed, out id))
{
return false;
}
return id != 0 && id <= maxId;
}
/// <summary>
/// Retrieve the ID associated with the passed in color. If the passed in color is black this service will return 0.
/// </summary>
/// <param name="color">The color to map to an ID.</param>
/// <returns>This value will be updated with the ID for the passed in color.</returns>
/// <exception cref="IndexOutOfRangeException">Thrown if the passed in color is mapped to an ID that is greater than the largest supported ID</exception>
/// <<exception cref="InvalidOperationException">Thrown if the passed in color cannot be mapped to an ID in the alpha 255 range<see cref="maxId"/></exception>
public static uint GetInstanceIdFromColor(Color32 color)
{
var id = GetIdForColor(GetPackedColorFromColor(color));
if (id > maxId) throw new IndexOutOfRangeException($"Passed in color: {color} maps to an ID: {id} which is greater than max ID: {maxId}");
return id;
}
}
}

3
com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs.meta


fileFormatVersion: 2
guid: ff8ada7e092f426680bbdc7f7af69fbb
timeCreated: 1603402608

186
com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs


using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Unity.Collections;
using Unity.Profiling;
using Unity.Simulation;
using UnityEditor;
using UnityEngine;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Profiling;
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Produces instance segmentation for each frame.
/// </summary>
[Serializable]
public sealed class InstanceSegmentationLabeler : CameraLabeler, IOverlayPanelProvider
{
///<inheritdoc/>
public override string description
{
get => "Produces an instance segmentation image for each frame. The image will render the pixels of each labeled object in a distinct color.";
protected set { }
}
/// <inheritdoc/>
protected override bool supportsVisualization => true;
static readonly string k_Directory = "InstanceSegmentation" + Guid.NewGuid().ToString();
const string k_FilePrefix = "Instance_";
/// <summary>
/// The GUID to associate with annotations produced by this labeler.
/// </summary>
[Tooltip("The id to associate with instance segmentation annotations in the dataset.")]
public string annotationId = "1ccebeb4-5886-41ff-8fe0-f911fa8cbcdf";
/// <summary>
/// The <see cref="idLabelConfig"/> which associates objects with labels.
/// </summary>
public IdLabelConfig idLabelConfig;
AnnotationDefinition m_AnnotationDefinition;
static ProfilerMarker s_OnObjectInfoReceivedCallback = new ProfilerMarker("OnInstanceSegmentationObjectInformationReceived");
static ProfilerMarker s_OnImageReceivedCallback = new ProfilerMarker("OnInstanceSegmentationImagesReceived");
Dictionary<int, AsyncAnnotation> m_AsyncAnnotations;
Texture m_CurrentTexture;
/// <inheritdoc cref="IOverlayPanelProvider"/>
// ReSharper disable once ConvertToAutoPropertyWithPrivateSetter
public Texture overlayImage => m_CurrentTexture;
/// <inheritdoc cref="IOverlayPanelProvider"/>
public string label => "InstanceSegmentation";
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "NotAccessedField.Local")]
struct InstanceColorValue
{
public uint instance_id;
public Color32 color;
}
string m_InstancePath;
List<InstanceColorValue> m_InstanceColorValues;
struct AsyncWrite
{
public NativeArray<Color32> data;
public int width;
public int height;
public string path;
}
/// <summary>
/// Creates a new InstanceSegmentationLabeler. Be sure to assign <see cref="idLabelConfig"/> before adding to a <see cref="PerceptionCamera"/>.
/// </summary>
public InstanceSegmentationLabeler() { }
/// <summary>
/// Creates a new InstanceSegmentationLabeler with the given <see cref="IdLabelConfig"/>.
/// </summary>
/// <param name="labelConfig">The label config for resolving the label for each object.</param>
public InstanceSegmentationLabeler(IdLabelConfig labelConfig)
{
this.idLabelConfig = labelConfig;
}
void OnRenderedObjectInfosCalculated(int frame, NativeArray<RenderedObjectInfo> renderedObjectInfos)
{
if (!m_AsyncAnnotations.TryGetValue(frame, out var annotation))
return;
m_AsyncAnnotations.Remove(frame);
using (s_OnObjectInfoReceivedCallback.Auto())
{
m_InstanceColorValues.Clear();
foreach (var objectInfo in renderedObjectInfos)
{
if (!idLabelConfig.TryGetLabelEntryFromInstanceId(objectInfo.instanceId, out var labelEntry))
continue;
m_InstanceColorValues.Add(new InstanceColorValue
{
instance_id = objectInfo.instanceId,
color = objectInfo.instanceColor
});
}
annotation.ReportFileAndValues(m_InstancePath, m_InstanceColorValues);
}
}
void OnImageCaptured(int frame, NativeArray<Color32> data, RenderTexture renderTexture)
{
using (s_OnImageReceivedCallback.Auto())
{
m_CurrentTexture = renderTexture;
m_InstancePath = $"{k_Directory}/{k_FilePrefix}{frame}.png";
var localPath = $"{Manager.Instance.GetDirectoryFor(k_Directory)}/{k_FilePrefix}{frame}.png";
var colors = new NativeArray<Color32>(data, Allocator.TempJob);
var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncWrite>>();
asyncRequest.data = new AsyncWrite
{
data = colors,
width = renderTexture.width,
height = renderTexture.height,
path = localPath
};
asyncRequest.Enqueue(r =>
{
Profiler.BeginSample("InstanceSegmentationEncode");
var pngBytes = ImageConversion.EncodeArrayToPNG(r.data.data.ToArray(), GraphicsFormat.R8G8B8A8_UNorm, (uint)r.data.width, (uint)r.data.height);
Profiler.EndSample();
Profiler.BeginSample("InstanceSegmentationWritePng");
File.WriteAllBytes(r.data.path, pngBytes);
Manager.Instance.ConsumerFileProduced(r.data.path);
Profiler.EndSample();
r.data.data.Dispose();
return AsyncRequest.Result.Completed;
});
asyncRequest.Execute();
}
}
/// <inheritdoc/>
protected override void OnBeginRendering()
{
m_AsyncAnnotations[Time.frameCount] = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
}
/// <inheritdoc/>
protected override void Setup()
{
if (idLabelConfig == null)
throw new InvalidOperationException("InstanceSegmentationLabeler's idLabelConfig field must be assigned");
m_InstanceColorValues = new List<InstanceColorValue>();
perceptionCamera.InstanceSegmentationImageReadback += OnImageCaptured;
perceptionCamera.RenderedObjectInfosCalculated += OnRenderedObjectInfosCalculated;
m_AsyncAnnotations = new Dictionary<int, AsyncAnnotation>();
m_AnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition(
"instance segmentation",
idLabelConfig.GetAnnotationSpecification(),
"pixel-wise instance segmentation label",
"PNG",
Guid.Parse(annotationId));
visualizationEnabled = supportsVisualization;
}
}
}

11
com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentationLabeler.cs.meta


fileFormatVersion: 2
guid: be2441bb87b3840b7b3ec637e06e903f
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

258
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/OverlayPanel.cs


using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine.UI;
namespace UnityEngine.Perception.GroundTruth
{
/// <summary>
/// Interface that should be defined by a class that wants to be able to provide a image to the overlay
/// panel.
/// </summary>
public interface IOverlayPanelProvider
{
/// <summary>
/// The image to the overlay panel.
/// </summary>
Texture overlayImage { get; }
/// <summary>
/// The label of the overlay panel.
/// </summary>
string label { get; }
}
/// <summary>
/// Some labeler's result in a full screen image per frame. The overlay panel controls which of these labeler's image
/// is currently shown.
/// </summary>
public class OverlayPanel : MonoBehaviour
{
internal PerceptionCamera perceptionCamera { get; set; }
bool m_Enabled;
GUIStyle m_LabelStyle;
GUIStyle m_SliderStyle;
GUIStyle m_SelectorToggleStyle;
GUIStyle m_WindowStyle;
float m_SegmentTransparency = 0.8f;
float m_BackgroundTransparency;
GameObject m_SegCanvas;
GameObject m_SegVisual;
RawImage m_OverlayImage;
int m_CachedHeight;
int m_CachedWidth;
bool m_ShowPopup = false;
Texture2D m_NormalDropDownTexture;
Texture2D m_HoverDropDownTexture;
void SetEnabled(bool isEnabled)
{
if (isEnabled == m_Enabled) return;
m_Enabled = isEnabled;
m_SegCanvas.SetActive(isEnabled);
foreach (var p in perceptionCamera.labelers)
{
if (p is IOverlayPanelProvider)
{
p.visualizationEnabled = isEnabled;
}
}
// Clear out the handle to the cached overlay texture if we are not isEnabled
if (!isEnabled)
m_OverlayImage.texture = null;
}
void SetupVisualizationElements()
{
m_Enabled = true;
m_SegmentTransparency = 0.8f;
m_BackgroundTransparency = 0.0f;
m_SegVisual = GameObject.Instantiate(Resources.Load<GameObject>("SegmentTexture"));
m_OverlayImage = m_SegVisual.GetComponent<RawImage>();
m_OverlayImage.material.SetFloat("_SegmentTransparency", m_SegmentTransparency);
m_OverlayImage.material.SetFloat("_BackTransparency", m_BackgroundTransparency);
if (m_SegVisual.transform is RectTransform rt)
{
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, Screen.width);
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, Screen.height);
}
if (m_SegCanvas == null)
{
m_SegCanvas = new GameObject("overlay_canvas");
m_SegCanvas.AddComponent<RectTransform>();
var canvas = m_SegCanvas.AddComponent<Canvas>();
canvas.renderMode = RenderMode.ScreenSpaceOverlay;
m_SegCanvas.AddComponent<CanvasScaler>();
m_SegVisual.transform.SetParent(m_SegCanvas.transform, false);
}
m_LabelStyle = new GUIStyle(GUI.skin.label) {padding = {left = 10}};
m_SliderStyle = new GUIStyle(GUI.skin.horizontalSlider) {margin = {left = 12}};
m_SelectorToggleStyle = new GUIStyle(GUI.skin.button);
if (m_NormalDropDownTexture == null)
{
m_NormalDropDownTexture = Resources.Load<Texture2D>("drop_down2");
m_HoverDropDownTexture = Resources.Load<Texture2D>("drop_down");
}
m_SelectorToggleStyle.normal.background = m_NormalDropDownTexture;
m_SelectorToggleStyle.border = new RectOffset(7, 70, 6, 6);
m_SelectorToggleStyle.alignment = TextAnchor.MiddleLeft;
m_SelectorToggleStyle.clipping = TextClipping.Clip;
m_SelectorToggleStyle.active.background = m_NormalDropDownTexture;
m_SelectorToggleStyle.hover.background = m_HoverDropDownTexture;
m_SelectorToggleStyle.focused.background = m_HoverDropDownTexture;
m_WindowStyle = new GUIStyle(GUI.skin.window);
var backTexture = Resources.Load<Texture2D>("instance_back");
m_WindowStyle.normal.background = backTexture;
}
IOverlayPanelProvider m_ActiveProvider = null;
// Make the contents of the window
void OnSelectorPopup(int windowID)
{
foreach (var labeler in perceptionCamera.labelers.Where(l => l is IOverlayPanelProvider && l.enabled))
{
var panel = labeler as IOverlayPanelProvider;
if (GUILayout.Button(panel.label))
{
m_ActiveProvider = panel;
m_ShowPopup = false;
}
}
}
internal void OnDrawGUI(float x, float y, float width, float height)
{
var any = perceptionCamera.labelers.Any(l => l is IOverlayPanelProvider && l.enabled);
// If there used to be active providers, but they have been turned off, remove
// the active provider and return null. If one has come online, then set it to the active
// provider
if (!any)
{
m_ActiveProvider = null;
}
else
{
var findNewProvider = m_ActiveProvider == null;
if (!findNewProvider)
{
if (m_ActiveProvider is CameraLabeler l)
{
findNewProvider = !l.enabled;
}
}
if (findNewProvider)
m_ActiveProvider= perceptionCamera.labelers.First(l => l is IOverlayPanelProvider && l.enabled) as IOverlayPanelProvider;
}
if (m_ActiveProvider == null)
{
if (m_SegCanvas != null)
m_SegCanvas.SetActive(false);
return;
}
if (m_OverlayImage == null)
{
SetupVisualizationElements();
}
// If all overlays were offline, but now one has come on line
// we need to set the canvas back to active
if (!m_SegCanvas.activeSelf)
m_SegCanvas.SetActive(true);
GUILayout.Label("Overlay");
GUILayout.BeginHorizontal();
GUILayout.Space(10);
GUILayout.Label("Enabled");
GUILayout.FlexibleSpace();
var isEnabled = GUILayout.Toggle(m_Enabled, "");
GUILayout.EndHorizontal();
SetEnabled(isEnabled);
if (!isEnabled)
{
return;
}
// Create the overlay button
GUILayout.Space(5);
GUILayout.BeginHorizontal();
GUILayout.Space(10);
// Truncate the label if it overflows the button size
var label = m_ActiveProvider?.label ?? "Not Selected";
var trunc = new StringBuilder(label.Substring(0, Math.Min(label.Length, 10)));
if (trunc.Length != label.Length)
trunc.Append("...");
GUILayout.Label("Overlay: ");
if (GUILayout.Button(trunc.ToString(), m_SelectorToggleStyle))
{
// If bottom is clicked we need to show the popup window
m_ShowPopup = true;
}
GUILayout.EndHorizontal();
if (m_ShowPopup)
{
var windowRect = new Rect(x, y, width, height);
GUILayout.Window(0, windowRect, OnSelectorPopup, "Choose Overlay", m_WindowStyle);
}
// Create the transparency sliders
GUILayout.Space(4);
GUILayout.Label("Object Alpha:", m_LabelStyle);
m_SegmentTransparency = GUILayout.HorizontalSlider(m_SegmentTransparency, 0.0f, 1.0f, m_SliderStyle, GUI.skin.horizontalSliderThumb);
GUILayout.Space(4);
GUILayout.Label("Background Alpha:", m_LabelStyle);
m_BackgroundTransparency = GUILayout.HorizontalSlider(m_BackgroundTransparency, 0.0f, 1.0f, m_SliderStyle, GUI.skin.horizontalSliderThumb);
GUI.skin.label.padding.left = 0;
// Grab the overlay image from the active provider
m_OverlayImage.texture = m_ActiveProvider?.overlayImage;
var rt = m_SegVisual.transform as RectTransform;
if (rt != null && m_CachedHeight != Screen.height)
{
m_CachedHeight = Screen.height;
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Vertical, m_CachedHeight);
}
if (rt != null && m_CachedWidth != Screen.width)
{
m_CachedWidth = Screen.width;
rt.SetSizeWithCurrentAnchors(RectTransform.Axis.Horizontal, Screen.width);
}
if (!GUI.changed) return;
m_OverlayImage.material.SetFloat("_SegmentTransparency", m_SegmentTransparency);
m_OverlayImage.material.SetFloat("_BackTransparency", m_BackgroundTransparency);
}
}
}

3
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/OverlayPanel.cs.meta


fileFormatVersion: 2
guid: 4e6b301436ac4cc7b23da1b72894c26f
timeCreated: 1603482449

104
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down.png

之前 之后
宽度: 88  |  高度: 36  |  大小: 6.6 KiB

106
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down.png.meta


fileFormatVersion: 2
guid: a87fa739a804d4dd8a9a65dba790bdd6
TextureImporter:
internalIDToNameTable: []
externalObjects: {}
serializedVersion: 11
mipmaps:
mipMapMode: 0
enableMipMap: 0
sRGBTexture: 1
linearTexture: 0
fadeOut: 0
borderMipMap: 0
mipMapsPreserveCoverage: 0
alphaTestReferenceValue: 0.5
mipMapFadeDistanceStart: 1
mipMapFadeDistanceEnd: 3
bumpmap:
convertToNormalMap: 0
externalNormalMap: 0
heightScale: 0.25
normalMapFilter: 0
isReadable: 0
streamingMipmaps: 0
streamingMipmapsPriority: 0
vTOnly: 0
grayScaleToAlpha: 0
generateCubemap: 6
cubemapConvolution: 0
seamlessCubemap: 0
textureFormat: 1
maxTextureSize: 2048
textureSettings:
serializedVersion: 2
filterMode: -1
aniso: -1
mipBias: -100
wrapU: 1
wrapV: 1
wrapW: -1
nPOTScale: 0
lightmap: 0
compressionQuality: 50
spriteMode: 1
spriteExtrude: 1
spriteMeshType: 0
alignment: 0
spritePivot: {x: 0.5, y: 0.5}
spritePixelsToUnits: 100
spriteBorder: {x: 7, y: 6, z: 29, w: 6}
spriteGenerateFallbackPhysicsShape: 1
alphaUsage: 1
alphaIsTransparency: 1
spriteTessellationDetail: -1
textureType: 8
textureShape: 1
singleChannelComponent: 0
maxTextureSizeSet: 0
compressionQualitySet: 0
textureFormatSet: 0
ignorePngGamma: 1
applyGammaDecoding: 0
platformSettings:
- serializedVersion: 3
buildTarget: DefaultTexturePlatform
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
- serializedVersion: 3
buildTarget: Standalone
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
spriteSheet:
serializedVersion: 2
sprites: []
outline: []
physicsShape: []
bones: []
spriteID: 5e97eb03825dee720800000000000000
internalID: 0
vertices: []
indices:
edges: []
weights: []
secondaryTextures: []
spritePackingTag:
pSDRemoveMatte: 0
pSDShowRemoveMatteOption: 0
userData:
assetBundleName:
assetBundleVariant:

99
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down2.png

之前 之后
宽度: 88  |  高度: 36  |  大小: 6.4 KiB

106
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/drop_down2.png.meta


fileFormatVersion: 2
guid: 0f330e04ba4604456890c268c8a18459
TextureImporter:
internalIDToNameTable: []
externalObjects: {}
serializedVersion: 11
mipmaps:
mipMapMode: 0
enableMipMap: 0
sRGBTexture: 1
linearTexture: 0
fadeOut: 0
borderMipMap: 0
mipMapsPreserveCoverage: 0
alphaTestReferenceValue: 0.5
mipMapFadeDistanceStart: 1
mipMapFadeDistanceEnd: 3
bumpmap:
convertToNormalMap: 0
externalNormalMap: 0
heightScale: 0.25
normalMapFilter: 0
isReadable: 0
streamingMipmaps: 0
streamingMipmapsPriority: 0
vTOnly: 0
grayScaleToAlpha: 0
generateCubemap: 6
cubemapConvolution: 0
seamlessCubemap: 0
textureFormat: 1
maxTextureSize: 2048
textureSettings:
serializedVersion: 2
filterMode: -1
aniso: -1
mipBias: -100
wrapU: 1
wrapV: 1
wrapW: -1
nPOTScale: 0
lightmap: 0
compressionQuality: 50
spriteMode: 1
spriteExtrude: 1
spriteMeshType: 1
alignment: 0
spritePivot: {x: 0.5, y: 0.5}
spritePixelsToUnits: 100
spriteBorder: {x: 0, y: 0, z: 0, w: 0}
spriteGenerateFallbackPhysicsShape: 1
alphaUsage: 1
alphaIsTransparency: 1
spriteTessellationDetail: -1
textureType: 8
textureShape: 1
singleChannelComponent: 0
maxTextureSizeSet: 0
compressionQualitySet: 0
textureFormatSet: 0
ignorePngGamma: 0
applyGammaDecoding: 0
platformSettings:
- serializedVersion: 3
buildTarget: DefaultTexturePlatform
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
- serializedVersion: 3
buildTarget: Standalone
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
spriteSheet:
serializedVersion: 2
sprites: []
outline: []
physicsShape: []
bones: []
spriteID: 5e97eb03825dee720800000000000000
internalID: 0
vertices: []
indices:
edges: []
weights: []
secondaryTextures: []
spritePackingTag:
pSDRemoveMatte: 0
pSDShowRemoveMatteOption: 0
userData:
assetBundleName:
assetBundleVariant:

5
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/instance_back.png

之前 之后
宽度: 32  |  高度: 32  |  大小: 575 B

104
com.unity.perception/Runtime/GroundTruth/Labelers/Visualization/Resources/instance_back.png.meta


fileFormatVersion: 2
guid: eeacd5d43d45649b8995200742000a22
TextureImporter:
internalIDToNameTable: []
externalObjects: {}
serializedVersion: 11
mipmaps:
mipMapMode: 0
enableMipMap: 0
sRGBTexture: 1
linearTexture: 0
fadeOut: 0
borderMipMap: 0
mipMapsPreserveCoverage: 0
alphaTestReferenceValue: 0.5
mipMapFadeDistanceStart: 1
mipMapFadeDistanceEnd: 3
bumpmap:
convertToNormalMap: 0
externalNormalMap: 0
heightScale: 0.25
normalMapFilter: 0
isReadable: 0
streamingMipmaps: 0
streamingMipmapsPriority: 0
grayScaleToAlpha: 0
generateCubemap: 6
cubemapConvolution: 0
seamlessCubemap: 0
textureFormat: 1
maxTextureSize: 2048
textureSettings:
serializedVersion: 2
filterMode: -1
aniso: -1
mipBias: -100
wrapU: 0
wrapV: 0
wrapW: 0
nPOTScale: 0
lightmap: 0
compressionQuality: 50
spriteMode: 1
spriteExtrude: 1
spriteMeshType: 1
alignment: 0
spritePivot: {x: 0.5, y: 0.5}
spritePixelsToUnits: 100
spriteBorder: {x: 0, y: 0, z: 0, w: 0}
spriteGenerateFallbackPhysicsShape: 1
alphaUsage: 1
alphaIsTransparency: 1
spriteTessellationDetail: -1
textureType: 8
textureShape: 1
singleChannelComponent: 0
maxTextureSizeSet: 0
compressionQualitySet: 0
textureFormatSet: 0
applyGammaDecoding: 0
platformSettings:
- serializedVersion: 3
buildTarget: DefaultTexturePlatform
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
- serializedVersion: 3
buildTarget: Standalone
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
forceMaximumCompressionQuality_BC6H_BC7: 0
spriteSheet:
serializedVersion: 2
sprites: []
outline: []
physicsShape: []
bones: []
spriteID: 5e97eb03825dee720800000000000000
internalID: 0
vertices: []
indices:
edges: []
weights: []
secondaryTextures: []
spritePackingTag:
pSDRemoveMatte: 0
pSDShowRemoveMatteOption: 0
userData:
assetBundleName:
assetBundleVariant:

135
com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceIdToColorMappingTests.cs


using System;
using NUnit.Framework;
using UnityEngine;
using UnityEngine.Perception.GroundTruth;
namespace GroundTruthTests
{
[TestFixture]
public class InstanceIdToColorMappingTests
{
[Test]
public void InstanceIdToColorMappingTests_TestHslColors()
{
for (var i = 1u; i <= 64u; i++)
{
Assert.IsTrue(InstanceIdToColorMapping.TryGetColorFromInstanceId(i, out var color));
Assert.IsTrue(InstanceIdToColorMapping.TryGetInstanceIdFromColor(color, out var id));
Assert.AreEqual(i, id);
color = InstanceIdToColorMapping.GetColorFromInstanceId(i);
id = InstanceIdToColorMapping.GetInstanceIdFromColor(color);
Assert.AreEqual(i, id);
}
}
[Test]
[TestCase(0, 0, 0, 255, 255u)]
[TestCase(255, 0, 0, 255, 4278190335u)]
[TestCase(0, 255, 0, 255, 16711935u)]
[TestCase(0, 0, 255, 255, 65535u)]
[TestCase(255, 255, 255, 255, 4294967295u)]
[TestCase(0, 0, 1, 255, 511u)]
[TestCase(127, 64, 83, 27, 2134922011u)]
public void InstanceIdToColorMappingTests_ToAndFromPackedColor(byte r, byte g, byte b, byte a, uint expected)
{
var color = new Color32(r, g, b, a);
var packed = InstanceIdToColorMapping.GetPackedColorFromColor(color);
Assert.AreEqual(packed, expected);
var c = InstanceIdToColorMapping.GetColorFromPackedColor(packed);
Assert.AreEqual(r, c.r);
Assert.AreEqual(g, c.g);
Assert.AreEqual(b, c.b);
Assert.AreEqual(a, c.a);
}
[Test]
[TestCase(1u, 255,0,0,255)]
[TestCase(2u,0,74,255,255)]
[TestCase(3u,149,255,0,255)]
[TestCase(4u,255,0,223,255)]
[TestCase(5u,0,255,212,255)]
[TestCase(6u,255,138,0,255)]
[TestCase(64u,195,0,75,255)]
[TestCase(65u,0,0,1,254)]
[TestCase(66u,0,0,2,254)]
[TestCase(64u + 256u,0,1,0,254)]
[TestCase(65u + 256u,0,1,1,254)]
[TestCase(64u + 65536u,1,0,0,254)]
[TestCase(16777216u,255,255,192,254)]
[TestCase(64u + 16777216u,0,0,0,253)]
[TestCase(64u + (16777216u * 2),0,0,0,252)]
public void InstanceIdToColorMappingTests_TestColorForId(uint id, byte r, byte g, byte b, byte a)
{
Assert.IsTrue(InstanceIdToColorMapping.TryGetColorFromInstanceId(id, out var color));
Assert.AreEqual(color.r, r);
Assert.AreEqual(color.g, g);
Assert.AreEqual(color.b, b);
Assert.AreEqual(color.a, a);
Assert.IsTrue(InstanceIdToColorMapping.TryGetInstanceIdFromColor(color, out var id2));
Assert.AreEqual(id, id2);
color = InstanceIdToColorMapping.GetColorFromInstanceId(id);
Assert.AreEqual(color.r, r);
Assert.AreEqual(color.g, g);
Assert.AreEqual(color.b, b);
Assert.AreEqual(color.a, a);
id2 = InstanceIdToColorMapping.GetInstanceIdFromColor(color);
Assert.AreEqual(id, id2);
}
[Test]
[TestCase(0u)]
[TestCase(255u)]
public void InstanceIdToColorMappingTests_GetBlackForId(uint id)
{
Assert.IsFalse(InstanceIdToColorMapping.TryGetColorFromInstanceId(id, out var color));
Assert.AreEqual(color.r, 0);
Assert.AreEqual(color.g, 0);
Assert.AreEqual(color.b, 0);
Assert.AreEqual(color.a, 255);
Assert.IsFalse(InstanceIdToColorMapping.TryGetInstanceIdFromColor(color, out var id2));
Assert.AreEqual(0, id2);
color = InstanceIdToColorMapping.GetColorFromInstanceId(id);
Assert.AreEqual(color.r, 0);
Assert.AreEqual(color.g, 0);
Assert.AreEqual(color.b, 0);
Assert.AreEqual(color.a, 255);
id2 = InstanceIdToColorMapping.GetInstanceIdFromColor(color);
Assert.AreEqual(0, id2);
}
[Test]
public void InstanceIdToColorMappingTests_ThrowExceptionIdTooLarge()
{
Assert.Throws<IndexOutOfRangeException>(() => InstanceIdToColorMapping.GetColorFromInstanceId(uint.MaxValue));
var c = new Color32(255, 255, 255, 0);
Assert.Throws<IndexOutOfRangeException>(() => InstanceIdToColorMapping.GetInstanceIdFromColor(c));
}
[Test]
public void InstanceIdToColorMappingTests_TryGetReturnsFalseIdTooLarge()
{
Assert.IsFalse(InstanceIdToColorMapping.TryGetColorFromInstanceId(uint.MaxValue, out var color));
color = new Color32(255, 255, 255, 0);
Assert.IsFalse(InstanceIdToColorMapping.TryGetInstanceIdFromColor(color, out var id));
}
[Test]
public void InstanceIdToColorMappingTests_ThrowExceptionIdNotMapped()
{
var c = new Color32(28,92,14,255);
Assert.Throws<InvalidOperationException>(() => InstanceIdToColorMapping.GetInstanceIdFromColor(c));
}
[Test]
public void InstanceIdToColorMappingTests_TryGetReturnsFalseIdNotMapped()
{
var c = new Color32(28,92,14,255);
Assert.IsFalse(InstanceIdToColorMapping.TryGetInstanceIdFromColor(c, out var id));
}
}
}

3
com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceIdToColorMappingTests.cs.meta


fileFormatVersion: 2
guid: 2977b6a5f1074f3f90e8a8c64ffb16a3
timeCreated: 1604077566
正在加载...
取消
保存