using System;
using Unity.Burst;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Jobs;
using Unity.Mathematics;
using Unity.Profiling;
using UnityEngine.UIElements;
namespace UnityEngine.Perception.GroundTruth
{
///
/// A CPU-based pass which computes bounding box and pixel counts per-object from instance segmentation images
///
public class RenderedObjectInfoGenerator : IGroundTruthGenerator, IDisposable
{
static ProfilerMarker s_LabelJobs = new ProfilerMarker("Label Jobs");
static ProfilerMarker s_LabelMerge = new ProfilerMarker("Label Merge");
const int k_StartingObjectCount = 1 << 8;
struct Object1DSpan
{
public int instanceId;
public int row;
public int left;
public int right;
}
[BurstCompile]
struct ComputeHistogramPerRowJob : IJob
{
[ReadOnly]
public NativeSlice segmentationImageData;
public int width;
public int rows;
public int rowStart;
[NativeDisableContainerSafetyRestriction]
public NativeList boundingBoxes;
public void Execute()
{
for (var row = 0; row < rows; row++)
{
var rowSlice = new NativeSlice(segmentationImageData, width * row, width);
var currentBB = new Object1DSpan
{
instanceId = -1,
row = row + rowStart
};
for (var i = 0; i < rowSlice.Length; i++)
{
var value = rowSlice[i];
if (value != currentBB.instanceId)
{
if (currentBB.instanceId > 0)
{
//save off currentBB
currentBB.right = i - 1;
boundingBoxes.Add(currentBB);
}
currentBB = new Object1DSpan
{
instanceId = (int)value,
left = i,
row = row + rowStart
};
}
}
if (currentBB.instanceId > 0)
{
//save off currentBB
currentBB.right = width - 1;
boundingBoxes.Add(currentBB);
}
}
}
}
NativeList m_InstanceIdToLabelEntryIndexLookup;
LabelingConfiguration m_LabelingConfiguration;
// ReSharper disable once InvalidXmlDocComment
///
/// Create a new CpuRenderedObjectInfoPass with the given LabelingConfiguration.
///
/// The LabelingConfiguration to use to determine labelId. Should match the
/// one used by the generating the input image. See
public RenderedObjectInfoGenerator(LabelingConfiguration labelingConfiguration)
{
m_LabelingConfiguration = labelingConfiguration;
m_InstanceIdToLabelEntryIndexLookup = new NativeList(k_StartingObjectCount, Allocator.Persistent);
}
///
public void SetupMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
if (m_LabelingConfiguration.TryGetMatchingConfigurationEntry(labeling, out _, out var index))
{
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
{
m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
}
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = index;
}
}
// ReSharper disable once InvalidXmlDocComment
///
/// Compute RenderedObjectInfo for each visible object in the given instance segmentation image.
/// InstanceSegmentationRawData should be the raw data from a texture filled by or
/// using the same LabelingConfiguration that was passed into this object.
///
/// The raw instance segmentation image.
/// Stride of the image data. Should be equal to the width of the image.
/// Whether bounding boxes should be top-left or bottom-right-based.
/// When this method returns, filled with RenderedObjectInfo entries for each object visible in the frame.
/// When the method returns, filled with a NativeArray with the count of objects for each entry in in the LabelingConfiguration passed into the constructor.
/// The allocator to use for allocating renderedObjectInfos and perLabelEntryObjectCount.
public void Compute(NativeArray instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray renderedObjectInfos, out NativeArray perLabelEntryObjectCount, Allocator allocator)
{
const int jobCount = 24;
var height = instanceSegmentationRawData.Length / stride;
//special math to round up
var rowsPerJob = height / jobCount;
var rowRemainder = height % jobCount;
var handles = new NativeArray(jobCount, Allocator.Temp);
var jobBoundingBoxLists = new NativeList[jobCount];
using (s_LabelJobs.Auto())
{
for (int row = 0, jobIndex = 0; row < height; row += rowsPerJob, jobIndex++)
{
jobBoundingBoxLists[jobIndex] = new NativeList(10, Allocator.TempJob);
var rowsThisJob = math.min(height - row, rowsPerJob);
if (jobIndex < rowRemainder)
rowsThisJob++;
handles[jobIndex] = new ComputeHistogramPerRowJob
{
segmentationImageData = new NativeSlice(instanceSegmentationRawData, row * stride, stride * rowsThisJob),
width = stride,
rowStart = row,
rows = rowsThisJob,
boundingBoxes = jobBoundingBoxLists[jobIndex]
}.Schedule();
if (jobIndex < rowRemainder)
row++;
}
JobHandle.CompleteAll(handles);
}
perLabelEntryObjectCount = new NativeArray(m_LabelingConfiguration.LabelEntries.Count, allocator);
var boundingBoxMap = new NativeHashMap(100, Allocator.Temp);
using (s_LabelMerge.Auto())
{
foreach (var boundingBoxList in jobBoundingBoxLists)
{
if (!boundingBoxList.IsCreated)
continue;
foreach (var info1D in boundingBoxList)
{
var objectInfo = new RenderedObjectInfo
{
boundingBox = new Rect(info1D.left, info1D.row, info1D.right - info1D.left + 1, 1),
instanceId = info1D.instanceId,
pixelCount = info1D.right - info1D.left + 1
};
if (boundingBoxMap.TryGetValue(info1D.instanceId, out var info))
{
objectInfo.boundingBox = Rect.MinMaxRect(
math.min(info.boundingBox.xMin, objectInfo.boundingBox.xMin),
math.min(info.boundingBox.yMin, objectInfo.boundingBox.yMin),
math.max(info.boundingBox.xMax, objectInfo.boundingBox.xMax),
math.max(info.boundingBox.yMax, objectInfo.boundingBox.yMax));
objectInfo.pixelCount += info.pixelCount;
}
boundingBoxMap[info1D.instanceId] = objectInfo;
}
}
var keyValueArrays = boundingBoxMap.GetKeyValueArrays(Allocator.Temp);
renderedObjectInfos = new NativeArray(keyValueArrays.Keys.Length, allocator);
for (var i = 0; i < keyValueArrays.Keys.Length; i++)
{
var instanceId = keyValueArrays.Keys[i];
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
continue;
var labelIndex = m_InstanceIdToLabelEntryIndexLookup[instanceId];
var labelId = m_LabelingConfiguration.LabelEntries[labelIndex].id;
perLabelEntryObjectCount[labelIndex]++;
var renderedObjectInfo = keyValueArrays.Values[i];
var boundingBox = renderedObjectInfo.boundingBox;
if (boundingBoxOrigin == BoundingBoxOrigin.TopLeft)
{
var y = height - boundingBox.yMax;
boundingBox = new Rect(boundingBox.x, y, boundingBox.width, boundingBox.height);
}
renderedObjectInfos[i] = new RenderedObjectInfo
{
instanceId = instanceId,
labelId = labelId,
boundingBox = boundingBox,
pixelCount = renderedObjectInfo.pixelCount
};
}
keyValueArrays.Dispose();
}
boundingBoxMap.Dispose();
foreach (var rowBoundingBox in jobBoundingBoxLists)
{
if (rowBoundingBox.IsCreated)
rowBoundingBox.Dispose();
}
handles.Dispose();
}
///
/// Attempts to find the label id for the given instance id using the LabelingConfiguration passed into the constructor.
///
/// The instanceId of the object for which the labelId should be found
/// The LabelEntry associated with the object. default if not found
/// True if a labelId is found for the given instanceId.
public bool TryGetLabelEntryFromInstanceId(int instanceId, out LabelEntry labelEntry)
{
labelEntry = default;
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
return false;
var labelEntryIndex = m_InstanceIdToLabelEntryIndexLookup[instanceId];
labelEntry = m_LabelingConfiguration.LabelEntries[labelEntryIndex];
return true;
}
///
public void Dispose()
{
m_InstanceIdToLabelEntryIndexLookup.Dispose();
}
}
}