浏览代码

Partially implemented refactor to address labeling bug.

/main
Jon Hogins 4 年前
当前提交
66eaf180
共有 10 个文件被更改,包括 90 次插入60 次删除
  1. 7
      com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs
  2. 4
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  3. 10
      com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs
  4. 13
      com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs
  5. 7
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs
  6. 38
      com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs
  7. 53
      com.unity.perception/Runtime/GroundTruth/RenderTextureReader.cs
  8. 2
      com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs
  9. 4
      com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs
  10. 12
      com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

7
com.unity.perception/Runtime/GroundTruth/Labelers/CameraLabeler.cs


using System;
using Unity.Simulation;
using UnityEngine.EventSystems;
using UnityEngine.Rendering;
using UnityEngine.UI;
using UnityEngine.UIElements;

/// Called just before the camera renders each frame the the labeler is enabled and <see cref="SensorHandle.ShouldCaptureThisFrame"/> is true.
/// </summary>
protected virtual void OnBeginRendering() {}
protected virtual void OnEndRendering() {}
/// <param name="scriptableRenderContext"></param>
protected virtual void OnEndRendering(ScriptableRenderContext scriptableRenderContext) {}
/// <summary>
/// Labeling pass to display labeler's visualization components, if applicable. Important note, all labeler's visualizations need
/// to use Unity's Immediate Mode GUI (IMGUI) <see cref="https://docs.unity3d.com/Manual/GUIScriptingGuide.html"/> system.

}
internal void InternalOnUpdate() => OnUpdate();
internal void InternalOnBeginRendering() => OnBeginRendering();
internal void InternalOnEndRendering() => OnEndRendering();
internal void InternalOnEndRendering(ScriptableRenderContext context) => OnEndRendering(context);
internal void InternalCleanup() => Cleanup();
internal void InternalVisualize() => OnVisualize();

4
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs


using System.Collections.Generic;
using System.Linq;
using Unity.Collections;
using UnityEngine.Rendering;
namespace UnityEngine.Perception.GroundTruth
{

asyncAnnotation.annotation.ReportValues(m_ToReport);
}
/// <param name="scriptableRenderContext"></param>
protected override void OnEndRendering()
protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
{
m_CurrentFrame = Time.frameCount;

10
com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs


"PNG",
id: Guid.Parse(annotationId));
m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture, myCamera,
(frameCount, data, tex) => OnSemanticSegmentationImageRead(frameCount, data));
m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture);
visualizationEnabled = supportsVisualization;
}

asyncRequest.Execute();
}
/// <inheritdoc/>
protected override void OnBeginRendering()
protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
m_SemanticSegmentationTextureReader.Capture(scriptableRenderContext,
(frameCount, data, renderTexture) => OnSemanticSegmentationImageRead(frameCount, data));
}
/// <inheritdoc/>

13
com.unity.perception/Runtime/GroundTruth/Labeling/IdLabelConfig.cs


using System.Linq;
using JetBrains.Annotations;
using Newtonsoft.Json;
using Unity.Collections;
namespace UnityEngine.Perception.GroundTruth {
/// <summary>

label_id = l.id,
label_name = l.label,
}).ToArray();
}
public static IdLabelMap GetIdLabelCache(Allocator allocator = Allocator.Temp)
{
return new IdLabelMap();
}
}
public struct IdLabelMap : IDisposable
{
public void Dispose()
{
}
}
}

7
com.unity.perception/Runtime/GroundTruth/PerceptionCamera.cs


}
}
void OnBeginCameraRendering(ScriptableRenderContext _, Camera cam)
void OnBeginCameraRendering(ScriptableRenderContext scriptableRenderContext, Camera cam)
{
if (!ShouldCallLabelers(cam, m_LastFrameCaptured))
return;

CallOnLabelers(l => l.InternalOnBeginRendering());
}
void OnEndCameraRendering(ScriptableRenderContext _, Camera cam)
void OnEndCameraRendering(ScriptableRenderContext scriptableRenderContext, Camera cam)
CallOnLabelers(l => l.InternalOnEndRendering());
CaptureInstanceSegmentation(scriptableRenderContext);
CallOnLabelers(l => l.InternalOnEndRendering(scriptableRenderContext));
}
void CallOnLabelers(Action<CameraLabeler> action)

38
com.unity.perception/Runtime/GroundTruth/PerceptionCamera_InstanceSegmentation.cs


using System;
using Unity.Collections;
using Unity.Simulation;
using UnityEngine.Rendering;
#if HDRP_PRESENT
using UnityEngine.Rendering.HighDefinition;
#elif URP_PRESENT

/// Invoked when instance segmentation images are read back from the graphics system. The first parameter is the
/// Time.frameCount at which the objects were rendered. May be invoked many frames after the objects were rendered.
/// </summary>
public event Action<int, NativeArray<Color32>, RenderTexture> InstanceSegmentationImageReadback;
public event Action<int, NativeArray<Color32>, RenderTexture/*, List<IdLabelEntry>*/> InstanceSegmentationImageReadback;
public event Action<int, NativeArray<RenderedObjectInfo>> RenderedObjectInfosCalculated;
public event Action<int, NativeArray<RenderedObjectInfo>/*, List<IdLabelEntry>*/> RenderedObjectInfosCalculated;
RenderedObjectInfoGenerator m_RenderedObjectInfoGenerator;
RenderTexture m_InstanceSegmentationTexture;

m_LensDistortionIntensityOverride;
#endif
m_InstanceSegmentationReader = new RenderTextureReader<Color32>(m_InstanceSegmentationTexture, myCamera, (frameCount, data, tex) =>
m_InstanceSegmentationReader = new RenderTextureReader<Color32>(m_InstanceSegmentationTexture);
}
void CaptureInstanceSegmentation(ScriptableRenderContext scriptableRenderContext)
{
//var myCache = new List<IdLabelEntry>();
var cache = IdLabelConfig.GetIdLabelCache();
var width = m_InstanceSegmentationTexture.width;
m_InstanceSegmentationReader.Capture(scriptableRenderContext, (frameCount, data, renderTexture) =>
InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
if (RenderedObjectInfosCalculated != null)
InstanceSegmentationImageReadback?.Invoke(frameCount, data, m_InstanceSegmentationTexture/*, myCache*/);
if(RenderedObjectInfosCalculated != null)
m_RenderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var renderedObjectInfos, Allocator.Temp);
RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
m_RenderedObjectInfoGenerator.Compute(data, width,
BoundingBoxOrigin.TopLeft, cache, out var renderedObjectInfos, Allocator.Temp);
RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos/*, myCache*/);
cache.Dispose();
m_InstanceSegmentationReader?.WaitForAllImages();
m_InstanceSegmentationReader?.Dispose();
m_InstanceSegmentationReader = null;
m_InstanceSegmentationReader?.WaitForAllImages();
m_InstanceSegmentationReader?.Dispose();
m_InstanceSegmentationReader = null;
}
}
}

53
com.unity.perception/Runtime/GroundTruth/RenderTextureReader.cs


/// RenderTextureReader reads a RenderTexture from the GPU each frame and passes the data back through a provided callback.
/// </summary>
/// <typeparam name="T">The type of the raw texture data to be provided.</typeparam>
public class RenderTextureReader<T> : IDisposable where T : struct
class RenderTextureReader<T> : IDisposable where T : struct
Action<int, NativeArray<T>, RenderTexture> m_ImageReadCallback;
int m_NextFrameToCapture;
Camera m_CameraRenderingToSource;
/// <param name="cameraRenderingToSource">The <see cref="Camera"/> which renders to the given renderTexture. This is used to determine when to read from the texture.</param>
/// <param name="imageReadCallback">The callback to call after reading the texture</param>
public RenderTextureReader(RenderTexture source, Camera cameraRenderingToSource, Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
public RenderTextureReader(RenderTexture source)
m_ImageReadCallback = imageReadCallback;
m_CameraRenderingToSource = cameraRenderingToSource;
m_NextFrameToCapture = Time.frameCount;
RenderPipelineManager.endFrameRendering += OnEndFrameRendering;
void OnEndFrameRendering(ScriptableRenderContext context, Camera[] cameras)
public void Capture(ScriptableRenderContext context, Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
#if UNITY_EDITOR
if (UnityEditor.EditorApplication.isPaused)
return;
#endif
if (!cameras.Contains(m_CameraRenderingToSource))
return;
if (m_NextFrameToCapture > Time.frameCount)
return;
m_NextFrameToCapture = Time.frameCount + 1;
// #if UNITY_EDITOR
// if (UnityEditor.EditorApplication.isPaused)
// return;
// #endif
// if (!cameras.Contains(m_CameraRenderingToSource))
// return;
//
// if (m_NextFrameToCapture > Time.frameCount)
// return;
//
// m_NextFrameToCapture = Time.frameCount + 1;
if (!GraphicsUtilities.SupportsAsyncReadback())
{

0, 0);
RenderTexture.active = null;
var data = m_CpuTexture.GetRawTextureData<T>();
m_ImageReadCallback(Time.frameCount, data, m_Source);
imageReadCallback(Time.frameCount, data, m_Source);
commandBuffer.RequestAsyncReadback(m_Source, r => OnGpuReadback(r, frameCount));
commandBuffer.RequestAsyncReadback(m_Source, r => OnGpuReadback(r, frameCount, imageReadCallback));
context.ExecuteCommandBuffer(commandBuffer);
context.Submit();
CommandBufferPool.Release(commandBuffer);

void OnGpuReadback(AsyncGPUReadbackRequest request, int frameCount)
void OnGpuReadback(AsyncGPUReadbackRequest request, int frameCount,
Action<int, NativeArray<T>, RenderTexture> imageReadCallback)
else if (request.done && m_ImageReadCallback != null)
else if (request.done && imageReadCallback != null)
m_ImageReadCallback(frameCount, request.GetData<T>(), m_Source);
imageReadCallback(frameCount, request.GetData<T>(), m_Source);
}
}

public void Dispose()
{
WaitForAllImages();
RenderPipelineManager.endFrameRendering -= OnEndFrameRendering;
if (m_CpuTexture != null)
{
Object.Destroy(m_CpuTexture);

2
com.unity.perception/Runtime/GroundTruth/RenderedObjectInfoGenerator.cs


/// <param name="boundingBoxOrigin">Whether bounding boxes should be top-left or bottom-right-based.</param>
/// <param name="renderedObjectInfos">When this method returns, filled with RenderedObjectInfo entries for each object visible in the frame.</param>
/// <param name="allocator">The allocator to use for allocating renderedObjectInfos and perLabelEntryObjectCount.</param>
public void Compute(NativeArray<Color32> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
public void Compute(NativeArray<Color32> instanceSegmentationRawData, int stride, BoundingBoxOrigin boundingBoxOrigin, IdLabelMap idLabelMap, out NativeArray<RenderedObjectInfo> renderedObjectInfos, Allocator allocator)
{
const int jobCount = 24;
var height = instanceSegmentationRawData.Length / stride;

4
com.unity.perception/Tests/Runtime/GroundTruthTests/RenderedObjectInfoTests.cs


var dataNativeArray = new NativeArray<Color32>(producesCorrectObjectInfoData.data, Allocator.Persistent);
renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, out var boundingBoxes, Allocator.Temp);
var idLabelMap = IdLabelConfig.GetIdLabelCache();
renderedObjectInfoGenerator.Compute(dataNativeArray, producesCorrectObjectInfoData.stride, producesCorrectObjectInfoData.boundingBoxOrigin, idLabelMap, out var boundingBoxes, Allocator.Temp);
idLabelMap.Dispose();
}
}
}

12
com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs


void Awake()
{
m_Reader = new RenderTextureReader<Color32>(source, cameraSource, ImageReadCallback);
m_Reader = new RenderTextureReader<Color32>(source);
RenderPipelineManager.endCameraRendering += (context, camera) =>
m_Reader.Capture(context,
(frameCount, data, renderTexture) => ImageReadCallback(frameCount, data, renderTexture));
}
void ImageReadCallback(int frameCount, NativeArray<Color32> data, RenderTexture renderTexture)

if (frames < 10)
return;
var idLabelMap = IdLabelConfig.GetIdLabelCache();
// Calculate the bounding box
if (fLensDistortionEnabled == false)
{

renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp);
renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, idLabelMap, out var boundingBoxes, Allocator.Temp);
boundingBoxWithoutLensDistortion = boundingBoxes[0].boundingBox;

{
var renderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var boundingBoxes, Allocator.Temp);
renderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, idLabelMap, out var boundingBoxes, Allocator.Temp);
boundingBoxWithLensDistortion = boundingBoxes[0].boundingBox;

fDone = true;
}
idLabelMap.Dispose();
}
cameraObject = SetupCamera(out perceptionCamera, false);

正在加载...
取消
保存