unity-tech-cn
/
com.unity.perception
29
0
派生 7
代码 合并请求 版本发布
浏览代码

Semantic segmentation supported

/solo_support
Steve Borkman 3 年前
当前提交
b0d3bfb5
共有 4 个文件被更改，包括 138 次插入 和 78 次删除
  1. 35
      com.unity.perception/Runtime/GroundTruth/Consumers/SoloConsumer.cs
  2. 6
      com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs
  3. 162
      com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs
  4. 13
      com.unity.perception/Runtime/GroundTruth/SimulationState.cs

35
com.unity.perception/Runtime/GroundTruth/Consumers/SoloConsumer.cs
查看文件


case InstanceSegmentationLabeler.InstanceSegmentation seg:
annotations.Add(ConvertAnnotation(frame, seg));
break;
case SemanticSegmentationLabeler.SemanticSegmentation seg:
annotations.Add(ConvertAnnotation(frame, seg));
break;
}
}

// write out the png data
var path = GetSequenceDirectoryPath(frame);
path = Path.Combine(path,$"step{frame.step}.segmentation.{segmentation.imageFormat}");
path = Path.Combine(path,$"step{frame.step}.instance.segmentation.{segmentation.imageFormat}");
var file = File.Create(path, 4096);
file.Write(segmentation.buffer, 0, segmentation.buffer.Length);
file.Close();

{
["instance_id"] = i.instanceId,
["rgba"] = FromColor32(i.rgba)
});
}
outSeg["imageFormat"] = segmentation.imageFormat;
outSeg["dimension"] = FromVector2(segmentation.dimension);
outSeg["imagePath"] = path;
outSeg["instances"] = values;
return outSeg;
}
static JToken ConvertAnnotation(Frame frame, SemanticSegmentationLabeler.SemanticSegmentation segmentation)
{
// write out the png data
var path = GetSequenceDirectoryPath(frame);
path = Path.Combine(path,$"step{frame.step}.semantic.segmentation.{segmentation.imageFormat}");
var file = File.Create(path, 4096);
file.Write(segmentation.buffer, 0, segmentation.buffer.Length);
file.Close();
var outSeg = ToAnnotationHeader(frame, segmentation);
var values = new JArray();
foreach (var i in segmentation.instances)
{
values.Add(new JObject
{
["label_name"] = i.labelName,
["rgba"] = FromColor32(i.pixelValue)
});
}

6
com.unity.perception/Runtime/GroundTruth/Labelers/BoundingBoxLabeler.cs
查看文件


using System;
using System.Collections.Generic;
using System.Linq;
using Unity.Collections;
using Unity.Profiling;
using UnityEngine.Serialization;

}
}
AnnotationDefinition m_AnnotationDefinition = new BoundingBoxAnnotationDefinition();
BoundingBoxAnnotationDefinition m_AnnotationDefinition;
/// <summary>
/// Bounding boxes for all of the labeled objects in a capture

m_AsyncData = new Dictionary<int, (AsyncAnnotationFuture annotation, LabelEntryMatchCache labelEntryMatchCache)>();
m_BoundingBoxValues = new List<BoundingBoxAnnotation.Entry>();
var spec = idLabelConfig.GetAnnotationSpecification().Select(i => new BoundingBoxAnnotationDefinition.DefinitionEntry { labelId = i.label_id, labelName = i.label_name });
m_AnnotationDefinition = new BoundingBoxAnnotationDefinition(spec);
DatasetCapture.RegisterAnnotationDefinition(m_AnnotationDefinition);
#if false

162
com.unity.perception/Runtime/GroundTruth/Labelers/SemanticSegmentationLabeler.cs
查看文件


using JetBrains.Annotations;
using Unity.Collections;
using Unity.Simulation;
using UnityEditor;
using UnityEngine.Perception.GroundTruth.DataModel;
using UnityEngine.Perception.GroundTruth.Exporters.Solo;
using UnityEngine.Profiling;
using UnityEngine.Rendering;

[Serializable]
public sealed class SemanticSegmentationLabeler : CameraLabeler, IOverlayPanelProvider
{
///<inheritdoc/>
public override string description
[Serializable]
public class SemanticSegmentationDefinition : AnnotationDefinition
{
static readonly string k_Id = "semantic segmentation";
static readonly string k_Description = "Generates a semantic segmentation image for each captured frame. " +
"Each object is rendered to the semantic segmentation image using the color associated with it based on " +
"this labeler's associated semantic segmentation label configuration. Semantic segmentation images are saved " +
"to the dataset in PNG format. Please note that only one SemanticSegmentationLabeler can render at once across all cameras.";
static readonly string k_AnnotationType = "semantic segmentation";
public IEnumerable<DefinitionEntry> spec;
public SemanticSegmentationDefinition() : base(k_Id, k_Description, k_AnnotationType) { }
public SemanticSegmentationDefinition(IEnumerable<DefinitionEntry> spec)
: base(k_Id, k_Description, k_AnnotationType)
{
this.spec = spec;
}
public struct DefinitionEntry : IMessageProducer
{
public DefinitionEntry(string name, Color pixelValue)
{
labelName = name;
this.pixelValue = pixelValue;
}
public string labelName;
public Color pixelValue;
public void ToMessage(IMessageBuilder builder)
{
builder.AddString("label_name", labelName);
builder.AddIntVector("pixel_value", Utils.ToIntVector(pixelValue));
}
}
}
SemanticSegmentationDefinition m_AnnotationDefinition;
[Serializable]
public class SemanticSegmentation : Annotation
get => "Generates a semantic segmentation image for each captured frame. Each object is rendered to the semantic segmentation image using the color associated with it based on this labeler's associated semantic segmentation label configuration. " +
"Semantic segmentation images are saved to the dataset in PNG format. " +
"Please note that only one " + GetType().Name + " can render at once across all cameras.";
protected set {}
public IEnumerable<SemanticSegmentationDefinition.DefinitionEntry> instances;
public string imageFormat;
public Vector2 dimension;
public byte[] buffer;
public override void ToMessage(IMessageBuilder builder)
{
base.ToMessage(builder);
builder.AddString("image_format", imageFormat);
builder.AddFloatVector("dimension", new[] { dimension.x, dimension.y });
builder.AddPngImage("semantic_segmentation", buffer);
var nested = builder.AddNestedMessage("instances");
foreach (var i in instances)
{
i.ToMessage(nested);
}
}
const string k_SemanticSegmentationDirectory = "SemanticSegmentation";
const string k_SegmentationFilePrefix = "segmentation_";
internal string semanticSegmentationDirectory;
public static string annotationId = "semantic segmentation";
/// <summary>
/// The id to associate with semantic segmentation annotations in the dataset.
/// </summary>
[Tooltip("The id to associate with semantic segmentation annotations in the dataset.")]
public string annotationId = "12f94d8d-5425-4deb-9b21-5e53ad957d66";
/// <summary>
/// The SemanticSegmentationLabelConfig which maps labels to pixel values.
/// </summary>

m_TargetTextureOverride = targetTextureOverride;
}
[SuppressMessage("ReSharper", "InconsistentNaming")]
struct SemanticSegmentationSpec
{
[UsedImplicitly]
public string label_name;
[UsedImplicitly]
public Color pixel_value;
}
public AsyncAnnotationFuture future;
public string path;
}
public override string description
{
get => string.Empty;
protected set { }
}
/// <inheritdoc/>

targetTexture.Create();
targetTexture.name = "Labeling";
semanticSegmentationDirectory = k_SemanticSegmentationDirectory + Guid.NewGuid();
#if HDRP_PRESENT
var gameObject = perceptionCamera.gameObject;

m_LensDistortionPass = new LensDistortionUrpPass(myCamera, targetTexture);
perceptionCamera.AddScriptableRenderPass(m_LensDistortionPass);
#endif
var specs = labelConfig.labelEntries.Select((l) => new SemanticSegmentationSpec()
var specs = labelConfig.labelEntries.Select(l => new SemanticSegmentationDefinition.DefinitionEntry
label_name = l.label,
pixel_value = l.color
labelName = l.label,
pixelValue = l.color
specs = specs.Append(new SemanticSegmentationSpec()
specs = specs.Append(new SemanticSegmentationDefinition.DefinitionEntry
label_name = "sky",
pixel_value = labelConfig.skyColor
labelName = "sky",
pixelValue = labelConfig.skyColor
#if false
m_SemanticSegmentationAnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition(
"semantic segmentation",
specs.ToArray(),
"pixel-wise semantic segmentation label",
"PNG",
id: Guid.Parse(annotationId));
#else
// m_SemanticSegmentationAnnotationDefinition = new AnnotationDefinition();
#endif
m_AnnotationDefinition = new SemanticSegmentationDefinition(specs);
DatasetCapture.RegisterAnnotationDefinition(m_AnnotationDefinition);
m_SemanticSegmentationTextureReader = new RenderTextureReader<Color32>(targetTexture);
visualizationEnabled = supportsVisualization;
}

if (!m_AsyncAnnotations.TryGetValue(frameCount, out var annotation))
if (!m_AsyncAnnotations.TryGetValue(frameCount, out var future))
var info = new SegmentationValue[]
{
new SegmentationValue
{
frame = frameCount
}
};
var datasetRelativePath = $"{semanticSegmentationDirectory}/{k_SegmentationFilePrefix}{frameCount}.png";
var localPath = $"{Manager.Instance.GetDirectoryFor(semanticSegmentationDirectory)}/{k_SegmentationFilePrefix}{frameCount}.png";
// annotation.ReportFileAndValues(datasetRelativePath, info);
m_AsyncAnnotations.Remove(frameCount);
var asyncRequest = Manager.Instance.CreateRequest<AsyncRequest<AsyncSemanticSegmentationWrite>>();

});
asyncRequest.data = new AsyncSemanticSegmentationWrite
{
future = future,
path = localPath
};
asyncRequest.Enqueue((r) =>
{

Profiler.BeginSample("WritePng");
File.WriteAllBytes(r.data.path, pngBytes);
Manager.Instance.ConsumerFileProduced(r.data.path);
Profiler.EndSample();
var toReport = new SemanticSegmentation
{
instances = m_AnnotationDefinition.spec,
sensorId = perceptionCamera.ID,
Id = m_AnnotationDefinition.id,
annotationType = m_AnnotationDefinition.annotationType,
description = m_AnnotationDefinition.description,
imageFormat = "png",
dimension = new Vector2(r.data.width, r.data.height),
buffer = pngBytes
};
r.data.future.Report(toReport);
r.data.data.Dispose();
return AsyncRequest.Result.Completed;
});

/// <inheritdoc/>
protected override void OnEndRendering(ScriptableRenderContext scriptableRenderContext)
{
#if false
m_AsyncAnnotations[Time.frameCount] = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_SemanticSegmentationAnnotationDefinition);
m_AsyncAnnotations[Time.frameCount] = perceptionCamera.SensorHandle.ReportAnnotationAsync(m_AnnotationDefinition);
#endif
}
/// <inheritdoc/>

13
com.unity.perception/Runtime/GroundTruth/SimulationState.cs
查看文件


}
case FutureType.Metric:
{
#if true
if (!(asyncFuture.GetId() is SPendingCaptureId captureId))
{
return false;
}
var b = Metrics.ContainsKey(captureId);
if (!b) return false;
var c = Metrics[captureId];
return c == null;
#else
#endif
}
default:
throw new ArgumentOutOfRangeException();

撰写 预览
正在加载...
取消
保存