浏览代码

Adding sky color control to semantic segmentation (#274)

* Adding background color for semantic segmentation and bad UI. Need to come back to this later.

* Updating UI

* Changing 'background' to 'sky' and improving tests.

* some UI polish

* Updating label config and PerceptionURP

Co-authored-by: Mohsen Kamalzadeh <>
/main
GitHub 4 年前
当前提交
1464ac81
共有 11 个文件被更改,包括 76 次插入11 次删除
  1. 1
      TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset
  2. 1
      TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset
  3. 4
      TestProjects/PerceptionURP/Packages/packages-lock.json
  4. 2
      com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs
  5. 7
      com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs
  6. 10
      com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs
  7. 2
      com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml
  8. 14
      com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml
  9. 6
      com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs
  10. 2
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs
  11. 38
      com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs

1
TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset


color: {r: 0, g: 1, b: 0.16973758, a: 1}
- label: Terrain
color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
skyColor: {r: 0, g: 0, b: 0, a: 1}

1
TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset


color: {r: 0, g: 1, b: 0.16973758, a: 1}
- label: Terrain
color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
skyColor: {r: 0, g: 0, b: 0, a: 1}

4
TestProjects/PerceptionURP/Packages/packages-lock.json


"com.unity.collections": "0.9.0-preview.6",
"com.unity.nuget.newtonsoft-json": "1.1.2",
"com.unity.render-pipelines.core": "7.1.6",
"com.unity.simulation.capture": "0.0.10-preview.19",
"com.unity.simulation.capture": "0.0.10-preview.20",
"com.unity.simulation.client": "0.0.10-preview.10",
"com.unity.simulation.core": "0.0.10-preview.22"
}

"url": "https://packages.unity.com"
},
"com.unity.simulation.capture": {
"version": "0.0.10-preview.19",
"version": "0.0.10-preview.20",
"depth": 1,
"source": "registry",
"dependencies": {

2
com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs


m_StartingIdEnumField.SetEnabled(AutoAssign);
m_SkyColorUi.style.display = DisplayStyle.None;
AutoAssignIdsIfNeeded();
m_MoveDownButton.clicked += MoveSelectedItemDown;
m_MoveUpButton.clicked += MoveSelectedItemUp;

7
com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs


protected Button m_MoveDownButton;
protected VisualElement m_MoveButtons;
protected VisualElement m_IdSpecificUi;
protected VisualElement m_SkyColorUi;
protected ColorField m_SkyColorField;
protected Label m_SkyHexLabel;
public void OnEnable()

m_StartingIdEnumField = m_Root.Q<EnumField>("starting-id-dropdown");
m_AutoIdToggle = m_Root.Q<Toggle>("auto-id-toggle");
m_IdSpecificUi = m_Root.Q<VisualElement>("id-specific-ui");
m_SkyColorUi = m_Root.Q<VisualElement>("sky-color-ui");
m_SkyColorField = m_Root.Q<ColorField>("sky-color-value");
m_SkyHexLabel = m_Root.Q<Label>("sky-color-hex");
m_SaveButton.SetEnabled(false);

10
com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs


{
m_MoveButtons.style.display = DisplayStyle.None;
m_IdSpecificUi.style.display = DisplayStyle.None;
var skyColorProperty = serializedObject.FindProperty(nameof(SemanticSegmentationLabelConfig.skyColor));
m_SkyColorField.BindProperty(skyColorProperty);
m_SkyColorField.RegisterValueChangedCallback(e => UpdateSkyHexLabel(e.newValue));
UpdateSkyHexLabel(skyColorProperty.colorValue);
}
private void UpdateSkyHexLabel(Color colorValue)
{
m_SkyHexLabel.text = "#" + ColorUtility.ToHtmlStringRGBA(colorValue);
}
public override void PostRemoveOperations()

2
com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml


<UXML xmlns="UnityEngine.UIElements" xmlns:editor="UnityEditor.UIElements">
<VisualElement class="added-label" style="padding-top: 3px;">
<Button name="remove-button" class="labeling__remove-item-button"/>
<TextField name="label-value" class="labeling__added-label-value"/>
<TextField name="label-value" class="labeling__added-label-value"/>
<VisualElement style="min-width:20px; flex-direction: row; display:none">
<Button name="move-up-button" class="move-label-in-config-button move-up" style="margin-right:-2px"/>
<Button name="move-down-button" class="move-label-in-config-button move-down"/>

14
com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml


<VisualElement class="outer-container" name="outer-container">
<Style src="../Uss/Styles.uss"/>
<VisualElement class="inner-container" name="id-specific-ui">
<Label text="Options" name="options-title" class="title-label"/>
</VisualElement>
</VisualElement>
<VisualElement class="inner-container" name="sky-color-ui">
<Label text="Options" name="options-title" class="title-label"/>
<VisualElement style="flex-direction: row; flex-grow: 1;">
<Label text="Sky Color" name="added-labels-title"
style="flex-grow: 1; font-size: 11; min-width : 60px; align-self:center;"/>
<VisualElement class="generic-hover"
style="min-width : 137px; flex-direction: row; padding: 3px 5px 3px 5px; margin-left: 3px; margin-right: 3px; border-width: 1px; border-color: #666666; border-radius: 4px;">
<editor:ColorField name="sky-color-value" style="min-width : 60px; max-width: 60px; align-self:center;"/>
<Label name="sky-color-hex"
style="font-size: 11; min-width : 60px; max-width: 60px; align-self:center; margin: 2px"/>
</VisualElement>
</VisualElement>
</VisualElement>
<VisualElement name="added-labels" class="inner-container" style="margin-top:5px">

6
com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs


Color.yellow,
Color.gray
};
/// <summary>
/// The color to use for the sky in semantic segmentation images
/// </summary>
public Color skyColor = Color.black;
}
/// <summary>

2
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs


s_LastFrameExecuted = Time.frameCount;
var renderList = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, -1);
cmd.ClearRenderTarget(true, true, Color.black);
cmd.ClearRenderTarget(true, true, m_LabelConfig.skyColor);
DrawRendererList(renderContext, cmd, RendererList.Create(renderList));
}

38
com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs


{
static readonly Color32 k_SemanticPixelValue = new Color32(10, 20, 30, Byte.MaxValue);
private static readonly Color32 k_InstanceSegmentationPixelValue = new Color32(255,0,0, 255);
private static readonly Color32 k_SkyValue = new Color32(10, 20, 30, 40);
public enum SegmentationKind
{

switch (segmentationKind)
{
case SegmentationKind.Instance:
//expectedPixelValue = new Color32(0, 74, 255, 255);
expectedPixelValue = k_InstanceSegmentationPixelValue;
cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived);
break;

CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
}
var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(label: "non-matching"));
yield return null;

CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
}
var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
var gameObject = TestHelper.CreateLabeledPlane();
gameObject.GetComponent<Labeling>().enabled = false;

}
[UnityTest]
public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesBlack([Values(false, true)] bool showVisualizations)
public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesSky([Values(false, true)] bool showVisualizations)
var expectedPixelValue = new Color32(0, 0, 0, 255);
var expectedPixelValue = k_SkyValue;
void OnSegmentationImageReceived(NativeArray<Color32> data)
{
timesSegmentationImageReceived++;

var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations);
var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations, expectedPixelValue);
//TestHelper.LoadAndStartRenderDocCapture(out var gameView);
yield return null;

}
[UnityTest]
public IEnumerator SemanticSegmentationPass_WithNoObjects_ProducesSky()
{
int timesSegmentationImageReceived = 0;
var expectedPixelValue = k_SkyValue;
void OnSegmentationImageReceived(NativeArray<Color32> data)
{
timesSegmentationImageReceived++;
CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
}
var cameraObject = SetupCameraSemanticSegmentation(
a => OnSegmentationImageReceived(a.data), false, expectedPixelValue);
yield return null;
//destroy the object to force all pending segmented image readbacks to finish and events to be fired.
DestroyTestObject(cameraObject);
Assert.AreEqual(1, timesSegmentationImageReceived);
}
[UnityTest]
public IEnumerator SemanticSegmentationPass_WithTextureOverride_RendersToOverride([Values(true, false)] bool showVisualizations)
{
var expectedPixelValue = new Color32(0, 0, 255, 255);

return cameraObject;
}
GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations)
GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations, Color? backgroundColor = null)
{
var cameraObject = SetupCamera(out var perceptionCamera, showVisualizations);
var labelConfig = ScriptableObject.CreateInstance<SemanticSegmentationLabelConfig>();

color = k_SemanticPixelValue
}
});
if (backgroundColor != null)
{
labelConfig.skyColor = backgroundColor.Value;
}
var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
semanticSegmentationLabeler.imageReadback += onSegmentationImageReceived;
perceptionCamera.AddLabeler(semanticSegmentationLabeler);

正在加载...
取消
保存