浏览代码

addressed feedback

- new test for keypoint labeler
- added ClearMaterialProperties method to IGroundTruthGenerator
- Other fixes
/0.8.0-preview.1_staging
Mohsen Kamalzadeh 4 年前
当前提交
6bd2ce6a
共有 10 个文件被更改,包括 231 次插入113 次删除
  1. 19
      com.unity.perception/Runtime/GroundTruth/IGroundTruthGenerator.cs
  2. 5
      com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs
  3. 135
      com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs
  4. 30
      com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs
  5. 33
      com.unity.perception/Runtime/GroundTruth/Labeling/LabelManager.cs
  6. 3
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/GroundTruthCrossPipelinePass.cs
  7. 10
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/InstanceSegmentationCrossPipelinePass.cs
  8. 5
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LensDistortionCrossPipelinePass.cs
  9. 23
      com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs
  10. 81
      com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

19
com.unity.perception/Runtime/GroundTruth/IGroundTruthGenerator.cs


public interface IGroundTruthGenerator
{
/// <summary>
/// Called by <see cref="LabelManager"/> when first registered or when a Labeling is created at runtime.
/// Called by <see cref="LabelManager"/> when a <see cref="Labeling"/> component is first registered, created at runtime, or enabled at runtime.
/// <param name="mpb">The MaterialPropertyBlock for the given meshRenderer. Can be used to set properties for custom rendering.</param>
/// <param name="renderer">The Renderer under the given Labeling.</param>
/// <param name="labeling">The Labeling component created</param>
/// <param name="instanceId">The instanceId assigned to the given Labeling instance.</param>
/// <param name="mpb">The MaterialPropertyBlock for the given <see cref="MeshRenderer"/>. Can be used to set properties for custom rendering.</param>
/// <param name="renderer">The <see cref="Renderer"/> under the given <see cref="LabelManager"/>.</param>
/// <param name="labeling">The <see cref="LabelManager"/> component that was registered, created, or enabled</param>
/// <param name="instanceId">The instanceId assigned to the given <see cref="LabelManager"/> instance.</param>
/// <summary>
/// Called by <see cref="LabelManager"/> when a <see cref="Labeling"/> component is disabled, causing it to not be included in ground-truth generation.
/// </summary>
/// <param name="mpb">The MaterialPropertyBlock for the given <see cref="MeshRenderer"/>. Can be used to set properties for custom rendering.</param>
/// <param name="renderer">The <see cref="Renderer"/> under the given <see cref="LabelManager"/>.</param>
/// <param name="labeling">The <see cref="LabelManager"/> component for which ground-truth generation should stop.</param>
/// <param name="instanceId">The instanceId assigned to the given <see cref="LabelManager"/> instance.</param>
void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId);
}
}

5
com.unity.perception/Runtime/GroundTruth/InstanceIdToColorMapping.cs


const uint k_HslCount = 64;
const uint k_ColorsPerAlpha = 256 * 256 * 256;
const uint k_InvalidPackedColor = 255; // packed uint for color (0, 0, 0, 255);
public static readonly Color32 invalidColor = new Color(0, 0, 0, 255);
/// <summary>
/// The color returned when an instanceId is not mapped to any color, and for clearing ground truth material properties on a <see cref="MaterialPropertyBlock"/>.
/// </summary>
public static readonly Color32 invalidColor = new Color(0, 0, 0, 255);
static void InitializeMaps()
{

135
com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs


void ProcessLabel(Labeling labeledEntity)
{
if (idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
if (!idLabelConfig.TryGetLabelEntryFromInstanceId(labeledEntity.instanceId, out var labelEntry))
return;
// Cache out the data of a labeled game object the first time we see it, this will
// save performance each frame. Also checks to see if a labeled game object can be annotated.
if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId))
// Cache out the data of a labeled game object the first time we see it, this will
// save performance each frame. Also checks to see if a labeled game object can be annotated.
if (!m_KnownStatus.ContainsKey(labeledEntity.instanceId))
var cached = new CachedData()
var cached = new CachedData()
{
status = false,
animator = null,
keypoints = new KeypointEntry(),
overrides = new List<(JointLabel, int)>()
};
status = false,
animator = null,
keypoints = new KeypointEntry(),
overrides = new List<(JointLabel, int)>()
};
var entityGameObject = labeledEntity.gameObject;
var entityGameObject = labeledEntity.gameObject;
cached.keypoints.instance_id = labeledEntity.instanceId;
cached.keypoints.label_id = labelEntry.id;
cached.keypoints.template_guid = activeTemplate.templateID;
cached.keypoints.instance_id = labeledEntity.instanceId;
cached.keypoints.label_id = labelEntry.id;
cached.keypoints.template_guid = activeTemplate.templateID;
cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];
for (var i = 0; i < cached.keypoints.keypoints.Length; i++)
{
cached.keypoints.keypoints[i] = new Keypoint { index = i, state = 0 };
}
cached.keypoints.keypoints = new Keypoint[activeTemplate.keypoints.Length];
for (var i = 0; i < cached.keypoints.keypoints.Length; i++)
{
cached.keypoints.keypoints[i] = new Keypoint { index = i, state = 0 };
}
var animator = entityGameObject.transform.GetComponentInChildren<Animator>();
if (animator != null)
{
cached.animator = animator;
cached.status = true;
}
var animator = entityGameObject.transform.GetComponentInChildren<Animator>();
if (animator != null)
foreach (var joint in entityGameObject.transform.GetComponentsInChildren<JointLabel>())
{
if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx))
cached.animator = animator;
cached.overrides.Add((joint, idx));
}
foreach (var joint in entityGameObject.transform.GetComponentsInChildren<JointLabel>())
{
if (TryToGetTemplateIndexForJoint(activeTemplate, joint, out var idx))
{
cached.overrides.Add((joint, idx));
cached.status = true;
}
}
m_KnownStatus[labeledEntity.instanceId] = cached;
}
m_KnownStatus[labeledEntity.instanceId] = cached;
}
var cachedData = m_KnownStatus[labeledEntity.instanceId];
var cachedData = m_KnownStatus[labeledEntity.instanceId];
if (cachedData.status)
{
var animator = cachedData.animator;
var keypoints = cachedData.keypoints.keypoints;
if (cachedData.status)
// Go through all of the rig keypoints and get their location
for (var i = 0; i < activeTemplate.keypoints.Length; i++)
var animator = cachedData.animator;
var keypoints = cachedData.keypoints.keypoints;
// Go through all of the rig keypoints and get their location
for (var i = 0; i < activeTemplate.keypoints.Length; i++)
var pt = activeTemplate.keypoints[i];
if (pt.associateToRig)
var pt = activeTemplate.keypoints[i];
if (pt.associateToRig)
var bone = animator.GetBoneTransform(pt.rigLabel);
if (bone != null)
var bone = animator.GetBoneTransform(pt.rigLabel);
if (bone != null)
{
var loc = ConvertToScreenSpace(bone.position);
keypoints[i].index = i;
keypoints[i].x = loc.x;
keypoints[i].y = loc.y;
keypoints[i].state = 2;
}
var loc = ConvertToScreenSpace(bone.position);
keypoints[i].index = i;
keypoints[i].x = loc.x;
keypoints[i].y = loc.y;
keypoints[i].state = 2;
}
// Go through all of the additional or override points defined by joint labels and get
// their locations
foreach (var (joint, idx) in cachedData.overrides)
{
var loc = ConvertToScreenSpace(joint.transform.position);
keypoints[idx].index = idx;
keypoints[idx].x = loc.x;
keypoints[idx].y = loc.y;
keypoints[idx].state = 2;
}
// Go through all of the additional or override points defined by joint labels and get
// their locations
foreach (var (joint, idx) in cachedData.overrides)
{
var loc = ConvertToScreenSpace(joint.transform.position);
keypoints[idx].index = idx;
keypoints[idx].x = loc.x;
keypoints[idx].y = loc.y;
keypoints[idx].state = 2;
}
cachedData.keypoints.pose = "unset";
if (cachedData.animator != null)
{
cachedData.keypoints.pose = GetPose(cachedData.animator);
}
cachedData.keypoints.pose = "unset";
m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = cachedData.keypoints;
if (cachedData.animator != null)
{
cachedData.keypoints.pose = GetPose(cachedData.animator);
m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = cachedData.keypoints;
}
}

30
com.unity.perception/Runtime/GroundTruth/Labeling/LabelEntryMatchCache.cs


if (m_IdLabelConfig.TryGetMatchingConfigurationEntry(labeling, out _, out var index))
{
Debug.Assert(index < k_DefaultValue, "Too many entries in the label config");
if (labeling.enabled)
{
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
{
var oldLength = m_InstanceIdToLabelEntryIndexLookup.Length;
m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
for (var i = oldLength; i < instanceId; i++)
m_InstanceIdToLabelEntryIndexLookup[i] = k_DefaultValue;
}
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = (ushort)index;
}
else if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
if (m_InstanceIdToLabelEntryIndexLookup.Length <= instanceId)
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = k_DefaultValue;
var oldLength = m_InstanceIdToLabelEntryIndexLookup.Length;
m_InstanceIdToLabelEntryIndexLookup.Resize((int)instanceId + 1, NativeArrayOptions.ClearMemory);
for (var i = oldLength; i < instanceId; i++)
m_InstanceIdToLabelEntryIndexLookup[i] = k_DefaultValue;
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = (ushort)index;
}
else if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
{

/// <inheritdoc/>
void IGroundTruthGenerator.ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
if (m_InstanceIdToLabelEntryIndexLookup.Length > instanceId)
{
m_InstanceIdToLabelEntryIndexLookup[(int)instanceId] = k_DefaultValue;
}
}
/// <inheritdoc/>
public void Dispose()

33
com.unity.perception/Runtime/GroundTruth/Labeling/LabelManager.cs


{
terrain.GetSplatMaterialPropertyBlock(mpb);
foreach (var pass in m_ActiveGenerators)
pass.SetupMaterialProperties(mpb, null, labeling, instanceId);
{
if (labeling.enabled)
{
pass.SetupMaterialProperties(mpb, null, labeling, instanceId);
}
else
{
pass.ClearMaterialProperties(mpb, null, labeling, instanceId);
}
}
terrain.SetSplatMaterialPropertyBlock(mpb);
}

{
renderer.GetPropertyBlock(mpb);
foreach (var pass in m_ActiveGenerators)
pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
{
if (labeling.enabled)
{
pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
}
else
{
pass.ClearMaterialProperties(mpb, renderer, labeling, instanceId);
}
}
renderer.SetPropertyBlock(mpb);

if (!mpb.isEmpty)
{
foreach (var pass in m_ActiveGenerators)
pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
{
if (labeling.enabled)
{
pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
}
else
{
pass.ClearMaterialProperties(mpb, renderer, labeling, instanceId);
}
}
renderer.SetPropertyBlock(mpb, i);
}

3
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/GroundTruthCrossPipelinePass.cs


public abstract void SetupMaterialProperties(
MaterialPropertyBlock mpb, Renderer meshRenderer, Labeling labeling, uint instanceId);
public abstract void ClearMaterialProperties(
MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId);
}
}

10
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/InstanceSegmentationCrossPipelinePass.cs


if (!found)
Debug.LogError($"Could not get a unique color for {instanceId}");
if (labeling.enabled)
mpb.SetVector(k_SegmentationIdProperty, (Color)color);
else
mpb.SetVector(k_SegmentationIdProperty, (Color) InstanceIdToColorMapping.invalidColor);
mpb.SetVector(k_SegmentationIdProperty, (Color)color);
}
public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
mpb.SetVector(k_SegmentationIdProperty, (Color) InstanceIdToColorMapping.invalidColor);
}
}
}

5
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/LensDistortionCrossPipelinePass.cs


{
SetLensDistortionShaderParameters();
}
public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
SetLensDistortionShaderParameters();
}
}
}

23
com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs


{
var entry = new SemanticSegmentationLabelEntry();
var found = false;
if (labeling.enabled)
foreach (var l in m_LabelConfig.labelEntries)
foreach (var l in m_LabelConfig.labelEntries)
if (labeling.labels.Contains(l.label))
if (labeling.labels.Contains(l.label))
{
entry = l;
found = true;
break;
}
entry = l;
found = true;
break;
if (found)
mpb.SetVector(k_LabelingId, entry.color);
else
mpb.SetVector(k_LabelingId, Color.black);
mpb.SetVector(k_LabelingId, found ? entry.color : Color.black);
}
public override void ClearMaterialProperties(MaterialPropertyBlock mpb, Renderer renderer, Labeling labeling, uint instanceId)
{
mpb.SetVector(k_LabelingId, Color.black);
}
}
}

81
com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs


texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

}
[UnityTest]
public IEnumerator Keypoint_TestStaticLabeledCube_WithDisabledLabeling_AndSwitchingLabelingState()
{
var incoming = new List<List<KeypointLabeler.KeypointEntry>>();
var template = CreateTestTemplate(Guid.NewGuid(), "TestTemplate");
var texture = new RenderTexture(1024, 768, 16);
texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
SetupCubeJoints(cube, template);
var labeling = cube.GetComponent<Labeling>();
cube.SetActive(true);
cam.SetActive(true);
AddTestObjectForCleanup(cam);
AddTestObjectForCleanup(cube);
labeling.enabled = false;
yield return null;
labeling.enabled = true;
yield return null;
labeling.enabled = false;
yield return null;
//force all async readbacks to complete
DestroyTestObject(cam);
texture.Release();
var testCase = incoming[0];
Assert.AreEqual(0, testCase.Count);
testCase = incoming[1];
Assert.AreEqual(1, testCase.Count);
var t = testCase.First();
Assert.NotNull(t);
Assert.AreEqual(1, t.instance_id);
Assert.AreEqual(1, t.label_id);
Assert.AreEqual(template.templateID.ToString(), t.template_guid);
Assert.AreEqual(9, t.keypoints.Length);
Assert.AreEqual(t.keypoints[0].x, t.keypoints[1].x);
Assert.AreEqual(t.keypoints[2].x, t.keypoints[3].x);
Assert.AreEqual(t.keypoints[4].x, t.keypoints[5].x);
Assert.AreEqual(t.keypoints[6].x, t.keypoints[7].x);
Assert.AreEqual(t.keypoints[0].y, t.keypoints[3].y);
Assert.AreEqual(t.keypoints[1].y, t.keypoints[2].y);
Assert.AreEqual(t.keypoints[4].y, t.keypoints[7].y);
Assert.AreEqual(t.keypoints[5].y, t.keypoints[6].y);
for (var i = 0; i < 9; i++) Assert.AreEqual(i, t.keypoints[i].index);
for (var i = 0; i < 8; i++) Assert.AreEqual(2, t.keypoints[i].state);
Assert.Zero(t.keypoints[8].state);
Assert.Zero(t.keypoints[8].x);
Assert.Zero(t.keypoints[8].y);
testCase = incoming[2];
Assert.AreEqual(0, testCase.Count);
}
[UnityTest]
public IEnumerator Keypoint_TestAllOffScreen()
{
var incoming = new List<List<KeypointLabeler.KeypointEntry>>();

{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
});
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

var texture = new RenderTexture(1024, 768, 16);
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);

texture.Create();
var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
{
incoming.Add(data);
incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
}, texture);
var cameraComponent = cam.GetComponent<Camera>();

正在加载...
取消
保存