Merge pull request #51 from unity/cpu-image

Update sample for "cpu image" API changes

.gitignore

 *.booproj
 *.svd
 *.pdb
+*.idea
 
 # Unity3D generated meta files
 *.pidb.meta

Assets/Scenes/CpuImages.unity

   m_ReflectionIntensity: 1
   m_CustomReflection: {fileID: 0}
   m_Sun: {fileID: 0}
-  m_IndirectSpecularColor: {r: 0.44657874, g: 0.49641275, b: 0.5748172, a: 1}
+  m_IndirectSpecularColor: {r: 0.44657898, g: 0.49641287, b: 0.5748173, a: 1}
   m_UseRadianceAmbientProbe: 0
 --- !u!157 &3
 LightmapSettings:
   m_Script: {fileID: 11500000, guid: fa850fbd5b8aded44846f96e35f1a9f5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+--- !u!1 &456510980
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 456510981}
+  - component: {fileID: 456510983}
+  - component: {fileID: 456510982}
+  m_Layer: 5
+  m_Name: RawHumanStencilImage
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &456510981
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 456510980}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 485666366}
+  m_RootOrder: 2
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMax: {x: 0, y: 1}
+  m_AnchoredPosition: {x: 640, y: -480}
+  m_SizeDelta: {x: 640, y: 480}
+  m_Pivot: {x: 0, y: 1}
+--- !u!114 &456510982
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 456510980}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 1, g: 1, b: 1, a: 1}
+  m_RaycastTarget: 1
+  m_Maskable: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+  m_Texture: {fileID: 0}
+  m_UVRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+--- !u!222 &456510983
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 456510980}
+  m_CullTransparentMesh: 0
 --- !u!1 &485666362
 GameObject:
   m_ObjectHideFlags: 0
   m_LocalScale: {x: 0, y: 0, z: 0}
   m_Children:
   - {fileID: 1474869892}
+  - {fileID: 1218579280}
+  - {fileID: 456510981}
   - {fileID: 901906344}
   - {fileID: 642201223}
   - {fileID: 1756864493}
   - {fileID: 1791068495}
   - {fileID: 773643825}
   m_Father: {fileID: 485666366}
-  m_RootOrder: 2
+  m_RootOrder: 4
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0, y: 0}
   m_AnchorMax: {x: 0, y: 0}
   m_LocalScale: {x: 1, y: 1, z: 1}
   m_Children: []
   m_Father: {fileID: 485666366}
-  m_RootOrder: 1
+  m_RootOrder: 3
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 1, y: 1}
   m_AnchorMax: {x: 1, y: 1}
     - target: {fileID: 2304810881573073946, guid: 4ee22c1f54b174122b6a766fe09e0de8,
         type: 3}
       propertyPath: m_RootOrder
-      value: 3
+      value: 5
       objectReference: {fileID: 0}
     - target: {fileID: 2304810881573073946, guid: 4ee22c1f54b174122b6a766fe09e0de8,
         type: 3}
   - component: {fileID: 1141330383}
   - component: {fileID: 1141330382}
   m_Layer: 0
-  m_Name: CameraImage
+  m_Name: CpuImages
   m_TagString: Untagged
   m_Icon: {fileID: 0}
   m_NavMeshLayer: 0
   m_Name: 
   m_EditorClassIdentifier: 
   m_CameraManager: {fileID: 282840816}
-  m_RawImage: {fileID: 1474869893}
+  m_RawCameraImage: {fileID: 1474869893}
+  m_OcclusionManager: {fileID: 1335839536}
+  m_RawHumanDepthImage: {fileID: 1218579281}
+  m_RawHumanStencilImage: {fileID: 456510982}
   m_ImageInfo: {fileID: 901906345}
 --- !u!4 &1141330383
 Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 5
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &1218579279
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 1218579280}
+  - component: {fileID: 1218579282}
+  - component: {fileID: 1218579281}
+  m_Layer: 5
+  m_Name: RawHumanDepthImage
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &1218579280
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1218579279}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 485666366}
+  m_RootOrder: 1
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMax: {x: 0, y: 1}
+  m_AnchoredPosition: {x: 0, y: -480}
+  m_SizeDelta: {x: 640, y: 480}
+  m_Pivot: {x: 0, y: 1}
+--- !u!114 &1218579281
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1218579279}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 1, g: 1, b: 1, a: 1}
+  m_RaycastTarget: 1
+  m_Maskable: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+  m_Texture: {fileID: 0}
+  m_UVRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+--- !u!222 &1218579282
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1218579279}
+  m_CullTransparentMesh: 0
 --- !u!1 &1248549346
 GameObject:
   m_ObjectHideFlags: 0
   m_Component:
   - component: {fileID: 1335839535}
   - component: {fileID: 1335839534}
+  - component: {fileID: 1335839536}
   m_Layer: 0
   m_Name: AR Session Origin
   m_TagString: Untagged
   m_Father: {fileID: 0}
   m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &1335839536
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1335839533}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: b15f82cc229284894964d2d30806969d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_HumanSegmentationStencilMode: 1
+  m_HumanSegmentationDepthMode: 1
 --- !u!1 &1449726330
 GameObject:
   m_ObjectHideFlags: 0
   - component: {fileID: 1474869894}
   - component: {fileID: 1474869893}
   m_Layer: 5
-  m_Name: RawImage
+  m_Name: RawCameraImage
   m_TagString: Untagged
   m_Icon: {fileID: 0}
   m_NavMeshLayer: 0

Assets/Scripts/CpuImageSample.cs

     [Tooltip("The ARCameraManager which will produce frame events.")]
     ARCameraManager m_CameraManager;
 
-     /// <summary>
+    /// <summary>
-        get { return m_CameraManager; }
-        set { m_CameraManager = value; }
+        get => m_CameraManager;
+        set => m_CameraManager = value;
-    RawImage m_RawImage;
+    RawImage m_RawCameraImage;
-    public RawImage rawImage
+    public RawImage rawCameraImage
-        get { return m_RawImage; }
-        set { m_RawImage = value; }
+        get => m_RawCameraImage;
+        set => m_RawCameraImage = value;
+     [SerializeField]
+     [Tooltip("The AROcclusionManager which will produce human depth and stencil textures.")]
+     AROcclusionManager m_OcclusionManager;
+
+     public AROcclusionManager occlusionManager
+     {
+         get => m_OcclusionManager;
+         set => m_OcclusionManager = value;
+     }
+
+     [SerializeField]
+     RawImage m_RawHumanDepthImage;
+
+     /// <summary>
+     /// The UI RawImage used to display the image on screen.
+     /// </summary>
+     public RawImage rawHumanDepthImage
+     {
+         get => m_RawHumanDepthImage;
+         set => m_RawHumanDepthImage = value;
+     }
+
+     [SerializeField]
+     RawImage m_RawHumanStencilImage;
+
+     /// <summary>
+     /// The UI RawImage used to display the image on screen.
+     /// </summary>
+     public RawImage rawHumanStencilImage
+     {
+         get => m_RawHumanStencilImage;
+         set => m_RawHumanStencilImage = value;
+     }
+
     [SerializeField]
     Text m_ImageInfo;
 
     public Text imageInfo
     {
-        get { return m_ImageInfo; }
-        set { m_ImageInfo = value; }
+        get => m_ImageInfo;
+        set => m_ImageInfo = value;
     }
 
     void OnEnable()
         }
     }
 
-    unsafe void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
+    unsafe void UpdateCameraImage()
-        XRCameraImage image;
-        if (!cameraManager.TryGetLatestImage(out image))
+        if (!cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
         {
             return;
         }
             "Image info:\n\twidth: {0}\n\theight: {1}\n\tplaneCount: {2}\n\ttimestamp: {3}\n\tformat: {4}",
             image.width, image.height, image.planeCount, image.timestamp, image.format);
 
-        // Once we have a valid XRCameraImage, we can access the individual image "planes"
-        // (the separate channels in the image). XRCameraImage.GetPlane provides
+        // Once we have a valid XRCpuImage, we can access the individual image "planes"
+        // (the separate channels in the image). XRCpuImage.GetPlane provides
-        // See XRCameraImage.FormatSupported for a complete list of supported formats.
+        // See XRCpuImage.FormatSupported for a complete list of supported formats.
-        if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
+        if (m_CameraTexture == null || m_CameraTexture.width != image.width || m_CameraTexture.height != image.height)
-            m_Texture = new Texture2D(image.width, image.height, format, false);
+            m_CameraTexture = new Texture2D(image.width, image.height, format, false);
-        var conversionParams = new XRCameraImageConversionParams(image, format, CameraImageTransformation.MirrorY);
+        var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorY);
-        var rawTextureData = m_Texture.GetRawTextureData<byte>();
+        var rawTextureData = m_CameraTexture.GetRawTextureData<byte>();
         try
         {
             image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
-            // We must dispose of the XRCameraImage after we're finished
+            // We must dispose of the XRCpuImage after we're finished
-        m_Texture.Apply();
+        m_CameraTexture.Apply();
-        m_RawImage.texture = m_Texture;
+        m_RawCameraImage.texture = m_CameraTexture;
+    }
+
+    void UpdateHumanDepthImage()
+    {
+        if (m_RawHumanDepthImage == null)
+            return;
+
+        // Attempt to get the latest human depth image. If this method succeeds,
+        // it acquires a native resource that must be disposed (see below).
+        if (occlusionManager && occlusionManager.TryAcquireHumanDepthCpuImage(out XRCpuImage image))
+        {
+            using (image)
+            {
+                UpdateRawImage(m_RawHumanDepthImage, image);
+            }
+        }
+        else
+        {
+            m_RawHumanDepthImage.enabled = false;
+        }
-    Texture2D m_Texture;
+    void UpdateHumanStencilImage()
+    {
+        if (m_RawHumanStencilImage == null)
+            return;
+
+        // Attempt to get the latest human stencil image. If this method succeeds,
+        // it acquires a native resource that must be disposed (see below).
+        if (occlusionManager && occlusionManager.TryAcquireHumanStencilCpuImage(out XRCpuImage image))
+        {
+            using (image)
+            {
+                UpdateRawImage(m_RawHumanStencilImage, image);
+            }
+        }
+        else
+        {
+            m_RawHumanStencilImage.enabled = false;
+        }
+    }
+
+    static void UpdateRawImage(RawImage rawImage, XRCpuImage cpuImage)
+    {
+        // Get the texture associated with the UI.RawImage that we wish to display on screen.
+        var texture = rawImage.texture as Texture2D;
+
+        // If the texture hasn't yet been created, or if its dimensions have changed, (re)create the texture.
+        // Note: Although texture dimensions do not normally change frame-to-frame, they can change in response to
+        //    a change in the camera resolution (for camera images) or changes to the quality of the human depth
+        //    and human stencil buffers.
+        if (texture == null || texture.width != cpuImage.width || texture.height != cpuImage.height)
+        {
+            texture = new Texture2D(cpuImage.width, cpuImage.height, cpuImage.format.AsTextureFormat(), false);
+            rawImage.texture = texture;
+        }
+
+        // For display, we need to mirror about the vertical access.
+        var conversionParams = new XRCpuImage.ConversionParams(cpuImage, cpuImage.format.AsTextureFormat(), XRCpuImage.Transformation.MirrorY);
+
+        // Get the Texture2D's underlying pixel buffer.
+        var rawTextureData = texture.GetRawTextureData<byte>();
+
+        // Make sure the destination buffer is large enough to hold the converted data (they should be the same size)
+        Debug.Assert(rawTextureData.Length == cpuImage.GetConvertedDataSize(conversionParams.outputDimensions, conversionParams.outputFormat),
+            "The Texture2D is not the same size as the converted data.");
+
+        // Perform the conversion.
+        cpuImage.Convert(conversionParams, rawTextureData);
+
+        // "Apply" the new pixel data to the Texture2D.
+        texture.Apply();
+
+        // Make sure it's enabled.
+        rawImage.enabled = true;
+    }
+
+    void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
+    {
+        UpdateCameraImage();
+        UpdateHumanDepthImage();
+        UpdateHumanStencilImage();
+    }
+
+    Texture2D m_CameraTexture;
 }

README.md

 
 The relevant script is [`MakeAppearOnPlane.cs`](https://github.com/Unity-Technologies/arfoundation-samples/blob/master/Assets/Scripts/MakeAppearOnPlane.cs).
 
-## CameraImage
+## CpuImage
-This samples shows how to manipulate the camera textures on the CPU. The video feed for pass through cameras involves GPU-only textures, and manipulating them on the CPU (e.g., for computer vision algorithms) would require an expensive GPU read. Fortunately, ARFoundation provides an API for obtaining the camera image on the CPU for further processing.
+This samples shows how to acquire and manipulate textures obtained from ARFoundation on the CPU. Most textures in ARFoundation (e.g., the pass-through video supplied by the `ARCameraManager`, and the human depth and human stencil buffers provided by the `AROcclusionManager`) are GPU textures. Computer vision or other CPU-based applications often require the pixel buffers on the CPU, which would normally involve an expensive GPU readback. ARFoundation provides an API for obtaining these textures on the CPU for further processing, without incurring the costly GPU readback.
-The relevant script is [`TestCameraImage.cs`](https://github.com/Unity-Technologies/arfoundation-samples/blob/master/Assets/Scripts/TestCameraImage.cs).
+The relevant script is [`CpuImageSample.cs`](https://github.com/Unity-Technologies/arfoundation-samples/blob/master/Assets/Scripts/CpuImageSample.cs).
+
+The resolution of the camera image is affected by the camera's configuration. The current configuration is indicated at the bottom left of the screen inside a dropdown box which lets you select one of the supported camera configurations. The [`CameraConfigController.cs`](https://github.com/Unity-Technologies/arfoundation-samples/blob/master/Assets/Scripts/CameraConfigController.cs) demonstrates enumerating and selecting a camera configuration. It is on the `CameraConfigs` GameObject.
-The resolution of the CPU image is affected by the camera's configuration. The current configuration is indicated at the bottom left of the screen inside a dropdown box which lets you select one of the supported camera configurations. The [`CameraConfigController.cs`](https://github.com/Unity-Technologies/arfoundation-samples/blob/master/Assets/Scripts/CameraConfigController.cs) demonstrates enumerating and selecting a camera configuration. It is on the `CameraConfigs` GameObject.
+Where available (currently iOS 13+ only), the human depth and human stencil textures are also available on the CPU. These appear inside two additional boxes underneath the camera's image.
 
 ## TogglePlaneDetection