(wip) distortion rejection with stencil buffer

SampleScenes/HDTest/DistortionTest.unity

     debug:
       m_Flags: 0
   m_NavMeshData: {fileID: 0}
+--- !u!1 &577211144
+GameObject:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 5
+  m_Component:
+  - component: {fileID: 577211145}
+  - component: {fileID: 577211147}
+  - component: {fileID: 577211146}
+  m_Layer: 0
+  m_Name: Sphere Reject
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &577211145
+Transform:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 577211144}
+  m_LocalRotation: {x: 0.7071068, y: 1.7763568e-15, z: -1.7763568e-15, w: 0.7071068}
+  m_LocalPosition: {x: 1.84, y: 4.661, z: 2.956}
+  m_LocalScale: {x: 0.4341928, y: 0.4341931, z: 0.43419296}
+  m_Children: []
+  m_Father: {fileID: 1885303981}
+  m_RootOrder: 5
+  m_LocalEulerAnglesHint: {x: 90, y: 0, z: 0}
+--- !u!23 &577211146
+MeshRenderer:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 577211144}
+  m_Enabled: 1
+  m_CastShadows: 0
+  m_ReceiveShadows: 0
+  m_DynamicOccludee: 1
+  m_MotionVectors: 1
+  m_LightProbeUsage: 0
+  m_ReflectionProbeUsage: 0
+  m_Materials:
+  - {fileID: 2100000, guid: 8bef593041e9c7b4e9bf34832917e46a, type: 2}
+  m_StaticBatchInfo:
+    firstSubMesh: 0
+    subMeshCount: 0
+  m_StaticBatchRoot: {fileID: 0}
+  m_ProbeAnchor: {fileID: 0}
+  m_LightProbeVolumeOverride: {fileID: 0}
+  m_ScaleInLightmap: 1
+  m_PreserveUVs: 1
+  m_IgnoreNormalsForChartDetection: 0
+  m_ImportantGI: 0
+  m_StitchLightmapSeams: 0
+  m_SelectedEditorRenderState: 3
+  m_MinimumChartSize: 4
+  m_AutoUVMaxDistance: 0.5
+  m_AutoUVMaxAngle: 89
+  m_LightmapParameters: {fileID: 0}
+  m_SortingLayerID: 0
+  m_SortingLayer: 0
+  m_SortingOrder: 0
+--- !u!33 &577211147
+MeshFilter:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 577211144}
+  m_Mesh: {fileID: 10207, guid: 0000000000000000e000000000000000, type: 0}
 --- !u!1 &581096758
 GameObject:
   m_ObjectHideFlags: 0
   m_PrefabInternal: {fileID: 0}
   m_GameObject: {fileID: 867827819}
   m_LocalRotation: {x: 0.7071068, y: 0, z: 0, w: 0.7071068}
-  m_LocalPosition: {x: 1.08, y: 4.628, z: 3.05}
+  m_LocalPosition: {x: 1.382, y: 4.628, z: 3.05}
   m_LocalScale: {x: 1.0000005, y: 1.0000005, z: 1.0000005}
   m_Children: []
   m_Father: {fileID: 1885303981}
   - {fileID: 581096759}
   - {fileID: 1972037646}
   - {fileID: 867827820}
+  - {fileID: 577211145}
   m_Father: {fileID: 0}
   m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}

SampleScenes/HDTest/GraphicTest/Common/Material/Mat_Distorsion_Ripple.mat

     - _DetailSmoothnessScale: 1
     - _DistortionDepthTest: 1
     - _DistortionEnable: 1
+    - _DistortionNullify: 0
+    - _DistortionStencilRef: 1
     - _DoubleSidedEnable: 0
     - _DoubleSidedNormalMode: 1
     - _Drag: 1

ScriptableRenderPipeline/HDRenderPipeline/HDRenderPipeline.cs

 
         Material m_CopyStencilForSplitLighting;
         Material m_CopyStencilForRegularLighting;
+        Material m_CopyStencilForDistortion;
         GPUCopy m_GPUCopy;
 
         // Various set of material use in render loop
             m_CopyStencilForRegularLighting = CoreUtils.CreateEngineMaterial("Hidden/HDRenderPipeline/CopyStencilBuffer");
             m_CopyStencilForRegularLighting.DisableKeyword("EXPORT_HTILE");
             m_CopyStencilForRegularLighting.SetInt(HDShaderIDs._StencilRef, (int)StencilLightingUsage.RegularLighting);
+            m_CopyStencilForDistortion = CoreUtils.CreateEngineMaterial("Hidden/HDRenderPipeline/CopyStencilBuffer");
+            m_CopyStencilForDistortion.DisableKeyword("EXPORT_HTILE");
+            m_CopyStencilForDistortion.SetInt(HDShaderIDs._StencilRef, 3); // Distortion mask
             m_CameraMotionVectorsMaterial   = CoreUtils.CreateEngineMaterial("Hidden/HDRenderPipeline/CameraMotionVectors");
 
             InitializeDebugMaterials();
         {
             // Currently, Unity does not offer a way to bind the stencil buffer as a texture in a compute shader.
             // Therefore, it's manually copied using a pixel shader.
-            return m_CurrentDebugDisplaySettings.renderingDebugSettings.enableSSSAndTransmission || LightLoop.GetFeatureVariantsEnabled(m_Asset.tileSettings);
+            return m_CurrentDebugDisplaySettings.renderingDebugSettings.enableSSSAndTransmission
+                || m_CurrentDebugDisplaySettings.renderingDebugSettings.enableDistortion 
+                || LightLoop.GetFeatureVariantsEnabled(m_Asset.tileSettings);
         }
 
         bool NeedHTileCopy()
                     // Our method of exporting the stencil requires one pass per unique stencil value.
                     CoreUtils.DrawFullScreen(cmd, m_CopyStencilForSplitLighting,   m_CameraStencilBufferCopyRT, m_CameraDepthStencilBufferRT);
                     CoreUtils.DrawFullScreen(cmd, m_CopyStencilForRegularLighting, m_CameraStencilBufferCopyRT, m_CameraDepthStencilBufferRT);
+                    CoreUtils.DrawFullScreen(cmd, m_CopyStencilForDistortion,      m_CameraStencilBufferCopyRT, m_CameraDepthStencilBufferRT);
                     cmd.ClearRandomWriteTargets();
                 }
             }
 
             RenderPyramidDepth(camera, cmd);
 
+            // TODO: Check with VFX team.
+            // Rendering distortion here have off course lot of artifact.
+            // But resolving at each objects that write in distortion is not possible (need to sort transparent, render those that do not distort, then resolve, then etc...)
+            // Instead we chose to apply distortion at the end after we cumulate distortion vector and desired blurriness. This
+            RenderDistortion(m_CullResults, camera, renderContext, cmd);
+
             // Required for the SSS and the shader feature classification pass.
             PrepareAndBindStencilTexture(cmd);
 
                 else
                 {
                     RenderVelocity(m_CullResults, hdCamera, renderContext, cmd); // Note we may have to render velocity earlier if we do temporalAO, temporal volumetric etc... Mean we will not take into account forward opaque in case of deferred rendering ?
-
-                    // TODO: Check with VFX team.
-                    // Rendering distortion here have off course lot of artifact.
-                    // But resolving at each objects that write in distortion is not possible (need to sort transparent, render those that do not distort, then resolve, then etc...)
-                    // Instead we chose to apply distortion at the end after we cumulate distortion vector and desired blurriness. This
-                    RenderDistortion(m_CullResults, camera, renderContext, cmd);
 
                     RenderGaussianPyramidColor(camera, cmd);
                     ApplyDistortion(cmd, m_Asset.renderPipelineResources);
                 resources.applyDistortionCS.GetKernelThreadGroupSizes(resources.applyDistortionKernel, out x, out y, out z);
                 cmd.SetComputeTextureParam(resources.applyDistortionCS, resources.applyDistortionKernel, HDShaderIDs._DistortionTexture, m_DistortionBufferRT);
                 cmd.SetComputeTextureParam(resources.applyDistortionCS, resources.applyDistortionKernel, HDShaderIDs._GaussianPyramidColorTexture, m_GaussianPyramidColorBufferRT);
-                cmd.SetComputeTextureParam(resources.applyDistortionCS, resources.applyDistortionKernel, HDShaderIDs._DepthPyramidTexture, m_DepthPyramidBufferRT);
+                cmd.SetComputeTextureParam(resources.applyDistortionCS, resources.applyDistortionKernel, HDShaderIDs._StencilTexture, GetStencilTexture());
                 cmd.SetComputeVectorParam(resources.applyDistortionCS, HDShaderIDs._Size, size);
                 cmd.SetComputeVectorParam(resources.applyDistortionCS, HDShaderIDs._GaussianPyramidColorMipSize, Shader.GetGlobalVector(HDShaderIDs._GaussianPyramidColorMipSize));
                 cmd.DispatchCompute(resources.applyDistortionCS, resources.applyDistortionKernel, (int)(size.x) / (int)x, (int)(size.y) / (int)y, 1);

ScriptableRenderPipeline/HDRenderPipeline/Material/Lit/Lit.shader

         [ToggleOff] _DistortionOnly("Distortion Only", Float) = 0.0
         [ToggleOff] _DistortionDepthTest("Distortion Depth Test Enable", Float) = 0.0
         [ToggleOff] _DepthOffsetEnable("Depth Offset View space", Float) = 0.0
+        [HideInInspector] _DistortionStencilRef("Distortion Stencil Comparison", Int) = 1 // UnityEngine.Rendering.CompareFunction.Never
+        [ToggleOff] _DistortionNullify("Nullify distortion", Float) = 0.0
 
         [ToggleOff]  _AlphaCutoffEnable("Alpha Cutoff Enable", Float) = 0.0
         _AlphaCutoff("Alpha Cutoff", Range(0.0, 1.0)) = 0.5
                 Ref  [_StencilRef]
                 Comp Always
                 Pass Replace
+                ReadMask 3
+                WriteMask 3
             }
 
             HLSLPROGRAM
                 Ref  [_StencilRef]
                 Comp Always
                 Pass Replace
+                ReadMask 3
+                WriteMask 3
             }
 
             HLSLPROGRAM
                 Ref  [_StencilRef]
                 Comp Always
                 Pass Replace
+                ReadMask 3
+                WriteMask 3
             }
 
             HLSLPROGRAM
             ZTest [_ZTestMode]
             ZWrite off
             Cull [_CullMode]
+
+            Stencil
+            {
+                Ref  4
+                Comp Always
+                Pass Replace
+                ReadMask 4
+                WriteMask 4
+            }
 
             HLSLPROGRAM
 

ScriptableRenderPipeline/HDRenderPipeline/Material/Unlit/Editor/BaseUnlitUI.cs

             public static GUIContent transparentDepthPrepassEnableText = new GUIContent("Enable transparent depth prepass", "It allow to ");	
 
             public static GUIContent doubleSidedEnableText = new GUIContent("Double Sided", "This will render the two face of the objects (disable backface culling) and flip/mirror normal");
-            public static GUIContent distortionEnableText = new GUIContent("Distortion", "Enable distortion on this shader");
+            public static GUIContent distortionEnableText = new GUIContent("Distortion", "Enable distortion on this shader"); 
+            public static GUIContent distortionNullifyText = new GUIContent("Nullify Distortion", "Nullify distortion");
 
             public static string advancedText = "Advanced Options";
         }
         protected const string kDistortionEnable = "_DistortionEnable";
         protected MaterialProperty distortionOnly = null;
         protected const string kDistortionOnly = "_DistortionOnly";
+        protected MaterialProperty distortionNullify = null;
+        protected const string kDistortionNullify = "_DistortionNullify";
         protected MaterialProperty distortionDepthTest = null;
         protected const string kDistortionDepthTest = "_DistortionDepthTest";
         protected MaterialProperty distortionVectorMap = null;
             distortionOnly = FindProperty(kDistortionOnly, props, false);
             distortionDepthTest = FindProperty(kDistortionDepthTest, props, false);
             distortionVectorMap = FindProperty(kDistortionVectorMap, props, false);
+            distortionNullify = FindProperty(kDistortionNullify, props, false);
         }
 
         void SurfaceTypePopup()
                         m_MaterialEditor.TexturePropertySingleLine(StylesBaseUnlit.distortionVectorMapText, distortionVectorMap);
                         m_MaterialEditor.ShaderProperty(distortionOnly, StylesBaseUnlit.distortionOnlyText);
                         m_MaterialEditor.ShaderProperty(distortionDepthTest, StylesBaseUnlit.distortionDepthTestText);
+                        m_MaterialEditor.ShaderProperty(distortionNullify, StylesBaseUnlit.distortionNullifyText);
                         EditorGUI.indentLevel--;
                     }
                 }
                 else
                 {
                     material.SetInt("_ZTestMode", (int)UnityEngine.Rendering.CompareFunction.Always);
+                }
+
+                bool distortionNullify = material.GetFloat(kDistortionNullify) > 0.0f;
+                if (distortionNullify)
+                {
+                    material.SetInt("_DistortionStencilRef", (int)UnityEngine.Rendering.CompareFunction.Always);
+                }
+                else
+                {
+                    material.SetInt("_DistortionStencilRef", (int)UnityEngine.Rendering.CompareFunction.Never);
                 }
             }
 

ScriptableRenderPipeline/HDRenderPipeline/RenderPipelineResources/ApplyDistorsion.compute

 #include "../../Core/ShaderLibrary/Common.hlsl"
+#include "../../Core/ShaderLibrary/Packing.hlsl"
+Texture2D<float4> _StencilTexture;
 Texture2D<float4> _GaussianPyramidColorTexture;
 RWTexture2D<float4> _CameraColorTexture;
 
 [numthreads(8, 8, 1)]
 void KMain(uint2 groupId : SV_GroupID, uint2 groupThreadId : SV_GroupThreadID, uint2 dispatchThreadId : SV_DispatchThreadID)
 {
+    // Reject pixels based on stencil buffer
+    // 3rd bit is 1 for rejected pixels
+    uint stencil = UnpackByte(_StencilTexture.Load(uint3(dispatchThreadId, 0)).r);
+    bool isRejected = stencil & 4 != 0;
+    //bool isRejected = (uint(stencil) & 4) != 0;
+    /*if (isRejected)
+        return;*/
+
+    _CameraColorTexture[dispatchThreadId] = float4(isRejected, isRejected, isRejected, 1.0);
+    return;
     // We use a bias when fetching distortion source pixels
     // This avoid artifacts when a distortion is overlapped by an opaque object
     const float _FetchBias = 0.9;

SampleScenes/HDTest/GraphicTest/Common/Material/Mat_Distorsion_Reject.mat

+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!21 &2100000
+Material:
+  serializedVersion: 6
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_Name: Mat_Distorsion_Reject
+  m_Shader: {fileID: 4800000, guid: 6e4ae4064600d784cac1e41a9e6f2e59, type: 3}
+  m_ShaderKeywords: _ALBEDOAFFECTEMISSIVE_OFF _ALPHACUTOFFENABLE_OFF _BLENDMODE_LERP
+    _DEPTHOFFSETENABLE_OFF _DISTORTIONNULLIFY_OFF _DISTORTION_ON _DOUBLESIDEDENABLE_OFF
+    _ENABLEPERPIXELDISPLACEMENT_OFF _ENABLESPECULAROCCLUSION_OFF _ENABLEVERTEXDISPLACEMENT_OFF
+    _ENABLEWIND_OFF _NORMALMAP_TANGENT_SPACE
+  m_LightmapFlags: 4
+  m_EnableInstancingVariants: 0
+  m_DoubleSidedGI: 0
+  m_CustomRenderQueue: 3000
+  stringTagMap:
+    RenderType: Transparent
+  disabledShaderPasses:
+  - GBuffer
+  - META
+  - SHADOWCASTER
+  - DepthOnly
+  - MOTIONVECTORS
+  - Forward
+  m_SavedProperties:
+    serializedVersion: 3
+    m_TexEnvs:
+    - _AnisotropyMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _BaseColorMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _BentNormalMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _BentNormalMapOS:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DistortionVectorMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _EmissiveColorMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _HeightMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _MaskMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _NormalMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _NormalMapOS:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _SpecularColorMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _SubsurfaceRadiusMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _TangentMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _TangentMapOS:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _ThicknessMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    m_Floats:
+    - _ATDistance: 1
+    - _AlbedoAffectEmissive: 0
+    - _AlphaCutoff: 0.5
+    - _AlphaCutoffEnable: 0
+    - _Anisotropy: 0
+    - _BlendMode: 0
+    - _CoatCoverage: 1
+    - _CoatIOR: 0.5
+    - _CullMode: 2
+    - _DepthOffsetEnable: 0
+    - _DetailAlbedoScale: 1
+    - _DetailNormalScale: 1
+    - _DetailSmoothnessScale: 1
+    - _DistortionDepthTest: 1
+    - _DistortionEnable: 1
+    - _DistortionNullify: 0
+    - _DistortionOnly: 1
+    - _DistortionStencilRef: 1
+    - _DoubleSidedEnable: 0
+    - _DoubleSidedNormalMode: 1
+    - _Drag: 1
+    - _DstBlend: 10
+    - _EmissiveColorMode: 1
+    - _EmissiveIntensity: 0
+    - _EnablePerPixelDisplacement: 0
+    - _EnableSpecularOcclusion: 0
+    - _EnableVertexDisplacement: 0
+    - _EnableWind: 0
+    - _HeightAmplitude: 0.01
+    - _HeightCenter: 0.5
+    - _HeightMax: 1
+    - _HeightMin: -1
+    - _IOR: 1
+    - _InitialBend: 1
+    - _MaterialID: 1
+    - _Metallic: 0
+    - _NormalMapSpace: 0
+    - _NormalScale: 1
+    - _PPDLodThreshold: 5
+    - _PPDMaxSamples: 15
+    - _PPDMinSamples: 5
+    - _PerPixelDisplacementObjectScale: 1
+    - _RefractionMode: 0
+    - _ShiverDirectionality: 0.5
+    - _ShiverDrag: 0.2
+    - _Smoothness: 1
+    - _SmoothnessRemapMax: 1
+    - _SmoothnessRemapMin: 0
+    - _SrcBlend: 5
+    - _StencilRef: 2
+    - _Stiffness: 1
+    - _SubsurfaceProfile: 0
+    - _SubsurfaceRadius: 1
+    - _SurfaceType: 1
+    - _TexWorldScale: 1
+    - _Thickness: 1
+    - _ThicknessMultiplier: 1
+    - _UVBase: 0
+    - _UVDetail: 0
+    - _VertexDisplacementObjectScale: 1
+    - _VertexDisplacementTilingScale: 1
+    - _ZTestMode: 4
+    - _ZWrite: 0
+    m_Colors:
+    - _BaseColor: {r: 1, g: 1, b: 1, a: 1}
+    - _DoubleSidedConstants: {r: 1, g: 1, b: -1, a: 0}
+    - _EmissionColor: {r: 1, g: 1, b: 1, a: 1}
+    - _EmissiveColor: {r: 0, g: 0, b: 0, a: 1}
+    - _SpecularColor: {r: 1, g: 1, b: 1, a: 1}
+    - _TransmittanceColor: {r: 1, g: 1, b: 1, a: 1}
+    - _UVDetailsMappingMask: {r: 1, g: 0, b: 0, a: 0}
+    - _UVMappingMask: {r: 1, g: 0, b: 0, a: 0}

SampleScenes/HDTest/GraphicTest/Common/Material/Mat_Distorsion_Reject.mat.meta

+fileFormatVersion: 2
+guid: 8bef593041e9c7b4e9bf34832917e46a
+timeCreated: 1507195204
+licenseType: Pro
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 2100000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: