New ui and 3RT/4RT modes work for decals

com.unity.render-pipelines.high-definition/HDRP/Editor/Material/Decal/DecalUI.cs

             Metal_AO_Smoothness = 7
         }
 
-        enum MaskBlendFlags
+        [Flags] enum MaskBlendFlags
         {
             Metal = 1 << 0, 
             AO = 1 << 1,
         // All Setup Keyword functions must be static. It allow to create script to automatically update the shaders with a script if code change
         static public void SetupMaterialKeywordsAndPass(Material material)
         {
+            MaskBlendFlags blendMode = (MaskBlendFlags)material.GetFloat(kMaskBlendMode);
+
-            CoreUtils.SetKeyword(material, "_MASKMAP", material.GetTexture(kMaskMap));
+            CoreUtils.SetKeyword(material, "_MASKMAP", material.GetTexture(kMaskMap)); 
             CoreUtils.SetKeyword(material, "_NORMAL_BLEND_SRC_B", material.GetFloat(kNormalBlendSrc) == 1.0f);
             CoreUtils.SetKeyword(material, "_MASK_BLEND_SRC_B", material.GetFloat(kMaskBlendSrc) == 1.0f);
 
             material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsSStr, false);
             material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsMSStr, false);
             material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsAOSStr, false);
-            material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsMAOStr, false);
-
-            MaskBlendFlags blendMode = (MaskBlendFlags)material.GetFloat(kMaskBlendMode);
-            switch(blendMode)
+            material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsMAOSStr, false);
+            material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecals3RTStr, true);
+            switch (blendMode)
-                case 0:
-                    break;
-
                 case MaskBlendFlags.Metal:
                     material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsMStr, true);
                     break;
                 case MaskBlendFlags.Metal | MaskBlendFlags.AO | MaskBlendFlags.Smoothness:
                     material.SetShaderPassEnabled(HDShaderPassNames.s_MeshDecalsMAOSStr, true);
                     break;
-
             }
         }
 
                 normalBlendSrcValue = EditorGUILayout.Popup( "Normal blend source", (int)normalBlendSrcValue, blendSourceNames);               
                 m_MaterialEditor.TexturePropertySingleLine(Styles.maskMapText, maskMap);
                 maskBlendSrcValue = EditorGUILayout.Popup( "Mask blend source", (int)maskBlendSrcValue, blendSourceNames);
-                EditorGUILayout.HelpBox("Individual mask map channel blending mode can be enabled/disabled in pipeline asset.\nEnabling this feature incurs a performance cost.", MessageType.Warning);
+                EditorGUILayout.HelpBox("Individual mask map channel blending mode can be enabled/disabled in pipeline asset.\nEnabling this feature incurs a performance cost.", MessageType.Info);
-                    maskBlendFlags = (MaskBlendFlags)EditorGUILayout.EnumFlagsField( "Mask blend mode", maskBlendFlags);         
+                    maskBlendFlags = (MaskBlendFlags)EditorGUILayout.EnumFlagsField( "Mask blend mode", maskBlendFlags);
+                    if (maskBlendFlags == 0)
+                        maskBlendFlags = MaskBlendFlags.Smoothness; // can not have nothing, to achieve this effect remove the mask map from shader
+                    if (maskBlendFlags == (MaskBlendFlags)(-1)) // everything
+                        maskBlendFlags = MaskBlendFlags.Metal | MaskBlendFlags.AO | MaskBlendFlags.Smoothness;
                 }
                 m_MaterialEditor.ShaderProperty(decalBlend, Styles.decalBlendText);
                 EditorGUI.indentLevel--;

com.unity.render-pipelines.high-definition/HDRP/Material/Decal/Decal.shader

 		[ToggleUI] _AlbedoMode("_AlbedoMode", Range(0.0, 1.0)) = 1.0
 		[HideInInspector] _NormalBlendSrc("_NormalBlendSrc", Float) = 0.0
 		[HideInInspector] _MaskBlendSrc("_MaskBlendSrc", Float) = 1.0
-		[HideInInspector] _MaskBlendMode("_MaskBlendMode", Float) = 0.0
+		[HideInInspector] _MaskBlendMode("_MaskBlendMode", Float) = 4.0 // smoothness 3RT default
     }
 
     HLSLINCLUDE
 		// 7 - Metal
 		// 8 - AO
 		// 9 - Metal + AO
-		// 10 - Smoothness and also 3RT mode
+		// 10 - Smoothness 
+		// 14 - 3RT
+
-			// back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_AO"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_AO"} // AO only
-												 // back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_MAO"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_MAO"} // AO + Metalness
-												  // back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_S"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_S"} // Smoothness 
-			// back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_MS"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_MS"} // Smoothness and Metalness 
-			// back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_AOS"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_AOS"} // AO + Smoothness
-			// back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
 			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
 			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
 		{
 			Name "DBufferMesh_MAOS"  // Name is not used
 			Tags{"LightMode" = "DBufferMesh_MAOS"} // Metalness AO and Smoothness
-			// back faces with zfail, for cases when camera is inside the decal volume
-			Cull Front
-			ZTest Greater
+			ZTest LEqual
+
+			HLSLPROGRAM
+
+			#define SHADERPASS SHADERPASS_DBUFFER_MESH
+			#include "../../ShaderVariables.hlsl"
+			#include "Decal.hlsl"
+			#include "ShaderPass/DecalSharePass.hlsl"
+			#include "DecalData.hlsl"
+			#include "../../ShaderPass/ShaderPassDBuffer.hlsl"
+
+			ENDHLSL
+		}
+
+		Pass
+		{
+			Name "DBufferMesh_3RT"  // Name is not used
+			Tags{"LightMode" = "DBufferMesh_3RT"} // Smoothness 
+			ZWrite Off
+			ZTest LEqual
+			// using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
+			Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
+			Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
+			Blend 2 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
+
+			ColorMask BA 2	// smoothness/smoothness alpha
 
 			HLSLPROGRAM
 

com.unity.render-pipelines.high-definition/HDRP/Material/Decal/DecalSystem.cs

                             m_DecalDatas[m_DecalDatasCount].blendParams = m_BlendParams;
                             if(!perChannelMask)
                             {
-                                m_DecalDatas[m_DecalDatasCount].blendParams.z = 0.0f;
+                                m_DecalDatas[m_DecalDatasCount].blendParams.z = 4.0f; // smoothness
                             }
                                                         
                             // we have not allocated the textures in atlas yet, so only store references to them
                     return;
                 int batchIndex = 0;
                 int totalToDraw = m_NumResults;
-                int shaderPass = instance.perChannelMask ? (int)m_Material.GetFloat("_MaskBlendMode") : 0; // relies on the order shader passes are declared in decal.shader and decalUI.cs
+                HDRenderPipelineAsset hdrp = GraphicsSettings.renderPipelineAsset as HDRenderPipelineAsset;
+                bool perChannelMask = hdrp.renderPipelineSettings.decalSettings.perChannelMask;
+                int shaderPass = perChannelMask ? (int)m_Material.GetFloat("_MaskBlendMode") : 4; // relies on the order shader passes are declared in decal.shader and decalUI.cs
+                shaderPass--; // shader passes are 0 based 
+               
                 // enum BlendMode
                 // {
                 //		Metal_AO_Smoothness,

com.unity.render-pipelines.high-definition/HDRP/Material/Decal/DecalUtilities.hlsl

 	}
 	else if (blendParams.z == 1)
 	{
-		dbuffer2Mask = float4(1, 0, 1, 1);	// M, _, S, S alpha
+		dbuffer2Mask = float4(1, 0, 0, 0);	// M, _, _, _
-		dbuffer2Mask = float4(1, 0, 0, 0);	// M, _, _, _
-		dbuffer3Mask = float2(1, 0); // M alpha, _
+		dbuffer2Mask = float4(0, 1, 0, 0);	// _, AO, _, _
+		dbuffer3Mask = float2(0, 1); // _, AO alpha
+		dbuffer2Mask = float4(1, 1, 0, 0);	// M, AO, _, _
+		dbuffer3Mask = float2(1, 1); // M Alpha, AO alpha
+	}
+	else if (blendParams.z == 4)
+	{
-	else if (blendParams.z == 4)
+	else if (blendParams.z == 5)
+	{
+		dbuffer2Mask = float4(1, 0, 1, 1);	// M, _, S, S alpha
+		dbuffer3Mask = float2(1, 0); // M alpha, _
+	}
+	else if (blendParams.z == 6)
-		dbuffer2Mask = float4(0, 1, 0, 0);	// _, AO, _, _
+		dbuffer2Mask = float4(0, 1, 1, 1);	// _, AO, S, S alpha
+	}
+	else if (blendParams.z == 7)
+	{
+		dbuffer2Mask = float4(1, 1, 1, 1);	// M, AO, S, S alpha
+		dbuffer3Mask = float2(1, 1); // M alpha, AO alpha
 	}
 
 	dbuffer2.xyz = (dbuffer2Mask.xyz == 1) ? src.xyz * src.w + dbuffer2.xyz * (1.0f - src.w) : dbuffer2.xyz;

com.unity.render-pipelines.high-definition/HDRP/RenderPipeline/HDRenderPipeline.cs

                     drawSettings.SetShaderPassName(3, HDShaderPassNames.s_MeshDecalsSName);
                     drawSettings.SetShaderPassName(4, HDShaderPassNames.s_MeshDecalsMSName);
                     drawSettings.SetShaderPassName(5, HDShaderPassNames.s_MeshDecalsAOSName);
-                    drawSettings.SetShaderPassName(5, HDShaderPassNames.s_MeshDecalsMAOSName);
+                    drawSettings.SetShaderPassName(6, HDShaderPassNames.s_MeshDecalsMAOSName);
-                    drawSettings.SetShaderPassName(0, HDShaderPassNames.s_MeshDecalsSName);
+                    drawSettings.SetShaderPassName(0, HDShaderPassNames.s_MeshDecals3RTName);
                 }
                                                
                 FilterRenderersSettings filterRenderersSettings = new FilterRenderersSettings(true)

com.unity.render-pipelines.high-definition/HDRP/RenderPipeline/HDStringConstants.cs

         public static readonly string s_MeshDecalsMAOStr = "DBufferMesh_MAO";
         public static readonly string s_MeshDecalsAOSStr = "DBufferMesh_AOS";
         public static readonly string s_MeshDecalsMAOSStr = "DBufferMesh_MAOS";
+        public static readonly string s_MeshDecals3RTStr = "DBufferMesh_3RT";
        
         // ShaderPass name
         public static readonly ShaderPassName s_EmptyName = new ShaderPassName(s_EmptyStr);
         public static readonly ShaderPassName s_MeshDecalsMAOName = new ShaderPassName(s_MeshDecalsMAOStr);
         public static readonly ShaderPassName s_MeshDecalsAOSName = new ShaderPassName(s_MeshDecalsAOSStr);
         public static readonly ShaderPassName s_MeshDecalsMAOSName = new ShaderPassName(s_MeshDecalsMAOSStr);
+        public static readonly ShaderPassName s_MeshDecals3RTName = new ShaderPassName(s_MeshDecals3RTStr);
 
         // Legacy name
         public static readonly ShaderPassName s_AlwaysName = new ShaderPassName("Always");