Fix tessellation frustum culling, clean and optimize

ScriptableRenderPipeline/Core/ShaderLibrary/Common.hlsl

     return positionSS;
 }
 
+float2 ComputeScreenSpacePosition(float3 positionWS, float4x4 viewProjectionMatrix)
+{
+    float4 positionCS = mul(viewProjectionMatrix, float4(positionWS, 1.0));
+    return ComputeScreenSpacePosition(positionCS);
+}
+
 float3 ComputeViewSpacePosition(float2 positionSS, float depthRaw, float4x4 invProjMatrix)
 {
     float4 positionCS = ComputeClipSpacePosition(positionSS, depthRaw);

ScriptableRenderPipeline/Core/ShaderLibrary/GeometricTools.hlsl

 }
 
 //-----------------------------------------------------------------------------
-// Distance functions
+// Miscellaneous functions
 //-----------------------------------------------------------------------------
 
 // Box is AABB
+}
+
+float3 ProjectPointOnPlane(float3 position, float3 planePosition, float3 planeNormal)
+{
+    return position - (dot(position - planePosition, planeNormal) * planeNormal);
+}
+
+// Plane equation: {(a, b, c) = N, d = -dot(N, P)}.
+// Returns the distance from the plane to the point 'p' along the normal.
+// Positive -> in front (above), negative -> behind (below).
+float DistanceFromPlane(float3 p, float4 plane)
+{
+    return dot(float4(p, 1.0), plane);
+}
+
+// Returns 'true' if a triangle defined by 3 vertices is outside of the frustum.
+// 'epsilon' is the (negative) distance to (outside of) the frustum below which we cull the triangle.
+bool CullTriangleFrustum(float3 p0, float3 p1, float3 p2, float epsilon, float4 frustumPlanes[4])
+{
+    bool hidden = false;
+
+    for (int i = 0; i < 3; i++)
+    {
+        // If all 3 points are behind any of the planes, we cull.
+        hidden = hidden || Max3(DistanceFromPlane(p0, frustumPlanes[i]),
+                                DistanceFromPlane(p1, frustumPlanes[i]),
+                                DistanceFromPlane(p2, frustumPlanes[i])) < epsilon;
+    }
+
+    return hidden;
+}
+
+// Returns 'true' if a triangle defined by 3 vertices is back-facing.
+// 'epsilon' is the (negative) value of dot(N, V) below which we cull the triangle.
+bool CullTriangleBackFace(float3 p0, float3 p1, float3 p2, float epsilon, float3 viewPos)
+{
+    float3 edge1 = p1 - p0;
+    float3 edge2 = p2 - p0;
+
+    float3 N     = cross(edge1, edge2);
+    float3 V     = viewPos - p0;
+    float  NdotV = dot(N, V);
+
+    // Optimize:
+    // NdotV / (length(N) * length(V)) < Epsilon
+    // NdotV < Epsilon * length(N) * length(V)
+    // NdotV < Epsilon * sqrt(dot(N, N) * sqrt(dot(V, V)
+    // NdotV < Epsilon * sqrt(dot(N, N) * dot(V, V))
+    return NdotV < epsilon * sqrt(dot(N, N) * dot(V, V));
 }
 
 #endif // UNITY_GEOMETRICTOOLS_INCLUDED

ScriptableRenderPipeline/Core/ShaderLibrary/Tessellation.hlsl

 #define TESSELLATION_INTERPOLATE_BARY(name, bary) ouput.name = input0.name * bary.x +  input1.name * bary.y +  input2.name * bary.z
 
-// TODO: Move in geomtry.hlsl
-float3 ProjectPointOnPlane(float3 position, float3 planePosition, float3 planeNormal)
-{
-    return position - (dot(position - planePosition, planeNormal) * planeNormal);
-}
-
 // p0, p1, p2 triangle world position
 // p0, p1, p2 triangle world vertex normal
 float3 PhongTessellation(float3 positionWS, float3 p0, float3 p1, float3 p2, float3 n0, float3 n1, float3 n2, float3 baryCoords, float shape)
 
 // Reference: http://twvideo01.ubm-us.net/o1/vault/gdc10/slides/Bilodeau_Bill_Direct3D11TutorialTessellation.pdf
 
-// Return true if the triangle must be culled
-// backFaceCullEpsilon is the threshold of the dot product between view and normal ( < 0 mean we cull)
-bool BackFaceCullTriangle(float3 p0, float3 p1, float3 p2, float backFaceCullEpsilon, float3 cameraPosWS)
-{
-    float3 edge0 = p1 - p0;
-    float3 edge2 = p2 - p0;
-
-    float3 N = normalize(cross(edge0, edge2));
-    float3 midpoint = (p0 + p1 + p2) / 3.0;
-    float3 V = normalize(cameraPosWS - midpoint);
-
-    return (dot(V, N) < backFaceCullEpsilon) ? true : false;
-}
-
-float2 GetScreenSpacePosition(float3 positionWS, float4x4 viewProjectionMatrix, float4 screenSize)
-{
-    float4 positionCS = mul(viewProjectionMatrix, float4(positionWS, 1.0));
-    float2 positionSS = positionCS.xy / positionCS.w;
-
-    // TODO: Check if we need to invert y
-    return (positionSS * 0.5 + 0.5) * float2(screenSize.x, -screenSize.y);
-}
-
-    float2 edgeScreenPosition0 = GetScreenSpacePosition(p0, viewProjectionMatrix, screenSize);
-    float2 edgeScreenPosition1 = GetScreenSpacePosition(p1, viewProjectionMatrix, screenSize);
-    float2 edgeScreenPosition2 = GetScreenSpacePosition(p2, viewProjectionMatrix, screenSize);
+    float2 edgeScreenPosition0 = ComputeScreenSpacePosition(p0, viewProjectionMatrix) * screenSize;
+    float2 edgeScreenPosition1 = ComputeScreenSpacePosition(p1, viewProjectionMatrix) * screenSize;
+    float2 edgeScreenPosition2 = ComputeScreenSpacePosition(p2, viewProjectionMatrix) * screenSize;
 
     float EdgeScale = 1.0 / triangleSize; // Edge size in reality, but name is simpler
     float3 tessFactor;
     float3 edgePosition1 = 0.5 * (p0 + p2);
     float3 edgePosition2 = 0.5 * (p0 + p1);
 
-    // TODO: Move to camera relative and change distance to length
+    // In case camera-relative rendering is enabled, 'cameraPosWS' is statically known to be 0,
+    // so the compiler will be able to optimize distance() to length().
     float dist0 = distance(edgePosition0, cameraPosWS);
     float dist1 = distance(edgePosition1, cameraPosWS);
     float dist2 = distance(edgePosition2, cameraPosWS);
 
     return tess;
 }
-
-// TODO: Move in geomtry.hlsl
-float DistanceFromPlane(float3 pos, float4 plane)
-{
-    float d = dot(float4(pos, 1.0), plane);
-    return d;
-}
-
-// Returns true if triangle with given 3 world positions is outside of camera's view frustum.
-// cullEps is distance outside of frustum that is still considered to be inside (i.e. max displacement)
-bool WorldViewFrustumCull(float3 p0, float3 p1, float3 p2, float cullEps, float4 cameraWorldClipPlanes[4])
-{
-    float4 planeTest;
-
-    // left
-    planeTest.x =   ((DistanceFromPlane(p0, cameraWorldClipPlanes[0]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p1, cameraWorldClipPlanes[0]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p2, cameraWorldClipPlanes[0]) > -cullEps) ? 1.0 : 0.0);
-    // right
-    planeTest.y =   ((DistanceFromPlane(p0, cameraWorldClipPlanes[1]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p1, cameraWorldClipPlanes[1]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p2, cameraWorldClipPlanes[1]) > -cullEps) ? 1.0 : 0.0);
-    // top
-    planeTest.z =   ((DistanceFromPlane(p0, cameraWorldClipPlanes[2]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p1, cameraWorldClipPlanes[2]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p2, cameraWorldClipPlanes[2]) > -cullEps) ? 1.0 : 0.0);
-    // bottom
-    planeTest.w =   ((DistanceFromPlane(p0, cameraWorldClipPlanes[3]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p1, cameraWorldClipPlanes[3]) > -cullEps) ? 1.0 : 0.0) +
-                    ((DistanceFromPlane(p2, cameraWorldClipPlanes[3]) > -cullEps) ? 1.0 : 0.0);
-
-    // has to pass all 4 plane tests to be visible
-    return !all(planeTest);
-}

ScriptableRenderPipeline/HDRenderPipeline/Camera/HDCamera.cs

             var gpuNonJitteredProj = GL.GetGPUProjectionMatrix(nonJitteredCameraProj, true);
 
             var pos = camera.transform.position;
+            var relPos = pos; // World-origin-relative
+                relPos = Vector3.zero; // Camera-relative
             }
 
             var gpuVP = gpuNonJitteredProj * gpuView;
             cameraPos = pos;
             screenSize = new Vector4(camera.pixelWidth, camera.pixelHeight, 1.0f / camera.pixelWidth, 1.0f / camera.pixelHeight);
 
+            // Warning: near and far planes appear to be broken.
-            for (int i = 0; i < 6; i++)
+            for (int i = 0; i < 4; i++)
+                // Left, right, top, bottom.
+
+            // Near, far.
+            frustumPlaneEquations[4] = new Vector4( camera.transform.forward.x,  camera.transform.forward.y,  camera.transform.forward.z, -Vector3.Dot(camera.transform.forward, relPos) - camera.nearClipPlane);
+            frustumPlaneEquations[5] = new Vector4(-camera.transform.forward.x, -camera.transform.forward.y, -camera.transform.forward.z,  Vector3.Dot(camera.transform.forward, relPos) + camera.farClipPlane);
 
             m_LastFrameActive = Time.frameCount;
         }

ScriptableRenderPipeline/HDRenderPipeline/Material/LayeredLit/LayeredLitTessellation.shader

 
     #include "../../../Core/ShaderLibrary/common.hlsl"
     #include "../../../Core/ShaderLibrary/Wind.hlsl"
+    #include "../../../Core/ShaderLibrary/GeometricTools.hlsl"
     #include "../../../Core/ShaderLibrary/tessellation.hlsl"
     #include "../../ShaderPass/FragInputs.hlsl"
     #include "../../ShaderPass/ShaderPass.cs.hlsl"

ScriptableRenderPipeline/HDRenderPipeline/Material/Lit/LitDataMeshModification.hlsl

 {
     float maxDisplacement = GetMaxDisplacement();
 
-
+    float frustumCullEps = -maxDisplacement;
+
-    bool frustumCulledCurrentView = WorldViewFrustumCull(p0, p1, p2, maxDisplacement, (float4[4])_FrustumPlanes); // _FrustumPlanes are primary camera planes
+    bool frustumCulledCurrentView = CullTriangleFrustum(p0, p1, p2, frustumCullEps, (float4[4])_FrustumPlanes); // _FrustumPlanes are primary camera planes
-    bool frustumCulledCurrentView = WorldViewFrustumCull(p0, p1, p2, maxDisplacement, (float4[4])unity_CameraWorldClipPlanes); // unity_CameraWorldClipPlanes is set by legacy Unity in case of shadow and contain shadow view plan
+    // 'unity_CameraWorldClipPlanes' are set by the legacy Unity and are not aware of camera-relative rendering.
+    bool frustumCulledCurrentView = CullTriangleFrustum(GetAbsolutePositionWS(p0), GetAbsolutePositionWS(p1), GetAbsolutePositionWS(p2), frustumCullEps, (float4[4])unity_CameraWorldClipPlanes);
-    bool frustumCulledMainView = WorldViewFrustumCull(p0, p1, p2, maxDisplacement, (float4[4])_FrustumPlanes);
+    bool frustumCulledMainView = CullTriangleFrustum(p0, p1, p2, frustumCullEps, (float4[4])_FrustumPlanes);
-    if (_TessellationBackFaceCullEpsilon > -0.99) // Is backface culling enabled ?
+    if (_TessellationBackFaceCullEpsilon > -1) // Is backface culling enabled ?
     {
         // Handle transform mirroring (like negative scaling)
         // Caution: don't change p1/p2 directly as it is use later
-        faceCull = BackFaceCullTriangle(p0, backfaceP1, backfaceP2, _TessellationBackFaceCullEpsilon, GetCurrentViewPosition()); // Use shadow view
+        faceCull = CullTriangleBackFace(p0, backfaceP1, backfaceP2, _TessellationBackFaceCullEpsilon, GetCurrentViewPosition()); // Use shadow view
     }
 #endif
 
     // During shadow passes, we decide that anything outside the main view frustum should not be tessellated.
     if (frustumCulledMainView)
     {
+        // Warning: currently, having different (discontinuous) tessellation factors causes
+        // cracks in tessellation just outside of the primary camera (view) frustum.
+        // The issue doesn't show up on geometry on-screen, but can become visible in shadows.
         return float4(1.0, 1.0, 1.0, 1.0);
     }
 

ScriptableRenderPipeline/HDRenderPipeline/Material/Lit/LitTessellation.shader

 
     #include "../../../Core/ShaderLibrary/common.hlsl"
     #include "../../../Core/ShaderLibrary/Wind.hlsl"
+    #include "../../../Core/ShaderLibrary/GeometricTools.hlsl"
     #include "../../../Core/ShaderLibrary/tessellation.hlsl"
     #include "../../ShaderPass/FragInputs.hlsl"
     #include "../../ShaderPass/ShaderPass.cs.hlsl"

ScriptableRenderPipeline/HDRenderPipeline/ShaderPass/TessellationShare.hlsl

     output.edge[0] = min(tf.x, MAX_TESSELLATION_FACTORS);
     output.edge[1] = min(tf.y, MAX_TESSELLATION_FACTORS);
     output.edge[2] = min(tf.z, MAX_TESSELLATION_FACTORS);
-    output.inside = min(tf.w, MAX_TESSELLATION_FACTORS);
+    output.inside  = min(tf.w, MAX_TESSELLATION_FACTORS);
 
     return output;
 }

ScriptableRenderPipeline/HDRenderPipeline/ShaderVariables.hlsl

 float4x4 _InvViewMatrix;
 float4x4 _InvProjMatrix;
 float4   _InvProjParam;
-float4   _ScreenSize;       // (w, h, 1/w, 1/h)
-float4   _FrustumPlanes[6]; // (N, -dot(N, P))
+float4   _ScreenSize;       // {w, h, 1/w, 1/h}
+float4   _FrustumPlanes[6]; // {(a, b, c) = N, d = -dot(N, P)} [L, R, T, B, N, F]
 CBUFFER_END
 
 #ifdef USE_LEGACY_UNITY_MATRIX_VARIABLES

ScriptableRenderPipeline/HDRenderPipeline/ShaderVariablesFunctions.hlsl

 // Note: '_WorldSpaceCameraPos' is set by the legacy Unity code.
 float3 GetPrimaryCameraPosition()
 {
-    return GetCameraRelativePositionWS(_WorldSpaceCameraPos);
+#if (SHADEROPTIONS_CAMERA_RELATIVE_RENDERING != 0)
+    return float3(0, 0, 0);
+#else
+    return _WorldSpaceCameraPos;
+#endif
 }
 
 // Could be e.g. the position of a primary camera or a shadow-casting light.