Set edge tess factors of edges outside the main view to 1

ScriptableRenderPipeline/Core/ShaderLibrary/GeometricTools.hlsl

     return dot(float4(p, 1.0), plane);
 }
 
-// Returns 'true' if a triangle defined by 3 vertices is outside of the frustum.
+// Returns 'true' if the triangle is outside of the frustum.
-bool CullTriangleFrustum(float3 p0, float3 p1, float3 p2, float epsilon, float4 frustumPlanes[4])
+bool CullTriangleFrustum(float3 p0, float3 p1, float3 p2, float epsilon, float4 frustumPlanes[6], int numPlanes)
-    for (int i = 0; i < 4; i++)
+    for (int i = 0; i < numPlanes; i++)
     {
         // If all 3 points are behind any of the planes, we cull.
         outside = outside || Max3(DistanceFromPlane(p0, frustumPlanes[i]),
     return outside;
 }
 
+// Returns 'true' if the edge of the triangle is outside of the frustum.
+// The edges are defined s.t. they are on the opposite side of the point with the given index.
+// 'epsilon' is the (negative) distance to (outside of) the frustum below which we cull the triangle.
+bool3 CullTriangleEdgesFrustum(float3 p0, float3 p1, float3 p2, float epsilon, float4 frustumPlanes[6], int numPlanes)
+{
+    bool3 edgesOutside = false;
+
+    for (int i = 0; i < numPlanes; i++)
+    {
+        bool3 pointsOutside = bool3(DistanceFromPlane(p0, frustumPlanes[i]) < epsilon,
+                                    DistanceFromPlane(p1, frustumPlanes[i]) < epsilon,
+                                    DistanceFromPlane(p2, frustumPlanes[i]) < epsilon);
+
+        // If both points of the edge are behind any of the planes, we cull.
+        edgesOutside.x = edgesOutside.x || (pointsOutside.y && pointsOutside.z);
+        edgesOutside.y = edgesOutside.y || (pointsOutside.x && pointsOutside.z);
+        edgesOutside.z = edgesOutside.z || (pointsOutside.x && pointsOutside.y);
+    }
+
+    return edgesOutside;
+}
+
-bool CullTriangleBackFace(float3 p0, float3 p1, float3 p2, float epsilon, float3 viewPos,
-                          float winding)
+bool CullTriangleBackFace(float3 p0, float3 p1, float3 p2, float epsilon, float3 viewPos, float winding)
 {
     float3 edge1 = p1 - p0;
     float3 edge2 = p2 - p0;

ScriptableRenderPipeline/Core/ShaderLibrary/Tessellation.hlsl

     return tessFactor;
 }
 
-float4 CalcTriEdgeTessFactors(float3 triVertexFactors)
+float4 CalcTriTessFactorsFromEdgeTessFactors(float3 triVertexFactors)
 {
     float4 tess;
     tess.x = triVertexFactors.x;

ScriptableRenderPipeline/HDRenderPipeline/Material/Lit/LitDataMeshModification.hlsl

     // For the culling part however we want to use the current view (shadow view).
     // Thus the following code play with both.
 
-    float frustumCullEps = -maxDisplacement;
+    float frustumEps = -maxDisplacement;
+    bool3 frustumCullEdgesMainView = CullTriangleEdgesFrustum(p0, p1, p2, frustumEps, _FrustumPlanes, 5); // Do not test the far plane
-    bool frustumCulledCurrentView = CullTriangleFrustum(p0, p1, p2, frustumCullEps, (float4[4])_FrustumPlanes); // _FrustumPlanes are primary camera planes
-    bool frustumCulledMainView = false;
+    bool frustumCullCurrView = all(frustumCullEdgesMainView);
-    // 'unity_CameraWorldClipPlanes' are set by the legacy Unity and are not aware of camera-relative rendering.
-    bool frustumCulledCurrentView = CullTriangleFrustum(GetAbsolutePositionWS(p0), GetAbsolutePositionWS(p1), GetAbsolutePositionWS(p2), frustumCullEps, (float4[4])unity_CameraWorldClipPlanes);
-    // In the case of shadow, we don't want to tessellate anything that is not seen by the main view frustum. It can result in minor popping of tessellation into a shadow but we can't afford it anyway.
-    bool frustumCulledMainView = CullTriangleFrustum(p0, p1, p2, frustumCullEps, (float4[4])_FrustumPlanes);
+    // 'unity_CameraWorldClipPlanes' are camera-relative rendering aware.
+    bool frustumCullCurrView = CullTriangleFrustum(p0, p1, p2, frustumEps, unity_CameraWorldClipPlanes, 4); // Do not test near/far planes
-    if (_TessellationBackFaceCullEpsilon > -1) // Is backface culling enabled ?
+    if (_TessellationBackFaceCullEpsilon > -1.0) // Is back-face culling enabled ?
     {
         // Handle transform mirroring (like negative scaling)
         float winding = unity_WorldTransformParams.w;
 
-    if (frustumCulledCurrentView || faceCull)
+    if (frustumCullCurrView || faceCull)
-        return float4(0.0, 0.0, 0.0, 0.0);
+        return 0;
-    // We use the parameters of the primary (scene view) camera in order
-    // to have identical tessellation levels for both the scene view and
-    // shadow views. Otherwise, depth comparisons become meaningless!
-    float3 tessFactor = float3(1.0, 1.0, 1.0);
+    // For performance reasons, we choose not to tessellate outside of the main camera view
+    // (we perform this test both during the regular scene rendering and the shadow pass).
+    // For edges not visible from the main view, our goal is to set the tessellation factor to 1.
+    // In this case, we set the tessellation factor to 0 here.
+    // That way, all scaling of this tessellation factor will still result in 0.
+    // Before we call CalcTriTessFactorsFromEdgeTessFactors(), all factors are clamped by max(f, 1),
+    // which achieves the desired effect.
+    float3 edgeTessFactors = float3(frustumCullEdgesMainView.x ? 0 : 1, frustumCullEdgesMainView.y ? 0 : 1, frustumCullEdgesMainView.z ? 0 : 1);
-        tessFactor *= GetScreenSpaceTessFactor( p0, p1, p2, _ViewProjMatrix, _ScreenSize, _TessellationFactorTriangleSize); // Use primary camera view
+        edgeTessFactors *= GetScreenSpaceTessFactor( p0, p1, p2, _ViewProjMatrix, _ScreenSize, _TessellationFactorTriangleSize); // Use primary camera view
     }
 
     // Distance based tessellation
         // We square the disance factor as it allow a better percptual descrease of vertex density.
-        tessFactor *= distFactor * distFactor;
+        edgeTessFactors *= distFactor * distFactor;
-    tessFactor *= _TessellationFactor;
+    edgeTessFactors *= _TessellationFactor;
-    tessFactor.xyz = float3(max(1.0, tessFactor.x), max(1.0, tessFactor.y), max(1.0, tessFactor.z));
-
-    float4 triTessFactors = CalcTriEdgeTessFactors(tessFactor);
+    edgeTessFactors = max(edgeTessFactors, float3(1.0, 1.0, 1.0));
-    // See comment above:
-    // During shadow passes, we decide that anything outside the main view frustum should not be tessellated.
-    if (frustumCulledMainView)
-    {
-        // Warning: we cannot modify the edge tessellation factors, as that creates cracks between patches.
-        // Therefore, we only modify the inside tessellation factor.
-        // TODO: find a way to modify the edge tessellation factors as well.
-        triTessFactors.w = 1.0;
-    }
-
-    return triTessFactors;
+    return CalcTriTessFactorsFromEdgeTessFactors(edgeTessFactors);
 }
 
 // tessellationFactors