Added a vertexLighting input to fragment PBR function. This way we modulate by material diffuse and do not apply ambient occlusion to it.

ScriptableRenderPipeline/LightweightPipeline/Shaders/LightweightLighting.cginc

     return vertexLightColor;
 }
 
-half4 LightweightFragmentPBR(float3 positionWS, half3 normalWS, half3 viewDirectionWS, half fogFactor, half3 diffuseGI, half3 albedo, half metallic, half3 specular, half smoothness, half occlusion, half3 emission, half alpha)
+half4 LightweightFragmentPBR(float3 positionWS, half3 normalWS, half3 viewDirectionWS, half fogFactor, half3 indirectDiffuse, half3 vertexLighting, half3 albedo, half metallic, half3 specular, half smoothness, half occlusion, half3 emission, half alpha)
 {
     BRDFData brdfData;
     InitializeBRDFData(albedo, metallic, specular, smoothness, alpha, brdfData);
     half3 reflectVec = reflect(-viewDirectionWS, normalWS);
     half roughness2 = brdfData.roughness * brdfData.roughness;
-    half3 indirectDiffuse = diffuseGI * occlusion;
+    indirectDiffuse *= occlusion;
     half3 indirectSpecular = GlossyEnvironment(reflectVec, brdfData.perceptualRoughness) * occlusion;
 
     // PBS
     half NdotL = saturate(dot(normalWS, lightDirectionWS));
     half3 radiance = light.color * (lightAtten * NdotL);
     color += LightweightBDRF(brdfData, roughness2, normalWS, lightDirectionWS, viewDirectionWS) * radiance;
+    color += vertexLighting * brdfData.diffuse;
 
 #ifdef _ADDITIONAL_LIGHTS
     int pixelLightCount = min(_AdditionalLightCount.x, unity_LightIndicesOffsetAndCount.y);

ScriptableRenderPipeline/LightweightPipeline/Shaders/LightweightPassLit.cginc

 #endif
 
 #if LIGHTMAP_ON
-    half3 diffuseGI = SampleLightmap(IN.uv01.zw, normalWS);
+    half3 indirectDiffuse = SampleLightmap(IN.uv01.zw, normalWS);
-    half3 diffuseGI = EvaluateSHPerPixel(normalWS, IN.vertexSH);
-#endif
-
-#if _VERTEX_LIGHTS
-    #if _SPECULAR_SETUP
-        // monochrome reflectivity for vertex to save ALU. Most metals are redish/yellowish
-        half3 diffuse = surfaceData.albedo * (1.0 - ReflectivitySpecular(surfaceData.specular));
-    #else
-        half3 diffuse = surfaceData.albedo * OneMinusReflectivityMetallic(surfaceData.metallic);
-    #endif
-    diffuseGI += (diffuse * IN.fogFactorAndVertexLight.yzw);
+    half3 indirectDiffuse = EvaluateSHPerPixel(normalWS, IN.vertexSH);
-    return LightweightFragmentPBR(IN.posWS, normalWS, IN.viewDir, fogFactor, diffuseGI, surfaceData.albedo, surfaceData.metallic, surfaceData.specular, surfaceData.smoothness, surfaceData.occlusion, surfaceData.emission, surfaceData.alpha);
+    return LightweightFragmentPBR(IN.posWS, normalWS, IN.viewDir, fogFactor, indirectDiffuse, IN.fogFactorAndVertexLight.yzw, surfaceData.albedo, surfaceData.metallic, surfaceData.specular, surfaceData.smoothness, surfaceData.occlusion, surfaceData.emission, surfaceData.alpha);
 }
 
 half4 LitPassFragmentSimple(LightweightVertexOutput IN) : SV_Target

ScriptableRenderPipeline/LightweightPipeline/Shaders/LightweightStandardParticles.shader

 #endif
 
                 half3 zero = half3(0.0, 0.0, 0.0);
-                return LightweightFragmentPBR(positionWS, normalWS, viewDirWS, fogFactor, /*diffuseGI*/ zero, surfaceData.Albedo,
+                return LightweightFragmentPBR(positionWS, normalWS, viewDirWS, fogFactor, /*indirectDiffuse*/ zero, /*vertex lighting*/ zero, surfaceData.Albedo,
                     surfaceData.Metallic, /* specularColor */ zero, surfaceData.Smoothness, surfaceData.Occlusion, surfaceData.Emission, surfaceData.Alpha);
             }
             ENDCG

ScriptableRenderPipeline/LightweightPipeline/Shaders/LightweightStandardTerrain.shader

                 half3 normalWS = normalize(IN.normal);
 #endif
 
+                half3 indirectDiffuse = half3(0, 0, 0);
-                half3 diffuseGI = SampleLightmap(lightmapUV, normalWS);
-#else
-                half3 diffuseGI = EvaluateSHPerPixel(normalWS);
-#endif
-
-#if _VERTEX_LIGHTS
-                half3 diffuse = surfaceData.albedo * OneMinusReflectivityMetallic(surfaceData.metallic);
-                diffuseGI += (diffuse * IN.fogFactorAndVertexLight.yzw);
+                half3 indirectDiffuse = SampleLightmap(lightmapUV, normalWS);
-                half4 color = LightweightFragmentPBR(IN.positionWS, normalWS, viewDirectionWS, fogFactor, diffuseGI, albedo, metallic, specular, smoothness, /* occlusion */ 1.0, half3(0, 0, 0), alpha);
+                half4 color = LightweightFragmentPBR(IN.positionWS, normalWS, viewDirectionWS, fogFactor, indirectDiffuse, 
+                    IN.fogFactorAndVertexLight.yzw, albedo, metallic, specular, smoothness, /* occlusion */ 1.0, /* emission */ half3(0, 0, 0), alpha);
                 return color;
             }
             ENDCG