From e1dfea0f121f3ee0e552fc010f0dde5ed9c7e783 Mon Sep 17 00:00:00 2001
From: Romain Guy <romainguy@curious-creature.com>
Date: Wed, 27 Sep 2023 21:36:28 -0700
Subject: [PATCH] Fix masked materials (#7215)

---
 NEW_RELEASE_NOTES.md         |  1 +
 shaders/src/main.fs          | 15 +++++++++++++++
 shaders/src/shading_lit.fs   | 17 -----------------
 shaders/src/shading_unlit.fs | 28 ++++++++++------------------
 4 files changed, 26 insertions(+), 35 deletions(-)

diff --git a/NEW_RELEASE_NOTES.md b/NEW_RELEASE_NOTES.md
index 4a1a9c7fa7..82b852576d 100644
--- a/NEW_RELEASE_NOTES.md
+++ b/NEW_RELEASE_NOTES.md
@@ -7,3 +7,4 @@ for next branch cut* header.
 appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md).
 
 ## Release notes for next branch cut
+- materials: fix alpha masked materials when MSAA is turned on [⚠️ **Recompile materials**]
diff --git a/shaders/src/main.fs b/shaders/src/main.fs
index 62ac9d65a7..07cf5bd24a 100644
--- a/shaders/src/main.fs
+++ b/shaders/src/main.fs
@@ -20,6 +20,19 @@ void blendPostLightingColor(const MaterialInputs material, inout vec4 color) {
 }
 #endif
 
+#if defined(BLEND_MODE_MASKED)
+void applyAlphaMask(inout vec4 baseColor) {
+    // Use derivatives to sharpen alpha tested edges, combined with alpha to
+    // coverage to smooth the result
+    baseColor.a = (baseColor.a - getMaskThreshold()) / max(fwidth(baseColor.a), 1e-3) + 0.5;
+    if (baseColor.a <= getMaskThreshold()) {
+        discard;
+    }
+}
+#else
+void applyAlphaMask(inout vec4 baseColor) {}
+#endif
+
 void main() {
     filament_lodBias = frameUniforms.lodBias;
 #if defined(FILAMENT_HAS_FEATURE_INSTANCING)
@@ -39,6 +52,8 @@ void main() {
     // Invoke user code
     material(inputs);
 
+    applyAlphaMask(inputs.baseColor);
+
     fragColor = evaluateMaterial(inputs);
 
 #if defined(MATERIAL_HAS_POST_LIGHTING_COLOR) && !defined(MATERIAL_HAS_REFLECTIONS)
diff --git a/shaders/src/shading_lit.fs b/shaders/src/shading_lit.fs
index 72b2b9ec33..a0d7081df8 100644
--- a/shaders/src/shading_lit.fs
+++ b/shaders/src/shading_lit.fs
@@ -3,11 +3,6 @@
 //------------------------------------------------------------------------------
 
 #if defined(BLEND_MODE_MASKED)
-float computeMaskedAlpha(float a) {
-    // Use derivatives to smooth alpha tested edges
-    return (a - getMaskThreshold()) / max(fwidth(a), 1e-3) + 0.5;
-}
-
 float computeDiffuseAlpha(float a) {
     // If we reach this point in the code, we already know that the fragment is not discarded due
     // to the threshold factor. Therefore we can just output 1.0, which prevents a "punch through"
@@ -15,14 +10,6 @@ float computeDiffuseAlpha(float a) {
     // of ALPHA_TO_COVERAGE.
     return (frameUniforms.needsAlphaChannel == 1.0) ? 1.0 : a;
 }
-
-void applyAlphaMask(inout vec4 baseColor) {
-    baseColor.a = computeMaskedAlpha(baseColor.a);
-    if (baseColor.a <= 0.0) {
-        discard;
-    }
-}
-
 #else // not masked
 
 float computeDiffuseAlpha(float a) {
@@ -32,9 +19,6 @@ float computeDiffuseAlpha(float a) {
     return 1.0;
 #endif
 }
-
-void applyAlphaMask(inout vec4 baseColor) {}
-
 #endif
 
 #if defined(GEOMETRIC_SPECULAR_AA)
@@ -65,7 +49,6 @@ float normalFiltering(float perceptualRoughness, const vec3 worldNormal) {
 
 void getCommonPixelParams(const MaterialInputs material, inout PixelParams pixel) {
     vec4 baseColor = material.baseColor;
-    applyAlphaMask(baseColor);
 
 #if defined(BLEND_MODE_FADE) && !defined(SHADING_MODEL_UNLIT)
     // Since we work in premultiplied alpha mode, we need to un-premultiply
diff --git a/shaders/src/shading_unlit.fs b/shaders/src/shading_unlit.fs
index a98cf60203..5202041609 100644
--- a/shaders/src/shading_unlit.fs
+++ b/shaders/src/shading_unlit.fs
@@ -9,12 +9,17 @@ void addEmissive(const MaterialInputs material, inout vec4 color) {
 #endif
 }
 
+vec4 fixupAlpha(vec4 color) {
 #if defined(BLEND_MODE_MASKED)
-float computeMaskedAlpha(float a) {
-    // Use derivatives to smooth alpha tested edges
-    return (a - getMaskThreshold()) / max(fwidth(a), 1e-3) + 0.5;
-}
+    // If we reach this point in the code, we already know that the fragment is not discarded due
+    // to the threshold factor. Therefore we can just output 1.0, which prevents a "punch through"
+    // effect from occuring. We do this only for TRANSLUCENT views in order to prevent breakage
+    // of ALPHA_TO_COVERAGE.
+    return vec4(color.rgb, (frameUniforms.needsAlphaChannel == 1.0) ? 1.0 : color.a);
+#else
+    return color;
 #endif
+}
 
 /**
  * Evaluates unlit materials. In this lighting model, only the base color and
@@ -33,19 +38,6 @@ float computeMaskedAlpha(float a) {
 vec4 evaluateMaterial(const MaterialInputs material) {
     vec4 color = material.baseColor;
 
-#if defined(BLEND_MODE_MASKED)
-    color.a = computeMaskedAlpha(color.a);
-    if (color.a <= 0.0) {
-        discard;
-    }
-
-    // Output 1.0 for translucent view to prevent "punch through" artifacts. We do not do this
-    // for opaque views to enable proper usage of ALPHA_TO_COVERAGE.
-    if (frameUniforms.needsAlphaChannel == 1.0) {
-        color.a = 1.0;
-    }
-#endif
-
 #if defined(VARIANT_HAS_DIRECTIONAL_LIGHTING)
 #if defined(VARIANT_HAS_SHADOWING)
     float visibility = 1.0;
@@ -71,5 +63,5 @@ vec4 evaluateMaterial(const MaterialInputs material) {
 
     addEmissive(material, color);
 
-    return color;
+    return fixupAlpha(color);
 }