Fix OpenGL stencil buffer writes (#6023)

RELEASE_NOTES.md

@@ -5,6 +5,8 @@ A new header is inserted each time a *tag* is created.
 
 ## main branch
 
+- engine: Fix stencil buffer writes with OpenGL backend.
+
 ## v1.27.0
 
 - WebGL: reduce max instance count to work around Chrome issues [⚠️ **Recompile Materials**]

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -309,9 +309,16 @@ void OpenGLDriver::setStencilState(StencilState ss) noexcept {
     mRenderPassStencilWrite |= ss.stencilWrite;
 
     // stencil test / operation
+    // GL_STENCIL_TEST must be enabled if we're testing OR writing to the stencil buffer.
     if (UTILS_LIKELY(
             ss.front.stencilFunc == StencilState::StencilFunction::A &&
-            ss.back.stencilFunc == StencilState::StencilFunction::A)) {
+            ss.back.stencilFunc == StencilState::StencilFunction::A &&
+            ss.front.stencilOpDepthFail == StencilOperation::KEEP &&
+            ss.back.stencilOpDepthFail == StencilOperation::KEEP &&
+            ss.front.stencilOpStencilFail == StencilOperation::KEEP &&
+            ss.back.stencilOpStencilFail == StencilOperation::KEEP &&
+            ss.front.stencilOpDepthStencilPass == StencilOperation::KEEP &&
+            ss.back.stencilOpDepthStencilPass == StencilOperation::KEEP)) {
         // that's equivalent to having the stencil test disabled
         gl.disable(GL_STENCIL_TEST);
     } else {