mesa: fix race conditions for CI's backend test (#9943)

- *Enable ASAN/UBSAN for CI's backend test*
 - Make local fix to mesa and embed patch in get-mesa.sh
 - Disable int128 instrinsics for abseil when asan+arm
 - Fix race conditions in PlatformOSMesa
 - Fix race in OpenGLDriver::terminate()
 - Delete mPlatform in BackendTest

build.sh

@@ -3,7 +3,7 @@ set -e
 
 # Host tools required by Android, WASM, and iOS builds
 MOBILE_HOST_TOOLS="matc resgen cmgen filamesh uberz"
-WEB_HOST_TOOLS="${MOBILE_HOST_TOOLS} mipgen filamesh"
+WEB_HOST_TOOLS="${MOBILE_HOST_TOOLS} mipgen filamesh glslminifier"
 
 function print_help {
     local self_name=$(basename "$0")

build/common/get-mesa.sh

@@ -134,7 +134,23 @@ pushd .
 cd ${MESA_DIR}
 
 # Need >= 24 to have llvmpipe for swrast.  llvmpipe is needed for GL >= 4.1.
-git checkout mesa-${MESA_VERSION}
+git checkout -f mesa-${MESA_VERSION}
+
+# Apply custom patch to fix a double-free in OSMesa
+git apply << 'EOF'
+diff --git a/src/mesa/program/program.c b/src/mesa/program/program.c
+index 74bd6a6c33b..a70814e53a1 100644
+--- a/src/mesa/program/program.c
++++ b/src/mesa/program/program.c
+@@ -130,6 +130,7 @@ _mesa_free_program_data(struct gl_context *ctx)
+       ctx->ATIFragmentShader.Current->RefCount--;
+       if (ctx->ATIFragmentShader.Current->RefCount <= 0) {
+          free(ctx->ATIFragmentShader.Current);
++         ctx->ATIFragmentShader.Current = NULL;
+       }
+    }
+ 
+EOF
 
 mkdir -p out
 

build/common/upload-release-assets/package-lock.json

@@ -13,12 +13,20 @@
       }
     },
     "node_modules/@actions/core": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
-      "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz",
+      "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==",
       "dependencies": {
-        "@actions/http-client": "^2.0.1",
-        "uuid": "^8.3.2"
+        "@actions/exec": "^1.1.1",
+        "@actions/http-client": "^2.0.1"
+      }
+    },
+    "node_modules/@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "dependencies": {
+        "@actions/io": "^1.0.1"
       }
     },
     "node_modules/@actions/glob": {
@@ -38,6 +46,11 @@
         "tunnel": "^0.0.6"
       }
     },
+    "node_modules/@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
     "node_modules/@octokit/auth-token": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-5.1.2.tgz",
@@ -257,24 +270,24 @@
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz",
       "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==",
       "license": "ISC"
-    },
-    "node_modules/uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
     }
   },
   "dependencies": {
     "@actions/core": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
-      "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz",
+      "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==",
       "requires": {
-        "@actions/http-client": "^2.0.1",
-        "uuid": "^8.3.2"
+        "@actions/exec": "^1.1.1",
+        "@actions/http-client": "^2.0.1"
+      }
+    },
+    "@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "requires": {
+        "@actions/io": "^1.0.1"
       }
     },
     "@actions/glob": {
@@ -294,6 +307,11 @@
         "tunnel": "^0.0.6"
       }
     },
+    "@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
     "@octokit/auth-token": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-5.1.2.tgz",
@@ -444,11 +462,6 @@
       "version": "7.0.2",
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz",
       "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q=="
-    },
-    "uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
     }
   }
 }

filament/backend/include/backend/platforms/PlatformOSMesa.h

@@ -75,7 +75,8 @@ private:
         OSMesaContext context;
         std::unique_ptr<uint8_t[]> buffer;
     };
-    std::unordered_map<std::thread::id, ContextInfo> mAdditionalContexts;
+    using ContextMap = std::unordered_map<std::thread::id, ContextInfo>;
+    ContextMap mAdditionalContexts;
     mutable std::shared_mutex mAdditionalContextsLock;
 };
 

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -403,6 +403,9 @@ void OpenGLDriver::terminate() {
     if (getJobWorker()) {
         getJobWorker()->terminate();
     }
+    // wait for the GPU again because JobWorker might have queued more work.
+    glFinish();
+
     if constexpr (UTILS_HAS_THREADING) {
         stopServiceThread();
     }

filament/backend/src/opengl/platforms/PlatformOSMesa.cpp

@@ -20,6 +20,7 @@
 #include <utils/Panic.h>
 #include <utils/ThreadUtils.h>
 
+#include <algorithm>
 #include <dlfcn.h>
 #include <memory>
 
@@ -203,6 +204,7 @@ void PlatformOSMesa::createContext(bool shared) {
 void PlatformOSMesa::releaseContext() noexcept {
     std::thread::id currentThreadId = utils::ThreadUtils::getThreadId();
     OSMesaContext context = nullptr;
+    std::unique_ptr<uint8_t[]> buffer;
 
     {
         std::lock_guard<std::shared_mutex> lock(mAdditionalContextsLock);
@@ -212,16 +214,15 @@ void PlatformOSMesa::releaseContext() noexcept {
             return;
         }
         context = it->second.context;
+        buffer = std::move(it->second.buffer);
+        mAdditionalContexts.erase(it);
     }
 
     OSMesaAPI* api = (OSMesaAPI*) mOsMesaApi;
-    // Passing NULL as the context is the standard way to unbind in OSMesa.
-    api->fOSMesaMakeCurrent(NULL, NULL, 0, 0, 0);
-    api->fOSMesaDestroyContext(context);
-
-    {
-        std::lock_guard<std::shared_mutex> lock(mAdditionalContextsLock);
-        mAdditionalContexts.erase(currentThreadId);
+    if (api) {
+        // Passing NULL as the context is the standard way to unbind in OSMesa.
+        api->fOSMesaMakeCurrent(NULL, NULL, 0, 0, 0);
+        api->fOSMesaDestroyContext(context);
     }
 }
 

filament/backend/test/BackendTest.cpp

@@ -83,6 +83,8 @@ BackendTest::~BackendTest() {
     driver->terminate();
     delete driver;
     recordFailedImages();
+
+    delete mPlatform;
 }
 
 void BackendTest::initializeDriver() {

filament/src/MaterialDefinition.cpp

@@ -574,13 +574,15 @@ void MaterialDefinition::processPushConstants() {
             [&](MaterialPushConstant const& constant) {
                 snprintf(buf, sizeof(buf), "%s.%s", structVarName.c_str(), constant.name.c_str());
 
+                CString const cs(buf, strlen(buf));
+
                 switch (constant.stage) {
                     case ShaderStage::VERTEX:
-                        vertexConstants.push_back({CString(buf), constant.type});
+                        vertexConstants.push_back({cs, constant.type});
                         vertexCount++;
                         break;
                     case ShaderStage::FRAGMENT:
-                        fragmentConstants.push_back({CString(buf), constant.type});
+                        fragmentConstants.push_back({cs, constant.type});
                         fragmentCount++;
                         break;
                     case ShaderStage::COMPUTE:

test/backend/test.sh

@@ -14,6 +14,7 @@ if [ ! -d "${PROJECT_ROOT_DIR}/mesa/out" ]; then
 fi
 
 BACKEND_TEST_TARGET=''
+ASAN_FLAG=''
 
 # Set environment variables to use Mesa drivers.
 os_name=$(uname -s)
@@ -27,15 +28,18 @@ if [[ "$os_name" == "Linux" ]]; then
         export VK_ICD_FILENAMES="${PROJECT_ROOT_DIR}/mesa/out/share/vulkan/icd.d/lvp_icd.x86_64.json"
     fi
     BACKEND_TEST_TARGET=backend_test_linux
+    ASAN_FLAG="-b"
 elif [[ "$os_name" == "Darwin" ]]; then
     export DYLD_LIBRARY_PATH="${PROJECT_ROOT_DIR}/mesa/out/lib"
     export VK_ICD_FILENAMES="${PROJECT_ROOT_DIR}/mesa/out/share/vulkan/icd.d/lvp_icd.aarch64.json"
     BACKEND_TEST_TARGET=backend_test_mac
+    # asan is too slow for macOs build of the backend test
+    ASAN_FLAG=""
 fi
 
-# Build backend_test_mac
-echo "Building backend_test_mac..."
-"${PROJECT_ROOT_DIR}/build.sh" -W -p desktop -X "${PROJECT_ROOT_DIR}/mesa" debug ${BACKEND_TEST_TARGET}
+# Build backend test
+echo "Building ${BACKEND_TEST_TARGET}..."
+"${PROJECT_ROOT_DIR}/build.sh" ${ASAN_FLAG} -W -y release -p desktop -X "${PROJECT_ROOT_DIR}/mesa" debug ${BACKEND_TEST_TARGET}
 
 set +e
 
@@ -51,10 +55,20 @@ do
     fi
 done
 
+# Mesa OSMesa is known to leak part of the context.
+LSAN_CMD_PREFIX=""
+if [[ "${ASAN_FLAG}" == "-b" ]]; then
+    echo "leak:PlatformOSMesa.cpp" > leak_skip.txt
+    export LSAN_OPTIONS=suppressions=leak_skip.txt
+fi
+
 FINAL_RESULT=0
 for BACKEND in ${BACKENDS[@]}; do
     echo "----- ${BACKEND} backend test -----"
-    ${PROJECT_ROOT_DIR}/out/cmake-debug/filament/backend/${BACKEND_TEST_TARGET} -a ${BACKEND} --ci --headless_only ${GTEST_FILTER_ARG}
+
+    ${PROJECT_ROOT_DIR}/out/cmake-debug/filament/backend/${BACKEND_TEST_TARGET} \
+                           -a ${BACKEND} --ci --headless_only ${GTEST_FILTER_ARG}
+
     RESULT=$(echo $?)
     if [ ${RESULT} -gt 0 ]; then
         echo "----- Error: backend ${BACKEND} test failed with result ${RESULT} -----"

third_party/abseil/absl/base/config.h

@@ -302,7 +302,19 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' ||
 #if (defined(__clang__) && !defined(_WIN32)) ||           \
     (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \
     (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__))
-#define ABSL_HAVE_INTRINSIC_INT128 1
+
+// Disable int128 instrinsic on arm when asan is enabled
+#if defined(__arm__) || defined(__aarch64__)
+    #if defined(__has_feature) || defined(__SANITIZE_ADDRESS__)
+        #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
+            #define ABSL_ASAN_ON_ARM_DISABLE_INT128 1
+        #endif
+    #endif
+#endif
+#ifndef ABSL_ASAN_ON_ARM_DISABLE_INT128
+    #define ABSL_HAVE_INTRINSIC_INT128 1
+#endif
+
 #elif defined(__CUDACC__)
 // __CUDACC_VER__ is a full version number before CUDA 9, and is defined to a
 // string explaining that it has been removed starting with CUDA 9. We use

third_party/abseil/tnt/README.md

@@ -32,3 +32,13 @@ This folder previously last updated as follows:
     rsync -r abseil_new/ abseil/ --delete --exclude tnt
     rm -rf master.zip abseil_new
     git add abseil ; git status
+
+## Custom Changes
+
+A custom patch has been applied to fix compilation on ARM architectures when AddressSanitizer (ASAN) is enabled (specifically regarding `ABSL_HAVE_INTRINSIC_INT128`).
+
+If you update the `abseil` folder, you may need to re-apply the patch. You can do so by running the following command from the repository root:
+
+```shell
+git apply third_party/abseil/tnt/asan_arm_int128.diff
+```