Merge branch 'rc/1.56.4' into release

If the shared osmesa lib is not found, then we assume that the
library has been compile-time linked via the linker.  This is to
accommodate build frameworks where compile-time linking is
preferred.

.github/actions/android-continuous/action.yml

@@ -0,0 +1,17 @@
+name: 'Android Continuous'
+inputs:
+  build-abi:
+    description: 'The target platform ABI'
+    required: true
+    default: 'armeabi-v7a'
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '17'
+    - name: Run build script
+      run: |
+        cd build/android && printf "y" | ./build.sh continuous ${{ inputs.build-abi }}
+      shell: bash

.github/actions/ubuntu-apt-add-src/action.yml

@@ -0,0 +1,9 @@
+name: 'ubuntu apt add deb-src'
+runs:
+  using: "composite"
+  steps:
+    - name: "ubuntu apt add deb-src"
+      run: |
+          echo "deb-src http://archive.ubuntu.com/ubuntu jammy main restricted universe" | sudo tee /etc/apt/sources.list.d/my.list
+          sudo apt-get update
+      shell: bash

.github/workflows/android-continuous.yml

@@ -10,30 +10,13 @@ on:
 jobs:
   build-android:
     name: build-android
-    runs-on: macos-14
+    # We intentially use a larger runner here to enable larger disk space
+    # (standard linux runner will fail on disk space and faster build time).
+    runs-on: ubuntu-22.04-32core
 
     steps:
-      - uses: actions/checkout@v3.3.0
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4.1.6
+      - name: Run Android Continuous
+        uses: ./.github/actions/android-continuous
         with:
-          distribution: 'temurin'
-          java-version: '17'
-      - name: Run build script
-        run: |
-          cd build/android && printf "y" | ./build.sh continuous
-      - uses: actions/upload-artifact@v1.0.0
-        with:
-          name: filament-android
-          path: out/filament-android-release.aar
-      - uses: actions/upload-artifact@v1.0.0
-        with:
-          name: filamat-android-full
-          path: out/filamat-android-release.aar
-      - uses: actions/upload-artifact@v1.0.0
-        with:
-          name: gltfio-android-release
-          path: out/gltfio-android-release.aar
-      - uses: actions/upload-artifact@v1.0.0
-        with:
-          name: filament-utils-android-release
-          path: out/filament-utils-android-release.aar
+          build-abi: armeabi-v7a,arm64-v8a,x86_64

.github/workflows/ios-continuous.yml

@@ -10,14 +10,14 @@ on:
 jobs:
   build-ios:
     name: build-ios
-    runs-on: macos-14
+    runs-on: macos-14-xlarge
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/ios && printf "y" | ./build.sh continuous
-      - uses: actions/upload-artifact@v1.0.0
+      - uses: actions/upload-artifact@v4
         with:
           name: filament-ios
           path: out/filament-release-ios.tgz

.github/workflows/linux-continuous.yml

@@ -10,14 +10,14 @@ on:
 jobs:
   build-linux:
     name: build-linux
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-22.04-16core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/linux && printf "y" | ./build.sh continuous
-      - uses: actions/upload-artifact@v1.0.0
+      - uses: actions/upload-artifact@v4
         with:
           name: filament-linux
           path: out/filament-release-linux.tgz

.github/workflows/mac-continuous.yml

@@ -10,14 +10,14 @@ on:
 jobs:
   build-mac:
     name: build-mac
-    runs-on: macos-14
+    runs-on: macos-14-xlarge
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/mac && printf "y" | ./build.sh continuous
-      - uses: actions/upload-artifact@v1.0.0
+      - uses: actions/upload-artifact@v4
         with:
           name: filament-mac
           path: out/filament-release-darwin.tgz

.github/workflows/npm-deploy.yml

@@ -13,7 +13,7 @@ jobs:
     name: npm-deploy
     runs-on: macos-14
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ github.event.inputs.release_tag }}
       # Setup .npmrc file to publish to npm

.github/workflows/presubmit.yml

@@ -15,10 +15,10 @@ jobs:
 
     strategy:
       matrix:
-        os: [macos-14, ubuntu-22.04]
+        os: [macos-14-xlarge, ubuntu-22.04-16core]
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           WORKFLOW_OS=`echo \`uname\` | sed "s/Darwin/mac/" | tr [:upper:] [:lower:]`
@@ -29,10 +29,10 @@ jobs:
 
   build-windows:
     name: build-windows
-    runs-on: windows-2019
+    runs-on: win-2019-16core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           build\windows\build-github.bat presubmit
@@ -40,24 +40,26 @@ jobs:
 
   build-android:
     name: build-android
-    runs-on: macos-14
+    runs-on: ubuntu-22.04-16core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
           java-version: '17'
       - name: Run build script
+        # Only build 1 64 bit target during presubmit to cut down build times during presubmit
+        # Continuous builds will build everything
         run: |
-          cd build/android && printf "y" | ./build.sh presubmit
+          cd build/android && printf "y" | ./build.sh presubmit arm64-v8a
 
   build-ios:
     name: build-iOS
-    runs-on: macos-14
+    runs-on: macos-14-xlarge
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/ios && printf "y" | ./build.sh presubmit
@@ -67,10 +69,25 @@ jobs:
 
   build-web:
     name: build-web
-    runs-on: macos-14
+    runs-on: ubuntu-22.04-16core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/web && printf "y" | ./build.sh presubmit
+
+  test-renderdiff:
+    name: test-renderdiff
+    runs-on: ubuntu-22.04-32core
+
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: ./.github/actions/ubuntu-apt-add-src
+      - name: Run script
+        run: |
+          source ./build/linux/ci-common.sh && bash test/renderdiff_tests.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: presubmit-renderdiff-result
+          path: ./out/renderdiff_tests

.github/workflows/release.yml

@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [macos-14, ubuntu-22.04]
+        os: [macos-14-xlarge, ubuntu-22.04-32core]
 
     steps:
       - name: Decide Git ref
@@ -41,7 +41,7 @@ jobs:
           TAG=${REF##*/}
           echo "ref=${REF}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ steps.git_ref.outputs.ref }}
       - name: Run build script
@@ -65,7 +65,7 @@ jobs:
 
   build-web:
     name: build-web
-    runs-on: macos-14
+    runs-on: ubuntu-22.04-16core
     if: github.event_name == 'release' || github.event.inputs.platform == 'web'
 
     steps:
@@ -76,7 +76,7 @@ jobs:
           TAG=${REF##*/}
           echo "ref=${REF}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ steps.git_ref.outputs.ref }}
       - name: Run build script
@@ -98,7 +98,7 @@ jobs:
 
   build-android:
     name: build-android
-    runs-on: macos-14
+    runs-on: ubuntu-22.04-16core
     if: github.event_name == 'release' || github.event.inputs.platform == 'android'
 
     steps:
@@ -109,7 +109,7 @@ jobs:
           TAG=${REF##*/}
           echo "ref=${REF}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ steps.git_ref.outputs.ref }}
       - uses: actions/setup-java@v3
@@ -120,7 +120,7 @@ jobs:
         env:
           TAG: ${{ steps.git_ref.outputs.tag }}
         run: |
-          cd build/android && printf "y" | ./build.sh release
+          cd build/android && printf "y" | ./build.sh release armeabi-v7a,arm64-v8a,x86,x86_64
           cd ../..
           mv out/filament-android-release.aar out/filament-${TAG}-android.aar
           mv out/filamat-android-release.aar out/filamat-${TAG}-android.aar
@@ -152,7 +152,7 @@ jobs:
 
   build-ios:
     name: build-ios
-    runs-on: macos-14
+    runs-on: macos-14-xlarge
     if: github.event_name == 'release' || github.event.inputs.platform == 'ios'
 
     steps:
@@ -163,7 +163,7 @@ jobs:
           TAG=${REF##*/}
           echo "ref=${REF}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ steps.git_ref.outputs.ref }}
       - name: Run build script
@@ -185,7 +185,7 @@ jobs:
 
   build-windows:
     name: build-windows
-    runs-on: windows-2019
+    runs-on: windows-2019-32core
     if: github.event_name == 'release' || github.event.inputs.platform == 'windows'
 
     steps:
@@ -197,7 +197,7 @@ jobs:
           echo "ref=${REF}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
         shell: bash
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
         with:
           ref: ${{ steps.git_ref.outputs.ref }}
       - name: Run build script
@@ -205,6 +205,7 @@ jobs:
           TAG: ${{ steps.git_ref.outputs.tag }}
         run: |
           build\windows\build-github.bat release
+          echo on
           move out\filament-windows.tgz out\filament-%TAG%-windows.tgz
         shell: cmd
       - uses: actions/github-script@v6

.github/workflows/web-continuous.yml

@@ -10,14 +10,14 @@ on:
 jobs:
   build-web:
     name: build-web
-    runs-on: macos-14
+    runs-on: ubuntu-22.04-16core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           cd build/web && printf "y" | ./build.sh continuous
-      - uses: actions/upload-artifact@v1.0.0
+      - uses: actions/upload-artifact@v4
         with:
           name: filament-web
           path: out/filament-release-web.tgz

.github/workflows/windows-continuous.yml

@@ -10,15 +10,15 @@ on:
 jobs:
   build-windows:
     name: build-windows
-    runs-on: windows-2019
+    runs-on: windows-2019-32core
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4.1.6
       - name: Run build script
         run: |
           build\windows\build-github.bat continuous
         shell: cmd
-      - uses: actions/upload-artifact@v1.0.0
+      - uses: actions/upload-artifact@v4
         with:
           name: filament-windows
           path: out/filament-windows.tgz

CMakeLists.txt

@@ -45,6 +45,8 @@ option(FILAMENT_ENABLE_FEATURE_LEVEL_0 "Enable Feature Level 0" ON)
 
 option(FILAMENT_ENABLE_MULTIVIEW "Enable multiview for Filament" OFF)
 
+option(FILAMENT_SUPPORTS_OSMESA "Enable OSMesa (headless GL context) for Filament" OFF)
+
 set(FILAMENT_NDK_VERSION "" CACHE STRING
     "Android NDK version or version prefix to be used when building for Android."
 )
@@ -73,6 +75,10 @@ set(FILAMENT_BACKEND_DEBUG_FLAG "" CACHE STRING
     "A debug flag meant for enabling/disabling backend debugging paths"
 )
 
+set(FILAMENT_OSMESA_PATH "" CACHE STRING
+    "Path to the OSMesa header and lib"
+)
+
 # Enable exceptions by default in spirv-cross.
 set(SPIRV_CROSS_EXCEPTIONS_TO_ASSERTIONS OFF)
 
@@ -132,12 +138,22 @@ else()
 endif()
 
 if (LINUX)
+    if (NOT FILAMENT_OSMESA_PATH STREQUAL "")
+        if (NOT EXISTS ${FILAMENT_OSMESA_PATH}/)
+            message(FATAL_ERROR "Cannot find specified OSMesa build directory: ${FILAMENT_OSMESA_PATH}")
+        endif()
+        set(FILAMENT_SUPPORTS_OSMESA TRUE)
+    endif()
+
     if (FILAMENT_SUPPORTS_WAYLAND)
         add_definitions(-DFILAMENT_SUPPORTS_WAYLAND)
         set(FILAMENT_SUPPORTS_X11 FALSE)
     elseif (FILAMENT_SUPPORTS_EGL_ON_LINUX)
         add_definitions(-DFILAMENT_SUPPORTS_EGL_ON_LINUX)
         set(FILAMENT_SUPPORTS_X11 FALSE)
+    elseif (FILAMENT_SUPPORTS_OSMESA)
+        set(FILAMENT_SUPPORTS_X11 FALSE)
+        add_definitions(-DFILAMENT_SUPPORTS_OSMESA)
     else ()
         if (FILAMENT_SUPPORTS_XCB)
             add_definitions(-DFILAMENT_SUPPORTS_XCB)

README.md

@@ -31,7 +31,7 @@ repositories {
 }
 
 dependencies {
-    implementation 'com.google.android.filament:filament-android:1.52.3'
+    implementation 'com.google.android.filament:filament-android:1.56.4'
 }
 ```
 
@@ -51,19 +51,9 @@ Here are all the libraries available in the group `com.google.android.filament`:
 iOS projects can use CocoaPods to install the latest release:
 
 ```shell
-pod 'Filament', '~> 1.52.3'
+pod 'Filament', '~> 1.56.4'
 ```
 
-### Snapshots
-
-If you prefer to live on the edge, you can download a continuous build by following the following
-steps:
-
-1. Find the [commit](https://github.com/google/filament/commits/main) you're interested in.
-2. Click the green check mark under the commit message.
-3. Click on the _Details_ link for the platform you're interested in.
-4. On the top left click _Summary_, then in the _Artifacts_ section choose the desired artifact.
-
 ## Documentation
 
 - [Filament](https://google.github.io/filament/Filament.html), an in-depth explanation of
@@ -176,6 +166,7 @@ steps:
   - [x] KHR_materials_unlit
   - [x] KHR_materials_variants
   - [x] KHR_materials_volume
+  - [x] KHR_materials_specular
   - [x] KHR_mesh_quantization
   - [x] KHR_texture_basisu
   - [x] KHR_texture_transform

RELEASE_NOTES.md

@@ -7,6 +7,72 @@ A new header is inserted each time a *tag* is created.
 Instead, if you are authoring a PR for the main branch, add your release note to
 [NEW_RELEASE_NOTES.md](./NEW_RELEASE_NOTES.md).
 
+## v1.56.4
+
+
+## v1.56.3
+
+
+## v1.56.2
+
+- vk: fix stage pool gc logic
+
+## v1.56.1
+
+## v1.56.0
+
+- backend: descriptor layouts distinguish samplers and external samplers (b/376089915) [⚠️ **New Material Version**]
+
+## v1.55.1
+
+
+## v1.55.0
+- Add descriptor sets to describe shader resources. [⚠️ **New Material Version**]
+
+## v1.54.5
+
+
+## v1.54.4
+
+- Add support for multi-layered render target with array textures.
+
+## v1.54.3
+
+
+## v1.54.2
+
+- Add a `name` API to Filament objects for debugging handle use-after-free assertions
+
+## v1.54.1
+
+
+## v1.54.0
+
+- materials: add a new `stereoscopicType` material parameter. [⚠️ **New Material Version**]
+- Fix a crash when compiling shaders on IMG devices
+
+## v1.53.5
+
+- engine: Fix bug causing certain sampler parameters to not be applied correctly in GLES 2.0 and on
+  certain GLES 3.0 drivers.
+
+## v1.53.4
+
+
+## v1.53.3
+
+- Add drag and drop support for IBL files for desktop gltf_viewer.
+
+## v1.53.2
+
+
+## v1.53.1
+
+
+## v1.53.0
+
+- engine: fix skinning normals with large transforms (b/342459864) [⚠️ **New Material Version**]
+
 ## v1.52.3
 
 
@@ -604,7 +670,7 @@ Instead, if you are authoring a PR for the main branch, add your release note to
 
 - engine: Binary size improvements.
 - engine: Add basic support for instanced renderables [**NEW API**].
-- engine: Fix, first imaged passsed to `Stream::SetAcquiredImage` is ignored and leaked.
+- engine: Fix, first imaged passed to `Stream::SetAcquiredImage` is ignored and leaked.
 - Vulkan: Robustness improvements.
 - Java: Fix, lookAt z axis negated.
 - gltfio: Be graceful when model has > 4 weights per vert.

android/build.gradle

@@ -83,12 +83,12 @@ buildscript {
         'minSdk': 21,
         'targetSdk': 34,
         'compileSdk': 34,
-        'kotlin': '1.9.21',
-        'kotlin_coroutines': '1.7.3',
-        'buildTools': '34.0.0',
-        'ndk': '26.1.10909125',
-        'androidx_core': '1.12.0',
-        'androidx_annotations': '1.7.0'
+        'kotlin': '2.0.21',
+        'kotlin_coroutines': '1.9.0',
+        'buildTools': '35.0.0',
+        'ndk': '27.0.11718014',
+        'androidx_core': '1.13.1',
+        'androidx_annotations': '1.9.0'
     ]
 
     ext.deps = [
@@ -104,7 +104,7 @@ buildscript {
     ]
 
     dependencies {
-        classpath 'com.android.tools.build:gradle:8.2.0'
+        classpath 'com.android.tools.build:gradle:8.6.1'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${versions.kotlin}"
     }
 

android/filament-android/src/main/cpp/Engine.cpp

@@ -420,6 +420,13 @@ Java_com_google_android_filament_Engine_nSetPaused(JNIEnv*, jclass,
     engine->setPaused(paused);
 }
 
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_Engine_nUnprotected(JNIEnv*, jclass,
+        jlong nativeEngine, jboolean paused) {
+    Engine* engine = (Engine*) nativeEngine;
+    engine->unprotected();
+}
+
 // Managers...
 
 extern "C" JNIEXPORT jlong JNICALL
@@ -492,6 +499,37 @@ Java_com_google_android_filament_Engine_nGetActiveFeatureLevel(JNIEnv *, jclass,
     return (jint)engine->getActiveFeatureLevel();
 }
 
+extern "C"
+JNIEXPORT jboolean JNICALL
+Java_com_google_android_filament_Engine_nHasFeatureFlag(JNIEnv *env, jclass clazz,
+        jlong nativeEngine, jstring name_) {
+    Engine* engine = (Engine*) nativeEngine;
+    const char *name = env->GetStringUTFChars(name_, 0);
+    std::optional<bool> result = engine->getFeatureFlag(name);
+    env->ReleaseStringUTFChars(name_, name);
+    return result.has_value();
+}
+extern "C"
+JNIEXPORT jboolean JNICALL
+Java_com_google_android_filament_Engine_nSetFeatureFlag(JNIEnv *env, jclass clazz,
+        jlong nativeEngine, jstring name_, jboolean value) {
+    Engine* engine = (Engine*) nativeEngine;
+    const char *name = env->GetStringUTFChars(name_, 0);
+    jboolean result = engine->setFeatureFlag(name, (bool)value);
+    env->ReleaseStringUTFChars(name_, name);
+    return result;
+}
+extern "C"
+JNIEXPORT jboolean JNICALL
+Java_com_google_android_filament_Engine_nGetFeatureFlag(JNIEnv *env, jclass clazz,
+        jlong nativeEngine, jstring name_) {
+    Engine* engine = (Engine*) nativeEngine;
+    const char *name = env->GetStringUTFChars(name_, 0);
+    std::optional<bool> result = engine->getFeatureFlag(name);
+    env->ReleaseStringUTFChars(name_, name);
+    return result.value_or(false); // we should never fail here
+}
+
 extern "C" JNIEXPORT jlong JNICALL Java_com_google_android_filament_Engine_nCreateBuilder(JNIEnv*,
         jclass) {
     Engine::Builder* builder = new Engine::Builder{};
@@ -513,13 +551,12 @@ extern "C" JNIEXPORT void JNICALL Java_com_google_android_filament_Engine_nSetBu
 extern "C" JNIEXPORT void JNICALL Java_com_google_android_filament_Engine_nSetBuilderConfig(JNIEnv*,
         jclass, jlong nativeBuilder, jlong commandBufferSizeMB, jlong perRenderPassArenaSizeMB,
         jlong driverHandleArenaSizeMB, jlong minCommandBufferSizeMB, jlong perFrameCommandsSizeMB,
-        jlong jobSystemThreadCount,
-        jlong textureUseAfterFreePoolSize, jboolean disableParallelShaderCompile,
+        jlong jobSystemThreadCount, jboolean disableParallelShaderCompile,
         jint stereoscopicType, jlong stereoscopicEyeCount,
         jlong resourceAllocatorCacheSizeMB, jlong resourceAllocatorCacheMaxAge,
         jboolean disableHandleUseAfterFreeCheck,
         jint preferredShaderLanguage,
-        jboolean forceGLES2Context) {
+        jboolean forceGLES2Context, jboolean assertNativeWindowIsValid) {
     Engine::Builder* builder = (Engine::Builder*) nativeBuilder;
     Engine::Config config = {
             .commandBufferSizeMB = (uint32_t) commandBufferSizeMB,
@@ -528,7 +565,6 @@ extern "C" JNIEXPORT void JNICALL Java_com_google_android_filament_Engine_nSetBu
             .minCommandBufferSizeMB = (uint32_t) minCommandBufferSizeMB,
             .perFrameCommandsSizeMB = (uint32_t) perFrameCommandsSizeMB,
             .jobSystemThreadCount = (uint32_t) jobSystemThreadCount,
-            .textureUseAfterFreePoolSize = (uint32_t) textureUseAfterFreePoolSize,
             .disableParallelShaderCompile = (bool) disableParallelShaderCompile,
             .stereoscopicType = (Engine::StereoscopicType) stereoscopicType,
             .stereoscopicEyeCount = (uint8_t) stereoscopicEyeCount,
@@ -537,6 +573,7 @@ extern "C" JNIEXPORT void JNICALL Java_com_google_android_filament_Engine_nSetBu
             .disableHandleUseAfterFreeCheck = (bool) disableHandleUseAfterFreeCheck,
             .preferredShaderLanguage = (Engine::Config::ShaderLanguage) preferredShaderLanguage,
             .forceGLES2Context = (bool) forceGLES2Context,
+            .assertNativeWindowIsValid = (bool) assertNativeWindowIsValid,
     };
     builder->config(&config);
 }
@@ -559,8 +596,24 @@ extern "C" JNIEXPORT void JNICALL Java_com_google_android_filament_Engine_nSetBu
     builder->paused((bool) paused);
 }
 
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_google_android_filament_Engine_nSetBuilderFeature(JNIEnv *env, jclass clazz,
+        jlong nativeBuilder, jstring name_, jboolean value) {
+    Engine::Builder* builder = (Engine::Builder*) nativeBuilder;
+    const char *name = env->GetStringUTFChars(name_, 0);
+    builder->feature(name, (bool)value);
+    env->ReleaseStringUTFChars(name_, name);
+}
+
 extern "C" JNIEXPORT jlong JNICALL
 Java_com_google_android_filament_Engine_nBuilderBuild(JNIEnv*, jclass, jlong nativeBuilder) {
     Engine::Builder* builder = (Engine::Builder*) nativeBuilder;
     return (jlong) builder->build();
 }
+
+extern "C"
+JNIEXPORT jlong JNICALL
+Java_com_google_android_filament_Engine_getSteadyClockTimeNano(JNIEnv *env, jclass clazz) {
+    return (jlong)Engine::getSteadyClockTimeNano();
+}

android/filament-android/src/main/cpp/EntityManager.cpp

@@ -38,7 +38,7 @@ Java_com_google_android_filament_EntityManager_nCreateArray(JNIEnv* env, jclass,
     // (which it is), but still.
     em->create((size_t) n, reinterpret_cast<Entity *>(entities));
 
-    env->ReleaseIntArrayElements(entities_, entities, JNI_ABORT);
+    env->ReleaseIntArrayElements(entities_, entities, 0);
 }
 
 extern "C" JNIEXPORT jint JNICALL

android/filament-android/src/main/cpp/IndirectLight.cpp

@@ -167,7 +167,7 @@ Java_com_google_android_filament_IndirectLight_nGetDirectionEstimateStatic(JNIEn
     jfloat *outDirection = env->GetFloatArrayElements(outDirection_, NULL);
     *reinterpret_cast<filament::math::float3*>(outDirection) = IndirectLight::getDirectionEstimate((filament::math::float3*)sh);
     env->ReleaseFloatArrayElements(outDirection_, outDirection, 0);
-    env->ReleaseFloatArrayElements(sh_, sh, JNI_ABORT);
+    env->ReleaseFloatArrayElements(sh_, sh, 0);
 }
 
 extern "C" JNIEXPORT void JNICALL
@@ -178,5 +178,5 @@ Java_com_google_android_filament_IndirectLight_nGetColorEstimateStatic(JNIEnv *e
     *reinterpret_cast<filament::math::float4*>(outColor) =
             IndirectLight::getColorEstimate((filament::math::float3*)sh, math::float3{x, y, z});
     env->ReleaseFloatArrayElements(outColor_, outColor, 0);
-    env->ReleaseFloatArrayElements(sh_, sh, JNI_ABORT);
+    env->ReleaseFloatArrayElements(sh_, sh, 0);
 }

android/filament-android/src/main/cpp/Material.cpp

@@ -25,12 +25,17 @@ using namespace filament;
 
 extern "C" JNIEXPORT jlong JNICALL
 Java_com_google_android_filament_Material_nBuilderBuild(JNIEnv *env, jclass,
-        jlong nativeEngine, jobject buffer_, jint size) {
+        jlong nativeEngine, jobject buffer_, jint size, jint shBandCount) {
     Engine* engine = (Engine*) nativeEngine;
     AutoBuffer buffer(env, buffer_, size);
-    Material* material = Material::Builder()
+    auto builder = Material::Builder();
+    if (shBandCount) {
+        builder.sphericalHarmonicsBandCount(shBandCount);
+    }
+    Material* material = builder
             .package(buffer.getData(), buffer.getSize())
             .build(*engine);
+
     return (jlong) material;
 }
 

android/filament-android/src/main/cpp/RenderableManager.cpp

@@ -253,16 +253,10 @@ Java_com_google_android_filament_RenderableManager_nBuilderMorphingStandard(JNIE
 }
 
 extern "C" JNIEXPORT void JNICALL
-Java_com_google_android_filament_RenderableManager_nBuilderSetMorphTargetBufferAt(JNIEnv*, jclass,
-        jlong nativeBuilder, int level, int primitiveIndex, jlong nativeMorphTargetBuffer,
-        int offset, int count) {
+Java_com_google_android_filament_RenderableManager_nBuilderSetMorphTargetBufferOffsetAt(JNIEnv*, jclass,
+        jlong nativeBuilder, int level, int primitiveIndex, int offset) {
     RenderableManager::Builder *builder = (RenderableManager::Builder *) nativeBuilder;
-    if (nativeMorphTargetBuffer) {
-        MorphTargetBuffer *morphTargetBuffer = (MorphTargetBuffer *) nativeMorphTargetBuffer;
-        builder->morphing(level, primitiveIndex, morphTargetBuffer, offset, count);
-    } else {
-        builder->morphing(level, primitiveIndex, offset, count);
-    }
+    builder->morphing(level, primitiveIndex, offset);
 }
 
 extern "C" JNIEXPORT void JNICALL
@@ -334,18 +328,12 @@ Java_com_google_android_filament_RenderableManager_nSetMorphWeights(JNIEnv* env,
 }
 
 extern "C" JNIEXPORT void JNICALL
-Java_com_google_android_filament_RenderableManager_nSetMorphTargetBufferAt(JNIEnv*,
+Java_com_google_android_filament_RenderableManager_nSetMorphTargetBufferOffsetAt(JNIEnv*,
         jclass, jlong nativeRenderableManager, jint i, int level, jint primitiveIndex,
-        jlong nativeMorphTargetBuffer, jint offset, jint count) {
+        jlong, jint offset) {
     RenderableManager *rm = (RenderableManager *) nativeRenderableManager;
-    if (nativeMorphTargetBuffer) {
-        MorphTargetBuffer *morphTargetBuffer = (MorphTargetBuffer *) nativeMorphTargetBuffer;
-        rm->setMorphTargetBufferAt((RenderableManager::Instance) i, (uint8_t) level,
-                (size_t) primitiveIndex, morphTargetBuffer, (size_t) offset, (size_t) count);
-    } else {
-        rm->setMorphTargetBufferAt((RenderableManager::Instance) i, (uint8_t) level,
-                (size_t) primitiveIndex, (size_t) offset, (size_t) count);
-    }
+    rm->setMorphTargetBufferOffsetAt((RenderableManager::Instance) i, (uint8_t) level,
+            (size_t) primitiveIndex, (size_t) offset);
 }
 
 extern "C" JNIEXPORT jint JNICALL

android/filament-android/src/main/cpp/Renderer.cpp

@@ -28,6 +28,14 @@
 using namespace filament;
 using namespace backend;
 
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_Renderer_nSkipFrame(JNIEnv *, jclass, jlong nativeRenderer,
+        jlong vsyncSteadyClockTimeNano) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    renderer->skipFrame(uint64_t(vsyncSteadyClockTimeNano));
+}
+
 extern "C" JNIEXPORT jboolean JNICALL
 Java_com_google_android_filament_Renderer_nBeginFrame(JNIEnv *, jclass, jlong nativeRenderer,
         jlong nativeSwapChain, jlong frameTimeNanos) {
@@ -187,3 +195,10 @@ Java_com_google_android_filament_Renderer_nSetPresentationTime(JNIEnv *, jclass
     Renderer *renderer = (Renderer *) nativeRenderer;
     renderer->setPresentationTime(monotonicClockNanos);
 }
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_Renderer_nSetVsyncTime(JNIEnv *, jclass,
+    jlong nativeRenderer, jlong steadyClockTimeNano) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    renderer->setVsyncTime(steadyClockTimeNano);
+}

android/filament-android/src/main/cpp/View.cpp

@@ -222,6 +222,20 @@ Java_com_google_android_filament_View_nIsFrontFaceWindingInverted(JNIEnv*,
     return static_cast<jboolean>(view->isFrontFaceWindingInverted());
 }
 
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_View_nSetTransparentPickingEnabled(JNIEnv*,
+        jclass, jlong nativeView, jboolean enabled) {
+    View* view = (View*) nativeView;
+    view->setTransparentPickingEnabled(enabled);
+}
+
+extern "C" JNIEXPORT jboolean JNICALL
+Java_com_google_android_filament_View_nIsTransparentPickingEnabled(JNIEnv*,
+        jclass, jlong nativeView) {
+    View* view = (View*) nativeView;
+    return static_cast<jboolean>(view->isTransparentPickingEnabled());
+}
+
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_View_nSetAmbientOcclusion(JNIEnv*, jclass, jlong nativeView, jint ordinal) {
     View* view = (View*) nativeView;
@@ -531,3 +545,12 @@ Java_com_google_android_filament_View_nGetFogEntity(JNIEnv *env, jclass clazz,
     View *view = (View *) nativeView;
     return (jint)view->getFogEntity().getId();
 }
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_google_android_filament_View_nClearFrameHistory(JNIEnv *env, jclass clazz,
+        jlong nativeView, jlong nativeEngine) {
+    View *view = (View *) nativeView;
+    Engine *engine = (Engine *) nativeEngine;
+    view->clearFrameHistory(*engine);
+}

android/filament-android/src/main/java/com/google/android/filament/Engine.java

@@ -224,13 +224,12 @@ public class Engine {
             nSetBuilderConfig(mNativeBuilder, config.commandBufferSizeMB,
                     config.perRenderPassArenaSizeMB, config.driverHandleArenaSizeMB,
                     config.minCommandBufferSizeMB, config.perFrameCommandsSizeMB,
-                    config.jobSystemThreadCount,
-                    config.textureUseAfterFreePoolSize, config.disableParallelShaderCompile,
+                    config.jobSystemThreadCount, config.disableParallelShaderCompile,
                     config.stereoscopicType.ordinal(), config.stereoscopicEyeCount,
                     config.resourceAllocatorCacheSizeMB, config.resourceAllocatorCacheMaxAge,
                     config.disableHandleUseAfterFreeCheck,
                     config.preferredShaderLanguage.ordinal(),
-                    config.forceGLES2Context);
+                    config.forceGLES2Context, config.assertNativeWindowIsValid);
             return this;
         }
 
@@ -259,6 +258,17 @@ public class Engine {
             return this;
         }
 
+        /**
+         * Set a feature flag value. This is the only way to set constant feature flags.
+         * @param name feature name
+         * @param value true to enable, false to disable
+         * @return A reference to this Builder for chaining calls.
+         */
+        public Builder feature(@NonNull String name, boolean value) {
+            nSetBuilderFeature(mNativeBuilder, name, value);
+            return this;
+        }
+
         /**
          * Creates an instance of Engine
          *
@@ -394,6 +404,7 @@ public class Engine {
         /**
          * Set to `true` to forcibly disable parallel shader compilation in the backend.
          * Currently only honored by the GL backend.
+         * @Deprecated use "backend.disable_parallel_shader_compile" feature flag instead
          */
         public boolean disableParallelShaderCompile = false;
 
@@ -419,22 +430,26 @@ public class Engine {
          */
         public long stereoscopicEyeCount = 2;
 
-        /*
+        /**
          * @Deprecated This value is no longer used.
          */
         public long resourceAllocatorCacheSizeMB = 64;
 
-        /*
-         * This value determines for how many frames are texture entries kept in the cache.
+        /**
+         * This value determines how many frames texture entries are kept for in the cache. This
+         * is a soft limit, meaning some texture older than this are allowed to stay in the cache.
+         * Typically only one texture is evicted per frame.
+         * The default is 1.
          */
-        public long resourceAllocatorCacheMaxAge = 2;
+        public long resourceAllocatorCacheMaxAge = 1;
 
-        /*
+        /**
          * Disable backend handles use-after-free checks.
+         * @Deprecated use "backend.disable_handle_use_after_free_check" feature flag instead
          */
         public boolean disableHandleUseAfterFreeCheck = false;
 
-        /*
+        /**
          * Sets a preferred shader language for Filament to use.
          *
          * The Metal backend supports two shader languages: MSL (Metal Shading Language) and
@@ -456,12 +471,20 @@ public class Engine {
         };
         public ShaderLanguage preferredShaderLanguage = ShaderLanguage.DEFAULT;
 
-        /*
+        /**
          * When the OpenGL ES backend is used, setting this value to true will force a GLES2.0
          * context if supported by the Platform, or if not, will have the backend pretend
          * it's a GLES2 context. Ignored on other backends.
          */
         public boolean forceGLES2Context = false;
+
+        /**
+         * Assert the native window associated to a SwapChain is valid when calling makeCurrent().
+         * This is only supported for:
+         *      - PlatformEGLAndroid
+         * @Deprecated use "backend.opengl.assert_native_window_is_valid" feature flag instead
+         */
+        public boolean assertNativeWindowIsValid = false;
     }
 
     private Engine(long nativeEngine, Config config) {
@@ -684,11 +707,11 @@ public class Engine {
 
     /**
      * Returns the maximum number of stereoscopic eyes supported by Filament. The actual number of
-     * eyes rendered is set at Engine creation time with the {@link
-     * Engine#Config#stereoscopicEyeCount} setting.
+     * eyes rendered is set at Engine creation time with the {@link Config#stereoscopicEyeCount}
+     * setting.
      *
      * @return the max number of stereoscopic eyes supported
-     * @see Engine#Config#stereoscopicEyeCount
+     * @see Config#stereoscopicEyeCount
      */
     public long getMaxStereoscopicEyes() {
         return nGetMaxStereoscopicEyes(getNativeObject());
@@ -885,7 +908,8 @@ public class Engine {
 
     /**
      * Returns whether the object is valid.
-     * @param object Object to check for validity
+     * @param ma Material
+     * @param mi MaterialInstance to check for validity
      * @return returns true if the specified object is valid.
      */
     public boolean isValidMaterialInstance(@NonNull Material ma, MaterialInstance mi) {
@@ -1289,6 +1313,57 @@ public class Engine {
         nSetPaused(getNativeObject(), paused);
     }
 
+    /**
+     * Switch the command queue to unprotected mode. Protected mode can be activated via
+     * Renderer::beginFrame() using a protected SwapChain.
+     * @see Renderer
+     * @see SwapChain
+     */
+    public void unprotected() {
+        nUnprotected(getNativeObject());
+    }
+
+    /**
+     * Get the current time. This is a convenience function that simply returns the
+     * time in nanosecond since epoch of std::chrono::steady_clock.
+     * @return current time in nanosecond since epoch of std::chrono::steady_clock.
+     * @see Renderer#beginFrame
+     */
+    public static native long getSteadyClockTimeNano();
+
+
+    /**
+     * Checks if a feature flag exists
+     * @param name name of the feature flag to check
+     * @return true if it exists false otherwise
+     */
+    public boolean hasFeatureFlag(@NonNull String name) {
+        return nHasFeatureFlag(mNativeObject, name);
+    }
+
+    /**
+     * Set the value of a non-constant feature flag.
+     * @param name name of the feature flag to set
+     * @param value value to set
+     * @return true if the value was set, false if the feature flag is constant or doesn't exist.
+     */
+    public boolean setFeatureFlag(@NonNull String name, boolean value) {
+        return nSetFeatureFlag(mNativeObject, name, value);
+    }
+
+    /**
+     * Retrieves the value of any feature flag.
+     * @param name name of the feature flag
+     * @return the value of the flag if it exists
+     * @exception IllegalArgumentException is thrown if the feature flag doesn't exist
+     */
+    public boolean getFeatureFlag(@NonNull String name) {
+        if (!hasFeatureFlag(name)) {
+            throw new IllegalArgumentException("The feature flag \"" + name + "\" doesn't exist");
+        }
+        return nGetFeatureFlag(mNativeObject, name);
+    }
+
     @UsedByReflection("TextureHelper.java")
     public long getNativeObject() {
         if (mNativeObject == 0) {
@@ -1366,6 +1441,7 @@ public class Engine {
     private static native void nFlush(long nativeEngine);
     private static native boolean nIsPaused(long nativeEngine);
     private static native void nSetPaused(long nativeEngine, boolean paused);
+    private static native void nUnprotected(long nativeEngine);
     private static native long nGetTransformManager(long nativeEngine);
     private static native long nGetLightManager(long nativeEngine);
     private static native long nGetRenderableManager(long nativeEngine);
@@ -1377,6 +1453,9 @@ public class Engine {
     private static native int nGetSupportedFeatureLevel(long nativeEngine);
     private static native int nSetActiveFeatureLevel(long nativeEngine, int ordinal);
     private static native int nGetActiveFeatureLevel(long nativeEngine);
+    private static native boolean nHasFeatureFlag(long nativeEngine, String name);
+    private static native boolean nSetFeatureFlag(long nativeEngine, String name, boolean value);
+    private static native boolean nGetFeatureFlag(long nativeEngine, String name);
 
     private static native long nCreateBuilder();
     private static native void nDestroyBuilder(long nativeBuilder);
@@ -1384,14 +1463,14 @@ public class Engine {
     private static native void nSetBuilderConfig(long nativeBuilder, long commandBufferSizeMB,
             long perRenderPassArenaSizeMB, long driverHandleArenaSizeMB,
             long minCommandBufferSizeMB, long perFrameCommandsSizeMB, long jobSystemThreadCount,
-            long textureUseAfterFreePoolSize, boolean disableParallelShaderCompile,
-            int stereoscopicType, long stereoscopicEyeCount,
+            boolean disableParallelShaderCompile, int stereoscopicType, long stereoscopicEyeCount,
             long resourceAllocatorCacheSizeMB, long resourceAllocatorCacheMaxAge,
             boolean disableHandleUseAfterFreeCheck,
             int preferredShaderLanguage,
-            boolean forceGLES2Context);
+            boolean forceGLES2Context, boolean assertNativeWindowIsValid);
     private static native void nSetBuilderFeatureLevel(long nativeBuilder, int ordinal);
     private static native void nSetBuilderSharedContext(long nativeBuilder, long sharedContext);
     private static native void nSetBuilderPaused(long nativeBuilder, boolean paused);
+    private static native void nSetBuilderFeature(long nativeBuilder, String name, boolean value);
     private static native long nBuilderBuild(long nativeBuilder);
 }

android/filament-android/src/main/java/com/google/android/filament/Material.java

@@ -346,6 +346,7 @@ public class Material {
     public static class Builder {
         private Buffer mBuffer;
         private int mSize;
+        private int mShBandCount = 0;
 
         /**
          * Specifies the material data. The material data is a binary blob produced by
@@ -361,6 +362,22 @@ public class Material {
             return this;
         }
 
+        /**
+         * Sets the quality of the indirect lights computations. This is only taken into account
+         * if this material is lit and in the surface domain. This setting will affect the
+         * IndirectLight computation if one is specified on the Scene and Spherical Harmonics
+         * are used for the irradiance.
+         *
+         * @param shBandCount Number of spherical harmonic bands. Must be 1, 2 or 3 (default).
+         * @return Reference to this Builder for chaining calls.
+         * @see IndirectLight
+         */
+        @NonNull
+        public Builder sphericalHarmonicsBandCount(@IntRange(from = 0) int shBandCount) {
+            mShBandCount = shBandCount;
+            return this;
+        }
+
         /**
          * Creates and returns the Material object.
          *
@@ -372,7 +389,8 @@ public class Material {
          */
         @NonNull
         public Material build(@NonNull Engine engine) {
-            long nativeMaterial = nBuilderBuild(engine.getNativeObject(), mBuffer, mSize);
+            long nativeMaterial = nBuilderBuild(engine.getNativeObject(),
+                mBuffer, mSize, mShBandCount);
             if (nativeMaterial == 0) throw new IllegalStateException("Couldn't create Material");
             return new Material(nativeMaterial);
         }
@@ -1023,7 +1041,7 @@ public class Material {
         mNativeObject = 0;
     }
 
-    private static native long nBuilderBuild(long nativeEngine, @NonNull Buffer buffer, int size);
+    private static native long nBuilderBuild(long nativeEngine, @NonNull Buffer buffer, int size, int shBandCount);
     private static native long nCreateInstance(long nativeMaterial);
     private static native long nCreateInstanceWithName(long nativeMaterial, @NonNull String name);
     private static native long nGetDefaultInstance(long nativeMaterial);

android/filament-android/src/main/java/com/google/android/filament/MorphTargetBuffer.java

@@ -74,7 +74,7 @@ public class MorphTargetBuffer {
          *
          * @exception IllegalStateException if the MorphTargetBuffer could not be created
          *
-         * @see #setMorphTargetBufferAt
+         * @see #setMorphTargetBufferOffsetAt
          */
         @NonNull
         public MorphTargetBuffer build(@NonNull Engine engine) {

android/filament-android/src/main/java/com/google/android/filament/RenderableManager.java

@@ -569,40 +569,13 @@ public class RenderableManager {
          *
          * @param level the level of detail (lod), only 0 can be specified
          * @param primitiveIndex zero-based index of the primitive, must be less than the count passed to Builder constructor
-         * @param morphTargetBuffer specifies the morph target buffer
          * @param offset specifies where in the morph target buffer to start reading (expressed as a number of vertices)
-         * @param count number of vertices in the morph target buffer to read, must equal the geometry's count (for triangles, this should be a multiple of 3)
          */
         @NonNull
         public Builder morphing(@IntRange(from = 0) int level,
                                 @IntRange(from = 0) int primitiveIndex,
-                                @IntRange(from = 0) int offset,
-                                @IntRange(from = 0) int count) {
-            nBuilderSetMorphTargetBufferAt(mNativeBuilder, level, primitiveIndex, 0, offset, count);
-            return this;
-        }
-
-        /** @deprecated */
-        @Deprecated
-        @NonNull
-        public Builder morphing(@IntRange(from = 0) int level,
-                                @IntRange(from = 0) int primitiveIndex,
-                                @NonNull MorphTargetBuffer morphTargetBuffer,
-                                @IntRange(from = 0) int offset,
-                                @IntRange(from = 0) int count) {
-            nBuilderSetMorphTargetBufferAt(mNativeBuilder, level, primitiveIndex,
-                    morphTargetBuffer.getNativeObject(), offset, count);
-            return this;
-        }
-
-        /** @deprecated */
-        @Deprecated
-        @NonNull
-        public Builder morphing(@IntRange(from = 0) int level,
-                                @IntRange(from = 0) int primitiveIndex,
-                                @NonNull MorphTargetBuffer morphTargetBuffer) {
-            nBuilderSetMorphTargetBufferAt(mNativeBuilder, level, primitiveIndex,
-                    morphTargetBuffer.getNativeObject(), 0, morphTargetBuffer.getVertexCount());
+                                @IntRange(from = 0) int offset) {
+            nBuilderSetMorphTargetBufferOffsetAt(mNativeBuilder, level, primitiveIndex, offset);
             return this;
         }
 
@@ -705,34 +678,11 @@ public class RenderableManager {
      *
      * @see Builder#morphing
      */
-    public void setMorphTargetBufferAt(@EntityInstance int i,
+    public void setMorphTargetBufferOffsetAt(@EntityInstance int i,
                                        @IntRange(from = 0) int level,
                                        @IntRange(from = 0) int primitiveIndex,
-                                       @IntRange(from = 0) int offset,
-                                       @IntRange(from = 0) int count) {
-        nSetMorphTargetBufferAt(mNativeObject, i, level, primitiveIndex, 0, offset, count);
-    }
-
-    /** @deprecated */
-    @Deprecated
-    public void setMorphTargetBufferAt(@EntityInstance int i,
-                                       @IntRange(from = 0) int level,
-                                       @IntRange(from = 0) int primitiveIndex,
-                                       @NonNull MorphTargetBuffer morphTargetBuffer,
-                                       @IntRange(from = 0) int offset,
-                                       @IntRange(from = 0) int count) {
-        nSetMorphTargetBufferAt(mNativeObject, i, level, primitiveIndex,
-                morphTargetBuffer.getNativeObject(), offset, count);
-    }
-
-    /** @deprecated */
-    @Deprecated
-    public void setMorphTargetBufferAt(@EntityInstance int i,
-                                       @IntRange(from = 0) int level,
-                                       @IntRange(from = 0) int primitiveIndex,
-                                       @NonNull MorphTargetBuffer morphTargetBuffer) {
-        nSetMorphTargetBufferAt(mNativeObject, i, level, primitiveIndex,
-                morphTargetBuffer.getNativeObject(), 0, morphTargetBuffer.getVertexCount());
+                                       @IntRange(from = 0) int offset) {
+        nSetMorphTargetBufferOffsetAt(mNativeObject, i, level, primitiveIndex, 0, offset);
     }
 
     /**
@@ -1033,7 +983,7 @@ public class RenderableManager {
     private static native void nBuilderSkinningBuffer(long nativeBuilder, long nativeSkinningBuffer, int boneCount, int offset);
     private static native void nBuilderMorphing(long nativeBuilder, int targetCount);
     private static native void nBuilderMorphingStandard(long nativeBuilder, long nativeMorphTargetBuffer);
-    private static native void nBuilderSetMorphTargetBufferAt(long nativeBuilder, int level, int primitiveIndex, long nativeMorphTargetBuffer, int offset, int count);
+    private static native void nBuilderSetMorphTargetBufferOffsetAt(long nativeBuilder, int level, int primitiveIndex, int offset);
     private static native void nBuilderEnableSkinningBuffers(long nativeBuilder, boolean enabled);
     private static native void nBuilderFog(long nativeBuilder, boolean enabled);
     private static native void nBuilderLightChannel(long nativeRenderableManager, int channel, boolean enable);
@@ -1043,7 +993,7 @@ public class RenderableManager {
     private static native int nSetBonesAsMatrices(long nativeObject, int i, Buffer matrices, int remaining, int boneCount, int offset);
     private static native int nSetBonesAsQuaternions(long nativeObject, int i, Buffer quaternions, int remaining, int boneCount, int offset);
     private static native void nSetMorphWeights(long nativeObject, int instance, float[] weights, int offset);
-    private static native void nSetMorphTargetBufferAt(long nativeObject, int i, int level, int primitiveIndex, long nativeMorphTargetBuffer, int offset, int count);
+    private static native void nSetMorphTargetBufferOffsetAt(long nativeObject, int i, int level, int primitiveIndex, long nativeMorphTargetBuffer, int offset);
     private static native int nGetMorphTargetCount(long nativeObject, int i);
     private static native void nSetAxisAlignedBoundingBox(long nativeRenderableManager, int i, float cx, float cy, float cz, float ex, float ey, float ez);
     private static native void nSetLayerMask(long nativeRenderableManager, int i, int select, int value);

android/filament-android/src/main/java/com/google/android/filament/Renderer.java

@@ -284,6 +284,33 @@ public class Renderer {
         nSetPresentationTime(getNativeObject(), monotonicClockNanos);
     }
 
+    /**
+     * The use of this method is optional. It sets the VSYNC time expressed as the duration in
+     * nanosecond since epoch of std::chrono::steady_clock.
+     * If called, passing 0 to frameTimeNanos in Renderer.BeginFrame will use this
+     * time instead.
+     * @param steadyClockTimeNano duration in nanosecond since epoch of std::chrono::steady_clock
+     * @see Engine#getSteadyClockTimeNano
+     * @see Renderer#beginFrame
+     */
+    public void setVsyncTime(long steadyClockTimeNano) {
+        nSetVsyncTime(getNativeObject(), steadyClockTimeNano);
+    }
+
+    /**
+     * Call skipFrame when momentarily skipping frames, for instance if the content of the
+     * scene doesn't change.
+     *
+     * @param vsyncSteadyClockTimeNano The time in nanoseconds when the frame started being rendered,
+     *                       in the {@link System#nanoTime()} timebase. Divide this value by 1000000 to
+     *                       convert it to the {@link android.os.SystemClock#uptimeMillis()}
+     *                       time base. This typically comes from
+     *                       {@link android.view.Choreographer.FrameCallback}.
+     */
+    public void skipFrame(long vsyncSteadyClockTimeNano) {
+        nSkipFrame(getNativeObject(), vsyncSteadyClockTimeNano);
+    }
+
     /**
      * Sets up a frame for this <code>Renderer</code>.
      * <p><code>beginFrame</code> manages frame pacing, and returns whether or not a frame should be
@@ -702,6 +729,8 @@ public class Renderer {
     }
 
     private static native void nSetPresentationTime(long nativeObject, long monotonicClockNanos);
+    private static native void nSetVsyncTime(long nativeObject, long steadyClockTimeNano);
+    private static native void nSkipFrame(long nativeObject, long vsyncSteadyClockTimeNano);
     private static native boolean nBeginFrame(long nativeRenderer, long nativeSwapChain, long frameTimeNanos);
     private static native void nEndFrame(long nativeRenderer);
     private static native void nRender(long nativeRenderer, long nativeView);

android/filament-android/src/main/java/com/google/android/filament/View.java

@@ -745,6 +745,33 @@ public class View {
         nSetFrontFaceWindingInverted(getNativeObject(), inverted);
     }
 
+    /**
+     * Returns true if transparent picking is enabled.
+     *
+     * @see #setTransparentPickingEnabled
+     */
+    public boolean isTransparentPickingEnabled() {
+        return nIsTransparentPickingEnabled(getNativeObject());
+    }
+
+    /**
+     * Enables or disables transparent picking. Disabled by default.
+     *
+     * When transparent picking is enabled, View::pick() will pick from both
+     * transparent and opaque renderables. When disabled, View::pick() will only
+     * pick from opaque renderables.
+     *
+     * <p>
+     * Transparent picking will create an extra pass for rendering depth
+     * from both transparent and opaque renderables. 
+     * </p>
+     *
+     * @param enabled true enables transparent picking, false disables it.
+     */
+    public void setTransparentPickingEnabled(boolean enabled) {
+        nSetTransparentPickingEnabled(getNativeObject(), enabled);
+    }
+
     /**
      * Sets options relative to dynamic lighting for this view.
      *
@@ -1233,6 +1260,18 @@ public class View {
         return nGetFogEntity(getNativeObject());
     }
 
+    /**
+     * When certain temporal features are used (e.g.: TAA or Screen-space reflections), the view
+     * keeps a history of previous frame renders associated with the Renderer the view was last
+     * used with. When switching Renderer, it may be necessary to clear that history by calling
+     * this method. Similarly, if the whole content of the screen change, like when a cut-scene
+     * starts, clearing the history might be needed to avoid artifacts due to the previous frame
+     * being very different.
+     */
+    public void clearFrameHistory(Engine engine) {
+        nClearFrameHistory(getNativeObject(), engine.getNativeObject());
+    }
+
     public long getNativeObject() {
         if (mNativeObject == 0) {
             throw new IllegalStateException("Calling method on destroyed View");
@@ -1269,6 +1308,8 @@ public class View {
     private static native boolean nIsPostProcessingEnabled(long nativeView);
     private static native void nSetFrontFaceWindingInverted(long nativeView, boolean inverted);
     private static native boolean nIsFrontFaceWindingInverted(long nativeView);
+    private static native void nSetTransparentPickingEnabled(long nativeView, boolean enabled);
+    private static native boolean nIsTransparentPickingEnabled(long nativeView);
     private static native void nSetAmbientOcclusion(long nativeView, int ordinal);
     private static native int nGetAmbientOcclusion(long nativeView);
     private static native void nSetAmbientOcclusionOptions(long nativeView, float radius, float bias, float power, float resolution, float intensity, float bilateralThreshold, int quality, int lowPassFilter, int upsampling, boolean enabled, boolean bentNormals, float minHorizonAngleRad);
@@ -1294,7 +1335,7 @@ public class View {
     private static native void nSetMaterialGlobal(long nativeView, int index, float x, float y, float z, float w);
     private static native void nGetMaterialGlobal(long nativeView, int index, float[] out);
     private static native int nGetFogEntity(long nativeView);
-
+    private static native void nClearFrameHistory(long nativeView, long nativeEngine);
 
     /**
      * List of available ambient occlusion techniques.

android/filament-utils-android/src/main/cpp/Manipulator.cpp

@@ -125,6 +125,11 @@ extern "C" JNIEXPORT void Java_com_google_android_filament_utils_Manipulator_nBu
     builder->groundPlane(a, b, c, d);
 }
 
+extern "C" JNIEXPORT void Java_com_google_android_filament_utils_Manipulator_nBuilderPanning(JNIEnv*, jclass, jlong nativeBuilder, jboolean enabled) {
+    Builder* builder = (Builder*) nativeBuilder;
+    builder->panning(enabled);
+}
+
 extern "C" JNIEXPORT long Java_com_google_android_filament_utils_Manipulator_nBuilderBuild(JNIEnv*, jclass, jlong nativeBuilder, jint mode) {
     Builder* builder = (Builder*) nativeBuilder;
     return (jlong) builder->build((Mode) mode);

android/filament-utils-android/src/main/java/com/google/android/filament/utils/Manipulator.java

@@ -274,6 +274,17 @@ public class Manipulator {
             return this;
         }
 
+        /**
+         * Sets whether panning is enabled in the manipulator.
+         *
+         * @return this <code>Builder</code> object for chaining calls
+         */
+        @NonNull
+        public Builder panning(Boolean enabled) {
+            nBuilderPanning(mNativeBuilder, enabled);
+            return this;
+        }
+
         /**
          * Creates and returns the <code>Manipulator</code> object.
          *
@@ -483,6 +494,7 @@ public class Manipulator {
     private static native void nBuilderFlightPanSpeed(long nativeBuilder, float x, float y);
     private static native void nBuilderFlightMoveDamping(long nativeBuilder, float damping);
     private static native void nBuilderGroundPlane(long nativeBuilder, float a, float b, float c, float d);
+    private static native void nBuilderPanning(long nativeBuilder, Boolean enabled);
     private static native long nBuilderBuild(long nativeBuilder, int mode);
 
     private static native void nDestroyManipulator(long nativeManip);

android/gradle.properties

@@ -1,5 +1,5 @@
 GROUP=com.google.android.filament
-VERSION_NAME=1.52.3
+VERSION_NAME=1.56.4
 
 POM_DESCRIPTION=Real-time physically based rendering engine for Android.
 

android/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 #Wed Nov 17 10:40:18 PST 2021
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

android/samples/sample-gltf-viewer/build.gradle

@@ -6,6 +6,10 @@ plugins {
 
 project.ext.isSample = true
 
+kotlin {
+    jvmToolchain(versions.jdk)
+}
+
 filamentTools {
     cmgenArgs = "-q --format=ktx --size=256 --extract-blur=0.1 --deploy=src/main/assets/envs/default_env"
     iblInputFile = project.layout.projectDirectory.file("../../../third_party/environments/lightroom_14b.hdr")
@@ -13,7 +17,7 @@ filamentTools {
 }
 
 // don't forget to update MainACtivity.kt when/if changing this.
-task copyMesh(type: Copy) {
+tasks.register('copyMesh', Copy) {
     from "../../../third_party/models/BusterDrone"
     into "src/main/assets/models"
 }

android/samples/sample-hello-triangle/src/main/java/com/google/android/filament/hellotriangle/MainActivity.kt

@@ -112,7 +112,13 @@ class MainActivity : Activity() {
     }
 
     private fun setupFilament() {
-        engine = Engine.Builder().featureLevel(Engine.FeatureLevel.FEATURE_LEVEL_0).build()
+        val config = Engine.Config()
+        //config.forceGLES2Context = true
+
+        engine = Engine.Builder()
+            .config(config)
+            .featureLevel(Engine.FeatureLevel.FEATURE_LEVEL_0)
+            .build()
         renderer = engine.createRenderer()
         scene = engine.createScene()
         view = engine.createView()
@@ -123,7 +129,9 @@ class MainActivity : Activity() {
         scene.skybox = Skybox.Builder().color(0.035f, 0.035f, 0.035f, 1.0f).build(engine)
 
         // post-processing is not supported at feature level 0
-        view.isPostProcessingEnabled = false
+        if (engine.activeFeatureLevel == Engine.FeatureLevel.FEATURE_LEVEL_0) {
+            view.isPostProcessingEnabled = false
+        }
 
         // Tell the view which camera we want to use
         view.camera = camera

build.sh

@@ -64,6 +64,9 @@ function print_help {
     echo "        enabling debug paths in the backend from the build script. For example, make a"
     echo "        systrace-enabled build without directly changing #defines. Remember to add -f when"
     echo "        changing this option."
+    echo "    -X osmesa_path"
+    echo "        Indicates a path to a completed OSMesa build. OSMesa is used to create an offscreen GL"
+    echo "        context for software rasterization"
     echo "    -S type"
     echo "        Enable stereoscopic rendering where type is one of [instanced|multiview]. This is only"
     echo "        meant for building the samples."
@@ -180,6 +183,8 @@ BACKEND_DEBUG_FLAG_OPTION=""
 
 STEREOSCOPIC_OPTION=""
 
+OSMESA_OPTION=""
+
 IOS_BUILD_SIMULATOR=false
 BUILD_UNIVERSAL_LIBRARIES=false
 
@@ -240,6 +245,7 @@ function build_desktop_target {
             ${ASAN_UBSAN_OPTION} \
             ${BACKEND_DEBUG_FLAG_OPTION} \
             ${STEREOSCOPIC_OPTION} \
+            ${OSMESA_OPTION} \
             ${architectures} \
             ../..
         ln -sf "out/cmake-${lc_target}/compile_commands.json" \
@@ -796,7 +802,7 @@ function check_debug_release_build {
 
 pushd "$(dirname "$0")" > /dev/null
 
-while getopts ":hacCfgijmp:q:uvslwedk:bx:S:" opt; do
+while getopts ":hacCfgijmp:q:uvslwedk:bx:S:X:" opt; do
     case ${opt} in
         h)
             print_help
@@ -950,6 +956,8 @@ while getopts ":hacCfgijmp:q:uvslwedk:bx:S:" opt; do
                     exit 1
             esac
             ;;
+        X)  OSMESA_OPTION="-DFILAMENT_OSMESA_PATH=${OPTARG}"
+            ;;
         \?)
             echo "Invalid option: -${OPTARG}" >&2
             echo ""

build/android/build.sh

@@ -57,15 +57,10 @@ FILAMENT_NDK_VERSION=${FILAMENT_NDK_VERSION:-$(cat `dirname $0`/ndk.version)}
 
 # Install the required NDK version specifically (if not present)
 if [[ ! -d "${ANDROID_HOME}/ndk/$FILAMENT_NDK_VERSION" ]]; then
-    ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager "ndk;$FILAMENT_NDK_VERSION" > /dev/null
+    yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --licenses
+    ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager "ndk;$FILAMENT_NDK_VERSION"
 fi
-
-# Only build 1 64 bit target during presubmit to cut down build times during presubmit
-# Continuous builds will build everything
 ANDROID_ABIS=
-if [[ "$TARGET" == "presubmit" ]]; then
-  ANDROID_ABIS="-q arm64-v8a"
-fi
 
 # Build the Android sample-gltf-viewer APK during release.
 BUILD_SAMPLES=
@@ -73,5 +68,19 @@ if [[ "$TARGET" == "release" ]]; then
     BUILD_SAMPLES="-k sample-gltf-viewer"
 fi
 
+function build_android() {
+    local ABI=$1
+
+    # Do the following in two steps so that we do not run out of space
+    if [[ -n "${BUILD_DEBUG}" ]]; then
+        FILAMENT_NDK_VERSION=${FILAMENT_NDK_VERSION} ./build.sh -p android -q ${ABI} -c ${BUILD_SAMPLES} ${GENERATE_ARCHIVES} ${BUILD_DEBUG}
+        rm -rf out/cmake-android-debug-*
+    fi
+    if [[ -n "${BUILD_RELEASE}" ]]; then
+        FILAMENT_NDK_VERSION=${FILAMENT_NDK_VERSION} ./build.sh -p android -q ${ABI} -c ${BUILD_SAMPLES} ${GENERATE_ARCHIVES} ${BUILD_RELEASE}
+        rm -rf out/cmake-android-release-*
+    fi
+}
+
 pushd `dirname $0`/../.. > /dev/null
-FILAMENT_NDK_VERSION=${FILAMENT_NDK_VERSION} ./build.sh -p android $ANDROID_ABIS -c $BUILD_SAMPLES $GENERATE_ARCHIVES $BUILD_DEBUG $BUILD_RELEASE
+build_android $2

build/android/ndk.version

@@ -1 +1 @@
-26.1.10909125
+27.0.11718014

build/web/ci-common.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 if [ `uname` == "Linux" ];then
-    curl -OL https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
-    unzip -q ninja-linux.zip
+    source `dirname $0`/../linux/ci-common.sh
 elif [ `uname` == "Darwin" ];then
     curl -OL https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
     unzip -q ninja-mac.zip
@@ -13,9 +12,6 @@ fi
 chmod +x ninja
 export PATH="$PWD:$PATH"
 
-# FIXME: kokoro machines have node and npm but currently they are symlinked to non-existent files
-# npm install -g typescript
-
 # Install emscripten.
 curl -L https://github.com/emscripten-core/emsdk/archive/refs/tags/3.1.15.zip > emsdk.zip
 unzip emsdk.zip ; mv emsdk-* emsdk ; cd emsdk

build/windows/build-github.bat

@@ -115,17 +115,23 @@ cmake ..\.. ^
     -DFILAMENT_SUPPORTS_VULKAN=ON ^
     || exit /b
 
+set build_flags=-j %NUMBER_OF_PROCESSORS%
+
+@echo on
+
+:: we've upgraded the windows machines, so the following are no longer accurate as of 09/19/24, but
+:: keeping around the comment for record.
+
 :: Attempt to fix "error C1060: compiler is out of heap space" seen on CI.
 :: Some resource libraries require significant heap space to compile, so first compile them serially.
-@echo on
-cmake --build . --target filagui --config %config% || exit /b
-cmake --build . --target uberarchive --config %config% || exit /b
-cmake --build . --target gltf-demo-resources --config %config% || exit /b
-cmake --build . --target filamentapp-resources --config %config% || exit /b
-cmake --build . --target sample-resources --config %config% || exit /b
-cmake --build . --target suzanne-resources --config %config% || exit /b
+:: cmake --build . --target filagui --config %config% %build_flags% || exit /b
+:: cmake --build . --target uberarchive --config %config% %build_flags% || exit /b
+:: cmake --build . --target gltf-demo-resources --config %config% %build_flags% || exit /b
+:: cmake --build . --target filamentapp-resources --config %config% %build_flags% || exit /b
+:: cmake --build . --target sample-resources --config %config% %build_flags% || exit /b
+:: cmake --build . --target suzanne-resources --config %config% %build_flags% || exit /b
 
-cmake --build . %INSTALL% --config %config% -- /m || exit /b
+cmake --build . %INSTALL% --config %config% %build_flags% -- /m || exit /b
 @echo off
 
 echo Disk info after building variant: %variant%

docs/Materials.html

@@ -1787,7 +1787,12 @@ non-shader data.
       declare a variable called <code>eyeDirection</code> you can access it in the fragment shader using
       <code>variable_eyeDirection</code>. In the vertex shader, the interpolant name is simply a member of
       the <code>MaterialVertexInputs</code> structure (<code>material.eyeDirection</code> in your example). Each
-      interpolant is of type <code>float4</code> (<code>vec4</code>) in the shaders.
+      interpolant is of type <code>float4</code> (<code>vec4</code>) in the shaders. By default the precision of the
+      interpolant is <code>highp</code> in <em class="asterisk">both</em> the vertex and fragment shaders.
+      An alternate syntax can be used to specify both the name and precision of the interpolant.
+      In this case the specified precision is used as-is in both fragment and vertex stages, in
+      particular if <code>default</code> is specified the default precision is used is the fragment shader
+      (<code>mediump</code>) and in the vertex shader (<code>highp</code>).
 
 </p></dd></dl><p></p><pre class="listing tilde"><code><span class="line">material {</span>
 <span class="line">    name : Skybox,</span>
@@ -1798,7 +1803,11 @@ non-shader data.
 <span class="line">        }</span>
 <span class="line">    ],</span>
 <span class="line">    variables : [</span>
-<span class="line">         eyeDirection</span>
+<span class="line">         eyeDirection,</span>
+<span class="line">         {</span>
+<span class="line">            name : eyeColor,</span>
+<span class="line">            precision : medium</span>
+<span class="line">         }</span>
 <span class="line">    ],</span>
 <span class="line">    vertexDomain : device,</span>
 <span class="line">    depthWrite : <span class="hljs-literal">false</span>,</span>

docs/Materials.md.html

@@ -1308,7 +1308,12 @@ Description
       declare a variable called `eyeDirection` you can access it in the fragment shader using
       `variable_eyeDirection`. In the vertex shader, the interpolant name is simply a member of
       the `MaterialVertexInputs` structure (`material.eyeDirection` in your example). Each
-      interpolant is of type `float4` (`vec4`) in the shaders.
+      interpolant is of type `float4` (`vec4`) in the shaders. By default the precision of the
+      interpolant is `highp` in *both* the vertex and fragment shaders.
+      An alternate syntax can be used to specify both the name and precision of the interpolant.
+      In this case the specified precision is used as-is in both fragment and vertex stages, in
+      particular if `default` is specified the default precision is used is the fragment shader
+      (`mediump`) and in the vertex shader (`highp`).
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ JSON
 material {
@@ -1320,7 +1325,11 @@ material {
         }
     ],
     variables : [
-         eyeDirection
+         eyeDirection,
+         {
+            name : eyeColor,
+            precision : medium
+         }
     ],
     vertexDomain : device,
     depthWrite : false,

filament/CMakeLists.txt

@@ -61,10 +61,12 @@ set(SRCS
         src/Engine.cpp
         src/Exposure.cpp
         src/Fence.cpp
+        src/FilamentBuilder.cpp
         src/FrameInfo.cpp
         src/FrameSkipper.cpp
         src/Froxelizer.cpp
         src/Frustum.cpp
+        src/HwDescriptorSetLayoutFactory.cpp
         src/HwRenderPrimitiveFactory.cpp
         src/HwVertexBufferInfoFactory.cpp
         src/IndexBuffer.cpp
@@ -75,8 +77,6 @@ set(SRCS
         src/MaterialInstance.cpp
         src/MaterialParser.cpp
         src/MorphTargetBuffer.cpp
-        src/PerViewUniforms.cpp
-        src/PerShadowMapUniforms.cpp
         src/PostProcessManager.cpp
         src/RenderPass.cpp
         src/RenderPrimitive.cpp
@@ -125,6 +125,12 @@ set(SRCS
         src/details/Texture.cpp
         src/details/VertexBuffer.cpp
         src/details/View.cpp
+        src/ds/ColorPassDescriptorSet.cpp
+        src/ds/DescriptorSet.cpp
+        src/ds/DescriptorSetLayout.cpp
+        src/ds/PostProcessDescriptorSet.cpp
+        src/ds/ShadowMapDescriptorSet.cpp
+        src/ds/SsrPassDescriptorSet.cpp
         src/fg/Blackboard.cpp
         src/fg/DependencyGraph.cpp
         src/fg/FrameGraph.cpp
@@ -148,23 +154,21 @@ set(PRIVATE_HDRS
         src/FrameInfo.h
         src/FrameSkipper.h
         src/Froxelizer.h
+        src/HwDescriptorSetLayoutFactory.h
         src/HwRenderPrimitiveFactory.h
         src/HwVertexBufferInfoFactory.h
         src/Intersections.h
         src/MaterialParser.h
-        src/PerViewUniforms.h
-        src/PerShadowMapUniforms.h
         src/PIDController.h
         src/PostProcessManager.h
-        src/RendererUtils.h
         src/RenderPass.h
         src/RenderPrimitive.h
+        src/RendererUtils.h
         src/ResourceAllocator.h
         src/ResourceList.h
         src/ShadowMap.h
         src/ShadowMapManager.h
         src/SharedHandle.h
-        src/TypedUniformBuffer.h
         src/UniformBuffer.h
         src/components/CameraManager.h
         src/components/LightManager.h
@@ -192,6 +196,14 @@ set(PRIVATE_HDRS
         src/details/Texture.h
         src/details/VertexBuffer.h
         src/details/View.h
+        src/downcast.h
+        src/ds/ColorPassDescriptorSet.h
+        src/ds/DescriptorSetLayout.h
+        src/ds/PostProcessDescriptorSet.h
+        src/ds/ShadowMapDescriptorSet.h
+        src/ds/SsrPassDescriptorSet.h
+        src/ds/TypedBuffer.h
+        src/ds/TypedUniformBuffer.h
         src/fg/Blackboard.h
         src/fg/FrameGraph.h
         src/fg/FrameGraphId.h
@@ -209,7 +221,6 @@ set(PRIVATE_HDRS
         src/materials/fsr/ffx_a.h
         src/materials/fsr/ffx_fsr1.h
         src/materials/fsr/ffx_fsr1_mobile.fs
-        src/downcast.h
 )
 
 set(MATERIAL_SRCS
@@ -297,6 +308,11 @@ if (FILAMENT_ENABLE_MULTIVIEW)
     add_definitions(-DFILAMENT_ENABLE_MULTIVIEW)
 endif()
 
+# Whether to force the profiling mode.
+if (FILAMENT_FORCE_PROFILING_MODE)
+    add_definitions(-DFILAMENT_FORCE_PROFILING_MODE)
+endif()
+
 # ==================================================================================================
 # Definitions
 # ==================================================================================================
@@ -596,6 +612,13 @@ else()
             -Wover-aligned
             -Werror
     )
+    if (CMAKE_CXX_STANDARD EQUAL 20)
+        # The lambdas for passes in PostProcessManager.cpp capture this
+        # implicitly in a way that's deprecated in c++20, but can't easily be
+        # fixed in a way that's backwards compatible with c++17:
+        # https://www.nextptr.com/tutorial/ta1430524603/capture-this-in-lambda-expression-timeline-of-change
+        list(APPEND FILAMENT_WARNINGS -Wno-deprecated-this-capture)
+    endif()
 endif()
 
 target_compile_options(${TARGET} PRIVATE

filament/backend/CMakeLists.txt

@@ -12,6 +12,7 @@ set(PUBLIC_HDRS
         include/backend/AcquiredImage.h
         include/backend/BufferDescriptor.h
         include/backend/CallbackHandler.h
+        include/backend/DescriptorSetOffsetArray.h
         include/backend/DriverApiForward.h
         include/backend/DriverEnums.h
         include/backend/Handle.h
@@ -69,9 +70,13 @@ set(PRIVATE_HDRS
 if (FILAMENT_SUPPORTS_OPENGL AND NOT FILAMENT_USE_EXTERNAL_GLES3)
     list(APPEND SRCS
             include/backend/platforms/OpenGLPlatform.h
+            src/opengl/BindingMap.h
             src/opengl/gl_headers.cpp
             src/opengl/gl_headers.h
             src/opengl/GLBufferObject.h
+            src/opengl/GLDescriptorSet.cpp
+            src/opengl/GLDescriptorSet.h
+            src/opengl/GLDescriptorSetLayout.h
             src/opengl/GLTexture.h
             src/opengl/GLUtils.cpp
             src/opengl/GLUtils.h
@@ -114,6 +119,8 @@ if (FILAMENT_SUPPORTS_OPENGL AND NOT FILAMENT_USE_EXTERNAL_GLES3)
             list(APPEND SRCS src/opengl/platforms/PlatformGLX.cpp)
         elseif (FILAMENT_SUPPORTS_EGL_ON_LINUX)
             list(APPEND SRCS src/opengl/platforms/PlatformEGLHeadless.cpp)
+        elseif (FILAMENT_SUPPORTS_OSMESA)
+            list(APPEND SRCS src/opengl/platforms/PlatformOSMesa.cpp)
         endif()
     elseif (WIN32)
         list(APPEND SRCS src/opengl/platforms/PlatformWGL.cpp)
@@ -172,11 +179,18 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/caching/VulkanDescriptorSetManager.h
             src/vulkan/caching/VulkanPipelineLayoutCache.cpp
             src/vulkan/caching/VulkanPipelineLayoutCache.h
+            src/vulkan/memory/ResourceManager.cpp
+            src/vulkan/memory/ResourceManager.h
+            src/vulkan/memory/ResourcePointer.h
+            src/vulkan/memory/Resource.cpp
+            src/vulkan/memory/Resource.h
             src/vulkan/platform/VulkanPlatform.cpp
             src/vulkan/platform/VulkanPlatformSwapChainImpl.cpp
             src/vulkan/platform/VulkanPlatformSwapChainImpl.h
             src/vulkan/spirv/VulkanSpirvUtils.cpp
             src/vulkan/spirv/VulkanSpirvUtils.h
+            src/vulkan/VulkanAsyncHandles.cpp
+            src/vulkan/VulkanAsyncHandles.h
             src/vulkan/VulkanBlitter.cpp
             src/vulkan/VulkanBlitter.h
             src/vulkan/VulkanBuffer.cpp
@@ -207,9 +221,6 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/VulkanSwapChain.h
             src/vulkan/VulkanReadPixels.cpp
             src/vulkan/VulkanReadPixels.h
-            src/vulkan/VulkanResourceAllocator.h
-            src/vulkan/VulkanResources.cpp
-            src/vulkan/VulkanResources.h
             src/vulkan/VulkanTexture.cpp
             src/vulkan/VulkanTexture.h
             src/vulkan/VulkanUtility.cpp
@@ -361,6 +372,11 @@ set(LINUX_LINKER_OPTIMIZATION_FLAGS
         -Wl,--exclude-libs,bluegl
 )
 
+if (LINUX AND FILAMENT_SUPPORTS_OSMESA)
+    set(OSMESA_COMPILE_FLAGS
+        -I${FILAMENT_OSMESA_PATH}/include/GL)
+endif()
+
 if (MSVC)
     set(FILAMENT_WARNINGS /W3)
 else()
@@ -372,6 +388,9 @@ else()
             -Wover-aligned
             -Werror
     )
+    if (CMAKE_CXX_STANDARD EQUAL 20)
+        list(APPEND FILAMENT_WARNINGS -Wno-deprecated-this-capture)
+    endif()
 endif()
 
 if (APPLE)
@@ -381,6 +400,7 @@ endif()
 
 target_compile_options(${TARGET} PRIVATE
         ${FILAMENT_WARNINGS}
+        ${OSMESA_COMPILE_FLAGS}
         $<$<CONFIG:Release>:${OPTIMIZATION_FLAGS}>
         $<$<AND:$<PLATFORM_ID:Darwin>,$<CONFIG:Release>>:${DARWIN_OPTIMIZATION_FLAGS}>
 )
@@ -390,6 +410,7 @@ if (FILAMENT_SUPPORTS_METAL)
 endif()
 
 target_link_libraries(${TARGET} PRIVATE
+        ${OSMESA_LINKER_FLAGS}
         $<$<AND:$<PLATFORM_ID:Linux>,$<CONFIG:Release>>:${LINUX_LINKER_OPTIMIZATION_FLAGS}>
 )
 
@@ -424,6 +445,7 @@ if (APPLE OR LINUX)
         test/test_StencilBuffer.cpp
         test/test_Scissor.cpp
         test/test_MipLevels.cpp
+        test/test_Handles.cpp
     )
     set(BACKEND_TEST_LIBS
         backend
@@ -495,21 +517,40 @@ endif()
 
 # ==================================================================================================
 # Compute tests
+#
+#if (NOT IOS AND NOT WEBGL)
+#
+#add_executable(compute_test
+#        test/ComputeTest.cpp
+#        test/Arguments.cpp
+#        test/test_ComputeBasic.cpp
+#        )
+#
+#target_link_libraries(compute_test PRIVATE
+#        backend
+#        getopt
+#        gtest
+#        )
+#
+#set_target_properties(compute_test PROPERTIES FOLDER Tests)
+#
+#endif()
 
-if (NOT IOS AND NOT WEBGL)
+# ==================================================================================================
+# Metal utils tests
 
-add_executable(compute_test
-        test/ComputeTest.cpp
-        test/Arguments.cpp
-        test/test_ComputeBasic.cpp
-        )
+if (APPLE AND NOT IOS)
 
-target_link_libraries(compute_test PRIVATE
+add_executable(metal_utils_test test/MetalTest.mm)
+
+target_compile_options(metal_utils_test PRIVATE "-fobjc-arc")
+
+target_link_libraries(metal_utils_test PRIVATE
         backend
         getopt
         gtest
         )
 
-set_target_properties(compute_test PROPERTIES FOLDER Tests)
+set_target_properties(metal_utils_test PROPERTIES FOLDER Tests)
 
 endif()

filament/backend/include/backend/DescriptorSetOffsetArray.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_COMMANDSTREAMVECTOR_H
+#define TNT_FILAMENT_BACKEND_COMMANDSTREAMVECTOR_H
+
+#include <backend/DriverApiForward.h>
+
+#include <initializer_list>
+#include <memory>
+
+#include <stddef.h>
+#include <stdint.h>
+
+
+namespace filament::backend {
+
+void* allocateFromCommandStream(DriverApi& driver, size_t size, size_t alignment) noexcept;
+
+class DescriptorSetOffsetArray {
+public:
+    using value_type = uint32_t;
+    using reference = value_type&;
+    using const_reference = value_type const&;
+    using size_type = uint32_t;
+    using difference_type = int32_t;
+    using pointer = value_type*;
+    using const_pointer = value_type const*;
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+
+    DescriptorSetOffsetArray() noexcept = default;
+
+    ~DescriptorSetOffsetArray() noexcept = default;
+
+    DescriptorSetOffsetArray(size_type size, DriverApi& driver) noexcept {
+        mOffsets = (value_type *)allocateFromCommandStream(driver,
+                size * sizeof(value_type), alignof(value_type));
+        std::uninitialized_fill_n(mOffsets, size, 0);
+    }
+
+    DescriptorSetOffsetArray(std::initializer_list<uint32_t> list, DriverApi& driver) noexcept {
+        mOffsets = (value_type *)allocateFromCommandStream(driver,
+                list.size() * sizeof(value_type), alignof(value_type));
+        std::uninitialized_copy(list.begin(), list.end(), mOffsets);
+    }
+
+    DescriptorSetOffsetArray(DescriptorSetOffsetArray const&) = delete;
+    DescriptorSetOffsetArray& operator=(DescriptorSetOffsetArray const&) = delete;
+
+    DescriptorSetOffsetArray(DescriptorSetOffsetArray&& rhs) noexcept
+            : mOffsets(rhs.mOffsets) {
+        rhs.mOffsets = nullptr;
+    }
+
+    DescriptorSetOffsetArray& operator=(DescriptorSetOffsetArray&& rhs) noexcept {
+        if (this != &rhs) {
+            mOffsets = rhs.mOffsets;
+            rhs.mOffsets = nullptr;
+        }
+        return *this;
+    }
+
+    bool empty() const noexcept { return mOffsets == nullptr; }
+
+    value_type* data() noexcept { return mOffsets; }
+    const value_type* data() const noexcept { return mOffsets; }
+
+
+    reference operator[](size_type n) noexcept {
+        return *(data() + n);
+    }
+
+    const_reference operator[](size_type n) const noexcept {
+        return *(data() + n);
+    }
+
+    void clear() noexcept {
+        mOffsets = nullptr;
+    }
+
+private:
+    value_type *mOffsets = nullptr;
+};
+
+} // namespace filament::backend
+
+#endif //TNT_FILAMENT_BACKEND_COMMANDSTREAMVECTOR_H

filament/backend/include/backend/DriverEnums.h

@@ -19,13 +19,16 @@
 #ifndef TNT_FILAMENT_BACKEND_DRIVERENUMS_H
 #define TNT_FILAMENT_BACKEND_DRIVERENUMS_H
 
-#include <utils/BitmaskEnum.h>
 #include <utils/unwindows.h> // Because we define ERROR in the FenceStatus enum.
 
 #include <backend/Platform.h>
 #include <backend/PresentCallable.h>
 
+#include <utils/BitmaskEnum.h>
+#include <utils/FixedCapacityVector.h>
 #include <utils/Invocable.h>
+#include <utils/compiler.h>
+#include <utils/debug.h>
 #include <utils/ostream.h>
 
 #include <math/vec4.h>
@@ -97,6 +100,8 @@ static constexpr size_t MAX_VERTEX_ATTRIBUTE_COUNT  = 16;   // This is guarantee
 static constexpr size_t MAX_SAMPLER_COUNT           = 62;   // Maximum needed at feature level 3.
 static constexpr size_t MAX_VERTEX_BUFFER_COUNT     = 16;   // Max number of bound buffer objects.
 static constexpr size_t MAX_SSBO_COUNT              = 4;    // This is guaranteed by OpenGL ES.
+static constexpr size_t MAX_DESCRIPTOR_SET_COUNT    = 4;    // This is guaranteed by Vulkan.
+static constexpr size_t MAX_DESCRIPTOR_COUNT        = 64;   // per set
 
 static constexpr size_t MAX_PUSH_CONSTANT_COUNT     = 32;   // Vulkan 1.1 spec allows for 128-byte
                                                             // of push constant (we assume 4-byte
@@ -191,6 +196,71 @@ static constexpr const char* shaderLanguageToString(ShaderLanguage shaderLanguag
     }
 }
 
+enum class ShaderStage : uint8_t {
+    VERTEX = 0,
+    FRAGMENT = 1,
+    COMPUTE = 2
+};
+
+static constexpr size_t PIPELINE_STAGE_COUNT = 2;
+enum class ShaderStageFlags : uint8_t {
+    NONE        =    0,
+    VERTEX      =    0x1,
+    FRAGMENT    =    0x2,
+    COMPUTE     =    0x4,
+    ALL_SHADER_STAGE_FLAGS = VERTEX | FRAGMENT | COMPUTE
+};
+
+static inline constexpr bool hasShaderType(ShaderStageFlags flags, ShaderStage type) noexcept {
+    switch (type) {
+        case ShaderStage::VERTEX:
+            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::VERTEX));
+        case ShaderStage::FRAGMENT:
+            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::FRAGMENT));
+        case ShaderStage::COMPUTE:
+            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::COMPUTE));
+    }
+}
+
+enum class DescriptorType : uint8_t {
+    UNIFORM_BUFFER,
+    SHADER_STORAGE_BUFFER,
+    SAMPLER,
+    INPUT_ATTACHMENT,
+    SAMPLER_EXTERNAL
+};
+
+enum class DescriptorFlags : uint8_t {
+    NONE = 0x00,
+    DYNAMIC_OFFSET = 0x01
+};
+
+using descriptor_set_t = uint8_t;
+
+using descriptor_binding_t = uint8_t;
+
+struct DescriptorSetLayoutBinding {
+    DescriptorType type;
+    ShaderStageFlags stageFlags;
+    descriptor_binding_t binding;
+    DescriptorFlags flags = DescriptorFlags::NONE;
+    uint16_t count = 0;
+
+    friend inline bool operator==(
+            DescriptorSetLayoutBinding const& lhs,
+            DescriptorSetLayoutBinding const& rhs) noexcept {
+        return lhs.type == rhs.type &&
+               lhs.flags == rhs.flags &&
+               lhs.count == rhs.count &&
+               lhs.stageFlags == rhs.stageFlags;
+    }
+};
+
+struct DescriptorSetLayout {
+    utils::FixedCapacityVector<DescriptorSetLayoutBinding> bindings;
+};
+
+
 /**
  * Bitmask for selecting render buffers
  */
@@ -249,8 +319,19 @@ struct Viewport {
     int32_t right() const noexcept { return left + int32_t(width); }
     //! get the top coordinate in window space of the viewport
     int32_t top() const noexcept { return bottom + int32_t(height); }
-};
 
+    friend bool operator==(Viewport const& lhs, Viewport const& rhs) noexcept {
+        // clang can do this branchless with xor/or
+        return lhs.left == rhs.left && lhs.bottom == rhs.bottom &&
+               lhs.width == rhs.width && lhs.height == rhs.height;
+    }
+
+    friend bool operator!=(Viewport const& lhs, Viewport const& rhs) noexcept {
+        // clang is being dumb and uses branches
+        return bool(((lhs.left ^ rhs.left) | (lhs.bottom ^ rhs.bottom)) |
+                    ((lhs.width ^ rhs.width) | (lhs.height ^ rhs.height)));
+    }
+};
 
 /**
  * Specifies the mapping of the near and far clipping plane to window coordinates.
@@ -270,15 +351,6 @@ enum class FenceStatus : int8_t {
     TIMEOUT_EXPIRED = 1,        //!< wait()'s timeout expired. The Fence condition is not satisfied.
 };
 
-/**
- * Status codes for sync objects
- */
-enum class SyncStatus : int8_t {
-    ERROR = -1,          //!< An error occurred. The Sync is not signaled.
-    SIGNALED = 0,        //!< The Sync is signaled.
-    NOT_SIGNALED = 1,    //!< The Sync is not signaled yet
-};
-
 static constexpr uint64_t FENCE_WAIT_FOR_EVER = uint64_t(-1);
 
 /**
@@ -368,6 +440,18 @@ enum class SamplerType : uint8_t {
     SAMPLER_CUBEMAP_ARRAY,  //!< Cube map array texture (feature level 2)
 };
 
+inline const char* stringify(SamplerType samplerType) {
+    switch (samplerType) {
+        case SamplerType::SAMPLER_2D: return "SAMPLER_2D";
+        case SamplerType::SAMPLER_2D_ARRAY: return "SAMPLER_2D_ARRAY";
+        case SamplerType::SAMPLER_CUBEMAP: return "SAMPLER_CUBEMAP";
+        case SamplerType::SAMPLER_EXTERNAL: return "SAMPLER_EXTERNAL";
+        case SamplerType::SAMPLER_3D: return "SAMPLER_3D";
+        case SamplerType::SAMPLER_CUBEMAP_ARRAY: return "SAMPLER_CUBEMAP_ARRAY";
+    }
+    return "UNKNOWN";
+}
+
 //! Subpass type
 enum class SubpassType : uint8_t {
     SUBPASS_INPUT
@@ -693,9 +777,27 @@ enum class TextureUsage : uint16_t {
     BLIT_SRC            = 0x0040,            //!< Texture can be used the source of a blit()
     BLIT_DST            = 0x0080,            //!< Texture can be used the destination of a blit()
     PROTECTED           = 0x0100,            //!< Texture can be used for protected content
-    DEFAULT             = UPLOADABLE | SAMPLEABLE   //!< Default texture usage
+    DEFAULT             = UPLOADABLE | SAMPLEABLE,   //!< Default texture usage
+    ALL_ATTACHMENTS     = COLOR_ATTACHMENT | DEPTH_ATTACHMENT | STENCIL_ATTACHMENT | SUBPASS_INPUT,   //!< Mask of all attachments
 };
 
+inline const char* stringify(TextureUsage usage) {
+    switch (usage) {
+        case TextureUsage::NONE: return "NONE";
+        case TextureUsage::COLOR_ATTACHMENT: return "COLOR_ATTACHMENT";
+        case TextureUsage::DEPTH_ATTACHMENT: return "DEPTH_ATTACHMENT";
+        case TextureUsage::STENCIL_ATTACHMENT: return "STENCIL_ATTACHMENT";
+        case TextureUsage::UPLOADABLE: return "UPLOADABLE";
+        case TextureUsage::SAMPLEABLE: return "SAMPLEABLE";
+        case TextureUsage::SUBPASS_INPUT: return "SUBPASS_INPUT";
+        case TextureUsage::BLIT_SRC: return "BLIT_SRC";
+        case TextureUsage::BLIT_DST: return "BLIT_DST";
+        case TextureUsage::PROTECTED: return "PROTECTED";
+        case TextureUsage::DEFAULT: return "DEFAULT";
+        default: return "UNKNOWN";
+    }
+}
+
 //! Texture swizzle
 enum class TextureSwizzle : uint8_t {
     SUBSTITUTE_ZERO,
@@ -887,6 +989,9 @@ struct SamplerParams { // NOLINT
 
     struct EqualTo {
         bool operator()(SamplerParams lhs, SamplerParams rhs) const noexcept {
+            assert_invariant(lhs.padding0 == 0);
+            assert_invariant(lhs.padding1 == 0);
+            assert_invariant(lhs.padding2 == 0);
             auto* pLhs = reinterpret_cast<uint32_t const*>(reinterpret_cast<char const*>(&lhs));
             auto* pRhs = reinterpret_cast<uint32_t const*>(reinterpret_cast<char const*>(&rhs));
             return *pLhs == *pRhs;
@@ -895,6 +1000,9 @@ struct SamplerParams { // NOLINT
 
     struct LessThan {
         bool operator()(SamplerParams lhs, SamplerParams rhs) const noexcept {
+            assert_invariant(lhs.padding0 == 0);
+            assert_invariant(lhs.padding1 == 0);
+            assert_invariant(lhs.padding2 == 0);
             auto* pLhs = reinterpret_cast<uint32_t const*>(reinterpret_cast<char const*>(&lhs));
             auto* pRhs = reinterpret_cast<uint32_t const*>(reinterpret_cast<char const*>(&rhs));
             return *pLhs == *pRhs;
@@ -902,6 +1010,12 @@ struct SamplerParams { // NOLINT
     };
 
 private:
+    friend inline bool operator == (SamplerParams lhs, SamplerParams rhs) noexcept {
+        return SamplerParams::EqualTo{}(lhs, rhs);
+    }
+    friend inline bool operator != (SamplerParams lhs, SamplerParams rhs) noexcept {
+        return  !SamplerParams::EqualTo{}(lhs, rhs);
+    }
     friend inline bool operator < (SamplerParams lhs, SamplerParams rhs) noexcept {
         return SamplerParams::LessThan{}(lhs, rhs);
     }
@@ -1058,7 +1172,7 @@ struct RasterState {
             bool inverseFrontFaces                      : 1;        // 31
 
             //! padding, must be 0
-            uint8_t padding                             : 1;        // 32
+            bool depthClamp                             : 1;        // 32
         };
         uint32_t u = 0;
     };
@@ -1069,32 +1183,6 @@ struct RasterState {
  * \privatesection
  */
 
-enum class ShaderStage : uint8_t {
-    VERTEX = 0,
-    FRAGMENT = 1,
-    COMPUTE = 2
-};
-
-static constexpr size_t PIPELINE_STAGE_COUNT = 2;
-enum class ShaderStageFlags : uint8_t {
-    NONE        =    0,
-    VERTEX      =    0x1,
-    FRAGMENT    =    0x2,
-    COMPUTE     =    0x4,
-    ALL_SHADER_STAGE_FLAGS = VERTEX | FRAGMENT | COMPUTE
-};
-
-static inline constexpr bool hasShaderType(ShaderStageFlags flags, ShaderStage type) noexcept {
-    switch (type) {
-        case ShaderStage::VERTEX:
-            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::VERTEX));
-        case ShaderStage::FRAGMENT:
-            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::FRAGMENT));
-        case ShaderStage::COMPUTE:
-            return bool(uint8_t(flags) & uint8_t(ShaderStageFlags::COMPUTE));
-    }
-}
-
 /**
  * Selects which buffers to clear at the beginning of the render pass, as well as which buffers
  * can be discarded at the beginning and end of the render pass.
@@ -1244,7 +1332,7 @@ enum class Workaround : uint16_t {
     ADRENO_UNIFORM_ARRAY_CRASH,
     // Workaround a Metal pipeline compilation error with the message:
     // "Could not statically determine the target of a texture". See light_indirect.fs
-    A8X_STATIC_TEXTURE_TARGET_ERROR,
+    METAL_STATIC_TEXTURE_TARGET_ERROR,
     // Adreno drivers sometimes aren't able to blit into a layer of a texture array.
     DISABLE_BLIT_INTO_TEXTURE_ARRAY,
     // Multiple workarounds needed for PowerVR GPUs
@@ -1259,6 +1347,8 @@ template<> struct utils::EnableBitMaskOperators<filament::backend::ShaderStageFl
         : public std::true_type {};
 template<> struct utils::EnableBitMaskOperators<filament::backend::TargetBufferFlags>
         : public std::true_type {};
+template<> struct utils::EnableBitMaskOperators<filament::backend::DescriptorFlags>
+        : public std::true_type {};
 template<> struct utils::EnableBitMaskOperators<filament::backend::TextureUsage>
         : public std::true_type {};
 template<> struct utils::EnableBitMaskOperators<filament::backend::StencilFace>

filament/backend/include/backend/Handle.h

@@ -23,6 +23,7 @@
 #include <utils/debug.h>
 
 #include <type_traits> // FIXME: STL headers are not allowed in public headers
+#include <utility>
 
 #include <stdint.h>
 
@@ -41,6 +42,8 @@ struct HwTexture;
 struct HwTimerQuery;
 struct HwVertexBufferInfo;
 struct HwVertexBuffer;
+struct HwDescriptorSetLayout;
+struct HwDescriptorSet;
 
 /*
  * A handle to a backend resource. HandleBase is for internal use only.
@@ -104,8 +107,18 @@ struct Handle : public HandleBase {
     Handle(Handle const& rhs) noexcept = default;
     Handle(Handle&& rhs) noexcept = default;
 
-    Handle& operator=(Handle const& rhs) noexcept = default;
-    Handle& operator=(Handle&& rhs) noexcept = default;
+    // Explicitly redefine copy/move assignment operators rather than just using default here.
+    // Because it doesn't make a call to the parent's method automatically during the std::move
+    // function call(https://en.cppreference.com/w/cpp/algorithm/move) in certain compilers like
+    // NDK 25.1.8937393 and below (see b/371980551)
+    Handle& operator=(Handle const& rhs) noexcept {
+        HandleBase::operator=(rhs);
+        return *this;
+    }
+    Handle& operator=(Handle&& rhs) noexcept {
+        HandleBase::operator=(std::move(rhs));
+        return *this;
+    }
 
     explicit Handle(HandleId id) noexcept : HandleBase(id) { }
 
@@ -130,19 +143,21 @@ private:
 
 // Types used by the command stream
 // (we use this renaming because the macro-system doesn't deal well with "<" and ">")
-using BufferObjectHandle     = Handle<HwBufferObject>;
-using FenceHandle            = Handle<HwFence>;
-using IndexBufferHandle      = Handle<HwIndexBuffer>;
-using ProgramHandle          = Handle<HwProgram>;
-using RenderPrimitiveHandle  = Handle<HwRenderPrimitive>;
-using RenderTargetHandle     = Handle<HwRenderTarget>;
-using SamplerGroupHandle     = Handle<HwSamplerGroup>;
-using StreamHandle           = Handle<HwStream>;
-using SwapChainHandle        = Handle<HwSwapChain>;
-using TextureHandle          = Handle<HwTexture>;
-using TimerQueryHandle       = Handle<HwTimerQuery>;
-using VertexBufferHandle     = Handle<HwVertexBuffer>;
-using VertexBufferInfoHandle = Handle<HwVertexBufferInfo>;
+using BufferObjectHandle        = Handle<HwBufferObject>;
+using FenceHandle               = Handle<HwFence>;
+using IndexBufferHandle         = Handle<HwIndexBuffer>;
+using ProgramHandle             = Handle<HwProgram>;
+using RenderPrimitiveHandle     = Handle<HwRenderPrimitive>;
+using RenderTargetHandle        = Handle<HwRenderTarget>;
+using SamplerGroupHandle        = Handle<HwSamplerGroup>;
+using StreamHandle              = Handle<HwStream>;
+using SwapChainHandle           = Handle<HwSwapChain>;
+using TextureHandle             = Handle<HwTexture>;
+using TimerQueryHandle          = Handle<HwTimerQuery>;
+using VertexBufferHandle        = Handle<HwVertexBuffer>;
+using VertexBufferInfoHandle    = Handle<HwVertexBufferInfo>;
+using DescriptorSetLayoutHandle = Handle<HwDescriptorSetLayout>;
+using DescriptorSetHandle       = Handle<HwDescriptorSet>;
 
 } // namespace filament::backend
 

filament/backend/include/backend/PipelineState.h

@@ -22,15 +22,23 @@
 
 #include <utils/ostream.h>
 
+#include <array>
+
 #include <stdint.h>
 
 namespace filament::backend {
 
 //! \privatesection
 
+struct PipelineLayout {
+    using SetLayout = std::array<Handle<HwDescriptorSetLayout>, MAX_DESCRIPTOR_SET_COUNT>;
+    SetLayout setLayout;      // 16
+};
+
 struct PipelineState {
     Handle<HwProgram> program;                                              //  4
     Handle<HwVertexBufferInfo> vertexBufferInfo;                            //  4
+    PipelineLayout pipelineLayout;                                          // 16
     RasterState rasterState;                                                //  4
     StencilState stencilState;                                              // 12
     PolygonOffset polygonOffset;                                            //  8

filament/backend/include/backend/Platform.h

@@ -68,12 +68,7 @@ public:
          */
         size_t handleArenaSize = 0;
 
-        /**
-         * This number of most-recently destroyed textures will be tracked for use-after-free.
-         * Throws an exception when a texture is freed but still bound to a SamplerGroup and used in
-         * a draw call. 0 disables completely. Currently only respected by the Metal backend.
-         */
-        size_t textureUseAfterFreePoolSize = 0;
+        size_t metalUploadBufferSizeBytes = 512 * 1024;
 
         /**
          * Set to `true` to forcibly disable parallel shader compilation in the backend.
@@ -96,6 +91,13 @@ public:
          * Sets the technique for stereoscopic rendering.
          */
         StereoscopicType stereoscopicType = StereoscopicType::NONE;
+
+        /**
+         * Assert the native window associated to a SwapChain is valid when calling makeCurrent().
+         * This is only supported for:
+         *      - PlatformEGLAndroid
+         */
+        bool assertNativeWindowIsValid = false;
     };
 
     Platform() noexcept;

filament/backend/include/backend/Program.h

@@ -24,9 +24,11 @@
 
 #include <backend/DriverEnums.h>
 
-#include <array>    // FIXME: STL headers are not allowed in public headers
-#include <utility>  // FIXME: STL headers are not allowed in public headers
-#include <variant>  // FIXME: STL headers are not allowed in public headers
+#include <array>
+#include <unordered_map>
+#include <tuple>
+#include <utility>
+#include <variant>
 
 #include <stddef.h>
 #include <stdint.h>
@@ -40,29 +42,36 @@ public:
     static constexpr size_t UNIFORM_BINDING_COUNT = CONFIG_UNIFORM_BINDING_COUNT;
     static constexpr size_t SAMPLER_BINDING_COUNT = CONFIG_SAMPLER_BINDING_COUNT;
 
-    struct Sampler {
-        utils::CString name = {};   // name of the sampler in the shader
-        uint32_t binding = 0;       // binding point of the sampler in the shader
+    struct Descriptor {
+        utils::CString name;
+        backend::DescriptorType type;
+        backend::descriptor_binding_t binding;
     };
 
-    struct SamplerGroupData {
-        utils::FixedCapacityVector<Sampler> samplers;
-        ShaderStageFlags stageFlags = ShaderStageFlags::ALL_SHADER_STAGE_FLAGS;
+    struct SpecializationConstant {
+        using Type = std::variant<int32_t, float, bool>;
+        uint32_t id;    // id set in glsl
+        Type value;     // value and type
     };
 
-    struct Uniform {
+    struct Uniform { // For ES2 support
         utils::CString name;    // full qualified name of the uniform field
         uint16_t offset;        // offset in 'uint32_t' into the uniform buffer
         uint8_t size;           // >1 for arrays
         UniformType type;       // uniform type
     };
 
-    using UniformBlockInfo = std::array<utils::CString, UNIFORM_BINDING_COUNT>;
-    using UniformInfo = utils::FixedCapacityVector<Uniform>;
-    using SamplerGroupInfo = std::array<SamplerGroupData, SAMPLER_BINDING_COUNT>;
+    using DescriptorBindingsInfo = utils::FixedCapacityVector<Descriptor>;
+    using DescriptorSetInfo = std::array<DescriptorBindingsInfo, MAX_DESCRIPTOR_SET_COUNT>;
+    using SpecializationConstantsInfo = utils::FixedCapacityVector<SpecializationConstant>;
     using ShaderBlob = utils::FixedCapacityVector<uint8_t>;
     using ShaderSource = std::array<ShaderBlob, SHADER_TYPE_COUNT>;
 
+    using AttributesInfo = utils::FixedCapacityVector<std::pair<utils::CString, uint8_t>>;
+    using UniformInfo = utils::FixedCapacityVector<Uniform>;
+    using BindingUniformsInfo = utils::FixedCapacityVector<
+            std::tuple<uint8_t, utils::CString, Program::UniformInfo>>;
+
     Program() noexcept;
 
     Program(const Program& rhs) = delete;
@@ -79,43 +88,19 @@ public:
     Program& diagnostics(utils::CString const& name,
             utils::Invocable<utils::io::ostream&(utils::io::ostream& out)>&& logger);
 
-    // sets one of the program's shader (e.g. vertex, fragment)
+    // Sets one of the program's shader (e.g. vertex, fragment)
     // string-based shaders are null terminated, consequently the size parameter must include the
     // null terminating character.
     Program& shader(ShaderStage shader, void const* data, size_t size);
 
-    // sets the language of the shader sources provided with shader() (defaults to ESSL3)
+    // Sets the language of the shader sources provided with shader() (defaults to ESSL3)
     Program& shaderLanguage(ShaderLanguage shaderLanguage);
 
-    // Note: This is only needed for GLES3.0 backends, because the layout(binding=) syntax is
-    //       not permitted in glsl. The backend needs a way to associate a uniform block
-    //       to a binding point.
-    Program& uniformBlockBindings(
-            utils::FixedCapacityVector<std::pair<utils::CString, uint8_t>> const& uniformBlockBindings) noexcept;
+    // Descriptor binding (set, binding, type -> shader name) info
+    Program& descriptorBindings(backend::descriptor_set_t set,
+            DescriptorBindingsInfo descriptorBindings) noexcept;
 
-    // Note: This is only needed for GLES2.0, this is used to emulate UBO. This function tells
-    // the program everything it needs to know about the uniforms at a given binding
-    Program& uniforms(uint32_t index, UniformInfo const& uniforms) noexcept;
-
-    // Note: This is only needed for GLES2.0.
-    Program& attributes(
-            utils::FixedCapacityVector<std::pair<utils::CString, uint8_t>> attributes) noexcept;
-
-    // sets the 'bindingPoint' sampler group descriptor for this program.
-    // 'samplers' can be destroyed after this call.
-    // This effectively associates a set of (BindingPoints, index) to a texture unit in the shader.
-    // Or more precisely, what layout(binding=) is set to in GLSL.
-    Program& setSamplerGroup(size_t bindingPoint, ShaderStageFlags stageFlags,
-            Sampler const* samplers, size_t count) noexcept;
-
-    struct SpecializationConstant {
-        using Type = std::variant<int32_t, float, bool>;
-        uint32_t id;                                // id set in glsl
-        Type value;                                 // value and type
-    };
-
-    Program& specializationConstants(
-            utils::FixedCapacityVector<SpecializationConstant> specConstants) noexcept;
+    Program& specializationConstants(SpecializationConstantsInfo specConstants) noexcept;
 
     struct PushConstant {
         utils::CString name;
@@ -129,33 +114,40 @@ public:
 
     Program& multiview(bool multiview) noexcept;
 
+    // For ES2 support only...
+    Program& uniforms(uint32_t index, utils::CString name, UniformInfo uniforms) noexcept;
+    Program& attributes(AttributesInfo attributes) noexcept;
+
+    //
+    // Getters for program construction...
+    //
+
     ShaderSource const& getShadersSource() const noexcept { return mShadersSource; }
     ShaderSource& getShadersSource() noexcept { return mShadersSource; }
 
-    UniformBlockInfo const& getUniformBlockBindings() const noexcept { return mUniformBlocks; }
-    UniformBlockInfo& getUniformBlockBindings() noexcept { return mUniformBlocks; }
-
-    SamplerGroupInfo const& getSamplerGroupInfo() const { return mSamplerGroups; }
-    SamplerGroupInfo& getSamplerGroupInfo() { return mSamplerGroups; }
-
-    auto const& getBindingUniformInfo() const { return mBindingUniformInfo; }
-    auto& getBindingUniformInfo() { return mBindingUniformInfo; }
-
-    auto const& getAttributes() const { return mAttributes; }
-    auto& getAttributes() { return mAttributes; }
-
     utils::CString const& getName() const noexcept { return mName; }
     utils::CString& getName() noexcept { return mName; }
 
     auto const& getShaderLanguage() const { return mShaderLanguage; }
 
-    utils::FixedCapacityVector<SpecializationConstant> const& getSpecializationConstants() const noexcept {
+    uint64_t getCacheId() const noexcept { return mCacheId; }
+
+    bool isMultiview() const noexcept { return mMultiview; }
+
+    CompilerPriorityQueue getPriorityQueue() const noexcept { return mPriorityQueue; }
+
+    SpecializationConstantsInfo const& getSpecializationConstants() const noexcept {
         return mSpecializationConstants;
     }
-    utils::FixedCapacityVector<SpecializationConstant>& getSpecializationConstants() noexcept {
+
+    SpecializationConstantsInfo& getSpecializationConstants() noexcept {
         return mSpecializationConstants;
     }
 
+    DescriptorSetInfo& getDescriptorBindings() noexcept {
+        return mDescriptorBindings;
+    }
+
     utils::FixedCapacityVector<PushConstant> const& getPushConstants(
             ShaderStage stage) const noexcept {
         return mPushConstants[static_cast<uint8_t>(stage)];
@@ -165,27 +157,29 @@ public:
         return mPushConstants[static_cast<uint8_t>(stage)];
     }
 
-    uint64_t getCacheId() const noexcept { return mCacheId; }
+    auto const& getBindingUniformInfo() const { return mBindingUniformsInfo; }
+    auto& getBindingUniformInfo() { return mBindingUniformsInfo; }
 
-    bool isMultiview() const noexcept { return mMultiview; }
-
-    CompilerPriorityQueue getPriorityQueue() const noexcept { return mPriorityQueue; }
+    auto const& getAttributes() const { return mAttributes; }
+    auto& getAttributes() { return mAttributes; }
 
 private:
     friend utils::io::ostream& operator<<(utils::io::ostream& out, const Program& builder);
 
-    UniformBlockInfo mUniformBlocks = {};
-    SamplerGroupInfo mSamplerGroups = {};
     ShaderSource mShadersSource;
     ShaderLanguage mShaderLanguage = ShaderLanguage::ESSL3;
     utils::CString mName;
     uint64_t mCacheId{};
-    utils::Invocable<utils::io::ostream&(utils::io::ostream& out)> mLogger;
-    utils::FixedCapacityVector<SpecializationConstant> mSpecializationConstants;
-    std::array<utils::FixedCapacityVector<PushConstant>, SHADER_TYPE_COUNT> mPushConstants;
-    utils::FixedCapacityVector<std::pair<utils::CString, uint8_t>> mAttributes;
-    std::array<UniformInfo, Program::UNIFORM_BINDING_COUNT> mBindingUniformInfo;
     CompilerPriorityQueue mPriorityQueue = CompilerPriorityQueue::HIGH;
+    utils::Invocable<utils::io::ostream&(utils::io::ostream& out)> mLogger;
+    SpecializationConstantsInfo mSpecializationConstants;
+    std::array<utils::FixedCapacityVector<PushConstant>, SHADER_TYPE_COUNT> mPushConstants;
+    DescriptorSetInfo mDescriptorBindings;
+
+    // For ES2 support only
+    AttributesInfo mAttributes;
+    BindingUniformsInfo mBindingUniformsInfo;
+
     // Indicates the current engine was initialized with multiview stereo, and the variant for this
     // program contains STE flag. This will be referred later for the OpenGL shader compiler to
     // determine whether shader code replacement for the num_views should be performed.

filament/backend/include/backend/TargetBufferInfo.h

@@ -30,10 +30,6 @@ namespace filament::backend {
 
 struct TargetBufferInfo {
     // note: the parameters of this constructor are not in the order of this structure's fields
-    TargetBufferInfo(Handle<HwTexture> handle, uint8_t level, uint16_t layer, uint8_t baseViewIndex) noexcept
-        : handle(handle), baseViewIndex(baseViewIndex), level(level), layer(layer) {
-    }
-
     TargetBufferInfo(Handle<HwTexture> handle, uint8_t level, uint16_t layer) noexcept
             : handle(handle), level(level), layer(layer) {
     }
@@ -51,14 +47,15 @@ struct TargetBufferInfo {
     // texture to be used as render target
     Handle<HwTexture> handle;
 
-    // Starting layer index for multiview. This value is only used when the `layerCount` for the
-    // render target is greater than 1.
-    uint8_t baseViewIndex = 0;
-
     // level to be used
     uint8_t level = 0;
 
-    // For cubemaps and 3D textures. See TextureCubemapFace for the face->layer mapping
+    // - For cubemap textures, this indicates the face of the cubemap. See TextureCubemapFace for
+    //   the face->layer mapping)
+    // - For 2d array, cubemap array, and 3d textures, this indicates an index of a single layer of
+    //   them.
+    // - For multiview textures (i.e., layerCount for the RenderTarget is greater than 1), this
+    //   indicates a starting layer index of the current 2d array texture for multiview.
     uint16_t layer = 0;
 };
 
@@ -103,7 +100,7 @@ public:
 
     // this is here for backward compatibility
     MRT(Handle<HwTexture> handle, uint8_t level, uint16_t layer) noexcept
-            : mInfos{{ handle, level, layer, 0 }} {
+            : mInfos{{ handle, level, layer }} {
     }
 };
 

filament/backend/include/backend/platforms/PlatformEGLAndroid.h

@@ -89,14 +89,28 @@ protected:
      */
     AcquiredImage transformAcquiredImage(AcquiredImage source) noexcept override;
 
+protected:
+    bool makeCurrent(ContextType type,
+            SwapChain* drawSwapChain,
+            SwapChain* readSwapChain) noexcept override;
+
 private:
+    struct InitializeJvmForPerformanceManagerIfNeeded {
+        InitializeJvmForPerformanceManagerIfNeeded();
+    };
+
     int mOSVersion;
     ExternalStreamManagerAndroid& mExternalStreamManager;
+    InitializeJvmForPerformanceManagerIfNeeded const mInitializeJvmForPerformanceManagerIfNeeded;
     utils::PerformanceHintManager mPerformanceHintManager;
     utils::PerformanceHintManager::Session mPerformanceHintSession;
 
     using clock = std::chrono::high_resolution_clock;
     clock::time_point mStartTimeOfActualWork;
+
+    void* mNativeWindowLib = nullptr;
+    int32_t (*ANativeWindow_getBuffersDefaultDataSpace)(ANativeWindow* window) = nullptr;
+    bool mAssertNativeWindowIsValid = false;
 };
 
 } // namespace filament::backend

filament/backend/include/backend/platforms/PlatformOSMesa.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_OPENGL_OPENGL_PLATFORM_OSMESA_H
+#define TNT_FILAMENT_BACKEND_OPENGL_OPENGL_PLATFORM_OSMESA_H
+
+#include <stdint.h>
+
+#include "bluegl/BlueGL.h"
+
+#include <osmesa.h>
+
+#include <backend/platforms/OpenGLPlatform.h>
+#include <backend/DriverEnums.h>
+
+namespace filament::backend {
+
+/**
+ * A concrete implementation of OpenGLPlatform that uses OSMesa, which is an offscreen
+ * context that can be used in conjunction with Mesa for software rasterization.
+ * See https://docs.mesa3d.org/osmesa.html for more information.
+ */
+class PlatformOSMesa : public OpenGLPlatform {
+protected:
+    // --------------------------------------------------------------------------------------------
+    // Platform Interface
+
+    Driver* createDriver(void* sharedGLContext, const DriverConfig& driverConfig) noexcept override;
+
+    int getOSVersion() const noexcept final override { return 0; }
+
+    // --------------------------------------------------------------------------------------------
+    // OpenGLPlatform Interface
+
+    void terminate() noexcept override;
+
+    SwapChain* createSwapChain(void* nativewindow, uint64_t flags) noexcept override;
+    SwapChain* createSwapChain(uint32_t width, uint32_t height, uint64_t flags) noexcept override;
+    void destroySwapChain(SwapChain* swapChain) noexcept override;
+    bool makeCurrent(ContextType type, SwapChain* drawSwapChain,
+            SwapChain* readSwapChain) noexcept override;
+    void commit(SwapChain* swapChain) noexcept override;
+
+private:
+    OSMesaContext mContext;
+    void* mOsMesaApi = nullptr;
+};
+
+} // namespace filament::backend
+
+#endif // TNT_FILAMENT_BACKEND_OPENGL_OPENGL_PLATFORM_OSMESA_H

filament/backend/include/backend/platforms/VulkanPlatform.h

@@ -88,6 +88,21 @@ public:
         VkFormat colorFormat = VK_FORMAT_UNDEFINED;
         VkFormat depthFormat = VK_FORMAT_UNDEFINED;
         VkExtent2D extent = {0, 0};
+        bool isProtected = false;
+    };
+
+    struct ImageSyncData {
+        static constexpr uint32_t INVALID_IMAGE_INDEX = UINT32_MAX;
+
+        // The index of the next image as returned by vkAcquireNextImage or equivalent.
+        uint32_t imageIndex = INVALID_IMAGE_INDEX;
+
+        // Semaphore to be signaled once the image is available.
+        VkSemaphore imageReadySemaphore = VK_NULL_HANDLE;
+
+        // A function called right before vkQueueSubmit. After this call, the image must be 
+        // available. This pointer can be null if imageReadySemaphore is not VK_NULL_HANDLE.
+        std::function<void(SwapChainPtr handle)> explicitImageReadyWait = nullptr;
     };
 
     VulkanPlatform();
@@ -127,6 +142,12 @@ public:
          * before recreating the swapchain. Default is true.
          */
         bool flushAndWaitOnWindowResize = true;
+
+        /**
+         * Whether the swapchain image should be transitioned to a layout suitable for
+         * presentation. Default is true.
+         */
+        bool transitionSwapChainImageLayoutForPresent = true;
     };
 
     /**
@@ -155,13 +176,10 @@ public:
      * corresponding VkImage will be used as the output color attachment. The client should signal
      * the `clientSignal` semaphore when the image is ready to be used by the backend.
      * @param handle         The handle returned by createSwapChain()
-     * @param clientSignal   The semaphore that the client will signal to indicate that the backend
-     *                       may render into the image.
-     * @param index          Pointer to memory that will be filled with the index that corresponding
-     *                       to an image in the `SwapChainBundle.colors` array.
+     * @param outImageSyncData The synchronization data used for image readiness
      * @return               Result of acquire
      */
-    virtual VkResult acquire(SwapChainPtr handle, VkSemaphore clientSignal, uint32_t* index);
+    virtual VkResult acquire(SwapChainPtr handle, ImageSyncData* outImageSyncData);
 
     /**
      * Present the image corresponding to `index` to the display. The client should wait on
@@ -182,6 +200,13 @@ public:
      */
     virtual bool hasResized(SwapChainPtr handle);
 
+    /**
+     * Check if the surface is protected.
+     * @param handle             The handle returned by createSwapChain()
+     * @return                   Whether the swapchain is protected
+     */
+    virtual bool isProtected(SwapChainPtr handle);
+
     /**
      * Carry out a recreation of the swapchain.
      * @param handle             The handle returned by createSwapChain()
@@ -250,6 +275,23 @@ public:
      */
     VkQueue getGraphicsQueue() const noexcept;
 
+    /**
+    * @return The family index of the protected graphics queue selected for the
+    *          Vulkan backend.
+    */
+    uint32_t getProtectedGraphicsQueueFamilyIndex() const noexcept;
+
+    /**
+     * @return The index of the protected graphics queue (if there are multiple
+     *          graphics queues) selected for the Vulkan backend.
+     */
+    uint32_t getProtectedGraphicsQueueIndex() const noexcept;
+
+    /**
+     * @return The protected queue that was selected for the Vulkan backend.
+     */
+    VkQueue getProtectedGraphicsQueue() const noexcept;
+
 private:
     static ExtensionSet getSwapchainInstanceExtensions();
 

filament/backend/include/private/backend/Driver.h

@@ -18,6 +18,7 @@
 #define TNT_FILAMENT_BACKEND_PRIVATE_DRIVER_H
 
 #include <backend/CallbackHandler.h>
+#include <backend/DescriptorSetOffsetArray.h>
 #include <backend/DriverApiForward.h>
 #include <backend/DriverEnums.h>
 #include <backend/Handle.h>

filament/backend/include/private/backend/DriverAPI.inc

@@ -139,7 +139,8 @@ DECL_DRIVER_API_N(beginFrame,
 DECL_DRIVER_API_N(setFrameScheduledCallback,
         backend::SwapChainHandle, sch,
         backend::CallbackHandler*, handler,
-        backend::FrameScheduledCallback&&, callback)
+        backend::FrameScheduledCallback&&, callback,
+        uint64_t, flags)
 
 DECL_DRIVER_API_N(setFrameCompletedCallback,
         backend::SwapChainHandle, sch,
@@ -162,6 +163,10 @@ DECL_DRIVER_API_0(finish)
 // reset state tracking, if the driver does any state tracking (e.g. GL)
 DECL_DRIVER_API_0(resetState)
 
+DECL_DRIVER_API_N(setDebugTag,
+        backend::HandleBase::HandleId, handleId,
+        utils::CString, tag)
+
 /*
  * Creating driver objects
  * -----------------------
@@ -196,20 +201,33 @@ DECL_DRIVER_API_R_N(backend::TextureHandle, createTexture,
         uint32_t, depth,
         backend::TextureUsage, usage)
 
-DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureSwizzled,
-        backend::SamplerType, target,
-        uint8_t, levels,
-        backend::TextureFormat, format,
-        uint8_t, samples,
-        uint32_t, width,
-        uint32_t, height,
-        uint32_t, depth,
-        backend::TextureUsage, usage,
+DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureView,
+        backend::TextureHandle, texture,
+        uint8_t, baseLevel,
+        uint8_t, levelCount)
+
+DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureViewSwizzle,
+        backend::TextureHandle, texture,
         backend::TextureSwizzle, r,
         backend::TextureSwizzle, g,
         backend::TextureSwizzle, b,
         backend::TextureSwizzle, a)
 
+DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureExternalImage,
+        backend::TextureFormat, format,
+        uint32_t, width,
+        uint32_t, height,
+        backend::TextureUsage, usage,
+        void*, image)
+
+DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureExternalImagePlane,
+        backend::TextureFormat, format,
+        uint32_t, width,
+        uint32_t, height,
+        backend::TextureUsage, usage,
+        void*, image,
+        uint32_t, plane)
+
 DECL_DRIVER_API_R_N(backend::TextureHandle, importTexture,
         intptr_t, id,
         backend::SamplerType, target,
@@ -221,9 +239,6 @@ DECL_DRIVER_API_R_N(backend::TextureHandle, importTexture,
         uint32_t, depth,
         backend::TextureUsage, usage)
 
-DECL_DRIVER_API_R_N(backend::SamplerGroupHandle, createSamplerGroup,
-        uint32_t, size, utils::FixedSizeString<32>, debugName)
-
 DECL_DRIVER_API_R_N(backend::RenderPrimitiveHandle, createRenderPrimitive,
         backend::VertexBufferHandle, vbh,
         backend::IndexBufferHandle, ibh,
@@ -257,25 +272,53 @@ DECL_DRIVER_API_R_N(backend::SwapChainHandle, createSwapChainHeadless,
 
 DECL_DRIVER_API_R_0(backend::TimerQueryHandle, createTimerQuery)
 
+DECL_DRIVER_API_R_N(backend::DescriptorSetLayoutHandle, createDescriptorSetLayout,
+        backend::DescriptorSetLayout&&, info)
+
+DECL_DRIVER_API_R_N(backend::DescriptorSetHandle, createDescriptorSet,
+        backend::DescriptorSetLayoutHandle, dslh)
+
+DECL_DRIVER_API_N(updateDescriptorSetBuffer,
+        backend::DescriptorSetHandle, dsh,
+        backend::descriptor_binding_t, binding,
+        backend::BufferObjectHandle, boh,
+        uint32_t, offset,
+        uint32_t, size
+)
+
+DECL_DRIVER_API_N(updateDescriptorSetTexture,
+        backend::DescriptorSetHandle, dsh,
+        backend::descriptor_binding_t, binding,
+        backend::TextureHandle, th,
+        SamplerParams, params
+)
+
+DECL_DRIVER_API_N(bindDescriptorSet,
+        backend::DescriptorSetHandle, dsh,
+        backend::descriptor_set_t, set,
+        backend::DescriptorSetOffsetArray&&, offsets
+)
+
 
 /*
  * Destroying driver objects
  * -------------------------
  */
 
-DECL_DRIVER_API_N(destroyVertexBuffer,    backend::VertexBufferHandle, vbh)
-DECL_DRIVER_API_N(destroyVertexBufferInfo,backend::VertexBufferInfoHandle, vbih)
-DECL_DRIVER_API_N(destroyIndexBuffer,     backend::IndexBufferHandle, ibh)
-DECL_DRIVER_API_N(destroyBufferObject,    backend::BufferObjectHandle, ibh)
-DECL_DRIVER_API_N(destroyRenderPrimitive, backend::RenderPrimitiveHandle, rph)
-DECL_DRIVER_API_N(destroyProgram,         backend::ProgramHandle, ph)
-DECL_DRIVER_API_N(destroySamplerGroup,    backend::SamplerGroupHandle, sbh)
-DECL_DRIVER_API_N(destroyTexture,         backend::TextureHandle, th)
-DECL_DRIVER_API_N(destroyRenderTarget,    backend::RenderTargetHandle, rth)
-DECL_DRIVER_API_N(destroySwapChain,       backend::SwapChainHandle, sch)
-DECL_DRIVER_API_N(destroyStream,          backend::StreamHandle, sh)
-DECL_DRIVER_API_N(destroyTimerQuery,      backend::TimerQueryHandle, sh)
-DECL_DRIVER_API_N(destroyFence,           backend::FenceHandle, fh)
+DECL_DRIVER_API_N(destroyVertexBuffer,          backend::VertexBufferHandle, vbh)
+DECL_DRIVER_API_N(destroyVertexBufferInfo,      backend::VertexBufferInfoHandle, vbih)
+DECL_DRIVER_API_N(destroyIndexBuffer,           backend::IndexBufferHandle, ibh)
+DECL_DRIVER_API_N(destroyBufferObject,          backend::BufferObjectHandle, ibh)
+DECL_DRIVER_API_N(destroyRenderPrimitive,       backend::RenderPrimitiveHandle, rph)
+DECL_DRIVER_API_N(destroyProgram,               backend::ProgramHandle, ph)
+DECL_DRIVER_API_N(destroyTexture,               backend::TextureHandle, th)
+DECL_DRIVER_API_N(destroyRenderTarget,          backend::RenderTargetHandle, rth)
+DECL_DRIVER_API_N(destroySwapChain,             backend::SwapChainHandle, sch)
+DECL_DRIVER_API_N(destroyStream,                backend::StreamHandle, sh)
+DECL_DRIVER_API_N(destroyTimerQuery,            backend::TimerQueryHandle, sh)
+DECL_DRIVER_API_N(destroyFence,                 backend::FenceHandle, fh)
+DECL_DRIVER_API_N(destroyDescriptorSetLayout,   backend::DescriptorSetLayoutHandle, dslh)
+DECL_DRIVER_API_N(destroyDescriptorSet,         backend::DescriptorSetHandle, dsh)
 
 /*
  * Synchronous APIs
@@ -305,6 +348,7 @@ DECL_DRIVER_API_SYNCHRONOUS_0(bool, isParallelShaderCompileSupported)
 DECL_DRIVER_API_SYNCHRONOUS_0(bool, isDepthStencilResolveSupported)
 DECL_DRIVER_API_SYNCHRONOUS_N(bool, isDepthStencilBlitSupported, backend::TextureFormat, format)
 DECL_DRIVER_API_SYNCHRONOUS_0(bool, isProtectedTexturesSupported)
+DECL_DRIVER_API_SYNCHRONOUS_0(bool, isDepthClampSupported)
 DECL_DRIVER_API_SYNCHRONOUS_0(uint8_t, getMaxDrawBuffers)
 DECL_DRIVER_API_SYNCHRONOUS_0(size_t, getMaxUniformBufferSize)
 DECL_DRIVER_API_SYNCHRONOUS_0(math::float2, getClipSpaceParams)
@@ -341,15 +385,6 @@ DECL_DRIVER_API_N(updateBufferObjectUnsynchronized,
 DECL_DRIVER_API_N(resetBufferObject,
         backend::BufferObjectHandle, ibh)
 
-DECL_DRIVER_API_N(updateSamplerGroup,
-        backend::SamplerGroupHandle, ubh,
-        backend::BufferDescriptor&&, data)
-
-DECL_DRIVER_API_N(setMinMaxLevels,
-        backend::TextureHandle, th,
-        uint32_t, minLevel,
-        uint32_t, maxLevel)
-
 DECL_DRIVER_API_N(update3DImage,
         backend::TextureHandle, th,
         uint32_t, level,
@@ -364,10 +399,12 @@ DECL_DRIVER_API_N(update3DImage,
 DECL_DRIVER_API_N(generateMipmaps,
         backend::TextureHandle, th)
 
+// Deprecated
 DECL_DRIVER_API_N(setExternalImage,
         backend::TextureHandle, th,
         void*, image)
 
+// Deprecated
 DECL_DRIVER_API_N(setExternalImagePlane,
         backend::TextureHandle, th,
         void*, image,
@@ -414,37 +451,16 @@ DECL_DRIVER_API_N(commit,
  * -----------------------
  */
 
-DECL_DRIVER_API_N(bindUniformBuffer,
-        uint32_t, index,
-        backend::BufferObjectHandle, ubh)
-
-DECL_DRIVER_API_N(bindBufferRange,
-        BufferObjectBinding, bindingType,
-        uint32_t, index,
-        backend::BufferObjectHandle, ubh,
-        uint32_t, offset,
-        uint32_t, size)
-
-DECL_DRIVER_API_N(unbindBuffer,
-        BufferObjectBinding, bindingType,
-        uint32_t, index)
-
-DECL_DRIVER_API_N(bindSamplers,
-        uint32_t, index,
-        backend::SamplerGroupHandle, sbh)
-
 DECL_DRIVER_API_N(setPushConstant,
         backend::ShaderStage, stage,
         uint8_t, index,
         backend::PushConstantVariant, value)
 
 DECL_DRIVER_API_N(insertEventMarker,
-        const char*, string,
-        uint32_t, len = 0)
+        const char*, string)
 
 DECL_DRIVER_API_N(pushGroupMarker,
-        const char*, string,
-        uint32_t, len = 0)
+        const char*, string)
 
 DECL_DRIVER_API_0(popGroupMarker)
 

filament/backend/include/private/backend/DriverApi.h

@@ -18,6 +18,17 @@
 #define TNT_FILAMENT_BACKEND_PRIVATE_DRIVERAPI_H
 
 #include "backend/DriverApiForward.h"
+
 #include "private/backend/CommandStream.h"
 
+#include <stddef.h>
+
+namespace filament::backend {
+
+inline void* allocateFromCommandStream(DriverApi& driver, size_t size, size_t alignment) noexcept {
+    return driver.allocate(size, alignment);
+}
+
+} // namespace filament::backend
+
 #endif // TNT_FILAMENT_BACKEND_PRIVATE_DRIVERAPI_H

filament/backend/include/private/backend/HandleAllocator.h

@@ -20,11 +20,12 @@
 #include <backend/Handle.h>
 
 #include <utils/Allocator.h>
+#include <utils/CString.h>
 #include <utils/Log.h>
+#include <utils/Panic.h>
 #include <utils/compiler.h>
 #include <utils/debug.h>
 #include <utils/ostream.h>
-#include <utils/Panic.h>
 
 #include <tsl/robin_map.h>
 
@@ -37,9 +38,9 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#define HandleAllocatorGL  HandleAllocator<32,  64, 136>    // ~4520 / pool / MiB
+#define HandleAllocatorGL  HandleAllocator<32,  96, 136>    // ~4520 / pool / MiB
 #define HandleAllocatorVK  HandleAllocator<64, 160, 312>    // ~1820 / pool / MiB
-#define HandleAllocatorMTL HandleAllocator<32,  48, 552>    // ~1660 / pool / MiB
+#define HandleAllocatorMTL HandleAllocator<32,  64, 552>    // ~1660 / pool / MiB
 
 namespace filament::backend {
 
@@ -168,19 +169,53 @@ public:
         auto [p, tag] = handleToPointer(handle.getId());
 
         if (isPoolHandle(handle.getId())) {
-            // check for use after free
+            // check for pool handle use-after-free
             if (UTILS_UNLIKELY(!mUseAfterFreeCheckDisabled)) {
                 uint8_t const age = (tag & HANDLE_AGE_MASK) >> HANDLE_AGE_SHIFT;
                 auto const pNode = static_cast<typename Allocator::Node*>(p);
                 uint8_t const expectedAge = pNode[-1].age;
-                FILAMENT_CHECK_POSTCONDITION(expectedAge == age) <<
-                        "use-after-free of Handle with id=" << handle.getId();
+                // getHandleTag() is only called if the check fails.
+                FILAMENT_CHECK_POSTCONDITION(expectedAge == age)
+                        << "use-after-free of Handle with id=" << handle.getId()
+                        << ", tag=" << getHandleTag(handle.getId()).c_str_safe();
+            }
+        } else {
+            // check for heap handle use-after-free
+            if (UTILS_UNLIKELY(!mUseAfterFreeCheckDisabled)) {
+                uint8_t const index = (handle.getId() & HANDLE_INDEX_MASK);
+                // if we've already handed out this handle index before, it's definitely a
+                // use-after-free, otherwise it's probably just a corrupted handle
+                if (index < mId) {
+                    FILAMENT_CHECK_POSTCONDITION(p != nullptr)
+                            << "use-after-free of heap Handle with id=" << handle.getId()
+                            << ", tag=" << getHandleTag(handle.getId()).c_str_safe();
+                } else {
+                    FILAMENT_CHECK_POSTCONDITION(p != nullptr)
+                            << "corrupted heap Handle with id=" << handle.getId()
+                            << ", tag=" << getHandleTag(handle.getId()).c_str_safe();
+                }
             }
         }
 
         return static_cast<Dp>(p);
     }
 
+    template<typename B>
+    bool is_valid(Handle<B>& handle) {
+        if (!handle) {
+            // null handles are invalid
+            return false;
+        }
+        auto [p, tag] = handleToPointer(handle.getId());
+        if (isPoolHandle(handle.getId())) {
+            uint8_t const age = (tag & HANDLE_AGE_MASK) >> HANDLE_AGE_SHIFT;
+            auto const pNode = static_cast<typename Allocator::Node*>(p);
+            uint8_t const expectedAge = pNode[-1].age;
+            return expectedAge == age;
+        }
+        return p != nullptr;
+    }
+
     template<typename Dp, typename B>
     inline typename std::enable_if_t<
             std::is_pointer_v<Dp> &&
@@ -189,6 +224,29 @@ public:
         return handle_cast<Dp>(const_cast<Handle<B>&>(handle));
     }
 
+    void associateTagToHandle(HandleBase::HandleId id, utils::CString&& tag) noexcept {
+        // TODO: for now, only pool handles check for use-after-free, so we only keep tags for
+        // those
+        if (isPoolHandle(id)) {
+            // Truncate the age to get the debug tag
+            uint32_t const key = id & ~(HANDLE_DEBUG_TAG_MASK ^ HANDLE_AGE_MASK);
+            // This line is the costly part. In the future, we could potentially use a custom
+            // allocator.
+            mDebugTags[key] = std::move(tag);
+        }
+    }
+
+    utils::CString getHandleTag(HandleBase::HandleId id) const noexcept {
+        if (!isPoolHandle(id)) {
+            return "(no tag)";
+        }
+        uint32_t const key = id & ~(HANDLE_DEBUG_TAG_MASK ^ HANDLE_AGE_MASK);
+        if (auto pos = mDebugTags.find(key); pos != mDebugTags.end()) {
+            return pos->second;
+        }
+        return "(no tag)";
+    }
+
 private:
 
     template<typename D>
@@ -306,12 +364,24 @@ private:
         }
     }
 
-    // we handle a 4 bits age per address
-    static constexpr uint32_t HANDLE_HEAP_FLAG      = 0x80000000u;      // pool vs heap handle
-    static constexpr uint32_t HANDLE_AGE_MASK       = 0x78000000u;      // handle's age
-    static constexpr uint32_t HANDLE_INDEX_MASK     = 0x07FFFFFFu;      // handle index
-    static constexpr uint32_t HANDLE_TAG_MASK       = HANDLE_AGE_MASK;
-    static constexpr uint32_t HANDLE_AGE_SHIFT      = 27;
+    // number if bits allotted to the handle's age (currently 4 max)
+    static constexpr uint32_t HANDLE_AGE_BIT_COUNT = 4;
+    // number if bits allotted to the handle's debug tag (HANDLE_AGE_BIT_COUNT max)
+    static constexpr uint32_t HANDLE_DEBUG_TAG_BIT_COUNT = 2;
+    // bit shift for both the age and debug tag
+    static constexpr uint32_t HANDLE_AGE_SHIFT = 27;
+    // mask for the heap (vs pool) flag
+    static constexpr uint32_t HANDLE_HEAP_FLAG = 0x80000000u;
+    // mask for the age
+    static constexpr uint32_t HANDLE_AGE_MASK =
+            ((1 << HANDLE_AGE_BIT_COUNT) - 1) << HANDLE_AGE_SHIFT;
+    // mask for the debug tag
+    static constexpr uint32_t HANDLE_DEBUG_TAG_MASK =
+            ((1 << HANDLE_DEBUG_TAG_BIT_COUNT) - 1) << HANDLE_AGE_SHIFT;
+    // mask for the index
+    static constexpr uint32_t HANDLE_INDEX_MASK = 0x07FFFFFFu;
+
+    static_assert(HANDLE_DEBUG_TAG_BIT_COUNT <= HANDLE_AGE_BIT_COUNT);
 
     static bool isPoolHandle(HandleBase::HandleId id) noexcept {
         return (id & HANDLE_HEAP_FLAG) == 0u;
@@ -326,7 +396,7 @@ private:
         // a non-pool handle.
         if (UTILS_LIKELY(isPoolHandle(id))) {
             char* const base = (char*)mHandleArena.getArea().begin();
-            uint32_t const tag = id & HANDLE_TAG_MASK;
+            uint32_t const tag = id & HANDLE_AGE_MASK;
             size_t const offset = (id & HANDLE_INDEX_MASK) * Allocator::getAlignment();
             return { static_cast<void*>(base + offset), tag };
         }
@@ -341,7 +411,7 @@ private:
         size_t const offset = (char*)p - base;
         assert_invariant((offset % Allocator::getAlignment()) == 0);
         auto id = HandleBase::HandleId(offset / Allocator::getAlignment());
-        id |= tag & HANDLE_TAG_MASK;
+        id |= tag & HANDLE_AGE_MASK;
         assert_invariant((id & HANDLE_HEAP_FLAG) == 0);
         return id;
     }
@@ -351,6 +421,7 @@ private:
     // Below is only used when running out of space in the HandleArena
     mutable utils::Mutex mLock;
     tsl::robin_map<HandleBase::HandleId, void*> mOverflowMap;
+    tsl::robin_map<HandleBase::HandleId, utils::CString> mDebugTags;
     HandleBase::HandleId mId = 0;
     bool mUseAfterFreeCheckDisabled = false;
 };

filament/backend/src/CircularBuffer.cpp

@@ -24,7 +24,7 @@
 #include <utils/debug.h>
 #include <utils/ostream.h>
 
-#if !defined(WIN32) && !defined(__EMSCRIPTEN__) && !defined(IOS)
+#if !defined(WIN32) && !defined(__EMSCRIPTEN__)
 #    include <sys/mman.h>
 #    include <unistd.h>
 #    define HAS_MMAP 1
@@ -32,10 +32,11 @@
 #    define HAS_MMAP 0
 #endif
 
-#include <stdint.h>
 #include <stddef.h>
-#include <stdlib.h>
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 using namespace utils;
 
@@ -81,6 +82,9 @@ void* CircularBuffer::alloc(size_t size) noexcept {
             // map the circular buffer once...
             vaddr = mmap(reserve_vaddr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
             if (vaddr != MAP_FAILED) {
+                // populate the address space with pages (because this is a circular buffer,
+                // all the pages will be allocated eventually, might as well do it now)
+                memset(vaddr, 0, size);
                 // and map the circular buffer again, behind the previous copy...
                 vaddr_shadow = mmap((char*)vaddr + size, size,
                         PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
@@ -101,7 +105,7 @@ void* CircularBuffer::alloc(size_t size) noexcept {
     if (UTILS_UNLIKELY(mAshmemFd < 0)) {
         // ashmem failed
         if (vaddr_guard != MAP_FAILED) {
-            munmap(vaddr_guard, size);
+            munmap(vaddr_guard, BLOCK_SIZE);
         }
 
         if (vaddr_shadow != MAP_FAILED) {
@@ -119,12 +123,11 @@ void* CircularBuffer::alloc(size_t size) noexcept {
         data = mmap(nullptr, size * 2 + BLOCK_SIZE,
                 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
-        FILAMENT_CHECK_POSTCONDITION(data) <<
+        FILAMENT_CHECK_POSTCONDITION(data != MAP_FAILED) <<
                 "couldn't allocate " << (size * 2 / 1024) <<
                 " KiB of virtual address space for the command buffer";
 
-        slog.d << "WARNING: Using soft CircularBuffer (" << (size * 2 / 1024) << " KiB)"
-               << io::endl;
+        slog.w << "Using 'soft' CircularBuffer (" << (size * 2 / 1024) << " KiB)" << io::endl;
 
         // guard page at the end
         void* guard = (void*)(uintptr_t(data) + size * 2);

filament/backend/src/CommandBufferQueue.cpp

@@ -101,9 +101,8 @@ void CommandBufferQueue::flush() noexcept {
     size_t const used = std::distance(
             static_cast<char const*>(begin), static_cast<char const*>(end));
 
+
     std::unique_lock<utils::Mutex> lock(mLock);
-    mCommandBuffersToExecute.push_back({ begin, end });
-    mCondition.notify_one();
 
     // circular buffer is too small, we corrupted the stream
     FILAMENT_CHECK_POSTCONDITION(used <= mFreeSpace) <<
@@ -112,10 +111,12 @@ void CommandBufferQueue::flush() noexcept {
             "Space used at this time: " << used <<
             " bytes, overflow: " << used - mFreeSpace << " bytes";
 
-    // wait until there is enough space in the buffer
     mFreeSpace -= used;
-    if (UTILS_UNLIKELY(mFreeSpace < requiredSize)) {
+    mCommandBuffersToExecute.push_back({ begin, end });
+    mCondition.notify_one();
 
+    // wait until there is enough space in the buffer
+    if (UTILS_UNLIKELY(mFreeSpace < requiredSize)) {
 
 #ifndef NDEBUG
         size_t const totalUsed = circularBuffer.size() - mFreeSpace;
@@ -153,8 +154,10 @@ std::vector<CommandBufferQueue::Range> CommandBufferQueue::waitForCommands() con
 }
 
 void CommandBufferQueue::releaseBuffer(CommandBufferQueue::Range const& buffer) {
+    size_t const used = std::distance(
+            static_cast<char const*>(buffer.begin), static_cast<char const*>(buffer.end));
     std::lock_guard<utils::Mutex> const lock(mLock);
-    mFreeSpace += uintptr_t(buffer.end) - uintptr_t(buffer.begin);
+    mFreeSpace += used;
     mCondition.notify_one();
 }
 

filament/backend/src/CommandStream.cpp

@@ -20,11 +20,16 @@
 #include <utils/CallStack.h>
 #endif
 
+#include <utils/compiler.h>
 #include <utils/Log.h>
+#include <utils/ostream.h>
 #include <utils/Profiler.h>
 #include <utils/Systrace.h>
 
+#include <cstddef>
 #include <functional>
+#include <string>
+#include <utility>
 
 #ifdef __ANDROID__
 #include <sys/system_properties.h>
@@ -74,8 +79,8 @@ CommandStream::CommandStream(Driver& driver, CircularBuffer& buffer) noexcept
 }
 
 void CommandStream::execute(void* buffer) {
-    SYSTRACE_CALL();
-    SYSTRACE_CONTEXT();
+    // NOTE: we can't use SYSTRACE_CALL() or similar here because, execute() below, also
+    // uses systrace BEGIN/END and the END is not guaranteed to be happening in this scope.
 
     Profiler profiler;
 
@@ -100,6 +105,7 @@ void CommandStream::execute(void* buffer) {
             // we want to remove all this when tracing is completely disabled
             profiler.stop();
             UTILS_UNUSED Profiler::Counters const counters = profiler.readCounters();
+            SYSTRACE_CONTEXT();
             SYSTRACE_VALUE32("GLThread (I)", counters.getInstructions());
             SYSTRACE_VALUE32("GLThread (C)", counters.getCpuCycles());
             SYSTRACE_VALUE32("GLThread (CPI x10)", counters.getCPI() * 10);

filament/backend/src/DriverBase.h

@@ -101,6 +101,14 @@ struct HwProgram : public HwBase {
     HwProgram() noexcept = default;
 };
 
+struct HwDescriptorSetLayout : public HwBase {
+    HwDescriptorSetLayout() noexcept = default;
+};
+
+struct HwDescriptorSet : public HwBase {
+    HwDescriptorSet() noexcept = default;
+};
+
 struct HwSamplerGroup : public HwBase {
     HwSamplerGroup() noexcept = default;
 };

filament/backend/src/HandleAllocator.cpp

@@ -80,6 +80,9 @@ HandleAllocator<P0, P1, P2>::HandleAllocator(const char* name, size_t size,
         bool disableUseAfterFreeCheck) noexcept
     : mHandleArena(name, size, disableUseAfterFreeCheck),
       mUseAfterFreeCheckDisabled(disableUseAfterFreeCheck) {
+    // Reserve initial space for debug tags. This prevents excessive calls to malloc when the first
+    // few tags are set.
+    mDebugTags.reserve(512);
 }
 
 template <size_t P0, size_t P1, size_t P2>

filament/backend/src/PlatformFactory.cpp

@@ -41,6 +41,10 @@
         #if defined(FILAMENT_SUPPORTS_OPENGL) && !defined(FILAMENT_USE_EXTERNAL_GLES3)
             #include "backend/platforms/PlatformEGLHeadless.h"
         #endif
+    #elif defined(FILAMENT_SUPPORTS_OSMESA)
+        #if defined(FILAMENT_SUPPORTS_OPENGL) && !defined(FILAMENT_USE_EXTERNAL_GLES3)
+            #include "backend/platforms/PlatformOSMesa.h"
+        #endif
     #endif
 #elif defined(WIN32)
     #if defined(FILAMENT_SUPPORTS_OPENGL) && !defined(FILAMENT_USE_EXTERNAL_GLES3)
@@ -124,6 +128,8 @@ Platform* PlatformFactory::create(Backend* backend) noexcept {
                 return new PlatformGLX();
             #elif defined(FILAMENT_SUPPORTS_EGL_ON_LINUX)
                 return new PlatformEGLHeadless();
+            #elif defined(FILAMENT_SUPPORTS_OSMESA)
+                return new PlatformOSMesa();
             #else
                 return nullptr;
             #endif

filament/backend/src/Program.cpp

@@ -14,7 +14,18 @@
  * limitations under the License.
  */
 
-#include "backend/Program.h"
+#include <backend/Program.h>
+#include <backend/DriverEnums.h>
+
+#include <utils/debug.h>
+#include <utils/CString.h>
+#include <utils/ostream.h>
+#include <utils/Invocable.h>
+
+#include <utility>
+
+#include <stddef.h>
+#include <stdint.h>
 
 namespace filament::backend {
 
@@ -52,41 +63,24 @@ Program& Program::shaderLanguage(ShaderLanguage shaderLanguage) {
     return *this;
 }
 
-Program& Program::uniformBlockBindings(
-        FixedCapacityVector<std::pair<utils::CString, uint8_t>> const& uniformBlockBindings) noexcept {
-    for (auto const& item : uniformBlockBindings) {
-        assert_invariant(item.second < UNIFORM_BINDING_COUNT);
-        mUniformBlocks[item.second] = item.first;
-    }
+Program& Program::descriptorBindings(backend::descriptor_set_t set,
+        DescriptorBindingsInfo descriptorBindings) noexcept {
+    mDescriptorBindings[set] = std::move(descriptorBindings);
     return *this;
 }
 
-Program& Program::uniforms(uint32_t index, UniformInfo const& uniforms) noexcept {
-    assert_invariant(index < UNIFORM_BINDING_COUNT);
-    mBindingUniformInfo[index] = uniforms;
+Program& Program::uniforms(uint32_t index, utils::CString name, UniformInfo uniforms) noexcept {
+    mBindingUniformsInfo.reserve(mBindingUniformsInfo.capacity() + 1);
+    mBindingUniformsInfo.emplace_back(index, std::move(name), std::move(uniforms));
     return *this;
 }
 
-
-Program& Program::attributes(
-        utils::FixedCapacityVector<std::pair<utils::CString, uint8_t>> attributes) noexcept {
+Program& Program::attributes(AttributesInfo attributes) noexcept {
     mAttributes = std::move(attributes);
     return *this;
 }
 
-Program& Program::setSamplerGroup(size_t bindingPoint, ShaderStageFlags stageFlags,
-        const Program::Sampler* samplers, size_t count) noexcept {
-    auto& groupData = mSamplerGroups[bindingPoint];
-    groupData.stageFlags = stageFlags;
-    auto& samplerList = groupData.samplers;
-    samplerList.reserve(count);
-    samplerList.resize(count);
-    std::copy_n(samplers, count, samplerList.data());
-    return *this;
-}
-
-Program& Program::specializationConstants(
-        FixedCapacityVector<SpecializationConstant> specConstants) noexcept {
+Program& Program::specializationConstants(SpecializationConstantsInfo specConstants) noexcept {
     mSpecializationConstants = std::move(specConstants);
     return *this;
 }

filament/backend/src/SystraceProfile.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_SYSTRACEPROFILE_H
+#define TNT_FILAMENT_BACKEND_SYSTRACEPROFILE_H
+
+#include <utils/Systrace.h>
+
+#define PROFILE_SCOPE(marker)       SYSTRACE_NAME(marker)
+
+#define PROFILE_NAME_BEGINFRAME    "backend::beginFrame"
+#define PROFILE_NAME_ENDFRAME      "backend::endFrame"
+
+#endif // TNT_FILAMENT_BACKEND_SYSTRACEPROFILE_H
+

filament/backend/src/VirtualMachineEnv.cpp

@@ -16,13 +16,23 @@
 
 #include "private/backend/VirtualMachineEnv.h"
 
+#include <utils/compiler.h>
 #include <utils/debug.h>
 
+#include <jni.h>
+
 namespace filament {
 
 JavaVM* VirtualMachineEnv::sVirtualMachine = nullptr;
 
-// This is called when the library is loaded. We need this to get a reference to the global VM
+/*
+ * This is typically called by filament_jni.so when it is loaded. If filament_jni.so is not used,
+ * then this must be called manually -- however, this is a problem because VirtualMachineEnv.h
+ * is currently private and part of backend.
+ * For now, we authorize this usage, but we will need to fix it; by making a proper public
+ * API for this.
+ */
+UTILS_PUBLIC
 UTILS_NOINLINE
 jint VirtualMachineEnv::JNI_OnLoad(JavaVM* vm) noexcept {
     JNIEnv* env = nullptr;

filament/backend/src/metal/MetalBuffer.h

@@ -73,10 +73,11 @@ public:
     enum class Type {
         NONE = 0,
         GENERIC = 1,
-        RING = 2,
+        RING = 2,       // deprecated
         STAGING = 3,
+        DESCRIPTOR_SET = 4,
     };
-    static constexpr size_t TypeCount = 3;
+    static constexpr size_t TypeCount = 4;
 
     static constexpr auto toIndex(Type t) {
         assert_invariant(t != Type::NONE);
@@ -88,6 +89,8 @@ public:
                 return 1;
             case Type::STAGING:
                 return 2;
+            case Type::DESCRIPTOR_SET:
+                return 3;
         }
     }
 
@@ -160,6 +163,8 @@ public:
          size_t size, bool forceGpuBuffer = false);
     ~MetalBuffer();
 
+    [[nodiscard]] bool wasAllocationSuccessful() const noexcept { return mBuffer || mCpuBuffer; }
+
     MetalBuffer(const MetalBuffer& rhs) = delete;
     MetalBuffer& operator=(const MetalBuffer& rhs) = delete;
 
@@ -169,8 +174,10 @@ public:
      * Update the buffer with data inside src. Potentially allocates a new buffer allocation to hold
      * the bytes which will be released when the current frame is finished.
      */
-    void copyIntoBuffer(void* src, size_t size, size_t byteOffset);
-    void copyIntoBufferUnsynchronized(void* src, size_t size, size_t byteOffset);
+    using TagResolver = utils::Invocable<const char*(void)>;
+    void copyIntoBuffer(void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag);
+    void copyIntoBufferUnsynchronized(
+            void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag);
 
     /**
      * Denotes that this buffer is used for a draw call ensuring that its allocation remains valid
@@ -180,7 +187,7 @@ public:
      * is no device allocation.
      *
      */
-    id<MTLBuffer> getGpuBufferForDraw(id<MTLCommandBuffer> cmdBuffer) noexcept;
+    id<MTLBuffer> getGpuBufferForDraw() noexcept;
 
     void* getCpuBuffer() const noexcept { return mCpuBuffer; }
 
@@ -204,6 +211,17 @@ public:
 
 private:
 
+    enum class UploadStrategy {
+        POOL,
+        BUMP_ALLOCATOR,
+    };
+
+    void uploadWithPoolBuffer(
+            void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) const;
+    void uploadWithBumpAllocator(
+            void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) const;
+
+    UploadStrategy mUploadStrategy;
     TrackedMetalBuffer mBuffer;
     size_t mBufferSize = 0;
     void* mCpuBuffer = nullptr;

filament/backend/src/metal/MetalBuffer.mm

@@ -27,16 +27,28 @@ MetalPlatform* TrackedMetalBuffer::platform = nullptr;
 MetalPlatform* ScopedAllocationTimer::platform = nullptr;
 
 MetalBuffer::MetalBuffer(MetalContext& context, BufferObjectBinding bindingType, BufferUsage usage,
-        size_t size, bool forceGpuBuffer) : mBufferSize(size), mContext(context) {
+        size_t size, bool forceGpuBuffer)
+    : mBufferSize(size), mContext(context) {
+    const MetalBumpAllocator& allocator = *mContext.bumpAllocator;
+    // VERTEX is also used for index buffers
+    if (allocator.getCapacity() > 0 && bindingType == BufferObjectBinding::VERTEX) {
+        mUploadStrategy = UploadStrategy::BUMP_ALLOCATOR;
+    } else {
+        mUploadStrategy = UploadStrategy::POOL;
+    }
+
     // If the buffer is less than 4K in size and is updated frequently, we don't use an explicit
     // buffer. Instead, we use immediate command encoder methods like setVertexBytes:length:atIndex:.
     // This won't work for SSBOs, since they are read/write.
+
+    /*
     if (size <= 4 * 1024 && bindingType != BufferObjectBinding::SHADER_STORAGE &&
             usage == BufferUsage::DYNAMIC && !forceGpuBuffer) {
         mBuffer = nil;
         mCpuBuffer = malloc(size);
         return;
     }
+    */
 
     // Otherwise, we allocate a private GPU buffer.
     {
@@ -44,8 +56,8 @@ MetalBuffer::MetalBuffer(MetalContext& context, BufferObjectBinding bindingType,
         mBuffer = { [context.device newBufferWithLength:size options:MTLResourceStorageModePrivate],
             TrackedMetalBuffer::Type::GENERIC };
     }
-    FILAMENT_CHECK_POSTCONDITION(mBuffer)
-            << "Could not allocate Metal buffer of size " << size << ".";
+    // mBuffer might fail to be allocated. Clients can check for this by calling
+    // wasAllocationSuccessful().
 }
 
 MetalBuffer::~MetalBuffer() {
@@ -54,49 +66,44 @@ MetalBuffer::~MetalBuffer() {
     }
 }
 
-void MetalBuffer::copyIntoBuffer(void* src, size_t size, size_t byteOffset) {
+void MetalBuffer::copyIntoBuffer(
+        void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) {
     if (size <= 0) {
         return;
     }
+
+    FILAMENT_CHECK_PRECONDITION(src)
+            << "copyIntoBuffer called with a null src, tag=" << getHandleTag();
     FILAMENT_CHECK_PRECONDITION(size + byteOffset <= mBufferSize)
             << "Attempting to copy " << size << " bytes into a buffer of size " << mBufferSize
-            << " at offset " << byteOffset;
+            << " at offset " << byteOffset << ", tag=" << getHandleTag();
+    // The copy blit requires that byteOffset be a multiple of 4.
+    FILAMENT_CHECK_PRECONDITION(!(byteOffset & 0x3))
+            << "byteOffset must be a multiple of 4, tag=" << getHandleTag();
 
-    // Either copy into the Metal buffer or into our cpu buffer.
+    // If we have a cpu buffer, we can directly copy into it.
     if (mCpuBuffer) {
         memcpy(static_cast<uint8_t*>(mCpuBuffer) + byteOffset, src, size);
         return;
     }
 
-    // Acquire a staging buffer to hold the contents of this update.
-    MetalBufferPool* bufferPool = mContext.bufferPool;
-    const MetalBufferPoolEntry* const staging = bufferPool->acquireBuffer(size);
-    memcpy(staging->buffer.get().contents, src, size);
-
-    // The blit below requires that byteOffset be a multiple of 4.
-    FILAMENT_CHECK_PRECONDITION(!(byteOffset & 0x3u)) << "byteOffset must be a multiple of 4";
-
-    // Encode a blit from the staging buffer into the private GPU buffer.
-    id<MTLCommandBuffer> cmdBuffer = getPendingCommandBuffer(&mContext);
-    id<MTLBlitCommandEncoder> blitEncoder = [cmdBuffer blitCommandEncoder];
-    blitEncoder.label = @"Buffer upload blit";
-    [blitEncoder copyFromBuffer:staging->buffer.get()
-                   sourceOffset:0
-                       toBuffer:mBuffer.get()
-              destinationOffset:byteOffset
-                           size:size];
-    [blitEncoder endEncoding];
-    [cmdBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
-        bufferPool->releaseBuffer(staging);
-    }];
+    switch (mUploadStrategy) {
+        case UploadStrategy::BUMP_ALLOCATOR:
+            uploadWithBumpAllocator(src, size, byteOffset, std::move(getHandleTag));
+            break;
+        case UploadStrategy::POOL:
+            uploadWithPoolBuffer(src, size, byteOffset, std::move(getHandleTag));
+            break;
+    }
 }
 
-void MetalBuffer::copyIntoBufferUnsynchronized(void* src, size_t size, size_t byteOffset) {
+void MetalBuffer::copyIntoBufferUnsynchronized(
+        void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) {
     // TODO: implement the unsynchronized version
-    copyIntoBuffer(src, size, byteOffset);
+    copyIntoBuffer(src, size, byteOffset, std::move(getHandleTag));
 }
 
-id<MTLBuffer> MetalBuffer::getGpuBufferForDraw(id<MTLCommandBuffer> cmdBuffer) noexcept {
+id<MTLBuffer> MetalBuffer::getGpuBufferForDraw() noexcept {
     // If there's a CPU buffer, then we return nil here, as the CPU-side buffer will be bound
     // separately.
     if (mCpuBuffer) {
@@ -139,7 +146,7 @@ void MetalBuffer::bindBuffers(id<MTLCommandBuffer> cmdBuffer, id<MTLCommandEncod
         }
         // getGpuBufferForDraw() might return nil, which means there isn't a device allocation for
         // this buffer. In this case, we'll bind the buffer below with the CPU-side memory.
-        id<MTLBuffer> gpuBuffer = buffer->getGpuBufferForDraw(cmdBuffer);
+        id<MTLBuffer> gpuBuffer = buffer->getGpuBufferForDraw();
         if (!gpuBuffer) {
             continue;
         }
@@ -199,5 +206,54 @@ void MetalBuffer::bindBuffers(id<MTLCommandBuffer> cmdBuffer, id<MTLCommandEncod
     }
 }
 
+void MetalBuffer::uploadWithPoolBuffer(
+        void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) const {
+    MetalBufferPool* bufferPool = mContext.bufferPool;
+    const MetalBufferPoolEntry* const staging = bufferPool->acquireBuffer(size);
+    FILAMENT_CHECK_POSTCONDITION(staging)
+            << "uploadWithPoolbuffer unable to acquire staging buffer of size " << size
+            << ", tag=" << getHandleTag();
+    memcpy(staging->buffer.get().contents, src, size);
+
+    // Encode a blit from the staging buffer into the private GPU buffer.
+    id<MTLCommandBuffer> cmdBuffer = getPendingCommandBuffer(&mContext);
+    id<MTLBlitCommandEncoder> blitEncoder = [cmdBuffer blitCommandEncoder];
+    blitEncoder.label = @"Buffer upload blit - pool buffer";
+    [blitEncoder copyFromBuffer:staging->buffer.get()
+                   sourceOffset:0
+                       toBuffer:mBuffer.get()
+              destinationOffset:byteOffset
+                           size:size];
+    [blitEncoder endEncoding];
+    [cmdBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+        bufferPool->releaseBuffer(staging);
+    }];
+}
+
+void MetalBuffer::uploadWithBumpAllocator(
+        void* src, size_t size, size_t byteOffset, TagResolver&& getHandleTag) const {
+    MetalBumpAllocator& allocator = *mContext.bumpAllocator;
+    auto [buffer, offset] = allocator.allocateStagingArea(size);
+    FILAMENT_CHECK_POSTCONDITION(buffer)
+            << "uploadWithBumpAllocator unable to acquire staging area of size " << size
+            << ", tag=" << getHandleTag();
+    void* const contents = buffer.contents;
+    FILAMENT_CHECK_POSTCONDITION(contents)
+            << "uploadWithBumpAllocator unable to acquire pointer to staging area, size " << size
+            << ", tag=" << getHandleTag();
+    memcpy(static_cast<char*>(contents) + offset, src, size);
+
+    // Encode a blit from the staging buffer into the private GPU buffer.
+    id<MTLCommandBuffer> cmdBuffer = getPendingCommandBuffer(&mContext);
+    id<MTLBlitCommandEncoder> blitEncoder = [cmdBuffer blitCommandEncoder];
+    blitEncoder.label = @"Buffer upload blit - bump allocator";
+    [blitEncoder copyFromBuffer:buffer
+                   sourceOffset:offset
+                       toBuffer:mBuffer.get()
+              destinationOffset:byteOffset
+                           size:size];
+    [blitEncoder endEncoding];
+}
+
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalBufferPool.h

@@ -38,6 +38,28 @@ struct MetalBufferPoolEntry {
     mutable uint32_t referenceCount;
 };
 
+class MetalBumpAllocator {
+public:
+    MetalBumpAllocator(id<MTLDevice> device, size_t capacity);
+
+    /**
+     * Allocates a staging area of the given size. Returns a pair of the buffer and the offset
+     * within the buffer. The buffer is guaranteed to be at least the given size, but may be larger.
+     * Clients must not write to the buffer beyond the returned offset + size.
+     * Clients are responsible for holding a reference to the returned buffer.
+     * Allocations are guaranteed to be aligned to 4 bytes.
+     */
+    std::pair<id<MTLBuffer>, size_t> allocateStagingArea(size_t size);
+
+    size_t getCapacity() const noexcept { return mCapacity; }
+
+private:
+    id<MTLDevice> mDevice;
+    TrackedMetalBuffer mCurrentUploadBuffer = nil;
+    size_t mHead = 0;
+    size_t mCapacity;
+};
+
 // Manages a pool of Metal buffers, periodically releasing ones that have been unused for awhile.
 class MetalBufferPool {
 public:

filament/backend/src/metal/MetalBufferPool.mm

@@ -116,5 +116,39 @@ void MetalBufferPool::reset() noexcept {
     mFreeStages.clear();
 }
 
+MetalBumpAllocator::MetalBumpAllocator(id<MTLDevice> device, size_t capacity)
+    : mDevice(device), mCapacity(capacity) {
+    if (mCapacity > 0) {
+        mCurrentUploadBuffer = { [device newBufferWithLength:capacity options:MTLStorageModeShared],
+            TrackedMetalBuffer::Type::STAGING };
+    }
+}
+
+std::pair<id<MTLBuffer>, size_t> MetalBumpAllocator::allocateStagingArea(size_t size) {
+    if (size == 0) {
+        return { nil, 0 };
+    }
+    if (size > mCapacity) {
+        return { [mDevice newBufferWithLength:size options:MTLStorageModeShared], 0 };
+    }
+    assert_invariant(mCurrentUploadBuffer);
+
+    // Align the head to a 4-byte boundary.
+    mHead = (mHead + 3) & ~3;
+
+    if (UTILS_LIKELY(mHead + size <= mCapacity)) {
+        const size_t oldHead = mHead;
+        mHead += size;
+        return { mCurrentUploadBuffer.get(), oldHead };
+    }
+
+    // We're finished with the current allocation.
+    mCurrentUploadBuffer = { [mDevice newBufferWithLength:mCapacity options:MTLStorageModeShared],
+        TrackedMetalBuffer::Type::STAGING };
+    mHead = size;
+
+    return { mCurrentUploadBuffer.get(), 0 };
+}
+
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalContext.h

@@ -21,6 +21,8 @@
 #include "MetalShaderCompiler.h"
 #include "MetalState.h"
 
+#include <backend/DriverEnums.h>
+
 #include <CoreVideo/CVMetalTextureCache.h>
 #include <Metal/Metal.h>
 #include <QuartzCore/QuartzCore.h>
@@ -44,14 +46,15 @@ namespace backend {
 class MetalDriver;
 class MetalBlitter;
 class MetalBufferPool;
+class MetalBumpAllocator;
 class MetalRenderTarget;
-class MetalSamplerGroup;
 class MetalSwapChain;
 class MetalTexture;
 class MetalTimerQueryInterface;
 struct MetalUniformBuffer;
 struct MetalIndexBuffer;
 struct MetalVertexBuffer;
+struct MetalDescriptorSet;
 
 constexpr static uint8_t MAX_SAMPLE_COUNT = 8;  // Metal devices support at most 8 MSAA samples
 
@@ -67,16 +70,64 @@ private:
     bool mDirty = false;
 };
 
-struct MetalContext {
-    explicit MetalContext(size_t metalFreedTextureListSize)
-        : texturesToDestroy(metalFreedTextureListSize) {}
+class MetalDynamicOffsets {
+public:
+    void setOffsets(uint32_t set, const uint32_t* offsets, uint32_t count) {
+        assert(set < MAX_DESCRIPTOR_SET_COUNT);
 
+        auto getStartIndexForSet = [&](uint32_t s) {
+            uint32_t startIndex = 0;
+            for (uint32_t i = 0; i < s; i++) {
+                startIndex += mCounts[i];
+            }
+            return startIndex;
+        };
+
+        const bool resizeNecessary = mCounts[set] != count;
+        if (UTILS_UNLIKELY(resizeNecessary)) {
+            int delta = count - mCounts[set];
+
+            auto thisSetStart = mOffsets.begin() + getStartIndexForSet(set);
+            if (delta > 0) {
+                mOffsets.insert(thisSetStart, delta, 0);
+            } else {
+                mOffsets.erase(thisSetStart, thisSetStart - delta);
+            }
+
+            mCounts[set] = count;
+        }
+
+        if (resizeNecessary ||
+                !std::equal(
+                        offsets, offsets + count, mOffsets.begin() + getStartIndexForSet(set))) {
+            std::copy(offsets, offsets + count, mOffsets.begin() + getStartIndexForSet(set));
+            mDirty = true;
+        }
+    }
+    bool isDirty() const { return mDirty; }
+    void setDirty(bool dirty) { mDirty = dirty; }
+
+    std::pair<uint32_t, const uint32_t*> getOffsets() const {
+        return { mOffsets.size(), mOffsets.data() };
+    }
+
+private:
+    std::array<uint32_t, MAX_DESCRIPTOR_SET_COUNT> mCounts = { 0 };
+    std::vector<uint32_t> mOffsets;
+    bool mDirty = false;
+};
+
+struct MetalContext {
     MetalDriver* driver;
     id<MTLDevice> device = nullptr;
     id<MTLCommandQueue> commandQueue = nullptr;
 
-    id<MTLCommandBuffer> pendingCommandBuffer = nullptr;
-    id<MTLRenderCommandEncoder> currentRenderPassEncoder = nullptr;
+    // The ID of pendingCommandBuffer (or the next command buffer, if pendingCommandBuffer is nil).
+    uint64_t pendingCommandBufferId = 1;
+    // read from driver thread, set from completion handlers
+    std::atomic<uint64_t> latestCompletedCommandBufferId = 0;
+    id<MTLCommandBuffer> pendingCommandBuffer = nil;
+    id<MTLRenderCommandEncoder> currentRenderPassEncoder = nil;
 
     std::atomic<bool> memorylessLimitsReached = false;
 
@@ -84,6 +135,7 @@ struct MetalContext {
     bool supportsTextureSwizzling = false;
     bool supportsAutoDepthResolve = false;
     bool supportsMemorylessRenderTargets = false;
+    bool supportsDepthClamp = false;
     uint8_t maxColorRenderTargets = 4;
     struct {
         uint8_t common;
@@ -92,7 +144,7 @@ struct MetalContext {
     } highestSupportedGpuFamily;
 
     struct {
-        bool a8xStaticTextureTargetError;
+        bool staticTextureTargetError;
     } bugs;
 
     // sampleCountLookup[requestedSamples] gives a <= sample count supported by the device.
@@ -107,10 +159,10 @@ struct MetalContext {
     // State trackers.
     PipelineStateTracker pipelineState;
     DepthStencilStateTracker depthStencilState;
-    std::array<BufferState, Program::UNIFORM_BINDING_COUNT> uniformState;
-    std::array<BufferState, MAX_SSBO_COUNT> ssboState;
     CullModeStateTracker cullModeState;
     WindingStateTracker windingState;
+    DepthClampStateTracker depthClampState;
+    ScissorRectStateTracker scissorRectState;
     Handle<HwRenderPrimitive> currentRenderPrimitive;
 
     // State caches.
@@ -123,24 +175,19 @@ struct MetalContext {
 
     std::array<MetalPushConstantBuffer, Program::SHADER_TYPE_COUNT> currentPushConstants;
 
-    MetalSamplerGroup* samplerBindings[Program::SAMPLER_BINDING_COUNT] = {};
+    // Keeps track of descriptor sets we've finalized for the current render pass.
+    tsl::robin_set<MetalDescriptorSet*> finalizedDescriptorSets;
+    std::array<MetalDescriptorSet*, MAX_DESCRIPTOR_SET_COUNT> currentDescriptorSets = {};
+    MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::VERTEX> vertexDescriptorBindings;
+    MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::FRAGMENT> fragmentDescriptorBindings;
+    MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::COMPUTE> computeDescriptorBindings;
+    MetalDynamicOffsets dynamicOffsets;
 
-    // Keeps track of sampler groups we've finalized for the current render pass.
-    tsl::robin_set<MetalSamplerGroup*> finalizedSamplerGroups;
-
-    // Keeps track of all alive sampler groups, textures.
-    tsl::robin_set<MetalSamplerGroup*> samplerGroups;
+    // Keeps track of all alive textures.
     tsl::robin_set<MetalTexture*> textures;
 
-    // This circular buffer implements delayed destruction for Metal texture handles. It keeps a
-    // handle to a fixed number of the most recently destroyed texture handles. When we're asked to
-    // destroy a texture handle, we free its texture memory, but keep the MetalTexture object alive,
-    // marking it as "terminated". If we later are asked to use that texture, we can check its
-    // terminated status and throw an Objective-C error instead of crashing, which is helpful for
-    // debugging use-after-free issues in release builds.
-    utils::FixedCircularBuffer<Handle<HwTexture>> texturesToDestroy;
-
     MetalBufferPool* bufferPool;
+    MetalBumpAllocator* bumpAllocator;
 
     MetalSwapChain* currentDrawSwapChain = nil;
     MetalSwapChain* currentReadSwapChain = nil;
@@ -151,6 +198,7 @@ struct MetalContext {
 
     // Empty texture used to prevent GPU errors when a sampler has been bound without a texture.
     id<MTLTexture> emptyTexture = nil;
+    id<MTLBuffer> emptyBuffer = nil;
 
     MetalBlitter* blitter = nullptr;
 

filament/backend/src/metal/MetalContext.mm

@@ -101,9 +101,14 @@ id<MTLCommandBuffer> getPendingCommandBuffer(MetalContext* context) {
     context->pendingCommandBuffer = [context->commandQueue commandBuffer];
     // It's safe for this block to capture the context variable. MetalDriver::terminate will ensure
     // all frames and their completion handlers finish before context is deallocated.
+    uint64_t thisCommandBufferId = context->pendingCommandBufferId;
     [context->pendingCommandBuffer addCompletedHandler:^(id <MTLCommandBuffer> buffer) {
         context->resourceTracker.clearResources((__bridge void*) buffer);
-        
+
+        // Command buffers should complete in order, so latestCompletedCommandBufferId will only
+        // ever increase.
+        context->latestCompletedCommandBufferId = thisCommandBufferId;
+
         auto errorCode = (MTLCommandBufferError)buffer.error.code;
         if (@available(macOS 11.0, *)) {
             if (errorCode == MTLCommandBufferErrorMemoryless) {
@@ -125,6 +130,7 @@ void submitPendingCommands(MetalContext* context) {
     assert_invariant(context->pendingCommandBuffer.status != MTLCommandBufferStatusCommitted);
     [context->pendingCommandBuffer commit];
     context->pendingCommandBuffer = nil;
+    context->pendingCommandBufferId++;
 }
 
 id<MTLTexture> getOrCreateEmptyTexture(MetalContext* context) {
@@ -167,7 +173,6 @@ void MetalPushConstantBuffer::setPushConstant(PushConstantVariant value, uint8_t
 
 void MetalPushConstantBuffer::setBytes(id<MTLCommandEncoder> encoder, ShaderStage stage) {
     constexpr size_t PUSH_CONSTANT_SIZE_BYTES = 4;
-    constexpr size_t PUSH_CONSTANT_BUFFER_INDEX = 26;
 
     static char buffer[MAX_PUSH_CONSTANT_COUNT * PUSH_CONSTANT_SIZE_BYTES];
     assert_invariant(mPushConstants.size() <= MAX_PUSH_CONSTANT_COUNT);

filament/backend/src/metal/MetalDriver.h

@@ -32,6 +32,7 @@
 #include <functional>
 #include <mutex>
 #include <vector>
+#include <deque>
 
 namespace filament {
 namespace backend {
@@ -57,11 +58,11 @@ class MetalDriver final : public DriverBase {
 
 public:
     static Driver* create(MetalPlatform* platform, const Platform::DriverConfig& driverConfig);
-    void runAtNextTick(const std::function<void()>& fn) noexcept;
 
 private:
 
     friend class MetalSwapChain;
+    friend struct MetalDescriptorSet;
 
     MetalPlatform& mPlatform;
     MetalContext* mContext;
@@ -73,10 +74,23 @@ private:
 
     /*
      * Tasks run regularly on the driver thread.
+     * Not thread-safe; tasks are run from the driver thead and must be enqueued from the driver
+     * thread.
      */
+    void runAtNextTick(const std::function<void()>& fn) noexcept;
     void executeTickOps() noexcept;
     std::vector<std::function<void()>> mTickOps;
-    std::mutex mTickOpsLock;
+
+    // Tasks regularly executed on the driver thread after a command buffer has completed
+    struct DeferredTask {
+        DeferredTask(uint64_t commandBufferId, utils::Invocable<void()>&& fn) noexcept
+            : commandBufferId(commandBufferId), fn(std::move(fn)) {}
+        uint64_t commandBufferId;     // after this command buffer completes
+        utils::Invocable<void()> fn;  // execute this task
+    };
+    void executeAfterCurrentCommandBufferCompletes(utils::Invocable<void()>&& fn) noexcept;
+    void executeDeferredOps() noexcept;
+    std::deque<DeferredTask> mDeferredTasks;
 
     /*
      * Driver interface
@@ -137,7 +151,6 @@ private:
     inline void setRenderPrimitiveBuffer(Handle<HwRenderPrimitive> rph, PrimitiveType pt,
             Handle<HwVertexBuffer> vbh, Handle<HwIndexBuffer> ibh);
 
-    void finalizeSamplerGroup(MetalSamplerGroup* sg);
     void enumerateBoundBuffers(BufferObjectBinding bindingType,
             const std::function<void(const BufferState&, MetalBuffer*, uint32_t)>& f);
 

filament/backend/src/metal/MetalEnums.h

@@ -71,7 +71,8 @@ constexpr inline MTLIndexType getIndexType(size_t elementSize) noexcept {
     } else if (elementSize == 4) {
         return MTLIndexTypeUInt32;
     }
-    FILAMENT_CHECK_POSTCONDITION(false) << "Index element size not supported.";
+    assert_invariant(false);
+    return MTLIndexTypeUInt16;
 }
 
 constexpr inline MTLVertexFormat getMetalFormat(ElementType type, bool normalized) noexcept {

filament/backend/src/metal/MetalExternalImage.h

@@ -32,100 +32,75 @@ struct MetalContext;
  * texture.
  */
 class MetalExternalImage {
-
 public:
+    MetalExternalImage() = default;
 
-    MetalExternalImage(MetalContext& context,
-            TextureSwizzle r = TextureSwizzle::CHANNEL_0,
-            TextureSwizzle g = TextureSwizzle::CHANNEL_1,
-            TextureSwizzle b = TextureSwizzle::CHANNEL_2,
-            TextureSwizzle a = TextureSwizzle::CHANNEL_3) noexcept;
+    MetalExternalImage(MetalExternalImage&&);
+    MetalExternalImage& operator=(MetalExternalImage&&);
+    ~MetalExternalImage() noexcept;
+
+    MetalExternalImage(const MetalExternalImage&) = delete;
+    MetalExternalImage& operator=(const MetalExternalImage&) = delete;
 
     /**
-     * @return true, if this MetalExternalImage is holding a live external image. Returns false
-     * until set has been called with a valid CVPixelBuffer. The image can be cleared via
-     * set(nullptr), and isValid will return false again.
+     * While the texture is used for rendering, this MetalExternalImage must be kept alive.
      */
-    bool isValid() const noexcept;
+    id<MTLTexture> getMtlTexture() const noexcept;
+
+    bool isValid() const noexcept {
+        return mImage != nil || mRgbTexture != nullptr;
+    }
+
+    NSUInteger getWidth() const noexcept;
+    NSUInteger getHeight() const noexcept;
 
     /**
-     * Set this external image to the passed-in CVPixelBuffer. Future calls to
-     * getMetalTextureForDraw will return a texture backed by this CVPixelBuffer. Previous
-     * CVPixelBuffers and related resources will be released when all GPU work using them has
-     * finished.
+     * Create an external image with the passed-in CVPixelBuffer.
      *
-     * Calling set with a YCbCr image will encode a compute pass to convert the image from YCbCr to
-     * RGB.
+     * Ownership is taken of the CVPixelBuffer, which will be released when the returned
+     * MetalExternalImage is destroyed (or, in the case of a YCbCr image, after the conversion has
+     * completed).
+     *
+     * Calling set with a YCbCr image will encode a compute pass to convert the image from
+     * YCbCr to RGB.
      */
-    void set(CVPixelBufferRef image) noexcept;
+    static MetalExternalImage createFromImage(MetalContext& context, CVPixelBufferRef image);
 
     /**
-     * Set this external image to a specific plane of the passed-in CVPixelBuffer. Future calls to
-     * getMetalTextureForDraw will return a texture backed by a single plane of this CVPixelBuffer.
-     * Previous CVPixelBuffers and related resources will be released when all GPU work using them
-     * has finished.
+     * Create an external image with a specific plane of the passed-in CVPixelBuffer.
+     *
+     * Ownership is taken of the CVPixelBuffer, which will be released when the returned
+     * MetalExternalImage is destroyed.
      */
-    void set(CVPixelBufferRef image, size_t plane) noexcept;
+    static MetalExternalImage createFromImagePlane(
+            MetalContext& context, CVPixelBufferRef image, uint32_t plane);
 
-    /**
-     * Returns the width of the external image, or 0 if one is not set. For YCbCr images, returns
-     * the width of the luminance plane.
-     */
-    size_t getWidth() const noexcept { return mWidth; }
-
-    /**
-     * Returns the height of the external image, or 0 if one is not set. For YCbCr images, returns
-     * the height of the luminance plane.
-     */
-    size_t getHeight() const noexcept { return mHeight; }
-
-    /**
-     * Get a Metal texture used to draw this image and denote that it is used for the current frame.
-     * For future frames that use this external image, getMetalTextureForDraw must be called again.
-     */
-    id<MTLTexture> getMetalTextureForDraw() const noexcept;
+    static void assertWritableImage(CVPixelBufferRef image);
 
     /**
      * Free resources. Should be called at least once when no further calls to set will occur.
      */
     static void shutdown(MetalContext& context) noexcept;
 
-    static void assertWritableImage(CVPixelBufferRef image);
-
 private:
+    MetalExternalImage(CVPixelBufferRef image, CVMetalTextureRef texture) noexcept
+        : mImage(image), mTexture(texture) {}
+    explicit MetalExternalImage(id<MTLTexture> texture) noexcept : mRgbTexture(texture) {}
 
-    void unset();
-
-    CVMetalTextureRef createTextureFromImage(CVPixelBufferRef image, MTLPixelFormat format,
-            size_t plane);
-    id<MTLTexture> createRgbTexture(size_t width, size_t height);
-    id<MTLTexture> createSwizzledTextureView(id<MTLTexture> texture) const;
-    id<MTLTexture> createSwizzledTextureView(CVMetalTextureRef texture) const;
-    void ensureComputePipelineState();
-    id<MTLCommandBuffer> encodeColorConversionPass(id<MTLTexture> inYPlane, id<MTLTexture>
-            inCbCrTexture, id<MTLTexture> outTexture);
+    static id<MTLTexture> createRgbTexture(id<MTLDevice> device, size_t width, size_t height);
+    static CVMetalTextureRef createTextureFromImage(CVMetalTextureCacheRef textureCache,
+            CVPixelBufferRef image, MTLPixelFormat format, size_t plane);
+    static void ensureComputePipelineState(MetalContext& context);
+    static id<MTLCommandBuffer> encodeColorConversionPass(MetalContext& context,
+            id<MTLTexture> inYPlane, id<MTLTexture> inCbCrTexture, id<MTLTexture> outTexture);
 
     static constexpr size_t Y_PLANE = 0;
     static constexpr size_t CBCR_PLANE = 1;
 
-    MetalContext& mContext;
-
-    // If the external image has a single plane, mImage and mTexture hold references to the image
-    // and created Metal texture, respectively.
-    // mTextureView is a view of mTexture with any swizzling applied.
+    // TODO: this could probably be a union.
     CVPixelBufferRef mImage = nullptr;
     CVMetalTextureRef mTexture = nullptr;
-    id<MTLTexture> mTextureView = nullptr;
-    size_t mWidth = 0;
-    size_t mHeight = 0;
-
-    // If the external image is in the YCbCr format, this holds the result of the converted RGB
-    // texture.
     id<MTLTexture> mRgbTexture = nil;
-
-    struct {
-        TextureSwizzle r, g, b, a;
-    } mSwizzle;
 };
 
 } // namespace backend

filament/backend/src/metal/MetalExternalImage.mm

@@ -34,10 +34,6 @@
 namespace filament {
 namespace backend {
 
-static const auto cvBufferDeleter = [](const void* buffer) {
-    CVBufferRelease((CVMetalTextureRef) buffer);
-};
-
 static const char* kernel = R"(
 #include <metal_stdlib>
 #include <simd/simd.h>
@@ -71,18 +67,30 @@ ycbcrToRgb(texture2d<half, access::read>  inYTexture    [[texture(0)]],
 }
 )";
 
-MetalExternalImage::MetalExternalImage(MetalContext& context, TextureSwizzle r, TextureSwizzle g,
-        TextureSwizzle b, TextureSwizzle a) noexcept : mContext(context), mSwizzle{r, g, b, a} { }
-
-bool MetalExternalImage::isValid() const noexcept {
-    return mRgbTexture != nil || mImage != nullptr;
+NSUInteger MetalExternalImage::getWidth() const noexcept {
+    if (mImage) {
+        return CVPixelBufferGetWidth(mImage);
+    }
+    if (mRgbTexture) {
+        return mRgbTexture.width;
+    }
+    return 0;
 }
 
-void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
-    unset();
+NSUInteger MetalExternalImage::getHeight() const noexcept {
+    if (mImage) {
+        return CVPixelBufferGetHeight(mImage);
+    }
+    if (mRgbTexture) {
+        return mRgbTexture.height;
+    }
+    return 0;
+}
 
+MetalExternalImage MetalExternalImage::createFromImage(
+        MetalContext& context, CVPixelBufferRef image) {
     if (!image) {
-        return;
+        return {};
     }
 
     OSType formatType = CVPixelBufferGetPixelFormatType(image);
@@ -96,30 +104,29 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
             << ".";
 
     if (planeCount == 0) {
-        mImage = image;
-        mTexture = createTextureFromImage(image, MTLPixelFormatBGRA8Unorm, 0);
-        mTextureView = createSwizzledTextureView(mTexture);
-        mWidth = CVPixelBufferGetWidth(image);
-        mHeight = CVPixelBufferGetHeight(image);
+        CVMetalTextureRef texture =
+                createTextureFromImage(context.textureCache, image, MTLPixelFormatBGRA8Unorm, 0);
+        return { CVPixelBufferRetain(image), texture };
     }
 
     if (planeCount == 2) {
-        CVMetalTextureRef yPlane = createTextureFromImage(image, MTLPixelFormatR8Unorm, Y_PLANE);
-        CVMetalTextureRef cbcrPlane = createTextureFromImage(image, MTLPixelFormatRG8Unorm,
-                CBCR_PLANE);
+        CVPixelBufferRetain(image);
+
+        CVMetalTextureRef yPlane =
+                createTextureFromImage(context.textureCache, image, MTLPixelFormatR8Unorm, Y_PLANE);
+        CVMetalTextureRef cbcrPlane =
+                createTextureFromImage(context.textureCache, image, MTLPixelFormatRG8Unorm, CBCR_PLANE);
 
         // Get the size of luminance plane.
-        mWidth = CVPixelBufferGetWidthOfPlane(image, Y_PLANE);
-        mHeight = CVPixelBufferGetHeightOfPlane(image, Y_PLANE);
+        NSUInteger width = CVPixelBufferGetWidthOfPlane(image, Y_PLANE);
+        NSUInteger height = CVPixelBufferGetHeightOfPlane(image, Y_PLANE);
 
-        id<MTLTexture> rgbTexture = createRgbTexture(mWidth, mHeight);
-        id<MTLCommandBuffer> commandBuffer = encodeColorConversionPass(
+        id<MTLTexture> rgbTexture = createRgbTexture(context.device, width, height);
+        id<MTLCommandBuffer> commandBuffer = encodeColorConversionPass(context,
                 CVMetalTextureGetTexture(yPlane),
                 CVMetalTextureGetTexture(cbcrPlane),
                 rgbTexture);
 
-        mRgbTexture = createSwizzledTextureView(rgbTexture);
-
         [commandBuffer addCompletedHandler:^(id <MTLCommandBuffer> o) {
             CVBufferRelease(yPlane);
             CVBufferRelease(cbcrPlane);
@@ -127,70 +134,83 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
         }];
 
         [commandBuffer commit];
+        return MetalExternalImage { rgbTexture };
     }
+
+    return {};
 }
 
-void MetalExternalImage::set(CVPixelBufferRef image, size_t plane) noexcept {
-    unset();
-
+MetalExternalImage MetalExternalImage::createFromImagePlane(
+        MetalContext& context, CVPixelBufferRef image, uint32_t plane) {
     if (!image) {
-        return;
+        return {};
     }
 
     const OSType formatType = CVPixelBufferGetPixelFormatType(image);
     FILAMENT_CHECK_POSTCONDITION(formatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
             << "Metal planar external images must be in the 420f format.";
+    FILAMENT_CHECK_POSTCONDITION(plane == 0 || plane == 1)
+            << "Metal planar external images must be created from planes 0 or 1.";
 
-    mImage = image;
-
-    auto getPlaneFormat = [] (size_t plane) {
-      // Right now Metal only supports kCVPixelFormatType_420YpCbCr8BiPlanarFullRange planar
-      // external images, so we can make the following assumptions about the format of each plane.
-      if (plane == 0) {
-        return MTLPixelFormatR8Unorm; // luminance
-      }
-      if (plane == 1) {
-        // CbCr
-        return MTLPixelFormatRG8Unorm; // CbCr
-      }
-      return MTLPixelFormatInvalid;
+    auto getPlaneFormat = [](size_t plane) {
+        // Right now Metal only supports kCVPixelFormatType_420YpCbCr8BiPlanarFullRange planar
+        // external images, so we can make the following assumptions about the format of each plane.
+        if (plane == 0) {
+            return MTLPixelFormatR8Unorm;  // luminance
+        }
+        if (plane == 1) {
+            return MTLPixelFormatRG8Unorm;  // CbCr
+        }
+        return MTLPixelFormatInvalid;
     };
 
     const MTLPixelFormat format = getPlaneFormat(plane);
     assert_invariant(format != MTLPixelFormatInvalid);
-    mTexture = createTextureFromImage(image, format, plane);
-    mTextureView = createSwizzledTextureView(mTexture);
+    CVMetalTextureRef mTexture = createTextureFromImage(context.textureCache, image, format, plane);
+    return { CVPixelBufferRetain(image), mTexture };
 }
 
-id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
+MetalExternalImage::MetalExternalImage(MetalExternalImage&& rhs) {
+    std::swap(mImage, rhs.mImage);
+    std::swap(mTexture, rhs.mTexture);
+    std::swap(mRgbTexture, rhs.mRgbTexture);
+}
+
+MetalExternalImage& MetalExternalImage::operator=(MetalExternalImage&& rhs) {
+    CVPixelBufferRelease(mImage);
+    CVBufferRelease(mTexture);
+    mImage = nullptr;
+    mTexture = nullptr;
+    mRgbTexture = nullptr;
+    std::swap(mImage, rhs.mImage);
+    std::swap(mTexture, rhs.mTexture);
+    std::swap(mRgbTexture, rhs.mRgbTexture);
+    return *this;
+}
+
+MetalExternalImage::~MetalExternalImage() noexcept {
+    CVPixelBufferRelease(mImage);
+    CVBufferRelease(mTexture);
+}
+
+id<MTLTexture> MetalExternalImage::getMtlTexture() const noexcept {
     if (mRgbTexture) {
         return mRgbTexture;
     }
-
-    // Retain the image and Metal texture until the GPU has finished with this frame. This does
-    // not need to be done for the RGB texture, because it is an Objective-C object whose
-    // lifetime is automatically managed by Metal.
-    auto& tracker = mContext.resourceTracker;
-    auto commandBuffer = getPendingCommandBuffer(&mContext);
-    if (tracker.trackResource((__bridge void*) commandBuffer, mImage, cvBufferDeleter)) {
-        CVPixelBufferRetain(mImage);
+    if (mTexture) {
+        return CVMetalTextureGetTexture(mTexture);
     }
-    if (tracker.trackResource((__bridge void*) commandBuffer, mTexture, cvBufferDeleter)) {
-        CVBufferRetain(mTexture);
-    }
-
-    assert_invariant(mTextureView);
-    return mTextureView;
+    return nil;
 }
 
-CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVPixelBufferRef image,
-        MTLPixelFormat format, size_t plane) {
+CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVMetalTextureCacheRef textureCache,
+        CVPixelBufferRef image, MTLPixelFormat format, size_t plane) {
     const size_t width = CVPixelBufferGetWidthOfPlane(image, plane);
     const size_t height = CVPixelBufferGetHeightOfPlane(image, plane);
 
     CVMetalTextureRef texture;
-    CVReturn result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
-            mContext.textureCache, image, nullptr, format, width, height, plane, &texture);
+    CVReturn result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache,
+            image, nullptr, format, width, height, plane, &texture);
     FILAMENT_CHECK_POSTCONDITION(result == kCVReturnSuccess)
             << "Could not create a CVMetalTexture from CVPixelBuffer.";
 
@@ -201,58 +221,19 @@ void MetalExternalImage::shutdown(MetalContext& context) noexcept {
     context.externalImageComputePipelineState = nil;
 }
 
-void MetalExternalImage::assertWritableImage(CVPixelBufferRef image) {
-    OSType formatType = CVPixelBufferGetPixelFormatType(image);
-    FILAMENT_CHECK_PRECONDITION(formatType == kCVPixelFormatType_32BGRA)
-            << "Metal SwapChain images must be in the 32BGRA format.";
-}
-
-void MetalExternalImage::unset() {
-    CVPixelBufferRelease(mImage);
-    CVBufferRelease(mTexture);
-
-    mImage = nullptr;
-    mTexture = nullptr;
-    mTextureView = nil;
-    mRgbTexture = nil;
-    mWidth = 0;
-    mHeight = 0;
-}
-
-id<MTLTexture> MetalExternalImage::createRgbTexture(size_t width, size_t height) {
+id<MTLTexture> MetalExternalImage::createRgbTexture(
+        id<MTLDevice> device, size_t width, size_t height) {
     MTLTextureDescriptor *descriptor =
             [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm
                                                                width:width
                                                               height:height
                                                            mipmapped:NO];
     descriptor.usage = MTLTextureUsageShaderWrite | MTLTextureUsageShaderRead;
-    return [mContext.device newTextureWithDescriptor:descriptor];
+    return [device newTextureWithDescriptor:descriptor];
 }
 
-id<MTLTexture> MetalExternalImage::createSwizzledTextureView(id<MTLTexture> texture) const {
-    const bool isDefaultSwizzle =
-            mSwizzle.r == TextureSwizzle::CHANNEL_0 &&
-            mSwizzle.g == TextureSwizzle::CHANNEL_1 &&
-            mSwizzle.b == TextureSwizzle::CHANNEL_2 &&
-            mSwizzle.a == TextureSwizzle::CHANNEL_3;
-    if (!isDefaultSwizzle && mContext.supportsTextureSwizzling) {
-        // Even though we've already checked supportsTextureSwizzling, we still need to guard these
-        // calls with @availability, otherwise the API usage will generate compiler warnings.
-        if (@available(iOS 13, *)) {
-            texture = createTextureViewWithSwizzle(texture,
-                    getSwizzleChannels(mSwizzle.r, mSwizzle.g, mSwizzle.b, mSwizzle.a));
-        }
-    }
-    return texture;
-}
-
-id<MTLTexture> MetalExternalImage::createSwizzledTextureView(CVMetalTextureRef ref) const {
-    id<MTLTexture> texture = CVMetalTextureGetTexture(ref);
-    return createSwizzledTextureView(texture);
-}
-
-void MetalExternalImage::ensureComputePipelineState() {
-    if (mContext.externalImageComputePipelineState != nil) {
+void MetalExternalImage::ensureComputePipelineState(MetalContext& context) {
+    if (context.externalImageComputePipelineState != nil) {
         return;
     }
 
@@ -260,29 +241,28 @@ void MetalExternalImage::ensureComputePipelineState() {
 
     NSString* objcSource = [NSString stringWithCString:kernel
                                               encoding:NSUTF8StringEncoding];
-    id<MTLLibrary> library = [mContext.device newLibraryWithSource:objcSource
-                                                            options:nil
-                                                              error:&error];
+    id<MTLLibrary> library = [context.device newLibraryWithSource:objcSource
+                                                          options:nil
+                                                            error:&error];
     NSERROR_CHECK("Unable to compile Metal shading library.");
 
     id<MTLFunction> kernelFunction = [library newFunctionWithName:@"ycbcrToRgb"];
 
-    mContext.externalImageComputePipelineState =
-            [mContext.device newComputePipelineStateWithFunction:kernelFunction
-                                                           error:&error];
+    context.externalImageComputePipelineState =
+            [context.device newComputePipelineStateWithFunction:kernelFunction error:&error];
     NSERROR_CHECK("Unable to create Metal compute pipeline state.");
 }
 
-id<MTLCommandBuffer> MetalExternalImage::encodeColorConversionPass(id<MTLTexture> inYPlane,
-        id<MTLTexture> inCbCrTexture, id<MTLTexture> outTexture) {
-    ensureComputePipelineState();
+id<MTLCommandBuffer> MetalExternalImage::encodeColorConversionPass(MetalContext& context,
+        id<MTLTexture> inYPlane, id<MTLTexture> inCbCrTexture, id<MTLTexture> outTexture) {
+    ensureComputePipelineState(context);
 
-    id<MTLCommandBuffer> commandBuffer = [mContext.commandQueue commandBuffer];
+    id<MTLCommandBuffer> commandBuffer = [context.commandQueue commandBuffer];
     commandBuffer.label = @"YCbCr to RGB conversion";
 
     id<MTLComputeCommandEncoder> computeEncoder = [commandBuffer computeCommandEncoder];
 
-    [computeEncoder setComputePipelineState:mContext.externalImageComputePipelineState];
+    [computeEncoder setComputePipelineState:context.externalImageComputePipelineState];
     [computeEncoder setTexture:inYPlane atIndex:0];
     [computeEncoder setTexture:inCbCrTexture atIndex:1];
     [computeEncoder setTexture:outTexture atIndex:2];
@@ -300,5 +280,11 @@ id<MTLCommandBuffer> MetalExternalImage::encodeColorConversionPass(id<MTLTexture
     return commandBuffer;
 }
 
+void MetalExternalImage::assertWritableImage(CVPixelBufferRef image) {
+    OSType formatType = CVPixelBufferGetPixelFormatType(image);
+    FILAMENT_CHECK_PRECONDITION(formatType == kCVPixelFormatType_32BGRA)
+            << "Metal SwapChain images must be in the 32BGRA format.";
+}
+
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalHandles.h

@@ -44,6 +44,7 @@
 #include <condition_variable>
 #include <memory>
 #include <type_traits>
+#include <vector>
 
 namespace filament {
 namespace backend {
@@ -73,7 +74,8 @@ public:
 
     void releaseDrawable();
 
-    void setFrameScheduledCallback(CallbackHandler* handler, FrameScheduledCallback&& callback);
+    void setFrameScheduledCallback(
+            CallbackHandler* handler, FrameScheduledCallback&& callback, uint64_t flags);
     void setFrameCompletedCallback(
             CallbackHandler* handler, utils::Invocable<void(void)>&& callback);
 
@@ -84,6 +86,8 @@ public:
     NSUInteger getSurfaceWidth() const;
     NSUInteger getSurfaceHeight() const;
 
+    bool isPixelBuffer() const { return type == SwapChainType::CVPIXELBUFFERREF; }
+
 private:
 
     enum class SwapChainType {
@@ -93,7 +97,6 @@ private:
     };
     bool isCaMetalLayer() const { return type == SwapChainType::CAMETALLAYER; }
     bool isHeadless() const { return type == SwapChainType::HEADLESS; }
-    bool isPixelBuffer() const { return type == SwapChainType::CVPIXELBUFFERREF; }
 
     void scheduleFrameScheduledCallback();
     void scheduleFrameCompletedCallback();
@@ -109,7 +112,7 @@ private:
     NSUInteger headlessWidth = 0;
     NSUInteger headlessHeight = 0;
     CAMetalLayer* layer = nullptr;
-    std::mutex layerDrawableMutex;
+    std::shared_ptr<std::mutex> layerDrawableMutex;
     MetalExternalImage externalImage;
     SwapChainType type;
 
@@ -121,6 +124,7 @@ private:
     struct {
         CallbackHandler* handler = nullptr;
         std::shared_ptr<FrameScheduledCallback> callback = nullptr;
+        uint64_t flags = 0;
     } frameScheduled;
 
     struct {
@@ -131,19 +135,17 @@ private:
 
 class MetalBufferObject : public HwBufferObject {
 public:
+
+    using TagResolver = MetalBuffer::TagResolver;
+
     MetalBufferObject(MetalContext& context, BufferObjectBinding bindingType, BufferUsage usage,
          uint32_t byteCount);
 
-    void updateBuffer(void* data, size_t size, uint32_t byteOffset);
-    void updateBufferUnsynchronized(void* data, size_t size, uint32_t byteOffset);
+    void updateBuffer(void* data, size_t size, uint32_t byteOffset, TagResolver&& getHandleTag);
+    void updateBufferUnsynchronized(
+            void* data, size_t size, uint32_t byteOffset, TagResolver&& getHandleTag);
     MetalBuffer* getBuffer() { return &buffer; }
 
-    // Tracks which uniform/ssbo buffers this buffer object is bound into.
-    static_assert(Program::UNIFORM_BINDING_COUNT <= 32);
-    static_assert(MAX_SSBO_COUNT <= 32);
-    utils::bitset32 boundUniformBuffers;
-    utils::bitset32 boundSsbos;
-
 private:
     MetalBuffer buffer;
 };
@@ -200,12 +202,11 @@ public:
     MetalProgram(MetalContext& context, Program&& program) noexcept;
 
     const MetalShaderCompiler::MetalFunctionBundle& getFunctions();
-    const Program::SamplerGroupInfo& getSamplerGroupInfo() { return samplerGroupInfo; }
+    const MetalShaderCompiler::MetalFunctionBundle& getFunctionsIfPresent() const;
 
 private:
     void initialize();
 
-    Program::SamplerGroupInfo samplerGroupInfo;
     MetalContext& mContext;
     MetalShaderCompiler::MetalFunctionBundle mFunctionBundle;
     MetalShaderCompiler::program_token_t mToken;
@@ -227,43 +228,42 @@ struct PixelBufferShape {
 class MetalTexture : public HwTexture {
 public:
     MetalTexture(MetalContext& context, SamplerType target, uint8_t levels, TextureFormat format,
-            uint8_t samples, uint32_t width, uint32_t height, uint32_t depth, TextureUsage usage,
-            TextureSwizzle r, TextureSwizzle g, TextureSwizzle b, TextureSwizzle a)
-            noexcept;
+            uint8_t samples, uint32_t width, uint32_t height, uint32_t depth,
+            TextureUsage usage) noexcept;
+
+    // constructors for creating texture views
+    MetalTexture(MetalContext& context, MetalTexture const* src, uint8_t baseLevel,
+            uint8_t levelCount) noexcept;
+    MetalTexture(MetalContext& context, MetalTexture const* src, TextureSwizzle r, TextureSwizzle g,
+            TextureSwizzle b, TextureSwizzle a) noexcept;
 
     // Constructor for importing an id<MTLTexture> outside of Filament.
     MetalTexture(MetalContext& context, SamplerType target, uint8_t levels, TextureFormat format,
             uint8_t samples, uint32_t width, uint32_t height, uint32_t depth, TextureUsage usage,
             id<MTLTexture> texture) noexcept;
 
-    ~MetalTexture();
+    // Constructors for importing external images.
+    MetalTexture(MetalContext& context, TextureFormat format, uint32_t width, uint32_t height,
+            TextureUsage usage, CVPixelBufferRef image) noexcept;
+    MetalTexture(MetalContext& context, TextureFormat format, uint32_t width, uint32_t height,
+            TextureUsage usage, CVPixelBufferRef image, uint32_t plane) noexcept;
 
-    // Returns an id<MTLTexture> suitable for reading in a shader, taking into account swizzle and
-    // LOD clamping.
-    id<MTLTexture> getMtlTextureForRead() noexcept;
+    // Returns an id<MTLTexture> suitable for reading in a shader, taking into account swizzle.
+    id<MTLTexture> getMtlTextureForRead() const noexcept;
 
     // Returns the id<MTLTexture> for attaching to a render pass.
-    id<MTLTexture> getMtlTextureForWrite() noexcept {
+    id<MTLTexture> getMtlTextureForWrite() const noexcept {
         return texture;
     }
 
+    std::shared_ptr<MetalExternalImage> getExternalImage() const noexcept { return externalImage; }
+
     void loadImage(uint32_t level, MTLRegion region, PixelBufferDescriptor& p) noexcept;
     void generateMipmaps() noexcept;
 
-    // A texture starts out with none of its mip levels (also referred to as LODs) available for
-    // reading. 4 actions update the range of LODs available:
-    // - calling loadImage
-    // - calling generateMipmaps
-    // - using the texture as a render target attachment
-    // - calling setMinMaxLevels
-    // A texture's available mips are consistent throughout a render pass.
-    void setLodRange(uint16_t minLevel, uint16_t maxLevel);
-    void extendLodRangeTo(uint16_t level);
-
     static MTLPixelFormat decidePixelFormat(MetalContext* context, TextureFormat format);
 
     MetalContext& context;
-    MetalExternalImage externalImage;
 
     // A "sidecar" texture used to implement automatic MSAA resolve.
     // This is created by MetalRenderTarget and stored here so it can be used with multiple
@@ -272,26 +272,6 @@ public:
 
     MTLPixelFormat devicePixelFormat;
 
-    // Frees memory associated with this texture and marks it as "terminated".
-    // Used to track "use after free" scenario.
-    void terminate() noexcept;
-    bool isTerminated() const noexcept { return terminated; }
-    inline void checkUseAfterFree(const char* samplerGroupDebugName, size_t textureIndex) const {
-        if (UTILS_LIKELY(!isTerminated())) {
-            return;
-        }
-        NSString* reason =
-                [NSString stringWithFormat:
-                                  @"Filament Metal texture use after free, sampler group = "
-                                  @"%s, texture index = %zu",
-                          samplerGroupDebugName, textureIndex];
-        NSException* useAfterFreeException =
-                [NSException exceptionWithName:@"MetalTextureUseAfterFree"
-                                        reason:reason
-                                      userInfo:nil];
-        [useAfterFreeException raise];
-    }
-
 private:
     void loadSlice(uint32_t level, MTLRegion region, uint32_t byteOffset, uint32_t slice,
             PixelBufferDescriptor const& data) noexcept;
@@ -302,95 +282,12 @@ private:
 
     id<MTLTexture> texture = nil;
 
+    std::shared_ptr<MetalExternalImage> externalImage;
+
     // If non-nil, a swizzled texture view to use instead of "texture".
     // Filament swizzling only affects texture reads, so this should not be used when the texture is
     // bound as a render target attachment.
     id<MTLTexture> swizzledTextureView = nil;
-    id<MTLTexture> lodTextureView = nil;
-
-    uint16_t minLod = std::numeric_limits<uint16_t>::max();
-    uint16_t maxLod = 0;
-
-    bool terminated = false;
-};
-
-class MetalSamplerGroup : public HwSamplerGroup {
-public:
-    explicit MetalSamplerGroup(size_t size, utils::FixedSizeString<32> name) noexcept
-        : size(size),
-          debugName(name),
-          textureHandles(size, Handle<HwTexture>()),
-          textures(size, nil),
-          samplers(size, nil) {}
-
-    inline void setTextureHandle(size_t index, Handle<HwTexture> th) {
-        assert_invariant(!finalized);
-        textureHandles[index] = th;
-    }
-
-    // This method is only used for debugging, to ensure all texture handles are alive.
-    const auto& getTextureHandles() const {
-        return textureHandles;
-    }
-
-    // Encode a MTLTexture into this SamplerGroup at the given index.
-    inline void setFinalizedTexture(size_t index, id<MTLTexture> t) {
-        assert_invariant(!finalized);
-        textures[index] = t;
-    }
-
-    // Encode a MTLSamplerState into this SamplerGroup at the given index.
-    inline void setFinalizedSampler(size_t index, id<MTLSamplerState> s) {
-        assert_invariant(!finalized);
-        samplers[index] = s;
-    }
-
-    // A SamplerGroup is "finalized" when all of its textures have been set and is ready for use in
-    // a draw call.
-    // Once a SamplerGroup is finalized, it must be reset or mutated to be written into again.
-    void finalize();
-    bool isFinalized() const noexcept { return finalized; }
-
-    // Both of these methods "unfinalize" a SamplerGroup, allowing it to be updated via calls to
-    // setFinalizedTexture or setFinalizedSampler. The difference is that when reset is called, all
-    // the samplers/textures must be rebound. The MTLArgumentEncoder must be specified, in case
-    // the texture types have changed.
-    // Mutate re-encodes the current set of samplers/textures into the new argument
-    // buffer.
-    void reset(id<MTLCommandBuffer> cmdBuffer, id<MTLArgumentEncoder> e, id<MTLDevice> device);
-    void mutate(id<MTLCommandBuffer> cmdBuffer);
-
-    id<MTLBuffer> getArgumentBuffer() const {
-        assert_invariant(finalized);
-        return argBuffer->getCurrentAllocation().first;
-    }
-
-    NSUInteger getArgumentBufferOffset() const {
-        return argBuffer->getCurrentAllocation().second;
-    }
-
-    inline std::pair<Handle<HwTexture>, id<MTLTexture>> getFinalizedTexture(size_t index) {
-        return {textureHandles[index], textures[index]};
-    }
-
-    // Calls the Metal useResource:usage:stages: method for all the textures in this SamplerGroup.
-    void useResources(id<MTLRenderCommandEncoder> renderPassEncoder);
-
-    size_t size;
-    utils::FixedSizeString<32> debugName;
-
-public:
-
-    // These vectors are kept in sync with one another.
-    utils::FixedCapacityVector<Handle<HwTexture>> textureHandles;
-    utils::FixedCapacityVector<id<MTLTexture>> textures;
-    utils::FixedCapacityVector<id<MTLSamplerState>> samplers;
-
-    id<MTLArgumentEncoder> encoder;
-
-    std::unique_ptr<MetalRingBuffer> argBuffer = nullptr;
-
-    bool finalized = false;
 };
 
 class MetalRenderTarget : public HwRenderTarget {
@@ -541,12 +438,67 @@ struct MetalTimerQuery : public HwTimerQuery {
 
     struct Status {
         std::atomic<bool> available {false};
-        uint64_t elapsed {0};   // only valid if available is true
+        std::atomic<uint64_t> elapsed {0};   // only valid if available is true
     };
 
     std::shared_ptr<Status> status;
 };
 
+class MetalDescriptorSetLayout : public HwDescriptorSetLayout {
+public:
+    MetalDescriptorSetLayout(DescriptorSetLayout&& layout) noexcept;
+
+    const auto& getBindings() const noexcept { return mLayout.bindings; }
+
+    size_t getDynamicOffsetCount() const noexcept { return mDynamicOffsetCount; }
+
+    /**
+     * Get an argument encoder for this descriptor set and shader stage.
+     * textureTypes should only include the textures present in the corresponding shader stage.
+     */
+    id<MTLArgumentEncoder> getArgumentEncoder(id<MTLDevice> device, ShaderStage stage,
+            utils::FixedCapacityVector<MTLTextureType> const& textureTypes);
+
+private:
+    id<MTLArgumentEncoder> getArgumentEncoderSlow(id<MTLDevice> device, ShaderStage stage,
+            utils::FixedCapacityVector<MTLTextureType> const& textureTypes);
+
+    DescriptorSetLayout mLayout;
+    size_t mDynamicOffsetCount = 0;
+    std::array<id<MTLArgumentEncoder>, Program::SHADER_TYPE_COUNT> mCachedArgumentEncoder = { nil };
+    std::array<utils::FixedCapacityVector<MTLTextureType>, Program::SHADER_TYPE_COUNT>
+            mCachedTextureTypes;
+};
+
+struct MetalDescriptorSet : public HwDescriptorSet {
+    MetalDescriptorSet(MetalDescriptorSetLayout* layout) noexcept;
+
+    void finalize(MetalDriver* driver);
+
+    id<MTLBuffer> finalizeAndGetBuffer(MetalDriver* driver, ShaderStage stage);
+
+    MetalDescriptorSetLayout* layout;
+
+    struct BufferBinding {
+        id<MTLBuffer> buffer;
+        uint32_t offset;
+        uint32_t size;
+    };
+    struct TextureBinding {
+        id<MTLTexture> texture;
+        SamplerParams sampler;
+    };
+    tsl::robin_map<descriptor_binding_t, BufferBinding> buffers;
+    tsl::robin_map<descriptor_binding_t, TextureBinding> textures;
+
+    std::vector<id<MTLResource>> vertexResources;
+    std::vector<id<MTLResource>> fragmentResources;
+
+    std::vector<std::shared_ptr<MetalExternalImage>> externalImages;
+
+    std::array<TrackedMetalBuffer, Program::SHADER_TYPE_COUNT> cachedBuffer = { nil };
+};
+
 } // namespace backend
 } // namespace filament
 

filament/backend/src/metal/MetalHandles.mm

@@ -73,7 +73,7 @@ MetalSwapChain::MetalSwapChain(MetalContext& context, CAMetalLayer* nativeWindow
     : context(context),
       depthStencilFormat(decideDepthStencilFormat(flags)),
       layer(nativeWindow),
-      externalImage(context),
+      layerDrawableMutex(std::make_shared<std::mutex>()),
       type(SwapChainType::CAMETALLAYER) {
 
     if (!(flags & SwapChain::CONFIG_TRANSPARENT) && !nativeWindow.opaque) {
@@ -99,17 +99,15 @@ MetalSwapChain::MetalSwapChain(MetalContext& context, int32_t width, int32_t hei
       depthStencilFormat(decideDepthStencilFormat(flags)),
       headlessWidth(width),
       headlessHeight(height),
-      externalImage(context),
       type(SwapChainType::HEADLESS) {}
 
 MetalSwapChain::MetalSwapChain(MetalContext& context, CVPixelBufferRef pixelBuffer, uint64_t flags)
     : context(context),
       depthStencilFormat(decideDepthStencilFormat(flags)),
-      externalImage(context),
+      externalImage(MetalExternalImage::createFromImage(context, pixelBuffer)),
       type(SwapChainType::CVPIXELBUFFERREF) {
     assert_invariant(flags & SWAP_CHAIN_CONFIG_APPLE_CVPIXELBUFFER);
     MetalExternalImage::assertWritableImage(pixelBuffer);
-    externalImage.set(pixelBuffer);
     assert_invariant(externalImage.isValid());
 }
 
@@ -120,7 +118,6 @@ MTLPixelFormat MetalSwapChain::decideDepthStencilFormat(uint64_t flags) {
 }
 
 MetalSwapChain::~MetalSwapChain() {
-    externalImage.set(nullptr);
 }
 
 NSUInteger MetalSwapChain::getSurfaceWidth() const {
@@ -170,7 +167,7 @@ id<MTLTexture> MetalSwapChain::acquireDrawable() {
     }
 
     if (isPixelBuffer()) {
-        return externalImage.getMetalTextureForDraw();
+        return externalImage.getMtlTexture();
     }
 
     assert_invariant(isCaMetalLayer());
@@ -179,7 +176,7 @@ id<MTLTexture> MetalSwapChain::acquireDrawable() {
     // calling -nextDrawable, or when releasing the last known reference
     // to any CAMetalDrawable returned from a previous -nextDrawable.
     {
-        std::lock_guard<std::mutex> lock(layerDrawableMutex);
+        std::lock_guard<std::mutex> lock(*layerDrawableMutex);
         drawable = [layer nextDrawable];
     }
 
@@ -188,8 +185,10 @@ id<MTLTexture> MetalSwapChain::acquireDrawable() {
 }
 
 void MetalSwapChain::releaseDrawable() {
-    std::lock_guard<std::mutex> lock(layerDrawableMutex);
-    drawable = nil;
+    if (drawable) {
+        std::lock_guard<std::mutex> lock(*layerDrawableMutex);
+        drawable = nil;
+    }
 }
 
 id<MTLTexture> MetalSwapChain::acquireDepthTexture() {
@@ -230,9 +229,10 @@ void MetalSwapChain::ensureDepthStencilTexture() {
 }
 
 void MetalSwapChain::setFrameScheduledCallback(
-        CallbackHandler* handler, FrameScheduledCallback&& callback) {
+        CallbackHandler* handler, FrameScheduledCallback&& callback, uint64_t flags) {
     frameScheduled.handler = handler;
     frameScheduled.callback = std::make_shared<FrameScheduledCallback>(std::move(callback));
+    frameScheduled.flags = flags;
 }
 
 void MetalSwapChain::setFrameCompletedCallback(
@@ -254,10 +254,6 @@ void MetalSwapChain::present() {
     }
 }
 
-#ifndef FILAMENT_RELEASE_PRESENT_DRAWABLE_MAIN_THREAD
-#define FILAMENT_RELEASE_PRESENT_DRAWABLE_MAIN_THREAD 1
-#endif
-
 class PresentDrawableData {
 public:
     PresentDrawableData() = delete;
@@ -265,10 +261,10 @@ public:
     PresentDrawableData& operator=(const PresentDrawableData&) = delete;
 
     static PresentDrawableData* create(id<CAMetalDrawable> drawable,
-            std::mutex* drawableMutex, MetalDriver* driver) {
+            std::shared_ptr<std::mutex> drawableMutex, MetalDriver* driver, uint64_t flags) {
         assert_invariant(drawableMutex);
         assert_invariant(driver);
-        return new PresentDrawableData(drawable, drawableMutex, driver);
+        return new PresentDrawableData(drawable, drawableMutex, driver, flags);
     }
 
     static void maybePresentAndDestroyAsync(PresentDrawableData* that, bool shouldPresent) {
@@ -276,34 +272,38 @@ public:
            [that->mDrawable present];
         }
 
-#if FILAMENT_RELEASE_PRESENT_DRAWABLE_MAIN_THREAD == 1
-        // mDrawable is acquired on the driver thread. Typically, we would release this object on
-        // the same thread, but after receiving consistent crash reports from within
-        // [CAMetalDrawable dealloc], we suspect this object requires releasing on the main thread.
-        dispatch_async(dispatch_get_main_queue(), ^{ cleanupAndDestroy(that); });
-#else
-        that->mDriver->runAtNextTick([that]() { cleanupAndDestroy(that); });
-#endif
+        if (that->mFlags & SwapChain::CALLBACK_DEFAULT_USE_METAL_COMPLETION_HANDLER) {
+            cleanupAndDestroy(that);
+        } else {
+            // mDrawable is acquired on the driver thread. Typically, we would release this object
+            // on the same thread, but after receiving consistent crash reports from within
+            // [CAMetalDrawable dealloc], we suspect this object requires releasing on the main
+            // thread.
+            dispatch_async(dispatch_get_main_queue(), ^{
+                cleanupAndDestroy(that);
+            });
+        }
     }
 
 private:
-    PresentDrawableData(id<CAMetalDrawable> drawable, std::mutex* drawableMutex,
-            MetalDriver* driver)
-        : mDrawable(drawable), mDrawableMutex(drawableMutex), mDriver(driver) {}
+    PresentDrawableData(id<CAMetalDrawable> drawable, std::shared_ptr<std::mutex> drawableMutex,
+            MetalDriver* driver, uint64_t flags)
+        : mDrawable(drawable), mDrawableMutex(drawableMutex), mDriver(driver), mFlags(flags) {}
 
     static void cleanupAndDestroy(PresentDrawableData *that) {
-        {
+        if (that->mDrawable) {
             std::lock_guard<std::mutex> lock(*(that->mDrawableMutex));
             that->mDrawable = nil;
         }
-        that->mDrawableMutex = nullptr;
+        that->mDrawableMutex.reset();
         that->mDriver = nullptr;
         delete that;
     }
 
     id<CAMetalDrawable> mDrawable;
-    std::mutex* mDrawableMutex = nullptr;
+    std::shared_ptr<std::mutex> mDrawableMutex;
     MetalDriver* mDriver = nullptr;
+    uint64_t mFlags = 0;
 };
 
 void presentDrawable(bool presentFrame, void* user) {
@@ -320,8 +320,8 @@ void MetalSwapChain::scheduleFrameScheduledCallback() {
 
     struct Callback {
         Callback(std::shared_ptr<FrameScheduledCallback> callback, id<CAMetalDrawable> drawable,
-                std::mutex* drawableMutex, MetalDriver* driver)
-            : f(callback), data(PresentDrawableData::create(drawable, drawableMutex, driver)) {}
+                 std::shared_ptr<std::mutex> drawableMutex, MetalDriver* driver, uint64_t flags)
+            : f(callback), data(PresentDrawableData::create(drawable, drawableMutex, driver, flags)) {}
         std::shared_ptr<FrameScheduledCallback> f;
         // PresentDrawableData* is destroyed by maybePresentAndDestroyAsync() later.
         std::unique_ptr<PresentDrawableData> data;
@@ -336,14 +336,19 @@ void MetalSwapChain::scheduleFrameScheduledCallback() {
 
     // This callback pointer will be captured by the block. Even if the scheduled handler is never
     // called, the unique_ptr will still ensure we don't leak memory.
+    uint64_t const flags = frameScheduled.flags;
     __block auto callback = std::make_unique<Callback>(
-        frameScheduled.callback, drawable, &layerDrawableMutex, context.driver);
+            frameScheduled.callback, drawable, layerDrawableMutex, context.driver, flags);
 
     backend::CallbackHandler* handler = frameScheduled.handler;
     MetalDriver* driver = context.driver;
     [getPendingCommandBuffer(&context) addScheduledHandler:^(id<MTLCommandBuffer> cb) {
         Callback* user = callback.release();
-        driver->scheduleCallback(handler, user, &Callback::func);
+        if (flags & SwapChain::CALLBACK_DEFAULT_USE_METAL_COMPLETION_HANDLER) {
+            Callback::func(user);
+        } else {
+            driver->scheduleCallback(handler, user, &Callback::func);
+        }
     }];
 }
 
@@ -378,12 +383,14 @@ MetalBufferObject::MetalBufferObject(MetalContext& context, BufferObjectBinding
         BufferUsage usage, uint32_t byteCount)
         : HwBufferObject(byteCount), buffer(context, bindingType, usage, byteCount) {}
 
-void MetalBufferObject::updateBuffer(void* data, size_t size, uint32_t byteOffset) {
-    buffer.copyIntoBuffer(data, size, byteOffset);
+void MetalBufferObject::updateBuffer(
+        void* data, size_t size, uint32_t byteOffset, TagResolver&& getHandleTag) {
+    buffer.copyIntoBuffer(data, size, byteOffset, std::move(getHandleTag));
 }
 
-void MetalBufferObject::updateBufferUnsynchronized(void* data, size_t size, uint32_t byteOffset) {
-    buffer.copyIntoBufferUnsynchronized(data, size, byteOffset);
+void MetalBufferObject::updateBufferUnsynchronized(
+        void* data, size_t size, uint32_t byteOffset, TagResolver&& getHandleTag) {
+    buffer.copyIntoBufferUnsynchronized(data, size, byteOffset, std::move(getHandleTag));
 }
 
 MetalVertexBufferInfo::MetalVertexBufferInfo(MetalContext& context, uint8_t bufferCount,
@@ -479,11 +486,6 @@ void MetalRenderPrimitive::setBuffers(MetalVertexBufferInfo const* const vbi,
 
 MetalProgram::MetalProgram(MetalContext& context, Program&& program) noexcept
     : HwProgram(program.getName()), mContext(context) {
-
-    // Save this program's SamplerGroupInfo, it's used during draw calls to bind sampler groups to
-    // the appropriate stage(s).
-    samplerGroupInfo = program.getSamplerGroupInfo();
-
     mToken = context.shaderCompiler->createProgram(program.getName(), std::move(program));
     assert_invariant(mToken);
 }
@@ -493,6 +495,10 @@ const MetalShaderCompiler::MetalFunctionBundle& MetalProgram::getFunctions() {
     return mFunctionBundle;
 }
 
+const MetalShaderCompiler::MetalFunctionBundle& MetalProgram::getFunctionsIfPresent() const {
+    return mFunctionBundle;
+}
+
 void MetalProgram::initialize() {
     if (!mToken) {
         return;
@@ -503,10 +509,9 @@ void MetalProgram::initialize() {
 
 MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t levels,
         TextureFormat format, uint8_t samples, uint32_t width, uint32_t height, uint32_t depth,
-        TextureUsage usage, TextureSwizzle r, TextureSwizzle g, TextureSwizzle b,
-        TextureSwizzle a) noexcept
-    : HwTexture(target, levels, samples, width, height, depth, format, usage), context(context),
-        externalImage(context, r, g, b, a) {
+        TextureUsage usage) noexcept
+    : HwTexture(target, levels, samples, width, height, depth, format, usage), context(context) {
+    assert_invariant(target != SamplerType::SAMPLER_EXTERNAL);
 
     devicePixelFormat = decidePixelFormat(&context, format);
     FILAMENT_CHECK_POSTCONDITION(devicePixelFormat != MTLPixelFormatInvalid)
@@ -553,8 +558,6 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
             descriptor.usage = getMetalTextureUsage(usage);
             descriptor.storageMode = MTLStorageModePrivate;
             texture = [context.device newTextureWithDescriptor:descriptor];
-            FILAMENT_CHECK_POSTCONDITION(texture != nil)
-                    << "Could not create Metal texture. Out of memory?";
             break;
         case SamplerType::SAMPLER_CUBEMAP:
         case SamplerType::SAMPLER_CUBEMAP_ARRAY:
@@ -569,8 +572,6 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
             descriptor.usage = getMetalTextureUsage(usage);
             descriptor.storageMode = MTLStorageModePrivate;
             texture = [context.device newTextureWithDescriptor:descriptor];
-            FILAMENT_CHECK_POSTCONDITION(texture != nil)
-                    << "Could not create Metal texture. Out of memory?";
             break;
         case SamplerType::SAMPLER_3D:
             descriptor = [MTLTextureDescriptor new];
@@ -583,8 +584,6 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
             descriptor.usage = getMetalTextureUsage(usage);
             descriptor.storageMode = MTLStorageModePrivate;
             texture = [context.device newTextureWithDescriptor:descriptor];
-            FILAMENT_CHECK_POSTCONDITION(texture != nil)
-                    << "Could not create Metal texture. Out of memory?";
             break;
         case SamplerType::SAMPLER_EXTERNAL:
             // If we're using external textures (CVPixelBufferRefs), we don't need to make any
@@ -593,15 +592,33 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
             break;
     }
 
-    // If swizzling is set, set up a swizzled texture view that we'll use when sampling this texture.
-    const bool isDefaultSwizzle =
-            r == TextureSwizzle::CHANNEL_0 &&
-            g == TextureSwizzle::CHANNEL_1 &&
-            b == TextureSwizzle::CHANNEL_2 &&
-            a == TextureSwizzle::CHANNEL_3;
-    // If texture is nil, then it must be a SAMPLER_EXTERNAL texture.
-    // Swizzling for external textures is handled inside MetalExternalImage.
-    if (!isDefaultSwizzle && texture && context.supportsTextureSwizzling) {
+    FILAMENT_CHECK_POSTCONDITION(target == SamplerType::SAMPLER_EXTERNAL || texture != nil)
+            << "Could not create Metal texture (SamplerType = " << int(target)
+            << ", levels = " << int(levels) << ", MTLPixelFormat = " << int(devicePixelFormat)
+            << ", width = " << width << ", height = " << height << ", depth = " << depth
+            << "). Out of memory?";
+}
+
+MetalTexture::MetalTexture(MetalContext& context, MetalTexture const* src, uint8_t baseLevel,
+        uint8_t levelCount) noexcept
+    : HwTexture(src->target, src->levels, src->samples, src->width, src->height, src->depth,
+              src->format, src->usage),
+      context(context),
+      devicePixelFormat(src->devicePixelFormat),
+      externalImage(src->externalImage) {
+    texture = createTextureViewWithLodRange(
+            src->getMtlTextureForRead(), baseLevel, baseLevel + levelCount - 1);
+}
+
+MetalTexture::MetalTexture(MetalContext& context, MetalTexture const* src, TextureSwizzle r,
+        TextureSwizzle g, TextureSwizzle b, TextureSwizzle a) noexcept
+    : HwTexture(src->target, src->levels, src->samples, src->width, src->height, src->depth,
+              src->format, src->usage),
+      context(context),
+      devicePixelFormat(src->devicePixelFormat),
+      externalImage(src->externalImage) {
+    texture = src->getMtlTextureForRead();
+    if (context.supportsTextureSwizzling) {
         // Even though we've already checked context.supportsTextureSwizzling, we still need to
         // guard these calls with @availability, otherwise the API usage will generate compiler
         // warnings.
@@ -615,44 +632,30 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
 MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t levels, TextureFormat format,
         uint8_t samples, uint32_t width, uint32_t height, uint32_t depth, TextureUsage usage,
         id<MTLTexture> metalTexture) noexcept
-    : HwTexture(target, levels, samples, width, height, depth, format, usage), context(context),
-        externalImage(context) {
+    : HwTexture(target, levels, samples, width, height, depth, format, usage), context(context) {
     texture = metalTexture;
-    setLodRange(0, levels - 1);
 }
 
-void MetalTexture::terminate() noexcept {
-    texture = nil;
-    swizzledTextureView = nil;
-    lodTextureView = nil;
-    msaaSidecar = nil;
-    externalImage.set(nullptr);
-    terminated = true;
+MetalTexture::MetalTexture(MetalContext& context, TextureFormat format, uint32_t width,
+        uint32_t height, TextureUsage usage, CVPixelBufferRef image) noexcept
+    : HwTexture(SamplerType::SAMPLER_EXTERNAL, 1, 1, width, height, 1, format, usage),
+      context(context),
+      externalImage(std::make_shared<MetalExternalImage>(
+              MetalExternalImage::createFromImage(context, image))) {
+    texture = externalImage->getMtlTexture();
 }
 
-MetalTexture::~MetalTexture() {
-    externalImage.set(nullptr);
+MetalTexture::MetalTexture(MetalContext& context, TextureFormat format, uint32_t width,
+        uint32_t height, TextureUsage usage, CVPixelBufferRef image, uint32_t plane) noexcept
+    : HwTexture(SamplerType::SAMPLER_EXTERNAL, 1, 1, width, height, 1, format, usage),
+      context(context),
+      externalImage(std::make_shared<MetalExternalImage>(
+              MetalExternalImage::createFromImagePlane(context, image, plane))) {
+    texture = externalImage->getMtlTexture();
 }
 
-id<MTLTexture> MetalTexture::getMtlTextureForRead() noexcept {
-    if (lodTextureView) {
-        return lodTextureView;
-    }
-    // The texture's swizzle remains constant throughout its lifetime, however its LOD range can
-    // change. We'll cache the LOD view, and set lodTextureView to nil if minLod or maxLod is
-    // updated.
-    id<MTLTexture> t = swizzledTextureView ? swizzledTextureView : texture;
-    if (!t) {
-        return nil;
-    }
-    if (UTILS_UNLIKELY(minLod > maxLod)) {
-        // If the texture does not have any available LODs, provide a view of only level 0.
-        // Filament should prevent this from ever occurring.
-        lodTextureView = createTextureViewWithLodRange(t, 0, 0);
-        return lodTextureView;
-    }
-    lodTextureView = createTextureViewWithLodRange(t, minLod, maxLod);
-    return lodTextureView;
+id<MTLTexture> MetalTexture::getMtlTextureForRead() const noexcept {
+    return swizzledTextureView ? swizzledTextureView : texture;
 }
 
 MTLPixelFormat MetalTexture::decidePixelFormat(MetalContext* context, TextureFormat format) {
@@ -771,15 +774,12 @@ void MetalTexture::loadImage(uint32_t level, MTLRegion region, PixelBufferDescri
             assert_invariant(false);
         }
     }
-
-    extendLodRangeTo(level);
 }
 
 void MetalTexture::generateMipmaps() noexcept {
     id <MTLBlitCommandEncoder> blitEncoder = [getPendingCommandBuffer(&context) blitCommandEncoder];
     [blitEncoder generateMipmapsForTexture:texture];
     [blitEncoder endEncoding];
-    setLodRange(0, texture.mipmapLevelCount - 1);
 }
 
 void MetalTexture::loadSlice(uint32_t level, MTLRegion region, uint32_t byteOffset, uint32_t slice,
@@ -903,98 +903,6 @@ void MetalTexture::loadWithBlit(uint32_t level, uint32_t slice, MTLRegion region
     context.blitter->blit(getPendingCommandBuffer(&context), args, "Texture upload blit");
 }
 
-void MetalTexture::extendLodRangeTo(uint16_t level) {
-    assert_invariant(!isInRenderPass(&context));
-    minLod = std::min(minLod, level);
-    maxLod = std::max(maxLod, level);
-    lodTextureView = nil;
-}
-
-void MetalTexture::setLodRange(uint16_t min, uint16_t max) {
-    assert_invariant(!isInRenderPass(&context));
-    assert_invariant(min <= max);
-    minLod = min;
-    maxLod = max;
-    lodTextureView = nil;
-}
-
-void MetalSamplerGroup::finalize() {
-    assert_invariant(encoder);
-    // TODO: we should be able to encode textures and samplers inside setFinalizedTexture and
-    // setFinalizedSampler as they become available, but Metal doesn't seem to like this; the arg
-    // buffer gets encoded incorrectly. This warrants more investigation.
-
-    auto [buffer, offset] = argBuffer->getCurrentAllocation();
-    [encoder setArgumentBuffer:buffer offset:offset];
-
-    // Encode all textures and samplers.
-    for (size_t s = 0; s < size; s++) {
-        [encoder setTexture:textures[s] atIndex:(s * 2 + 0)];
-        [encoder setSamplerState:samplers[s] atIndex:(s * 2 + 1)];
-    }
-
-    finalized = true;
-}
-
-void MetalSamplerGroup::reset(id<MTLCommandBuffer> cmdBuffer, id<MTLArgumentEncoder> e,
-        id<MTLDevice> device) {
-    encoder = e;
-
-    // The number of slots in the ring buffer we use to manage argument buffer allocations.
-    // This number was chosen to avoid running out of slots and having to allocate a "fallback"
-    // buffer when SamplerGroups are updated multiple times a frame. This value can reduced after
-    // auditing Filament's calls to updateSamplerGroup, which should be as few times as possible.
-    // For example, the bloom downsample pass should be refactored to maintain two separate
-    // MaterialInstances instead of "ping ponging" between two texture bindings, which causes a
-    // single SamplerGroup to be updated many times a frame.
-    static constexpr auto METAL_ARGUMENT_BUFFER_SLOTS = 32;
-
-    MTLSizeAndAlign argBufferLayout;
-    argBufferLayout.size = encoder.encodedLength;
-    argBufferLayout.align = encoder.alignment;
-    // Chances are, even though the MTLArgumentEncoder might change, the required size and alignment
-    // probably won't. So we can re-use the previous ring buffer.
-    if (UTILS_UNLIKELY(!argBuffer || !argBuffer->canAccomodateLayout(argBufferLayout))) {
-        argBuffer = std::make_unique<MetalRingBuffer>(device, MTLResourceStorageModeShared,
-                argBufferLayout, METAL_ARGUMENT_BUFFER_SLOTS);
-    } else {
-        argBuffer->createNewAllocation(cmdBuffer);
-    }
-
-    // Clear all textures and samplers.
-    assert_invariant(textureHandles.size() == textures.size());
-    assert_invariant(textures.size() == samplers.size());
-    for (size_t s = 0; s < textureHandles.size(); s++) {
-        textureHandles[s] = {};
-        textures[s] = nil;
-        samplers[s] = nil;
-    }
-
-    finalized = false;
-}
-
-void MetalSamplerGroup::mutate(id<MTLCommandBuffer> cmdBuffer) {
-    assert_invariant(finalized);    // only makes sense to mutate if this sampler group is finalized
-    assert_invariant(argBuffer);
-    argBuffer->createNewAllocation(cmdBuffer);
-    finalized = false;
-}
-
-void MetalSamplerGroup::useResources(id<MTLRenderCommandEncoder> renderPassEncoder) {
-    assert_invariant(finalized);
-    if (@available(iOS 13, *)) {
-        // TODO: pass only the appropriate stages to useResources.
-        [renderPassEncoder useResources:textures.data()
-                                  count:textures.size()
-                                  usage:MTLResourceUsageRead | MTLResourceUsageSample
-                                 stages:MTLRenderStageFragment | MTLRenderStageVertex];
-    } else {
-        [renderPassEncoder useResources:textures.data()
-                                  count:textures.size()
-                                  usage:MTLResourceUsageRead | MTLResourceUsageSample];
-    }
-}
-
 MetalRenderTarget::MetalRenderTarget(MetalContext* context, uint32_t width, uint32_t height,
         uint8_t samples, Attachment colorAttachments[MRT::MAX_SUPPORTED_RENDER_TARGET_COUNT],
         Attachment depthAttachment, Attachment stencilAttachment) :
@@ -1340,5 +1248,195 @@ FenceStatus MetalFence::wait(uint64_t timeoutNs) {
     return FenceStatus::ERROR;
 }
 
+MetalDescriptorSetLayout::MetalDescriptorSetLayout(DescriptorSetLayout&& l) noexcept
+    : mLayout(std::move(l)) {
+    size_t dynamicBindings = 0;
+    for (const auto& binding : mLayout.bindings) {
+        if (any(binding.flags & DescriptorFlags::DYNAMIC_OFFSET)) {
+            dynamicBindings++;
+        }
+    }
+    mDynamicOffsetCount = dynamicBindings;
+}
+
+id<MTLArgumentEncoder> MetalDescriptorSetLayout::getArgumentEncoder(id<MTLDevice> device, ShaderStage stage,
+        utils::FixedCapacityVector<MTLTextureType> const& textureTypes) {
+    auto const index = static_cast<size_t>(stage);
+    assert_invariant(index < mCachedArgumentEncoder.size());
+    if (mCachedArgumentEncoder[index] &&
+            std::equal(
+                    textureTypes.begin(), textureTypes.end(), mCachedTextureTypes[index].begin())) {
+        return mCachedArgumentEncoder[index];
+    }
+    mCachedArgumentEncoder[index] = getArgumentEncoderSlow(device, stage, textureTypes);
+    mCachedTextureTypes[index] = textureTypes;
+    return mCachedArgumentEncoder[index];
+}
+
+id<MTLArgumentEncoder> MetalDescriptorSetLayout::getArgumentEncoderSlow(id<MTLDevice> device,
+        ShaderStage stage, utils::FixedCapacityVector<MTLTextureType> const& textureTypes) {
+    auto const& bindings = getBindings();
+    NSMutableArray<MTLArgumentDescriptor*>* arguments = [NSMutableArray new];
+    // Important! The bindings must be sorted by binding number. This has already been done inside
+    // createDescriptorSetLayout.
+    size_t textureIndex = 0;
+    for (auto const& binding : bindings) {
+        if (!hasShaderType(binding.stageFlags, stage)) {
+            continue;
+        }
+        switch (binding.type) {
+            case DescriptorType::UNIFORM_BUFFER:
+            case DescriptorType::SHADER_STORAGE_BUFFER: {
+                MTLArgumentDescriptor* bufferArgument = [MTLArgumentDescriptor argumentDescriptor];
+                bufferArgument.index = binding.binding * 2;
+                bufferArgument.dataType = MTLDataTypePointer;
+                bufferArgument.access = MTLArgumentAccessReadOnly;
+                [arguments addObject:bufferArgument];
+                break;
+            }
+            case DescriptorType::SAMPLER:
+            case DescriptorType::SAMPLER_EXTERNAL: {
+                MTLArgumentDescriptor* textureArgument = [MTLArgumentDescriptor argumentDescriptor];
+                textureArgument.index = binding.binding * 2;
+                textureArgument.dataType = MTLDataTypeTexture;
+                MTLTextureType textureType = MTLTextureType2D;
+                if (textureIndex < textureTypes.size()) {
+                    textureType = textureTypes[textureIndex++];
+                }
+                textureArgument.textureType = textureType;
+                textureArgument.access = MTLArgumentAccessReadOnly;
+                [arguments addObject:textureArgument];
+
+                MTLArgumentDescriptor* samplerArgument = [MTLArgumentDescriptor argumentDescriptor];
+                samplerArgument.index = binding.binding * 2 + 1;
+                samplerArgument.dataType = MTLDataTypeSampler;
+                textureArgument.access = MTLArgumentAccessReadOnly;
+                [arguments addObject:samplerArgument];
+                break;
+            }
+            case DescriptorType::INPUT_ATTACHMENT:
+                // TODO: support INPUT_ATTACHMENT
+                assert_invariant(false);
+                break;
+        }
+    }
+    return [device newArgumentEncoderWithArguments:arguments];
+}
+
+MetalDescriptorSet::MetalDescriptorSet(MetalDescriptorSetLayout* layout) noexcept
+    : layout(layout) {}
+
+void MetalDescriptorSet::finalize(MetalDriver* driver) {
+    [driver->mContext->currentRenderPassEncoder useResource:driver->mContext->emptyBuffer
+                                                      usage:MTLResourceUsageRead];
+    [driver->mContext->currentRenderPassEncoder
+            useResource:getOrCreateEmptyTexture(driver->mContext)
+                  usage:MTLResourceUsageRead];
+
+    if (@available(iOS 13.0, *)) {
+        [driver->mContext->currentRenderPassEncoder useResources:vertexResources.data()
+                                                           count:vertexResources.size()
+                                                           usage:MTLResourceUsageRead
+                                                          stages:MTLRenderStageVertex];
+        [driver->mContext->currentRenderPassEncoder useResources:fragmentResources.data()
+                                                           count:fragmentResources.size()
+                                                           usage:MTLResourceUsageRead
+                                                          stages:MTLRenderStageFragment];
+    } else {
+        [driver->mContext->currentRenderPassEncoder useResources:vertexResources.data()
+                                                           count:vertexResources.size()
+                                                           usage:MTLResourceUsageRead];
+        [driver->mContext->currentRenderPassEncoder useResources:fragmentResources.data()
+                                                           count:fragmentResources.size()
+                                                           usage:MTLResourceUsageRead];
+    }
+}
+
+id<MTLBuffer> MetalDescriptorSet::finalizeAndGetBuffer(MetalDriver* driver, ShaderStage stage) {
+    auto const index = static_cast<size_t>(stage);
+    assert_invariant(index < cachedBuffer.size());
+    auto& buffer = cachedBuffer[index];
+
+    if (buffer) {
+        return buffer.get();
+    }
+
+    // Map all the texture bindings to their respective texture types.
+    auto const& bindings = layout->getBindings();
+    auto textureTypes = utils::FixedCapacityVector<MTLTextureType>::with_capacity(bindings.size());
+    for (auto const& binding : bindings) {
+        if (!hasShaderType(binding.stageFlags, stage)) {
+            continue;
+        }
+        MTLTextureType textureType = MTLTextureType2D;
+        if (auto found = textures.find(binding.binding); found != textures.end()) {
+            auto const& textureBinding = textures[binding.binding];
+            textureType = textureBinding.texture.textureType;
+        }
+        textureTypes.push_back(textureType);
+    }
+
+    MetalContext const& context = *driver->mContext;
+
+    id<MTLArgumentEncoder> encoder =
+            layout->getArgumentEncoder(context.device, stage, textureTypes);
+
+    {
+        ScopedAllocationTimer timer("descriptor_set");
+        buffer = { [context.device newBufferWithLength:encoder.encodedLength
+                                               options:MTLResourceStorageModeShared],
+            TrackedMetalBuffer::Type::DESCRIPTOR_SET };
+    }
+    [encoder setArgumentBuffer:buffer.get() offset:0];
+
+    for (auto const& binding : bindings) {
+        if (!hasShaderType(binding.stageFlags, stage)) {
+            continue;
+        }
+        switch (binding.type) {
+            case DescriptorType::UNIFORM_BUFFER:
+            case DescriptorType::SHADER_STORAGE_BUFFER: {
+                auto found = buffers.find(binding.binding);
+                if (found == buffers.end()) {
+                    [encoder setBuffer:driver->mContext->emptyBuffer
+                                offset:0
+                               atIndex:binding.binding * 2];
+                    continue;
+                }
+
+                auto const& bufferBinding = buffers[binding.binding];
+                [encoder setBuffer:bufferBinding.buffer
+                            offset:bufferBinding.offset
+                           atIndex:binding.binding * 2];
+                break;
+            }
+            case DescriptorType::SAMPLER:
+            case DescriptorType::SAMPLER_EXTERNAL: {
+                auto found = textures.find(binding.binding);
+                if (found == textures.end()) {
+                    [encoder setTexture:driver->mContext->emptyTexture atIndex:binding.binding * 2];
+                    id<MTLSamplerState> sampler =
+                            driver->mContext->samplerStateCache.getOrCreateState({});
+                    [encoder setSamplerState:sampler atIndex:binding.binding * 2 + 1];
+                    continue;
+                }
+
+                auto const& textureBinding = textures[binding.binding];
+                [encoder setTexture:textureBinding.texture atIndex:binding.binding * 2];
+                SamplerState samplerState { .samplerParams = textureBinding.sampler };
+                id<MTLSamplerState> sampler =
+                        driver->mContext->samplerStateCache.getOrCreateState(samplerState);
+                [encoder setSamplerState:sampler atIndex:binding.binding * 2 + 1];
+                break;
+            }
+            case DescriptorType::INPUT_ATTACHMENT:
+                assert_invariant(false);
+                break;
+        }
+    }
+
+    return buffer.get();
+}
+
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalShaderCompiler.h

@@ -83,6 +83,10 @@ public:
             return std::get<Compute>(mPrograms);
         }
 
+        bool isRaster() const { return std::holds_alternative<Raster>(mPrograms); }
+
+        bool isCompute() const { return std::holds_alternative<Compute>(mPrograms); }
+
         static MetalFunctionBundle none() {
             return MetalFunctionBundle(None{});
         }

filament/backend/src/metal/MetalShaderCompiler.mm

@@ -226,9 +226,11 @@ MetalShaderCompiler::program_token_t MetalShaderCompiler::createProgram(
             CompilerPriorityQueue const priorityQueue = program.getPriorityQueue();
             mCompilerThreadPool.queue(priorityQueue, token,
                     [this, name, device = mDevice, program = std::move(program), token]() {
-                        MetalFunctionBundle compiledProgram = compileProgram(program, device);
-                        token->set(compiledProgram);
-                        mCallbackManager.put(token->handle);
+                        @autoreleasepool {
+                            MetalFunctionBundle compiledProgram = compileProgram(program, device);
+                            token->set(compiledProgram);
+                            mCallbackManager.put(token->handle);
+                        }
                     });
 
             break;

filament/backend/src/metal/MetalState.h

@@ -28,37 +28,35 @@
 
 #include <memory>
 #include <tsl/robin_map.h>
+
 #include <utils/Hash.h>
+#include <utils/Invocable.h>
 
 namespace filament {
 namespace backend {
 
-inline bool operator==(const SamplerParams& lhs, const SamplerParams& rhs) {
-    return SamplerParams::EqualTo{}(lhs, rhs);
-}
-
 //   Rasterization Bindings
 //   ----------------------
 //   Bindings    Buffer name                          Count
 //   ------------------------------------------------------
 //   0           Zero buffer (placeholder vertex buffer)  1
 //   1-16        Filament vertex buffers                 16   limited by MAX_VERTEX_BUFFER_COUNT
-//   17-25       Uniform buffers                          9   Program::UNIFORM_BINDING_COUNT
-//   26          Push constants                           1
-//   27-30       Sampler groups (argument buffers)        4   Program::SAMPLER_BINDING_COUNT
+//   20          Push constants                           1
+//   21-24       Descriptor sets (argument buffers)       4   limited by MAX_DESCRIPTOR_SET_COUNT
+//   25          Dynamic offset buffer                    1
 //
-//   Total                                               31
+//   Total                                               23
 
 //   Compute Bindings
 //   ----------------------
 //   Bindings    Buffer name                          Count
 //   ------------------------------------------------------
 //   0-3         SSBO buffers                             4   MAX_SSBO_COUNT
-//   17-25       Uniform buffers                          9   Program::UNIFORM_BINDING_COUNT
-//   26          Push constants                           1
-//   27-30       Sampler groups (argument buffers)        4   Program::SAMPLER_BINDING_COUNT
+//   20          Push constants                           1
+//   21-24       Descriptor sets (argument buffers)       4   limited by MAX_DESCRIPTOR_SET_COUNT
+//   25          Dynamic offset buffer                    1
 //
-//   Total                                               18
+//   Total                                               10
 
 // The total number of vertex buffer "slots" that the Metal backend can bind.
 // + 1 to account for the zero buffer, a placeholder buffer used internally by the Metal backend.
@@ -71,10 +69,11 @@ static constexpr uint32_t ZERO_VERTEX_BUFFER_BINDING = 0u;
 
 static constexpr uint32_t USER_VERTEX_BUFFER_BINDING_START = 1u;
 
+
 // These constants must match the equivalent in CodeGenerator.h.
-static constexpr uint32_t UNIFORM_BUFFER_BINDING_START = 17u;
-static constexpr uint32_t SSBO_BINDING_START = 0u;
-static constexpr uint32_t SAMPLER_GROUP_BINDING_START = 27u;
+static constexpr uint32_t PUSH_CONSTANT_BUFFER_INDEX = 20u;
+static constexpr uint32_t DESCRIPTOR_SET_BINDING_START = 21u;
+static constexpr uint32_t DYNAMIC_OFFSET_BINDING = 25u;
 
 // Forward declarations necessary here, definitions at end of file.
 inline bool operator==(const MTLViewport& lhs, const MTLViewport& rhs);
@@ -163,6 +162,8 @@ template<typename StateType,
          typename HashFn = utils::hash::MurmurHashFn<StateType>>
 class StateCache {
 
+    using MapType = tsl::robin_map<StateType, MetalType, HashFn>;
+
 public:
 
     StateCache() = default;
@@ -172,6 +173,18 @@ public:
 
     void setDevice(id<MTLDevice> device) noexcept { mDevice = device; }
 
+    void removeIf(utils::Invocable<bool(const StateType&)> fn) noexcept {
+        typename MapType::const_iterator it = mStateCache.begin();
+        while (it != mStateCache.end()) {
+            const auto& [key, _] = *it;
+            if (UTILS_UNLIKELY(fn(key))) {
+                it = mStateCache.erase(it);
+            } else {
+                ++it;
+            }
+        }
+    }
+
     MetalType getOrCreateState(const StateType& state) noexcept {
         assert_invariant(mDevice);
 
@@ -199,7 +212,7 @@ private:
     StateCreator creator;
     id<MTLDevice> mDevice = nil;
 
-    tsl::robin_map<StateType, MetalType, HashFn> mStateCache;
+    MapType mStateCache;
 
 };
 
@@ -207,9 +220,8 @@ private:
 // Different kinds of state, like pipeline state, uniform buffer state, etc., are passed to the
 // current Metal command encoder and persist throughout the lifetime of the encoder (a frame).
 // StateTracker is used to prevent calling redundant state change methods.
-template<typename StateType>
+template <typename StateType, typename StateEqual = std::equal_to<StateType>>
 class StateTracker {
-
 public:
 
     // Call to force the state to dirty at the beginning of each frame, as all state must be
@@ -217,7 +229,7 @@ public:
     void invalidate() noexcept { mStateDirty = true; }
 
     void updateState(const StateType& newState) noexcept {
-        if (mCurrentState != newState) {
+        if (!StateEqual()(mCurrentState, newState)) {
             mCurrentState = newState;
             mStateDirty = true;
         }
@@ -238,7 +250,6 @@ private:
 
     bool mStateDirty = true;
     StateType mCurrentState = {};
-
 };
 
 // Pipeline state
@@ -346,6 +357,16 @@ using DepthStencilStateTracker = StateTracker<DepthStencilState>;
 using DepthStencilStateCache = StateCache<DepthStencilState, id<MTLDepthStencilState>,
         DepthStateCreator>;
 
+struct MtlScissorRectEqual {
+    bool operator()(const MTLScissorRect& lhs, const MTLScissorRect& rhs) const {
+        return lhs.height == rhs.height &&
+            lhs.width == rhs.width &&
+            lhs.x == rhs.x &&
+            lhs.y == rhs.y;
+    }
+};
+using ScissorRectStateTracker = StateTracker<MTLScissorRect, MtlScissorRectEqual>;
+
 // Uniform buffers
 
 class MetalBufferObject;
@@ -382,18 +403,22 @@ using SamplerStateCache = StateCache<SamplerState, id<MTLSamplerState>, SamplerS
 
 using CullModeStateTracker = StateTracker<MTLCullMode>;
 using WindingStateTracker = StateTracker<MTLWinding>;
+using DepthClampStateTracker = StateTracker<MTLDepthClipMode>;
 
 // Argument encoder
 
 struct ArgumentEncoderState {
+    NSUInteger bufferCount;
     utils::FixedCapacityVector<MTLTextureType> textureTypes;
 
-    explicit ArgumentEncoderState(utils::FixedCapacityVector<MTLTextureType>&& types)
-        : textureTypes(std::move(types)) {}
+    explicit ArgumentEncoderState(
+            NSUInteger bufferCount, utils::FixedCapacityVector<MTLTextureType>&& types)
+        : bufferCount(bufferCount), textureTypes(std::move(types)) {}
 
     bool operator==(const ArgumentEncoderState& rhs) const noexcept {
         return std::equal(textureTypes.begin(), textureTypes.end(), rhs.textureTypes.begin(),
-                rhs.textureTypes.end());
+                       rhs.textureTypes.end()) &&
+                bufferCount == rhs.bufferCount;
     }
 
     bool operator!=(const ArgumentEncoderState& rhs) const noexcept {
@@ -415,6 +440,30 @@ struct ArgumentEncoderCreator {
 using ArgumentEncoderCache = StateCache<ArgumentEncoderState, id<MTLArgumentEncoder>,
         ArgumentEncoderCreator, ArgumentEncoderHasher>;
 
+template <NSUInteger N, ShaderStage stage>
+class MetalBufferBindings {
+public:
+    MetalBufferBindings() { invalidate(); }
+
+    void invalidate() {
+        mDirtyBuffers.reset();
+        mDirtyOffsets.reset();
+        for (int i = 0; i < int(N); i++) {
+            mDirtyBuffers.set(i, true);
+            mDirtyOffsets.set(i, true);
+        }
+    }
+    void setBuffer(const id<MTLBuffer> buffer, NSUInteger offset, NSUInteger index);
+    void bindBuffers(id<MTLCommandEncoder> encoder, NSUInteger startIndex);
+
+private:
+    static_assert(N <= 8);
+    std::array<__weak id<MTLBuffer>, N> mBuffers = { nil };
+    std::array<NSUInteger, N> mOffsets = { 0 };
+    utils::bitset8 mDirtyBuffers;
+    utils::bitset8 mDirtyOffsets;
+};
+
 } // namespace backend
 } // namespace filament
 

filament/backend/src/metal/MetalState.mm

@@ -166,28 +166,40 @@ id<MTLSamplerState> SamplerStateCreator::operator()(id<MTLDevice> device,
 id<MTLArgumentEncoder> ArgumentEncoderCreator::operator()(id<MTLDevice> device,
         const ArgumentEncoderState &state) noexcept {
     const auto& textureTypes = state.textureTypes;
-    const auto& count = textureTypes.size();
-    assert_invariant(count > 0);
+    const auto& textureCount = textureTypes.size();
+    const auto& bufferCount = state.bufferCount;
+    assert_invariant(textureCount > 0);
 
     // Metal has separate data types for textures versus samplers, so the argument buffer layout
     // alternates between texture and sampler, i.e.:
+    // buffer0
+    // buffer1
     // textureA
     // samplerA
     // textureB
     // samplerB
     // etc
     NSMutableArray<MTLArgumentDescriptor*>* arguments =
-            [NSMutableArray arrayWithCapacity:(count * 2)];
-    for (size_t i = 0; i < count; i++) {
+            [NSMutableArray arrayWithCapacity:(bufferCount + textureCount * 2)];
+    size_t i = 0;
+    for (size_t j = 0; j < bufferCount; j++) {
+        MTLArgumentDescriptor* bufferArgument = [MTLArgumentDescriptor argumentDescriptor];
+        bufferArgument.index = i++;
+        bufferArgument.dataType = MTLDataTypePointer;
+        bufferArgument.access = MTLArgumentAccessReadOnly;
+        [arguments addObject:bufferArgument];
+    }
+
+    for (size_t j = 0; j < textureCount; j++) {
         MTLArgumentDescriptor* textureArgument = [MTLArgumentDescriptor argumentDescriptor];
-        textureArgument.index = i * 2 + 0;
+        textureArgument.index = i++;
         textureArgument.dataType = MTLDataTypeTexture;
         textureArgument.textureType = textureTypes[i];
         textureArgument.access = MTLArgumentAccessReadOnly;
         [arguments addObject:textureArgument];
 
         MTLArgumentDescriptor* samplerArgument = [MTLArgumentDescriptor argumentDescriptor];
-        samplerArgument.index = i * 2 + 1;
+        samplerArgument.index = i++;
         samplerArgument.dataType = MTLDataTypeSampler;
         textureArgument.access = MTLArgumentAccessReadOnly;
         [arguments addObject:samplerArgument];
@@ -196,5 +208,64 @@ id<MTLArgumentEncoder> ArgumentEncoderCreator::operator()(id<MTLDevice> device,
     return [device newArgumentEncoderWithArguments:arguments];
 }
 
+template <NSUInteger N, ShaderStage stage>
+void MetalBufferBindings<N, stage>::setBuffer(const id<MTLBuffer> buffer, NSUInteger offset, NSUInteger index) {
+    assert_invariant(offset + 1 <= N);
+
+    if (mBuffers[index] != buffer) {
+        mBuffers[index] = buffer;
+        mDirtyBuffers.set(index);
+    }
+
+    if (mOffsets[index] != offset) {
+        mOffsets[index] = offset;
+        mDirtyOffsets.set(index);
+    }
+}
+
+template <NSUInteger N, ShaderStage stage>
+void MetalBufferBindings<N, stage>::bindBuffers(
+        id<MTLCommandEncoder> encoder, NSUInteger startIndex) {
+    if (mDirtyBuffers.none() && mDirtyOffsets.none()) {
+        return;
+    }
+
+    utils::bitset8 onlyOffsetDirty = mDirtyOffsets & ~mDirtyBuffers;
+    onlyOffsetDirty.forEachSetBit([&](size_t i) {
+        if constexpr (stage == ShaderStage::VERTEX) {
+            [(id<MTLRenderCommandEncoder>)encoder setVertexBufferOffset:mOffsets[i]
+                                                                atIndex:i + startIndex];
+        } else if constexpr (stage == ShaderStage::FRAGMENT) {
+            [(id<MTLRenderCommandEncoder>)encoder setFragmentBufferOffset:mOffsets[i]
+                                                                  atIndex:i + startIndex];
+        } else if constexpr (stage == ShaderStage::COMPUTE) {
+            [(id<MTLComputeCommandEncoder>)encoder setBufferOffset:mOffsets[i]
+                                                           atIndex:i + startIndex];
+        }
+    });
+    mDirtyOffsets.reset();
+
+    mDirtyBuffers.forEachSetBit([&](size_t i) {
+        if constexpr (stage == ShaderStage::VERTEX) {
+            [(id<MTLRenderCommandEncoder>)encoder setVertexBuffer:mBuffers[i]
+                                                           offset:mOffsets[i]
+                                                          atIndex:i + startIndex];
+        } else if constexpr (stage == ShaderStage::FRAGMENT) {
+            [(id<MTLRenderCommandEncoder>)encoder setFragmentBuffer:mBuffers[i]
+                                                             offset:mOffsets[i]
+                                                            atIndex:i + startIndex];
+        } else if constexpr (stage == ShaderStage::COMPUTE) {
+            [(id<MTLComputeCommandEncoder>)encoder setBuffer:mBuffers[i]
+                                                      offset:mOffsets[i]
+                                                     atIndex:i + startIndex];
+        }
+    });
+    mDirtyBuffers.reset();
+}
+
+template class MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::VERTEX>;
+template class MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::FRAGMENT>;
+template class MetalBufferBindings<MAX_DESCRIPTOR_SET_COUNT, ShaderStage::COMPUTE>;
+
 } // namespace backend
 } // namespace filament