Update release notes

This is to allow direct rendering of 3D objects into the swapchain 
under certain circumstances (no msaa or postprocessing).

.github/workflows/android-continuous.yml

@@ -0,0 +1,17 @@
+name: Android
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-android:
+    name: build-android
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          cd build/android && ./build.sh continuous

.github/workflows/ios-continuous.yml

@@ -0,0 +1,17 @@
+name: iOS
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-ios:
+    name: build-ios
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          cd build/ios && ./build.sh continuous

.github/workflows/linux-continuous.yml

@@ -0,0 +1,17 @@
+name: Linux
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-linux:
+    name: build-linux
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          cd build/linux && ./build.sh continuous

.github/workflows/mac-continuous.yml

@@ -0,0 +1,17 @@
+name: macOS
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-mac:
+    name: build-mac
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          cd build/mac && ./build.sh continuous

.github/workflows/presubmit.yml

@@ -12,15 +12,7 @@ jobs:
         os: [macos-latest, ubuntu-latest]
 
     steps:
-      - name: Checkout Filament
-        run: |
-          git version
-          git init $GITHUB_WORKSPACE
-          cd $GITHUB_WORKSPACE
-          git remote add origin https://github.com/google/filament
-          git config gc.auto 0
-          git fetch --tags --prune --progress --no-recurse-submodules origin +${GITHUB_REF}:${GITHUB_REF/refs\//refs\/remote\/}
-          git checkout --progress --force ${GITHUB_REF/refs\//refs\/remote\/}
+      - uses: actions/checkout@v1.0.0
       - name: Run build script
         run: |
           WORKFLOW_OS=`echo \`uname\` | sed "s/Darwin/mac/" | tr [:upper:] [:lower:]`
@@ -33,34 +25,18 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - name: Checkout Filament
-        run: |
-          git version
-          git init %GITHUB_WORKSPACE%
-          cd %GITHUB_WORKSPACE%
-          git remote add origin https://github.com/google/filament
-          git config gc.auto 0
-          git fetch --tags --prune --progress --no-recurse-submodules origin +%GITHUB_REF%:%GITHUB_REF:refs/=refs/remote/%
-          git checkout --progress --force %GITHUB_REF:refs/=refs/remote/%
-        shell: cmd
+      - uses: actions/checkout@v1.0.0
       - name: Run build script
         run: |
-          build\windows\build-github.bat
+          build\windows\build-github.bat presubmit
+        shell: cmd
 
   build-android:
     name: build-android
     runs-on: macos-latest
 
     steps:
-      - name: Checkout Filament
-        run: |
-          git version
-          git init $GITHUB_WORKSPACE
-          cd $GITHUB_WORKSPACE
-          git remote add origin https://github.com/google/filament
-          git config gc.auto 0
-          git fetch --tags --prune --progress --no-recurse-submodules origin +${GITHUB_REF}:${GITHUB_REF/refs\//refs\/remote\/}
-          git checkout --progress --force ${GITHUB_REF/refs\//refs\/remote\/}
+      - uses: actions/checkout@v1.0.0
       - name: Run build script
         run: |
           cd build/android && ./build.sh ${TARGET}
@@ -72,15 +48,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-      - name: Checkout Filament
-        run: |
-          git version
-          git init $GITHUB_WORKSPACE
-          cd $GITHUB_WORKSPACE
-          git remote add origin https://github.com/google/filament
-          git config gc.auto 0
-          git fetch --tags --prune --progress --no-recurse-submodules origin +${GITHUB_REF}:${GITHUB_REF/refs\//refs\/remote\/}
-          git checkout --progress --force ${GITHUB_REF/refs\//refs\/remote\/}
+      - uses: actions/checkout@v1.0.0
       - name: Run build script
         run: |
           cd build/ios && ./build.sh ${TARGET}
@@ -92,15 +60,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-      - name: Checkout Filament
-        run: |
-          git version
-          git init $GITHUB_WORKSPACE
-          cd $GITHUB_WORKSPACE
-          git remote add origin https://github.com/google/filament
-          git config gc.auto 0
-          git fetch --tags --prune --progress --no-recurse-submodules origin +${GITHUB_REF}:${GITHUB_REF/refs\//refs\/remote\/}
-          git checkout --progress --force ${GITHUB_REF/refs\//refs\/remote\/}
+      - uses: actions/checkout@v1.0.0
       - name: Run build script
         run: |
           cd build/web && ./build.sh ${TARGET}

.github/workflows/release.yml

@@ -0,0 +1,175 @@
+name: Release
+
+# This Workflow can be triggered two ways:
+# 1. A GitHub release is created (using the GitHub web UI). This triggers all of the platforms to build and upload assets.
+# 2. A repository_dispatch API event is sent. This triggers a build for only the platform specified in the dispatch event.
+
+env:
+  RELEASE_TAG: ${{ github.event.client_payload.release_tag }}
+
+on:
+  repository_dispatch:
+  release:
+    types: [created]
+
+jobs:
+  build-desktop:
+    name: build-desktop
+    runs-on: ${{ matrix.os }}
+    if: github.event_name == 'release' || github.event.client_payload.platform == 'desktop'
+
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+      - name: Decide Git ref
+        id: git_ref
+        run: |
+          REF=${RELEASE_TAG:-${GITHUB_REF}}
+          TAG=${REF##*/}
+          echo ::set-output name=ref::${REF}
+          echo ::set-output name=tag::${TAG}
+      - uses: actions/checkout@v1.0.0
+        with:
+          ref: ${{ steps.git_ref.outputs.ref }}
+      - name: Run build script
+        run: |
+          WORKFLOW_OS=`echo \`uname\` | sed "s/Darwin/mac/" | tr [:upper:] [:lower:]`
+          cd build/$WORKFLOW_OS && ./build.sh release
+      - name: Upload release assets
+        run: |
+          pip3 install setuptools
+          pip3 install PyGithub
+          DATE=`date +%Y%m%d`
+          if [ -f out/filament-release-darwin.tgz ]; then mv out/filament-release-darwin.tgz out/filament-${DATE}-mac.tgz; fi;
+          if [ -f out/filament-release-linux.tgz ]; then mv out/filament-release-linux.tgz out/filament-${DATE}-linux.tgz; fi;
+          python3 build/common/upload-assets.py ${TAG} out/*.tgz
+        env:
+          TAG: ${{ steps.git_ref.outputs.tag }}
+          GITHUB_API_KEY: ${{ secrets.GITHUB_API_KEY }}
+
+  build-web:
+    name: build-web
+    runs-on: macos-latest
+    if: github.event_name == 'release' || github.event.client_payload.platform == 'web'
+
+    steps:
+      - name: Decide Git ref
+        id: git_ref
+        run: |
+          REF=${RELEASE_TAG:-${GITHUB_REF}}
+          TAG=${REF##*/}
+          echo ::set-output name=ref::${REF}
+          echo ::set-output name=tag::${TAG}
+      - uses: actions/checkout@v1.0.0
+        with:
+          ref: ${{ steps.git_ref.outputs.ref }}
+      - name: Run build script
+        run: |
+          cd build/web && ./build.sh release
+      - name: Upload release assets
+        run: |
+          pip3 install setuptools
+          pip3 install PyGithub
+          DATE=`date +%Y%m%d`
+          mv out/filament-release-web.tgz out/filament-${DATE}-web.tgz
+          python3 build/common/upload-assets.py ${TAG} out/*.tgz
+        env:
+          TAG: ${{ steps.git_ref.outputs.tag }}
+          GITHUB_API_KEY: ${{ secrets.GITHUB_API_KEY }}
+
+  build-android:
+    name: build-android
+    runs-on: macos-latest
+    if: github.event_name == 'release' || github.event.client_payload.platform == 'android'
+
+    steps:
+      - name: Decide Git ref
+        id: git_ref
+        run: |
+          REF=${RELEASE_TAG:-${GITHUB_REF}}
+          TAG=${REF##*/}
+          echo ::set-output name=ref::${REF}
+          echo ::set-output name=tag::${TAG}
+      - uses: actions/checkout@v1.0.0
+        with:
+          ref: ${{ steps.git_ref.outputs.ref }}
+      - name: Run build script
+        run: |
+          cd build/android && ./build.sh release
+      - name: Upload release assets
+        run: |
+          pip3 install setuptools
+          pip3 install PyGithub
+          DATE=`date +%Y%m%d`
+          mv out/filament-android-release.aar out/filament-${DATE}-android.aar
+          mv out/filamat-android-full-release.aar out/filamat-${DATE}-full-android.aar
+          mv out/filamat-android-lite-release.aar out/filamat-${DATE}-lite-android.aar
+          mv out/gltfio-android-release.aar out/gltfio-${DATE}-android.aar
+          python3 build/common/upload-assets.py ${TAG} out/*.aar
+        env:
+          TAG: ${{ steps.git_ref.outputs.tag }}
+          GITHUB_API_KEY: ${{ secrets.GITHUB_API_KEY }}
+
+  build-ios:
+    name: build-ios
+    runs-on: macos-latest
+    if: github.event_name == 'release' || github.event.client_payload.platform == 'ios'
+
+    steps:
+      - name: Decide Git ref
+        id: git_ref
+        run: |
+          REF=${RELEASE_TAG:-${GITHUB_REF}}
+          TAG=${REF##*/}
+          echo ::set-output name=ref::${REF}
+          echo ::set-output name=tag::${TAG}
+      - uses: actions/checkout@v1.0.0
+        with:
+          ref: ${{ steps.git_ref.outputs.ref }}
+      - name: Run build script
+        run: |
+          cd build/ios && ./build.sh release
+      - name: Upload release assets
+        run: |
+          pip3 install setuptools
+          pip3 install PyGithub
+          DATE=`date +%Y%m%d`
+          mv out/filament-release-ios.tgz out/filament-${DATE}-ios.tgz
+          python3 build/common/upload-assets.py ${TAG} out/*.tgz
+        env:
+          TAG: ${{ steps.git_ref.outputs.tag }}
+          GITHUB_API_KEY: ${{ secrets.GITHUB_API_KEY }}
+
+  build-windows:
+    name: build-windows
+    runs-on: windows-latest
+    if: github.event_name == 'release' || github.event.client_payload.platform == 'windows'
+
+    steps:
+      - name: Decide Git ref
+        id: git_ref
+        run: |
+          REF=${RELEASE_TAG:-${GITHUB_REF}}
+          TAG=${REF##*/}
+          echo ::set-output name=ref::${REF}
+          echo ::set-output name=tag::${TAG}
+        shell: bash
+      - uses: actions/checkout@v1.0.0
+        with:
+          ref: ${{ steps.git_ref.outputs.ref }}
+      - name: Run build script
+        run: |
+          build\windows\build-github.bat release
+        shell: cmd
+      - name: Upload release assets
+        run: |
+          pip3 install PyGithub
+          DATE=`date +%Y%m%d`
+          mv out/filament-windows.tgz out/filament-${DATE}-windows.tgz
+          python build/common/upload-assets.py ${TAG} out/*.tgz
+        shell: bash
+        env:
+          TAG: ${{ steps.git_ref.outputs.tag }}
+          GITHUB_API_KEY: ${{ secrets.GITHUB_API_KEY }}

.github/workflows/web-continuous.yml

@@ -0,0 +1,17 @@
+name: Web
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-web:
+    name: build-web
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          cd build/web && ./build.sh continuous

.github/workflows/windows-continuous.yml

@@ -0,0 +1,18 @@
+name: Windows
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-windows:
+    name: build-windows
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - name: Run build script
+        run: |
+          build\windows\build-github.bat continuous
+        shell: cmd

CMakeLists.txt

@@ -211,6 +211,13 @@ if (ANDROID OR WEBGL)
     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables -fno-rtti")
 endif()
 
+# With WebGL, we disable RTTI even for debug builds because we pass emscripten::val back and forth
+# between C++ and JavaScript in order to efficiently access typed arrays, which are unbound.
+# NOTE: This is not documented in emscripten so we should consider a different approach.
+if (WEBGL)
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti")
+endif()
+
 # ==================================================================================================
 # Debug compiler flags
 # ==================================================================================================
@@ -219,16 +226,24 @@ endif()
 #  -fsanitize=address causes a crash with assimp, which we can't explain for now
 #set(EXTRA_SANITIZE_OPTIONS "-fsanitize=undefined -fsanitize=address")
 # clang-cl.exe on Windows does not support -fstack-protector.
-if (NOT CLANG_CL AND NOT MSVC)
+if (NOT CLANG_CL AND NOT MSVC AND NOT WEBGL)
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fstack-protector")
 endif()
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${EXTRA_SANITIZE_OPTIONS}")
 
+# Disable the stack check for macOS to workaround a known issue in clang 11.0.0.
+# See: https://forums.developer.apple.com/thread/121887
+if (APPLE)
+    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}  -fno-stack-check")
+endif()
+
 # ==================================================================================================
 # Linker flags
 # ==================================================================================================
 # Strip unused sections
-set(GC_SECTIONS "-Wl,--gc-sections")
+if (NOT WEBGL)
+    set(GC_SECTIONS "-Wl,--gc-sections")
+endif()
 set(B_SYMBOLIC_FUNCTIONS "-Wl,-Bsymbolic-functions")
 
 if (APPLE)

README.md

@@ -1,11 +1,11 @@
 # Filament
 
-<img alt="Android" src="build/img/android.png" width="20px" height="20px" hspace="2px"/>[![Android Build Status](https://filament-build.storage.googleapis.com/badges/build_status_android.svg)](https://filament-build.storage.googleapis.com/badges/build_link_android.html)
-<img alt="iOS" src="build/img/macos.png" width="20px" height="20px" hspace="2px"/>[![iOS Build Status](https://filament-build.storage.googleapis.com/badges/build_status_ios.svg)](https://filament-build.storage.googleapis.com/badges/build_link_ios.html)
-<img alt="Linux" src="build/img/linux.png" width="20px" height="20px" hspace="2px"/>[![Linux Build Status](https://filament-build.storage.googleapis.com/badges/build_status_linux.svg)](https://filament-build.storage.googleapis.com/badges/build_link_linux.html)
-<img alt="macOS" src="build/img/macos.png" width="20px" height="20px" hspace="2px"/>[![MacOS Build Status](https://filament-build.storage.googleapis.com/badges/build_status_mac.svg)](https://filament-build.storage.googleapis.com/badges/build_link_mac.html)
-<img alt="Windows" src="build/img/windows.png" width="20px" height="20px" hspace="2px"/>[![Windows Build Status](https://filament-build.storage.googleapis.com/badges/build_status_windows.svg)](https://filament-build.storage.googleapis.com/badges/build_link_windows.html)
-<img alt="Web" src="build/img/web.png" width="20px" height="20px" hspace="2px"/>[![Web Build Status](https://filament-build.storage.googleapis.com/badges/build_status_web.svg)](https://filament-build.storage.googleapis.com/badges/build_link_web.html)
+![Android Build Status](https://github.com/google/filament/workflows/Android/badge.svg)
+![iOS Build Status](https://github.com/google/filament/workflows/iOS/badge.svg)
+![Linux Build Status](https://github.com/google/filament/workflows/Linux/badge.svg)
+![macOS Build Status](https://github.com/google/filament/workflows/macOS/badge.svg)
+![Windows Build Status](https://github.com/google/filament/workflows/Windows/badge.svg)
+![Web Build Status](https://github.com/google/filament/workflows/Web/badge.svg)
 
 Filament is a real-time physically based rendering engine for Android, iOS, Linux, macOS, Windows,
 and WebGL. It is designed to be as small as possible and as efficient as possible on Android.
@@ -21,9 +21,6 @@ Android devices and as the renderer inside the Android Studio plugin.
 Make sure you always use tools from the same release as the runtime library. This is particularly
 important for `matc` (material compiler).
 
-If you prefer to live on the edge, you can download a continuous build by clicking one of the build
-badges above.
-
 ## Documentation
 
 - [Filament](https://google.github.io/filament/Filament.html), an in-depth explanation of
@@ -717,13 +714,11 @@ same version that our continuous builds use.
 
 ```
 cd <your chosen parent folder for the emscripten SDK>
-curl -L https://github.com/emscripten-core/emsdk/archive/a77638d.zip > emsdk.zip
-unzip emsdk.zip
-mv emsdk-* emsdk
-cd emsdk
-./emsdk update
-./emsdk install sdk-1.38.28-64bit
-./emsdk activate sdk-1.38.28-64bit
+curl -L https://github.com/emscripten-core/emsdk/archive/1b1f08f.zip > emsdk.zip
+unzip emsdk.zip ; mv emsdk-* emsdk ; cd emsdk
+./emsdk install lastest
+./emsdk activate lastest
+source ./emsdk_env.sh
 ```
 
 After this you can invoke the [easy build](#easy-build) script as follows:

RELEASE_NOTES.md

@@ -3,6 +3,21 @@
 This file contains one line summaries of commits that are worthy of mentioning in release notes.
 A new header is inserted each time a *tag* is created.
 
+## Next release
+
+- Fixed an assertion when a parameter array occurs last in a material definition.
+- Fixed morph shapes not rendering in WebGL.
+- Added support for the latest version of emscripten.
+- gltfio: fixed blackness seen with default material.
+- Added ETC2 and BC compressed texture support to Metal backend.
+- Rendering a `SAMPLER_EXTERNAL` texture before setting an external image no longer results in GPU errors.
+- Fixed a normals issue when skinning without a normal map or anisotropy.
+- Fixed an issue where transparent views couldn't be used with post-processing.
+- Always use higher quality 3-bands SH for indirect lighting, even on mobile.
+- The Metal backend can now handle binding individual planes of YUV external images.
+- Added support for depth buffer when post-processing is turned off
+- Improved performance on GPUs that use tile-based rendering
+
 ## v1.4.2
 
 - Cleaned up the validation strategy in Engine (checks for use-after-destroy etc).

android/common/CallbackUtils.cpp

@@ -27,39 +27,6 @@ struct {
     jmethodID execute;
 } gCallbackUtils;
 
-JniCallback* JniCallback::make(filament::Engine* engine,
-        JNIEnv* env, jobject handler, jobject callback) {
-    void* that = engine->streamAlloc(sizeof(JniCallback), alignof(JniCallback));
-    return new (that) JniCallback(env, handler, callback);
-}
-
-JniCallback::JniCallback(JNIEnv* env, jobject handler, jobject callback)
-        : mEnv(env)
-        , mHandler(env->NewGlobalRef(handler))
-        , mCallback(env->NewGlobalRef(callback)) {
-}
-
-JniCallback::~JniCallback() {
-    if (mHandler && mCallback) {
-#ifdef ANDROID
-        if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) {
-            mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback);
-        }
-#endif
-        if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) {
-            mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback);
-        }
-    }
-    mEnv->DeleteGlobalRef(mHandler);
-    mEnv->DeleteGlobalRef(mCallback);
-}
-
-void JniCallback::invoke(void*, size_t, void* user) {
-    JniCallback* data = reinterpret_cast<JniCallback*>(user);
-    // don't call delete here, because we don't own the storage
-    data->~JniCallback();
-}
-
 JniBufferCallback* JniBufferCallback::make(filament::Engine* engine,
         JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer) {
     void* that = engine->streamAlloc(sizeof(JniBufferCallback), alignof(JniBufferCallback));
@@ -95,6 +62,37 @@ void JniBufferCallback::invoke(void*, size_t, void* user) {
     data->~JniBufferCallback();
 }
 
+JniImageCallback* JniImageCallback::make(filament::Engine* engine,
+        JNIEnv* env, jobject handler, jobject callback, long image) {
+    void* that = engine->streamAlloc(sizeof(JniImageCallback), alignof(JniImageCallback));
+    return new (that) JniImageCallback(env, handler, callback, image);
+}
+
+JniImageCallback::JniImageCallback(JNIEnv* env, jobject handler, jobject callback, long image)
+        : mEnv(env)
+        , mHandler(env->NewGlobalRef(handler))
+        , mCallback(env->NewGlobalRef(callback))
+        , mImage(image) { }
+
+JniImageCallback::~JniImageCallback() {
+    if (mHandler && mCallback) {
+#ifdef ANDROID
+        if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) {
+            mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback);
+        }
+#endif
+        if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) {
+            mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback);
+        }
+    }
+    mEnv->DeleteGlobalRef(mHandler);
+    mEnv->DeleteGlobalRef(mCallback);
+}
+
+void JniImageCallback::invoke(void* image, void* user) {
+    reinterpret_cast<JniImageCallback*>(user)->~JniImageCallback();
+}
+
 void registerCallbackUtils(JNIEnv *env) {
 #ifdef ANDROID
     gCallbackUtils.handlerClass = env->FindClass("android/os/Handler");

android/common/CallbackUtils.h

@@ -23,21 +23,6 @@
 
 #include <filament/Engine.h>
 
-struct JniCallback {
-    static JniCallback* make(filament::Engine* engine,
-            JNIEnv* env, jobject handler, jobject callback);
-
-    static void invoke(void* buffer, size_t n, void* user);
-
-private:
-    JniCallback(JNIEnv* env, jobject handler, jobject callback);
-    ~JniCallback();
-
-    JNIEnv* mEnv;
-    jobject mHandler;
-    jobject mCallback;
-};
-
 struct JniBufferCallback {
     static JniBufferCallback* make(filament::Engine* engine,
             JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer);
@@ -53,3 +38,19 @@ private:
     jobject mCallback;
     AutoBuffer mBuffer;
 };
+
+struct JniImageCallback {
+    static JniImageCallback* make(filament::Engine* engine, JNIEnv* env, jobject handler,
+            jobject runnable, long image);
+
+    static void invoke(void* image, void* user);
+
+private:
+    JniImageCallback(JNIEnv* env, jobject handler, jobject runnable, long image);
+    ~JniImageCallback();
+
+    JNIEnv* mEnv;
+    jobject mHandler;
+    jobject mCallback;
+    long mImage;
+};

android/filament-android/src/main/cpp/Engine.cpp

@@ -59,6 +59,13 @@ Java_com_google_android_filament_Engine_nCreateSwapChain(JNIEnv* env,
     return (jlong) engine->createSwapChain(win, (uint64_t) flags);
 }
 
+extern "C" JNIEXPORT jlong JNICALL
+Java_com_google_android_filament_Engine_nCreateSwapChainHeadless(JNIEnv* env,
+        jclass klass, jlong nativeEngine, jint width, jint height, jlong flags) {
+    Engine* engine = (Engine*) nativeEngine;
+    return (jlong) engine->createSwapChain(width, height, (uint64_t) flags);
+}
+
 extern "C" JNIEXPORT jlong JNICALL
 Java_com_google_android_filament_Engine_nCreateSwapChainFromRawPointer(JNIEnv*,
         jclass, jlong nativeEngine, jlong pointer, jlong flags) {

android/filament-android/src/main/cpp/Renderer.cpp

@@ -97,6 +97,42 @@ Java_com_google_android_filament_Renderer_nReadPixels(JNIEnv *env, jclass,
     return 0;
 }
 
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_android_filament_Renderer_nReadPixelsEx(JNIEnv *env, jclass,
+        jlong nativeRenderer, jlong nativeEngine, jlong nativeRenderTarget,
+        jint xoffset, jint yoffset, jint width, jint height,
+        jobject storage, jint remaining,
+        jint left, jint top, jint type, jint alignment, jint stride, jint format,
+        jobject handler, jobject runnable) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    Engine *engine = (Engine *) nativeEngine;
+    RenderTarget *renderTarget = (RenderTarget *) nativeRenderTarget;
+
+    stride = stride ? stride : width;
+    size_t sizeInBytes = PixelBufferDescriptor::computeDataSize(
+            (PixelDataFormat) format, (PixelDataType) type,
+            (size_t) stride, (size_t) (height + top), (size_t) alignment);
+
+    AutoBuffer nioBuffer(env, storage, 0);
+    if (sizeInBytes > (remaining << nioBuffer.getShift())) {
+        // BufferOverflowException
+        return -1;
+    }
+
+    void *buffer = nioBuffer.getData();
+    auto *callback = JniBufferCallback::make(engine, env, handler, runnable, std::move(nioBuffer));
+
+    PixelBufferDescriptor desc(buffer, sizeInBytes, (backend::PixelDataFormat) format,
+            (backend::PixelDataType) type, (uint8_t) alignment, (uint32_t) left, (uint32_t) top,
+            (uint32_t) stride, &JniBufferCallback::invoke, callback);
+
+    renderer->readPixels(renderTarget,
+            uint32_t(xoffset), uint32_t(yoffset), uint32_t(width), uint32_t(height),
+            std::move(desc));
+
+    return 0;
+}
+
 extern "C" JNIEXPORT jdouble JNICALL
 Java_com_google_android_filament_Renderer_nGetUserTime(JNIEnv*, jclass, jlong nativeRenderer) {
     Renderer *renderer = (Renderer *) nativeRenderer;

android/filament-android/src/main/cpp/Stream.cpp

@@ -24,6 +24,25 @@
 #include "common/NioUtils.h"
 #include "common/CallbackUtils.h"
 
+#ifdef ANDROID
+
+#if __has_include(<android/hardware_buffer_jni.h>)
+#include <android/hardware_buffer_jni.h>
+#else
+struct AHardwareBuffer;
+typedef struct AHardwareBuffer AHardwareBuffer;
+#endif
+
+#include <android/log.h>
+
+#include <dlfcn.h>
+
+using PFN_FROMHARDWAREBUFFER = AHardwareBuffer* (*)(JNIEnv*, jobject);
+static PFN_FROMHARDWAREBUFFER AHardwareBuffer_fromHardwareBuffer_fn = nullptr;
+static bool sHardwareBufferSupported = true;
+
+#endif
+
 using namespace filament;
 using namespace backend;
 
@@ -108,10 +127,10 @@ Java_com_google_android_filament_Stream_nBuilderBuild(JNIEnv*, jclass,
     return (jlong) builder->builder()->build(*engine);
 }
 
-extern "C" JNIEXPORT jboolean JNICALL
-Java_com_google_android_filament_Stream_nIsNative(JNIEnv*, jclass, jlong nativeStream) {
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_android_filament_Stream_nGetStreamType(JNIEnv*, jclass, jlong nativeStream) {
     Stream* stream = (Stream*) nativeStream;
-    return (jboolean) stream->isNativeStream();
+    return (jint) stream->getStreamType();
 }
 
 extern "C" JNIEXPORT void JNICALL
@@ -160,3 +179,44 @@ Java_com_google_android_filament_Stream_nGetTimestamp(JNIEnv*, jclass, jlong nat
     Stream *stream = (Stream *) nativeStream;
     return stream->getTimestamp();
 }
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_Stream_nSetAcquiredImage(JNIEnv* env, jclass, jlong nativeStream,
+        jlong nativeEngine, jobject hwbuffer, jobject handler, jobject runnable) {
+    Engine* engine = (Engine*) nativeEngine;
+    Stream* stream = (Stream*) nativeStream;
+
+#ifdef ANDROID
+
+    // This function is not available before NDK 15 or before Android 8.
+    if (UTILS_UNLIKELY(!AHardwareBuffer_fromHardwareBuffer_fn)) {
+        if (!sHardwareBufferSupported) {
+            return;
+        }
+        AHardwareBuffer_fromHardwareBuffer_fn = (PFN_FROMHARDWAREBUFFER) dlsym(RTLD_DEFAULT, "AHardwareBuffer_fromHardwareBuffer");
+         if (!AHardwareBuffer_fromHardwareBuffer_fn) {
+            __android_log_print(ANDROID_LOG_WARN, "Filament", "AHardwareBuffer_fromHardwareBuffer is not available.");
+            sHardwareBufferSupported = false;
+        }
+        return;
+    }
+
+    AHardwareBuffer* nativeBuffer = AHardwareBuffer_fromHardwareBuffer_fn(env, hwbuffer);
+    if (!nativeBuffer) {
+        __android_log_print(ANDROID_LOG_INFO, "Filament", "Unable to obtain native HardwareBuffer.");
+        return;
+    }
+
+    auto* callback = JniImageCallback::make(engine, env, handler, runnable, (long) nativeBuffer);
+
+#else
+
+    // TODO: for non-Android platforms, it is unclear how to go from "jobject" to "void*"
+    // For now this code is reserved for future use.
+    auto* callback = JniImageCallback::make(engine, env, handler, runnable, 0);
+    void* nativeBuffer = nullptr;
+
+#endif
+
+    stream->setAcquiredImage((void*) nativeBuffer, &JniImageCallback::invoke, callback);
+}

android/filament-android/src/main/cpp/View.cpp

@@ -241,8 +241,14 @@ Java_com_google_android_filament_View_nGetAmbientOcclusion(JNIEnv*, jclass, jlon
 
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_View_nSetAmbientOcclusionOptions(JNIEnv*, jclass,
-    jlong nativeView, jfloat radius, jfloat bias, jfloat power, jfloat resolution) {
+    jlong nativeView, jfloat radius, jfloat bias, jfloat power, jfloat resolution, jfloat intensity) {
     View* view = (View*) nativeView;
-    View::AmbientOcclusionOptions options = { .radius = radius, .bias = bias, .power = power, .resolution = resolution};
+    View::AmbientOcclusionOptions options = {
+            .radius = radius,
+            .bias = bias,
+            .power = power,
+            .resolution = resolution,
+            .intensity = intensity
+    };
     view->setAmbientOcclusionOptions(options);
 }

android/filament-android/src/main/java/com/google/android/filament/Engine.java

@@ -290,6 +290,32 @@ public class Engine {
         throw new IllegalArgumentException("Invalid surface " + surface);
     }
 
+    /**
+     * Creates a headless {@link SwapChain}
+     *
+     * @param width  width of the rendering buffer
+     * @param height height of the rendering buffer
+     * @param flags  configuration flags, see {@link SwapChain}
+     *
+     * @return a newly created {@link SwapChain} object
+     *
+     * @exception IllegalStateException can be thrown if the SwapChain couldn't be created
+     *
+     * @see SwapChain#CONFIG_DEFAULT
+     * @see SwapChain#CONFIG_TRANSPARENT
+     * @see SwapChain#CONFIG_READABLE
+     *
+     */
+    @NonNull
+    public SwapChain createSwapChain(int width, int height, long flags) {
+        if (width >= 0 && height >= 0) {
+            long nativeSwapChain = nCreateSwapChainHeadless(getNativeObject(), width, height, flags);
+            if (nativeSwapChain == 0) throw new IllegalStateException("Couldn't create SwapChain");
+            return new SwapChain(nativeSwapChain, null);
+        }
+        throw new IllegalArgumentException("Invalid parameters");
+    }
+
     /**
      * Creates a {@link SwapChain} from a {@link NativeSurface}.
      *
@@ -608,6 +634,7 @@ public class Engine {
     private static native void nDestroyEngine(long nativeEngine);
     private static native long nGetBackend(long nativeEngine);
     private static native long nCreateSwapChain(long nativeEngine, Object nativeWindow, long flags);
+    private static native long nCreateSwapChainHeadless(long nativeEngine, int width, int height, long flags);
     private static native long nCreateSwapChainFromRawPointer(long nativeEngine, long pointer, long flags);
     private static native void nDestroySwapChain(long nativeEngine, long nativeSwapChain);
     private static native long nCreateView(long nativeEngine);

android/filament-android/src/main/java/com/google/android/filament/Renderer.java

@@ -299,6 +299,94 @@ public class Renderer {
         }
     }
 
+    /**
+     * Reads back the content of a specified {@link RenderTarget}.
+     *
+     *<pre>
+     *
+     *  Framebuffer as seen on         User buffer (PixelBufferDescriptor)
+     *  screen
+     *  +--------------------+
+     *  |                    |                .stride         .alignment
+     *  |                    |         ----------------------->-->
+     *  |                    |         O----------------------+--+   low addresses
+     *  |                    |         |          |           |  |
+     *  |             w      |         |          | .top      |  |
+     *  |       <--------->  |         |          V           |  |
+     *  |       +---------+  |         |     +---------+      |  |
+     *  |       |     ^   |  | ======> |     |         |      |  |
+     *  |   x   |    h|   |  |         |.left|         |      |  |
+     *  +------>|     v   |  |         +---->|         |      |  |
+     *  |       +.........+  |         |     +.........+      |  |
+     *  |            ^       |         |                      |  |
+     *  |          y |       |         +----------------------+--+  high addresses
+     *  O------------+-------+
+     *
+     *</pre>
+     *
+     *
+     * <p>Typically <code>readPixels</code> will be called after {@link #render} and before
+     * {@link #endFrame}.</p>
+     * <br>
+     * <p>After calling this method, the callback associated with <code>buffer</code>
+     * will be invoked on the main thread, indicating that the read-back has completed.
+     * Typically, this will happen after multiple calls to {@link #beginFrame},
+     * {@link #render}, {@link #endFrame}.</p>
+     * <br>
+     * <p><code>readPixels</code> is intended for debugging and testing.
+     * It will impact performance significantly.</p>
+     *
+     * @param renderTarget  {@link RenderTarget} to read back from
+     * @param xoffset       left offset of the sub-region to read back
+     * @param yoffset       bottom offset of the sub-region to read back
+     * @param width         width of the sub-region to read back
+     * @param height        height of the sub-region to read back
+     * @param buffer        client-side buffer where the read-back will be written
+     *
+     *                  <p>
+     *                  The following format are always supported:
+     *                      <li>{@link Texture.Format#RGBA}</li>
+     *                      <li>{@link Texture.Format#RGBA_INTEGER}</li>
+     *                  </p>
+     *
+     *                  <p>
+     *                  The following types are always supported:
+     *                      <li>{@link Texture.Type#UBYTE}</li>
+     *                      <li>{@link Texture.Type#UINT}</li>
+     *                      <li>{@link Texture.Type#INT}</li>
+     *                      <li>{@link Texture.Type#FLOAT}</li>
+     *                  </p>
+     *
+     *                  <p>Other combination of format/type may be supported. If a combination is
+     *                  not supported, this operation may fail silently. Use a DEBUG build
+     *                  to get some logs about the failure.</p>
+     *
+     * @exception BufferOverflowException if the specified parameters would result in reading
+     * outside of <code>buffer</code>.
+     */
+    public void readPixels(
+            @NonNull RenderTarget renderTarget,
+            @IntRange(from = 0) int xoffset, @IntRange(from = 0) int yoffset,
+            @IntRange(from = 0) int width, @IntRange(from = 0) int height,
+            @NonNull Texture.PixelBufferDescriptor buffer) {
+
+        if (buffer.storage.isReadOnly()) {
+            throw new ReadOnlyBufferException();
+        }
+
+        int result = nReadPixelsEx(getNativeObject(), mEngine.getNativeObject(),
+                renderTarget.getNativeObject(),
+                xoffset, yoffset, width, height,
+                buffer.storage, buffer.storage.remaining(),
+                buffer.left, buffer.top, buffer.type.ordinal(), buffer.alignment,
+                buffer.stride, buffer.format.ordinal(),
+                buffer.handler, buffer.callback);
+
+        if (result < 0) {
+            throw new BufferOverflowException();
+        }
+    }
+
     /**
      * Returns a timestamp (in seconds) for the last call to {@link #beginFrame}. This value is
      * constant for all {@link View views} rendered during a frame. The epoch is set with
@@ -386,6 +474,12 @@ public class Renderer {
             Buffer storage, int remaining,
             int left, int top, int type, int alignment, int stride, int format,
             Object handler, Runnable callback);
+    private static native int nReadPixelsEx(long nativeRenderer, long nativeEngine,
+            long nativeRenderTarget,
+            int xoffset, int yoffset, int width, int height,
+            Buffer storage, int remaining,
+            int left, int top, int type, int alignment, int stride, int format,
+            Object handler, Runnable callback);
     private static native double nGetUserTime(long nativeRenderer);
     private static native void nResetUserTime(long nativeRenderer);
 }

android/filament-android/src/main/java/com/google/android/filament/Stream.java

@@ -33,6 +33,20 @@ public class Stream {
     private long mNativeObject;
     private long mNativeEngine;
 
+    /**
+     * Represents the immutable stream type.
+     */
+    public enum StreamType {
+        /** Not synchronized but copy-free. Good for video. */
+        NATIVE,
+
+        /** Synchronized, but GL-only and incurs copies. Good for AR on devices before API 26. */
+        TEXTURE_ID,
+
+        /** Synchronized, copy-free, and take a release callback. Good for AR but requires API 26+. */
+        ACQUIRED,
+    };
+
     Stream(long nativeStream, Engine engine) {
         mNativeObject = nativeStream;
         mNativeEngine = engine.getNativeObject();
@@ -40,6 +54,12 @@ public class Stream {
 
     /**
      * Use <code>Builder</code> to construct an Stream object instance.
+     *
+     * By default, Stream objects are {@link StreamType#ACQUIRED ACQUIRED} and must have external images pushed to them via
+     * {@line #setAcquiredImage}.
+     *
+     * To create a {@link StreamType#NATIVE NATIVE} or {@link StreamType#TEXTURE_ID TEXTURE_ID} stream, call one of the <pre>stream</pre> methods
+     * on the builder.
      */
     public static class Builder {
         @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) // Keep to finalize native resources
@@ -55,8 +75,8 @@ public class Stream {
         }
 
         /**
-         * Creates a native stream. Native streams can sample data directly from an
-         * opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture}
+         * Creates a {@link StreamType#NATIVE NATIVE} stream. Native streams can sample data
+         * directly from an opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture}
          * on Android.
          *
          * @param streamSource an opaque native stream handle, e.g.: on Android this must be a
@@ -74,7 +94,7 @@ public class Stream {
         }
 
         /**
-         * Creates a copy stream. A copy stream will sample data from the supplied
+         * Creates a {@link StreamType#TEXTURE_ID TEXTURE_ID} stream. A copy stream will sample data from the supplied
          * external texture and copy it into an internal private texture.
          *
          * <p>Currently only OpenGL external texture ids are supported.</p>
@@ -150,12 +170,24 @@ public class Stream {
     }
 
     /**
-     * Indicates whether this <code>Stream</code> is a native stream or a copy stream.
-     *
-     * @return true if this is a native  <code>Stream</code>, false otherwise.
+     * Indicates whether this <code>Stream</code> is NATIVE, TEXTURE_ID, or ACQUIRED.
      */
-    public boolean isNative() {
-        return nIsNative(getNativeObject());
+    public StreamType getStreamType() {
+        return StreamType.values()[nGetStreamType(getNativeObject())];
+    }
+
+    /**
+     * Updates an <pre>ACQUIRED</pre> stream with an image that is guaranteed to be used in the next frame.
+     *
+     * This method should be called on the same thread that calls {#link Renderer#beginFrame}, which is
+     * also where the callback is invoked.
+     *
+     * @param hwbuffer {@link android.hardware.HardwareBuffer HardwareBuffer} (requires API level 26)
+     * @param handler {@link java.util.concurrent.Executor Executor} or {@link android.os.Handler Handler}.
+     * @param callback a callback invoked by <code>handler</code> when the <code>hwbuffer</code> can be released.
+     */
+    public void setAcquiredImage(Object hwbuffer, Object handler, Runnable callback) {
+        nSetAcquiredImage(getNativeObject(), mNativeEngine, hwbuffer, handler, callback);
     }
 
     /**
@@ -286,6 +318,7 @@ public class Stream {
     private static native void nBuilderHeight(long nativeStreamBuilder, int height);
     private static native long nBuilderBuild(long nativeStreamBuilder, long nativeEngine);
 
+    private static native int nGetStreamType(long nativeStream);
     private static native void nSetDimensions(long nativeStream, int width, int height);
     private static native int nReadPixels(long nativeStream, long nativeEngine,
             int xoffset, int yoffset, int width, int height,
@@ -293,6 +326,6 @@ public class Stream {
             int left, int top, int type, int alignment, int stride, int format,
             Object handler, Runnable callback);
     private static native long nGetTimestamp(long nativeStream);
-
-    private static native boolean nIsNative(long nativeStream);
+    private static native void nSetAcquiredImage(long nativeStream, long nativeEngine,
+            Object hwbuffer, Object handler, Runnable callback);
 }

android/filament-android/src/main/java/com/google/android/filament/SwapChain.java

@@ -85,15 +85,15 @@ public class SwapChain {
      */
     public static final long CONFIG_READABLE = 0x2;
 
-    SwapChain(long nativeSwapChain, @NonNull Object surface) {
+    SwapChain(long nativeSwapChain, Object surface) {
         mNativeObject = nativeSwapChain;
         mSurface = surface;
     }
 
     /**
-     * @return the native <code>Object</code> this <code>SwapChain</code> was created from.
+     * @return the native <code>Object</code> this <code>SwapChain</code> was created from or null
+     *         for a headless SwapChain.
      */
-    @NonNull
     public Object getNativeWindow() {
         return mSurface;
     }

android/filament-android/src/main/java/com/google/android/filament/Texture.java

@@ -322,13 +322,26 @@ public class Texture {
         public CompressedFormat compressedFormat;
 
         @Nullable public Object handler;
-        @Nullable public Runnable callback;
 
         /**
-         * Valid handler types:
-         * - Android: Handler, Executor
-         * - Other: Executor
+         * Callback used to destroy the buffer data.
+         * <p>
+         * Guarantees:
+         * <ul>
+         *     <li>Called on the main filament thread.</li>
+         * </ul>
+         * </p>
+         *
+         * <p>
+         * Limitations:
+         * <ul>
+         *     <li>Must be lightweight.</li>
+         *     <li>Must not call filament APIs.</li>
+         * </ul>
+         * </p>
          */
+        @Nullable public Runnable callback;
+
 
         /**
          * Creates a <code>PixelBufferDescriptor</code>

android/filament-android/src/main/java/com/google/android/filament/View.java

@@ -149,6 +149,11 @@ public class View {
          * How each dimension of the AO buffer is scaled. Must be positive and <= 1.
          */
         public float resolution = 0.5f;
+
+        /**
+         * Strength of the Ambient Occlusion effect. Must be positive.
+         */
+        public float intensity = 1.0f;
     }
 
     /**
@@ -761,7 +766,8 @@ public class View {
      */
     public void setAmbientOcclusionOptions(@NonNull AmbientOcclusionOptions options) {
         mAmbientOcclusionOptions = options;
-        nSetAmbientOcclusionOptions(getNativeObject(), options.radius, options.bias, options.power, options.resolution);
+        nSetAmbientOcclusionOptions(getNativeObject(), options.radius, options.bias, options.power,
+                options.resolution, options.intensity);
     }
 
     /**
@@ -819,5 +825,5 @@ public class View {
     private static native boolean nIsFrontFaceWindingInverted(long nativeView);
     private static native void nSetAmbientOcclusion(long nativeView, int ordinal);
     private static native int nGetAmbientOcclusion(long nativeView);
-    private static native void nSetAmbientOcclusionOptions(long nativeView, float radius, float bias, float power, float resolution);
+    private static native void nSetAmbientOcclusionOptions(long nativeView, float radius, float bias, float power, float resolution, float intensity);
 }

android/gltfio-android/src/main/cpp/Animator.cpp

@@ -27,7 +27,7 @@ extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_gltfio_Animator_nApplyAnimation(JNIEnv*, jclass, jlong nativeAnimator,
         jint index, jfloat time) {
     Animator* animator = (Animator*) nativeAnimator;
-    animator->applyAnimation(index, time);
+    animator->applyAnimation(static_cast<size_t>(index), time);
 }
 
 extern "C" JNIEXPORT void JNICALL
@@ -46,14 +46,14 @@ extern "C" JNIEXPORT float JNICALL
 Java_com_google_android_filament_gltfio_Animator_nGetAnimationDuration(JNIEnv*, jclass,
         jlong nativeAnimator, jint index) {
     Animator* animator = (Animator*) nativeAnimator;
-    return animator->getAnimationDuration(index);
+    return animator->getAnimationDuration(static_cast<size_t>(index));
 }
 
 extern "C" JNIEXPORT jstring JNICALL
 Java_com_google_android_filament_gltfio_Animator_nGetAnimationName(JNIEnv* env, jclass,
         jlong nativeAnimator, jint index) {
     Animator* animator = (Animator*) nativeAnimator;
-    const char* val = animator->getAnimationName(index);
+    const char* val = animator->getAnimationName(static_cast<size_t>(index));
     return val ? env->NewStringUTF(val) : nullptr;
 
 }

android/gltfio-android/src/main/cpp/AssetLoader.cpp

@@ -32,7 +32,7 @@ using namespace utils;
 extern void registerCallbackUtils(JNIEnv*);
 extern void registerNioUtils(JNIEnv*);
 
-jint JNI_OnLoad(JavaVM* vm, void* reserved) {
+jint JNI_OnLoad(JavaVM* vm, void*) {
     JNIEnv* env;
     if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
         return -1;
@@ -69,6 +69,15 @@ Java_com_google_android_filament_gltfio_AssetLoader_nCreateAssetFromBinary(JNIEn
             buffer.getSize());
 }
 
+extern "C" JNIEXPORT jlong JNICALL
+Java_com_google_android_filament_gltfio_AssetLoader_nCreateAssetFromJson(JNIEnv* env, jclass,
+        jlong nativeLoader, jobject javaBuffer, jint remaining) {
+    AssetLoader* loader = (AssetLoader*) nativeLoader;
+    AutoBuffer buffer(env, javaBuffer, remaining);
+    return (jlong) loader->createAssetFromJson((const uint8_t *) buffer.getData(),
+            buffer.getSize());
+}
+
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_gltfio_AssetLoader_nEnableDiagnostics(JNIEnv*, jclass,
         jlong nativeLoader, jboolean enable) {

android/gltfio-android/src/main/cpp/FilamentAsset.cpp

@@ -49,7 +49,7 @@ extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_gltfio_FilamentAsset_nGetBoundingBox(JNIEnv* env, jclass,
         jlong nativeAsset, jfloatArray result) {
     FilamentAsset* asset = (FilamentAsset*) nativeAsset;
-    float* values = (float*) env->GetFloatArrayElements(result, nullptr);
+    float* values = env->GetFloatArrayElements(result, nullptr);
     const filament::Aabb box = asset->getBoundingBox();
     const float3 center = box.center();
     const float3 extent = box.extent();
@@ -59,13 +59,13 @@ Java_com_google_android_filament_gltfio_FilamentAsset_nGetBoundingBox(JNIEnv* en
     values[3] = extent.x;
     values[4] = extent.y;
     values[5] = extent.z;
-    env->ReleaseFloatArrayElements(result, (jfloat*) values, 0);
+    env->ReleaseFloatArrayElements(result, values, 0);
 }
 
 extern "C" JNIEXPORT jstring JNICALL
 Java_com_google_android_filament_gltfio_FilamentAsset_nGetName(JNIEnv* env, jclass,
         jlong nativeAsset, jint entityId) {
-    uint32_t id = entityId;
+    uint32_t id = static_cast<uint32_t>(entityId);
     Entity* entity = (Entity*) &id;
     FilamentAsset* asset = (FilamentAsset*) nativeAsset;
     const char* val = asset->getName(*entity);
@@ -73,8 +73,26 @@ Java_com_google_android_filament_gltfio_FilamentAsset_nGetName(JNIEnv* env, jcla
 }
 
 extern "C" JNIEXPORT jlong JNICALL
-Java_com_google_android_filament_gltfio_FilamentAsset_nGetAnimator(JNIEnv* env, jclass,
+Java_com_google_android_filament_gltfio_FilamentAsset_nGetAnimator(JNIEnv* , jclass,
         jlong nativeAsset) {
     FilamentAsset* asset = (FilamentAsset*) nativeAsset;
     return (jlong) asset->getAnimator();
 }
+
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_android_filament_gltfio_FilamentAsset_nGetResourceUriCount(JNIEnv*, jclass,
+                                                                           jlong nativeAsset) {
+    FilamentAsset* asset = (FilamentAsset*) nativeAsset;
+    return (jint) asset->getResourceUriCount();
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_gltfio_FilamentAsset_nGetResourceUris(JNIEnv* env, jclass,
+                                                                       jlong nativeAsset,
+                                                                       jobjectArray result) {
+    FilamentAsset* asset = (FilamentAsset*) nativeAsset;
+    auto resourceUris = asset->getResourceUris();
+    for (int i = 0; i < asset->getResourceUriCount(); ++i) {
+        env->SetObjectArrayElement(result, (jsize) i, env->NewStringUTF(resourceUris[i]));
+    }
+}

android/gltfio-android/src/main/cpp/ResourceLoader.cpp

@@ -28,7 +28,7 @@ using namespace filament;
 using namespace gltfio;
 using namespace utils;
 
-static void destroy(void* data, size_t size, void *userData) {
+static void destroy(void*, size_t, void *userData) {
     AutoBuffer* buffer = (AutoBuffer*) userData;
     delete buffer;
 }

android/gltfio-android/src/main/java/com/google/android/filament/gltfio/AssetLoader.java

@@ -37,7 +37,6 @@ import java.nio.Buffer;
  *
  * companion object {
  *     init {
- *        Filament.init()
  *        AssetLoader.init()
  *    }
  * }
@@ -54,11 +53,21 @@ import java.nio.Buffer;
  *         assetLoader.createAssetFromBinary(ByteBuffer.wrap(bytes))!!
  *     }
  *
- *     ResourceLoader(engine).loadResources(filamentAsset).destroy()
+ *     val resourceLoader = ResourceLoader(engine)
+ *     resourceLoader.loadResources(filamentAsset)
+ *     for (uri in filamentAsset.resourceUris) {
+ *         val buffer = loadResource(uri)
+ *         resourceLoader.addResourceData(uri, buffer)
+ *     }
+ *     resourceLoader.destroy()
  *     animator = asset.getAnimator()
  *
  *     scene.addEntities(filamentAsset.entities)
  * }
+ *
+ * private fun loadResource(uri: String): Buffer {
+ *     TODO("Load your asset here (e.g. using Android's AssetManager API)")
+ * }
  * </pre>
  *
  * @see Animator
@@ -113,6 +122,15 @@ public class AssetLoader {
         return new FilamentAsset(nativeAsset);
     }
 
+    /**
+     * Creates a {@link FilamentAsset} from the contents of a GLTF file.
+     */
+    @Nullable
+    public FilamentAsset createAssetFromJson(@NonNull Buffer buffer) {
+        long nativeAsset = nCreateAssetFromJson(mNativeObject, buffer, buffer.remaining());
+        return new FilamentAsset(nativeAsset);
+    }
+
     /**
      * Allows clients to enable diagnostic shading on newly-loaded assets.
      */
@@ -123,7 +141,7 @@ public class AssetLoader {
     /**
      * Frees all memory associated with the given {@link FilamentAsset}.
      */
-    public void destroyAsset(@Nullable FilamentAsset asset) {
+    public void destroyAsset(@NonNull FilamentAsset asset) {
         nDestroyAsset(mNativeObject, asset.getNativeObject());
         asset.clearNativeObject();
     }
@@ -132,6 +150,7 @@ public class AssetLoader {
             long nativeEntities);
     private static native void nDestroyAssetLoader(long nativeLoader);
     private static native long nCreateAssetFromBinary(long nativeLoader, Buffer buffer, int remaining);
+    private static native long nCreateAssetFromJson(long nativeLoader, Buffer buffer, int remaining);
     private static native void nEnableDiagnostics(long nativeLoader, boolean enable);
     private static native void nDestroyAsset(long nativeLoader, long nativeAsset);
 }

android/gltfio-android/src/main/java/com/google/android/filament/gltfio/FilamentAsset.java

@@ -67,7 +67,7 @@ public class FilamentAsset {
      * <p>All of these have a transform component. Some of the returned entities may also have a
      * renderable component.</p>
      */
-    public @Entity int[] getEntities() {
+    public @NonNull @Entity int[] getEntities() {
         int[] result = new int[nGetEntityCount(mNativeObject)];
         nGetEntities(mNativeObject, result);
         return result;
@@ -95,7 +95,7 @@ public class FilamentAsset {
      * <p>When calling this for the first time, this must be called after
      * {@see ResourceLoader#loadResources}.</p>
      */
-    public Animator getAnimator() {
+    public @NonNull Animator getAnimator() {
         if (mAnimator != null) {
             return mAnimator;
         }
@@ -103,6 +103,15 @@ public class FilamentAsset {
         return mAnimator;
     }
 
+    /**
+     * Gets resource URIs for all externally-referenced buffers.
+     */
+    public @NonNull String[] getResourceUris() {
+        String[] uris = new String[nGetResourceUriCount(mNativeObject)];
+        nGetResourceUris(mNativeObject, uris);
+        return uris;
+    }
+
     void clearNativeObject() {
         mNativeObject = 0;
     }
@@ -113,4 +122,6 @@ public class FilamentAsset {
     private static native void nGetBoundingBox(long nativeAsset, float[] box);
     private static native String nGetName(long nativeAsset, int entity);
     private static native long nGetAnimator(long nativeAsset);
+    private static native int nGetResourceUriCount(long nativeAsset);
+    private static native void nGetResourceUris(long nativeAsset, String[] result);
 }

android/samples/README.md

@@ -58,6 +58,14 @@ Demonstrates how to use `Stream` with Android's Camera2 API:
 
 ![Hello Camera](../../docs/images/samples/sample_hello_camera.jpg)
 
+### `stream-test`
+
+Tests the various ways to interact with `Stream` by drawing into an external texture using Canvas.
+See the following screenshot; if the two sets of stripes are perfectly aligned, then the Filament
+frame and the external texture are perfectly synchronized.
+
+![Stream Test](../../docs/images/samples/sample_stream_test.jpg)
+
 ## Prerequisites
 
 Before you start, make sure to read [Filament's README](../../README.md). You need to be able to

android/samples/stream-test/.gitignore

@@ -0,0 +1,12 @@
+*.iml
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+/.idea/caches
+/.idea/gradle.xml
+.DS_Store
+/build
+/captures
+/app/src/main/assets/materials/*.filamat
+.externalNativeBuild

android/samples/stream-test/app/.gitignore

@@ -0,0 +1 @@
+/build

android/samples/stream-test/app/build.gradle

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlin-android-extensions'
+
+apply from: '../../../build/filament-tasks.gradle'
+
+compileMaterials {
+    group 'Filament'
+    description 'Compile materials'
+
+    inputDir = file("src/main/materials")
+    outputDir = file("src/main/assets/materials")
+}
+
+preBuild.dependsOn compileMaterials
+
+clean.doFirst {
+    delete "src/main/assets"
+}
+
+android {
+    compileSdkVersion 29
+    defaultConfig {
+        applicationId "com.google.android.filament.streamtest"
+        minSdkVersion 23 // 21 is required for CameraDevice.StateCallback, 23 is required for shouldShowRequestPermissionRationale
+        targetSdkVersion 29
+        versionCode 1
+        versionName "1.0"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+
+    // Filament comes with native code, the following declarations
+    // can be used to generate architecture specific APKs
+    flavorDimensions 'cpuArch'
+    productFlavors {
+        arm8 {
+            dimension 'cpuArch'
+            ndk {
+                abiFilters 'arm64-v8a'
+            }
+        }
+        arm7 {
+            dimension 'cpuArch'
+            ndk {
+                abiFilters 'armeabi-v7a'
+            }
+        }
+        x86_64 {
+            dimension 'cpuArch'
+            ndk {
+                abiFilters 'x86_64'
+            }
+        }
+        x86 {
+            dimension 'cpuArch'
+            ndk {
+                abiFilters 'x86'
+            }
+        }
+        universal {
+            dimension 'cpuArch'
+        }
+    }
+
+    // We use the .filamat extension for materials compiled with matc
+    // Telling aapt to not compress them allows to load them efficiently
+    aaptOptions {
+        noCompress 'filamat'
+    }
+}
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+
+    // Depend on Filament
+    implementation 'com.google.android.filament:filament-android'
+    implementation 'com.android.support:support-compat:28.0.0'
+}

android/samples/stream-test/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/samples/stream-test/app/src/main/AndroidManifest.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+  Copyright (C) 2019 The Android Open Source Project
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ -->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.android.filament.streamtest">
+
+  <application
+      android:allowBackup="true"
+      android:icon="@mipmap/ic_launcher"
+      android:label="@string/app_name"
+      android:roundIcon="@mipmap/ic_launcher_round"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+    <activity android:name=".MainActivity">
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN"/>
+
+        <category android:name="android.intent.category.LAUNCHER"/>
+      </intent-filter>
+    </activity>
+  </application>
+
+</manifest>

android/samples/stream-test/app/src/main/java/com/google/android/filament/streamtest/MainActivity.kt

@@ -0,0 +1,450 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.filament.streamtest
+
+import android.annotation.SuppressLint
+import android.app.Activity
+import android.os.Bundle
+import android.support.v4.app.ActivityCompat
+import android.view.Choreographer
+import android.view.Surface
+import android.view.SurfaceView
+
+import com.google.android.filament.*
+import com.google.android.filament.RenderableManager.*
+import com.google.android.filament.VertexBuffer.*
+import com.google.android.filament.android.UiHelper
+
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.channels.Channels
+import android.opengl.*
+import android.view.MotionEvent
+
+
+class MainActivity : Activity(), ActivityCompat.OnRequestPermissionsResultCallback {
+    companion object {
+        init {
+            Filament.init()
+        }
+    }
+
+    private lateinit var surfaceView: SurfaceView
+    private lateinit var uiHelper: UiHelper
+    private lateinit var choreographer: Choreographer
+
+    private lateinit var engine: Engine
+    private lateinit var renderer: Renderer
+    private lateinit var scene: Scene
+    private lateinit var view: View
+
+    // This helper wraps the Android camera2 API and connects it to a Filament material.
+    private lateinit var streamHelper: StreamHelper
+
+    // This is the Filament camera, not the phone camera. :)
+    private lateinit var camera: Camera
+
+    // Other Filament objects:
+    private lateinit var material: Material
+    private lateinit var materialInstance: MaterialInstance
+    private lateinit var vertexBuffer: VertexBuffer
+    private lateinit var indexBuffer: IndexBuffer
+
+    // Filament entity representing a renderable object
+    @Entity private var renderable = 0
+    @Entity private var light = 0
+
+    // A swap chain is Filament's representation of a surface
+    private var swapChain: SwapChain? = null
+
+    // Performs the rendering and schedules new frames
+    private val frameScheduler = FrameCallback()
+
+    private var externalTextureID: Int = 0
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+
+        surfaceView = SurfaceView(this)
+        setContentView(surfaceView)
+
+        choreographer = Choreographer.getInstance()
+
+        setupSurfaceView()
+        setupFilament()
+        setupView()
+        setupScene()
+
+        externalTextureID = createExternalTexture()
+        streamHelper = StreamHelper(engine, materialInstance, windowManager.defaultDisplay, externalTextureID)
+        this.title = streamHelper.getTestName()
+    }
+
+    @SuppressLint("ClickableViewAccessibility")
+    private fun setupSurfaceView() {
+        uiHelper = UiHelper(UiHelper.ContextErrorPolicy.DONT_CHECK)
+        uiHelper.renderCallback = SurfaceCallback()
+        uiHelper.attachTo(surfaceView)
+
+        surfaceView.setOnTouchListener { _, event ->
+            when(event.action){
+                MotionEvent.ACTION_DOWN -> {
+                    streamHelper.nextTest()
+                    this.title = streamHelper.getTestName()
+                }
+            }
+            super.onTouchEvent(event)
+        }
+    }
+
+    private fun setupFilament() {
+        val eglContext = createEGLContext()
+        engine = Engine.create(eglContext)
+        renderer = engine.createRenderer()
+        scene = engine.createScene()
+        view = engine.createView()
+        camera = engine.createCamera()
+    }
+
+    private fun setupView() {
+        view.setClearColor(0.035f, 0.035f, 0.035f, 1.0f)
+        view.camera = camera
+        view.scene = scene
+    }
+
+    private fun setupScene() {
+        loadMaterial()
+        setupMaterial()
+        createMesh()
+
+        // To create a renderable we first create a generic entity
+        renderable = EntityManager.get().create()
+
+        // We then create a renderable component on that entity
+        // A renderable is made of several primitives; in this case we declare only 1
+        // If we wanted each face of the cube to have a different material, we could
+        // declare 6 primitives (1 per face) and give each of them a different material
+        // instance, setup with different parameters
+        RenderableManager.Builder(1)
+                // Overall bounding box of the renderable
+                .boundingBox(Box(0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f))
+                // Sets the mesh data of the first primitive, 6 faces of 6 indices each
+                .geometry(0, PrimitiveType.TRIANGLES, vertexBuffer, indexBuffer, 0, 6 * 6)
+                // Sets the material of the first primitive
+                .material(0, materialInstance)
+                .build(engine, renderable)
+
+        // Add the entity to the scene to render it
+        scene.addEntity(renderable)
+
+        // We now need a light, let's create a directional light
+        light = EntityManager.get().create()
+
+        // Create a color from a temperature (5,500K)
+        val (r, g, b) = Colors.cct(5_500.0f)
+        LightManager.Builder(LightManager.Type.DIRECTIONAL)
+                .color(r, g, b)
+                // Intensity of the sun in lux on a clear day
+                .intensity(110_000.0f)
+                // The direction is normalized on our behalf
+                .direction(0.0f, -0.5f, -1.0f)
+                .castShadows(true)
+                .build(engine, light)
+
+        // Add the entity to the scene to light it
+        scene.addEntity(light)
+
+        // Set the exposure on the camera, this exposure follows the sunny f/16 rule
+        // Since we've defined a light that has the same intensity as the sun, it
+        // guarantees a proper exposure
+        camera.setExposure(16.0f, 1.0f / 125.0f, 100.0f)
+
+        // Move the camera back to see the object
+        camera.lookAt(0.0, 0.0, 6.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)
+    }
+
+    private fun loadMaterial() {
+        readUncompressedAsset("materials/lit.filamat").let {
+            material = Material.Builder().payload(it, it.remaining()).build(engine)
+        }
+    }
+
+    private fun setupMaterial() {
+        materialInstance = material.createInstance()
+        materialInstance.setParameter("baseColor", Colors.RgbType.SRGB, 1.0f, 0.85f, 0.57f)
+        materialInstance.setParameter("roughness", 0.3f)
+    }
+
+    private fun createMesh() {
+        val floatSize = 4
+        val shortSize = 2
+        // A vertex is a position + a tangent frame:
+        // 3 floats for XYZ position, 4 floats for normal+tangents (quaternion)
+        val vertexSize = 3 * floatSize + 4 * floatSize
+
+        // Define a vertex and a function to put a vertex in a ByteBuffer
+        @Suppress("ArrayInDataClass")
+        data class Vertex(val x: Float, val y: Float, val z: Float, val tangents: FloatArray)
+        fun ByteBuffer.put(v: Vertex): ByteBuffer {
+            putFloat(v.x)
+            putFloat(v.y)
+            putFloat(v.z)
+            v.tangents.forEach { putFloat(it) }
+            return this
+        }
+
+        // 6 faces, 4 vertices per face
+        val vertexCount = 6 * 4
+
+        // Create tangent frames, one per face
+        val tfPX = FloatArray(4)
+        val tfNX = FloatArray(4)
+        val tfPY = FloatArray(4)
+        val tfNY = FloatArray(4)
+        val tfPZ = FloatArray(4)
+        val tfNZ = FloatArray(4)
+
+        MathUtils.packTangentFrame( 0.0f,  1.0f, 0.0f, 0.0f, 0.0f, -1.0f,  1.0f,  0.0f,  0.0f, tfPX)
+        MathUtils.packTangentFrame( 0.0f,  1.0f, 0.0f, 0.0f, 0.0f, -1.0f, -1.0f,  0.0f,  0.0f, tfNX)
+        MathUtils.packTangentFrame(-1.0f,  0.0f, 0.0f, 0.0f, 0.0f, -1.0f,  0.0f,  1.0f,  0.0f, tfPY)
+        MathUtils.packTangentFrame(-1.0f,  0.0f, 0.0f, 0.0f, 0.0f,  1.0f,  0.0f, -1.0f,  0.0f, tfNY)
+        MathUtils.packTangentFrame( 0.0f,  1.0f, 0.0f, 1.0f, 0.0f,  0.0f,  0.0f,  0.0f,  1.0f, tfPZ)
+        MathUtils.packTangentFrame( 0.0f, -1.0f, 0.0f, 1.0f, 0.0f,  0.0f,  0.0f,  0.0f, -1.0f, tfNZ)
+
+        val vertexData = ByteBuffer.allocate(vertexCount * vertexSize)
+                // It is important to respect the native byte order
+                .order(ByteOrder.nativeOrder())
+                // Face -Z
+                .put(Vertex(-1.5f, -1.5f, -1.0f, tfNZ))
+                .put(Vertex(-1.5f,  1.5f, -1.0f, tfNZ))
+                .put(Vertex( 1.5f,  1.5f, -1.0f, tfNZ))
+                .put(Vertex( 1.5f, -1.5f, -1.0f, tfNZ))
+                // Face +X
+                .put(Vertex( 1.5f, -1.5f, -1.0f, tfPX))
+                .put(Vertex( 1.5f,  1.5f, -1.0f, tfPX))
+                .put(Vertex( 1.0f,  1.0f,  1.0f, tfPX))
+                .put(Vertex( 1.0f, -1.0f,  1.0f, tfPX))
+                // Face +Z
+                .put(Vertex(-1.0f, -1.0f,  1.0f, tfPZ))
+                .put(Vertex( 1.0f, -1.0f,  1.0f, tfPZ))
+                .put(Vertex( 1.0f,  1.0f,  1.0f, tfPZ))
+                .put(Vertex(-1.0f,  1.0f,  1.0f, tfPZ))
+                // Face -X
+                .put(Vertex(-1.0f, -1.0f,  1.0f, tfNX))
+                .put(Vertex(-1.0f,  1.0f,  1.0f, tfNX))
+                .put(Vertex(-1.5f,  1.5f, -1.0f, tfNX))
+                .put(Vertex(-1.5f, -1.5f, -1.0f, tfNX))
+                // Face -Y
+                .put(Vertex(-1.0f, -1.0f,  1.0f, tfNY))
+                .put(Vertex(-1.5f, -1.5f, -1.0f, tfNY))
+                .put(Vertex( 1.5f, -1.5f, -1.0f, tfNY))
+                .put(Vertex( 1.0f, -1.0f,  1.0f, tfNY))
+                // Face +Y
+                .put(Vertex(-1.5f,  1.5f, -1.0f, tfPY))
+                .put(Vertex(-1.0f,  1.0f,  1.0f, tfPY))
+                .put(Vertex( 1.0f,  1.0f,  1.0f, tfPY))
+                .put(Vertex( 1.5f,  1.5f, -1.0f, tfPY))
+                // Make sure the cursor is pointing in the right place in the byte buffer
+                .flip()
+
+        // Declare the layout of our mesh
+        vertexBuffer = VertexBuffer.Builder()
+                .bufferCount(1)
+                .vertexCount(vertexCount)
+                // Because we interleave position and color data we must specify offset and stride
+                // We could use de-interleaved data by declaring two buffers and giving each
+                // attribute a different buffer index
+                .attribute(VertexAttribute.POSITION, 0, AttributeType.FLOAT3, 0,             vertexSize)
+                .attribute(VertexAttribute.TANGENTS, 0, AttributeType.FLOAT4, 3 * floatSize, vertexSize)
+                .build(engine)
+
+        // Feed the vertex data to the mesh
+        // We only set 1 buffer because the data is interleaved
+        vertexBuffer.setBufferAt(engine, 0, vertexData)
+
+        // Create the indices
+        val indexData = ByteBuffer.allocate(6 * 2 * 3 * shortSize)
+                .order(ByteOrder.nativeOrder())
+        repeat(6) {
+            val i = (it * 4).toShort()
+            indexData
+                    .putShort(i).putShort((i + 1).toShort()).putShort((i + 2).toShort())
+                    .putShort(i).putShort((i + 2).toShort()).putShort((i + 3).toShort())
+        }
+        indexData.flip()
+
+        // 6 faces, 2 triangles per face,
+        indexBuffer = IndexBuffer.Builder()
+                .indexCount(vertexCount * 2)
+                .bufferType(IndexBuffer.Builder.IndexType.USHORT)
+                .build(engine)
+        indexBuffer.setBuffer(engine, indexData)
+    }
+
+    override fun onResume() {
+        super.onResume()
+        choreographer.postFrameCallback(frameScheduler)
+    }
+
+    override fun onPause() {
+        super.onPause()
+        choreographer.removeFrameCallback(frameScheduler)
+    }
+
+    override fun onDestroy() {
+        super.onDestroy()
+
+        // Stop the animation and any pending frame
+        choreographer.removeFrameCallback(frameScheduler)
+
+        // Always detach the surface before destroying the engine
+        uiHelper.detach()
+
+        // Cleanup all resources
+        engine.destroyEntity(light)
+        engine.destroyEntity(renderable)
+        engine.destroyRenderer(renderer)
+        engine.destroyVertexBuffer(vertexBuffer)
+        engine.destroyIndexBuffer(indexBuffer)
+        engine.destroyMaterialInstance(materialInstance)
+        engine.destroyMaterial(material)
+        engine.destroyView(view)
+        engine.destroyScene(scene)
+        engine.destroyCamera(camera)
+
+        // Engine.destroyEntity() destroys Filament related resources only
+        // (components), not the entity itself
+        val entityManager = EntityManager.get()
+        entityManager.destroy(light)
+        entityManager.destroy(renderable)
+
+        // Destroying the engine will free up any resource you may have forgotten
+        // to destroy, but it's recommended to do the cleanup properly
+        engine.destroy()
+    }
+
+    inner class FrameCallback : Choreographer.FrameCallback {
+        override fun doFrame(frameTimeNanos: Long) {
+            // Schedule the next frame
+            choreographer.postFrameCallback(this)
+
+            // This check guarantees that we have a swap chain
+            if (uiHelper.isReadyToRender) {
+
+                // If beginFrame() returns false you should skip the frame
+                // This means you are sending frames too quickly to the GPU
+                if (renderer.beginFrame(swapChain!!)) {
+                    streamHelper.repaintCanvas()
+                    materialInstance.setParameter("uvOffset", streamHelper.uvOffset)
+
+                    renderer.render(view)
+                    renderer.endFrame()
+                }
+            }
+        }
+    }
+
+    inner class SurfaceCallback : UiHelper.RendererCallback {
+        override fun onNativeWindowChanged(surface: Surface) {
+            swapChain?.let { engine.destroySwapChain(it) }
+            swapChain = engine.createSwapChain(surface)
+        }
+
+        override fun onDetachedFromSurface() {
+            swapChain?.let {
+                engine.destroySwapChain(it)
+                // Required to ensure we don't return before Filament is done executing the
+                // destroySwapChain command, otherwise Android might destroy the Surface
+                // too early
+                engine.flushAndWait()
+                swapChain = null
+            }
+        }
+
+        override fun onResized(width: Int, height: Int) {
+            val aspect = width.toDouble() / height.toDouble()
+            camera.setProjection(45.0, aspect, 0.1, 20.0, Camera.Fov.VERTICAL)
+
+            view.viewport = Viewport(0, 0, width, height)
+        }
+    }
+
+    private fun readUncompressedAsset(@Suppress("SameParameterValue") assetName: String): ByteBuffer {
+        assets.openFd(assetName).use { fd ->
+            val input = fd.createInputStream()
+            val dst = ByteBuffer.allocate(fd.length.toInt())
+
+            val src = Channels.newChannel(input)
+            src.read(dst)
+            src.close()
+
+            return dst.apply { rewind() }
+        }
+    }
+
+    private fun createEGLContext(): EGLContext {
+        // Providing this constant here (rather than using EGL_OPENGL_ES3_BIT ) allows us to use a lower target API for this project.
+        val kEGLOpenGLES3Bit = 64
+
+        val shareContext = EGL14.EGL_NO_CONTEXT
+        val display = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
+
+        val minorMajor: IntArray? = null
+        EGL14.eglInitialize(display, minorMajor, 0, minorMajor, 0)
+        val configs = arrayOfNulls<EGLConfig>(1)
+        val numConfig = intArrayOf(0)
+        val attribs = intArrayOf(EGL14.EGL_RENDERABLE_TYPE, kEGLOpenGLES3Bit, EGL14.EGL_NONE)
+        EGL14.eglChooseConfig(display, attribs, 0, configs, 0, 1, numConfig, 0)
+
+        val contextAttribs = intArrayOf(EGL14.EGL_CONTEXT_CLIENT_VERSION, 3, EGL14.EGL_NONE)
+        val context = EGL14.eglCreateContext(display, configs[0], shareContext, contextAttribs, 0)
+
+        val surfaceAttribs = intArrayOf(EGL14.EGL_WIDTH, 1, EGL14.EGL_HEIGHT, 1, EGL14.EGL_NONE)
+
+        val surface = EGL14.eglCreatePbufferSurface(display, configs[0], surfaceAttribs, 0)
+
+        check(EGL14.eglMakeCurrent(display, surface, surface, context)) { "Error making GL context." }
+        return context
+    }
+
+    private  fun createExternalTexture(): Int {
+        val textures = IntArray(1)
+        GLES30.glGenTextures(1, textures, 0)
+        val result = textures[0]
+
+        val textureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES
+        GLES30.glBindTexture(textureTarget, result)
+        GLES30.glTexParameteri(textureTarget, GLES30.GL_TEXTURE_WRAP_S, GLES30.GL_CLAMP_TO_EDGE)
+        GLES30.glTexParameteri(textureTarget, GLES30.GL_TEXTURE_WRAP_T, GLES30.GL_CLAMP_TO_EDGE)
+        GLES30.glTexParameteri(textureTarget, GLES30.GL_TEXTURE_MIN_FILTER, GLES30.GL_NEAREST)
+        GLES30.glTexParameteri(textureTarget, GLES30.GL_TEXTURE_MAG_FILTER, GLES30.GL_NEAREST)
+
+        if (!GLES30.glIsTexture(result)) {
+            throw RuntimeException("OpenGL error: $result is an invalid texture.")
+        }
+
+        val error = GLES30.glGetError()
+        if (error != GLES30.GL_NO_ERROR) {
+            val errorString = GLU.gluErrorString(error)
+            throw RuntimeException("OpenGL error: $errorString!")
+        }
+
+        return result
+    }
+
+
+}

android/samples/stream-test/app/src/main/java/com/google/android/filament/streamtest/StreamHelper.kt

@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.filament.streamtest
+
+import android.graphics.LinearGradient
+import android.os.Handler
+import android.util.Size
+import android.view.Surface
+
+import android.graphics.*
+import android.media.ImageReader
+import android.opengl.Matrix
+import android.view.Display
+
+import com.google.android.filament.*
+
+
+/**
+ * Demonstrates Filament's various texture sharing mechanisms.
+ */
+class StreamHelper(private val filamentEngine: Engine, private val filamentMaterial: MaterialInstance, private val display: Display, private val externalTextureId: Int) {
+    /**
+     * The StreamSource configures the source data for the texture.
+     *
+     * All tests draw animated test stripes as follows:
+     *
+     *  - The left stripe uses texture-based animation via Android's 2D drawing API.
+     *  - The right stripe uses shader-based animation.
+     *
+     * Ideally these are perfectly in sync with each other.
+     */
+    enum class StreamSource {
+        CANVAS_STREAM_NATIVE,     // copy-free but does not guarantee synchronization
+        CANVAS_STREAM_TEXID,      // synchronized but incurs a copy
+        CANVAS_STREAM_ACQUIRED,   // synchronized and copy-free
+    }
+
+    private var streamSource = StreamSource.CANVAS_STREAM_NATIVE
+    private val directImageHandler = Handler()
+    private var resolution = Size(640, 480)
+    private var surfaceTexture: SurfaceTexture? = null
+    private var imageReader: ImageReader? = null
+    private var frameNumber = 0L
+    private var canvasSurface: Surface? = null
+    private var filamentTexture: Texture? = null
+    private var filamentStream: Stream? = null
+    var uvOffset = 0.0f
+        private set
+
+    private val kGradientSpeed = 5
+    private val kGradientCount = 5
+
+    private val kGradientColors = intArrayOf(
+            Color.RED, Color.RED,
+            Color.WHITE, Color.WHITE,
+            Color.GREEN, Color.GREEN,
+            Color.WHITE, Color.WHITE,
+            Color.BLUE, Color.BLUE)
+
+    private val kGradientStops = floatArrayOf(
+            0.0f, 0.1f,
+            0.1f, 0.5f,
+            0.5f, 0.6f,
+            0.6f, 0.9f,
+            0.9f, 1.0f)
+
+    // This seems a little high, but lower values cause occasional "client tried to acquire more than maxImages buffers" on a Pixel 3.
+    private val kImageReaderMaxImages = 7
+
+    init {
+        startTest()
+    }
+
+    fun repaintCanvas() {
+        val kGradientScale = resolution.width.toFloat() / kGradientCount
+        val kGradientOffset = (frameNumber.toFloat() * kGradientSpeed) % resolution.width
+        val surface = canvasSurface
+        if (surface != null) {
+            val canvas = surface.lockCanvas(null)
+
+            val movingPaint = Paint()
+            movingPaint.shader = LinearGradient(kGradientOffset, 0.0f, kGradientOffset + kGradientScale, 0.0f, kGradientColors, kGradientStops, Shader.TileMode.REPEAT)
+            canvas.drawRect(Rect(0, resolution.height / 2, resolution.width, resolution.height), movingPaint)
+
+            val staticPaint = Paint()
+            staticPaint.shader = LinearGradient(0.0f, 0.0f, kGradientScale, 0.0f, kGradientColors, kGradientStops, Shader.TileMode.REPEAT)
+            canvas.drawRect(Rect(0, 0, resolution.width, resolution.height / 2), staticPaint)
+
+            surface.unlockCanvasAndPost(canvas)
+
+            if (streamSource == StreamSource.CANVAS_STREAM_TEXID) {
+                surfaceTexture!!.updateTexImage()
+            }
+
+            if (streamSource == StreamSource.CANVAS_STREAM_ACQUIRED) {
+                val image = imageReader!!.acquireLatestImage()
+                filamentStream!!.setAcquiredImage(image.hardwareBuffer!!, directImageHandler) { image.close() }
+            }
+        }
+
+        frameNumber++
+        uvOffset = 1.0f - kGradientOffset / resolution.width.toFloat()
+    }
+
+    fun nextTest() {
+        stopTest()
+        streamSource = StreamSource.values()[(streamSource.ordinal + 1) % 3]
+        startTest()
+    }
+
+    fun getTestName(): String {
+        return streamSource.name
+    }
+
+    private fun startTest() {
+
+        // Create the Filament Texture and Sampler objects.
+        filamentTexture = Texture.Builder()
+                .sampler(Texture.Sampler.SAMPLER_EXTERNAL)
+                .format(Texture.InternalFormat.RGB8)
+                .build(filamentEngine)
+
+        val filamentTexture = this.filamentTexture!!
+
+        val sampler = TextureSampler(TextureSampler.MinFilter.LINEAR, TextureSampler.MagFilter.LINEAR, TextureSampler.WrapMode.REPEAT)
+
+        // We are texturing a front-facing square shape so we need to generate a matrix that transforms (u, v, 0, 1)
+        // into a new UV coordinate according to the screen rotation and the aspect ratio of the camera image.
+        val aspectRatio = resolution.width.toFloat() / resolution.height.toFloat()
+        val textureTransform = FloatArray(16)
+        Matrix.setIdentityM(textureTransform, 0)
+        when (display.rotation) {
+            Surface.ROTATION_0 -> {
+                Matrix.translateM(textureTransform, 0, 1.0f, 0.0f, 0.0f)
+                Matrix.rotateM(textureTransform, 0, 90.0f, 0.0f, 0.0f, 1.0f)
+                Matrix.translateM(textureTransform, 0, 1.0f, 0.0f, 0.0f)
+                Matrix.scaleM(textureTransform, 0, -1.0f, 1.0f / aspectRatio, 1.0f)
+            }
+            Surface.ROTATION_90 -> {
+                Matrix.translateM(textureTransform, 0, 1.0f, 1.0f, 0.0f)
+                Matrix.rotateM(textureTransform, 0, 180.0f, 0.0f, 0.0f, 1.0f)
+                Matrix.translateM(textureTransform, 0, 1.0f, 0.0f, 0.0f)
+                Matrix.scaleM(textureTransform, 0, -1.0f / aspectRatio, 1.0f, 1.0f)
+            }
+            Surface.ROTATION_270 -> {
+                Matrix.translateM(textureTransform, 0, 1.0f, 0.0f, 0.0f)
+                Matrix.scaleM(textureTransform, 0, -1.0f / aspectRatio, 1.0f, 1.0f)
+            }
+        }
+
+        // Connect the Stream to the Texture and the Texture to the MaterialInstance.
+        filamentMaterial.setParameter("videoTexture", filamentTexture, sampler)
+        filamentMaterial.setParameter("textureTransform", MaterialInstance.FloatElement.MAT4, textureTransform, 0, 1)
+
+        if (streamSource == StreamSource.CANVAS_STREAM_NATIVE) {
+
+            // Create the Android surface that will hold the canvas image.
+            surfaceTexture = SurfaceTexture(0)
+            surfaceTexture!!.setDefaultBufferSize(resolution.width, resolution.height)
+            surfaceTexture!!.detachFromGLContext()
+            canvasSurface = Surface(surfaceTexture)
+
+            // Create the Filament Stream object that gets bound to the Texture.
+            filamentStream = Stream.Builder()
+                    .stream(surfaceTexture!!)
+                    .build(filamentEngine)
+
+            filamentTexture.setExternalStream(filamentEngine, filamentStream!!)
+        }
+
+        if (streamSource == StreamSource.CANVAS_STREAM_TEXID) {
+
+            // Create the Android surface that will hold the canvas image.
+            surfaceTexture = SurfaceTexture(externalTextureId)
+            surfaceTexture!!.setDefaultBufferSize(resolution.width, resolution.height)
+            canvasSurface = Surface(surfaceTexture)
+
+            // Create the Filament Stream object that gets bound to the Texture.
+            filamentStream = Stream.Builder()
+                    .stream(externalTextureId.toLong())
+                    .width(resolution.width)
+                    .height(resolution.height)
+                    .build(filamentEngine)
+
+            filamentTexture.setExternalStream(filamentEngine, filamentStream!!)
+        }
+
+        if (streamSource == StreamSource.CANVAS_STREAM_ACQUIRED) {
+            filamentStream = Stream.Builder()
+                    .width(resolution.width)
+                    .height(resolution.height)
+                    .build(filamentEngine)
+
+            filamentTexture.setExternalStream(filamentEngine, filamentStream!!)
+
+            this.imageReader = ImageReader.newInstance(resolution.width, resolution.height, ImageFormat.RGB_565, kImageReaderMaxImages).apply {
+                canvasSurface = surface
+            }
+        }
+
+        // Draw the first frame now.
+        frameNumber = 0
+        repaintCanvas()
+    }
+
+    private fun stopTest() {
+        filamentTexture?.let { filamentEngine.destroyTexture(it) }
+        filamentStream?.let { filamentEngine.destroyStream(it) }
+        surfaceTexture?.release()
+    }
+}

android/samples/stream-test/app/src/main/materials/lit.mat

@@ -0,0 +1,88 @@
+// Simple lit material that defines 3 parameters:
+// - baseColor
+// - roughness
+// - metallic
+//
+// These parameters can be used by the application to change the appearance of the material.
+//
+// This source material must be compiled to a binary material using the matc tool.
+// The command used to compile this material is:
+// matc -p mobile -a opengl -o app/src/main/assets/lit.filamat app/src/materials/lit.mat
+//
+// See build.gradle for an example of how to compile materials automatically
+// Please refer to the documentation for more information about matc and the materials system.
+
+material {
+    name : lit,
+
+    // Dynamic lighting is enabled on this material
+    shadingModel : lit,
+
+    // We don't need to declare a "requires" array, lit materials
+    // always requires the "tangents" vertex attribute (the normal
+    // is required for lighting, tangent/bitangent for normal mapping
+    // and anisotropy)
+
+    // Custom vertex shader outputs
+    variables : [
+        uv
+    ],
+
+    // List of parameters exposed by this material
+    parameters : [
+        // The color must be passed in linear space, not sRGB
+        {
+            type : float3,
+            name : baseColor
+        },
+        {
+            type : float,
+            name : roughness
+        },
+        {
+            type : float,
+            name : uvOffset
+        },
+        {
+            type : samplerExternal,
+            name : videoTexture
+        },
+        {
+            type : mat4,
+            name : textureTransform
+        }
+    ],
+}
+
+vertex {
+    void materialVertex(inout MaterialVertexInputs material) {
+        material.uv = 0.5 * (getPosition() + vec4(1));
+    }
+}
+
+fragment {
+    void material(inout MaterialInputs material) {
+        prepareMaterial(material);
+        material.roughness = materialParams.roughness;
+        material.metallic = 0.0;
+
+        // Apply the video stream to the +Z face on the cube.
+        if (variable_uv.z >= 1.0) {
+            vec2 uv = (materialParams.textureTransform * vec4(variable_uv.xy, 0, 1)).xy;
+
+            // If in right-hand side (in portrait mode), apply an offset.
+            if (uv.y < 0.5) {
+                uv.x += materialParams.uvOffset;
+            }
+
+            material.baseColor.rgb = inverseTonemapSRGB(texture(materialParams_videoTexture, uv).rgb);
+
+            if (uv.y < 0.5) {
+                material.baseColor.rgb *= 0.3;
+            }
+
+        } else {
+            material.baseColor.rgb = materialParams.baseColor;
+        }
+    }
+}

android/samples/stream-test/app/src/main/res/drawable-v24/ic_launcher_foreground.xml

@@ -0,0 +1,34 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:aapt="http://schemas.android.com/aapt"
+    android:width="108dp"
+    android:height="108dp"
+    android:viewportHeight="108"
+    android:viewportWidth="108">
+  <path
+      android:fillType="evenOdd"
+      android:pathData="M32,64C32,64 38.39,52.99 44.13,50.95C51.37,48.37 70.14,49.57 70.14,49.57L108.26,87.69L108,109.01L75.97,107.97L32,64Z"
+      android:strokeColor="#00000000"
+      android:strokeWidth="1">
+    <aapt:attr name="android:fillColor">
+      <gradient
+          android:endX="78.5885"
+          android:endY="90.9159"
+          android:startX="48.7653"
+          android:startY="61.0927"
+          android:type="linear">
+        <item
+            android:color="#44000000"
+            android:offset="0.0"/>
+        <item
+            android:color="#00000000"
+            android:offset="1.0"/>
+      </gradient>
+    </aapt:attr>
+  </path>
+  <path
+      android:fillColor="#FFFFFF"
+      android:fillType="nonZero"
+      android:pathData="M66.94,46.02L66.94,46.02C72.44,50.07 76,56.61 76,64L32,64C32,56.61 35.56,50.11 40.98,46.06L36.18,41.19C35.45,40.45 35.45,39.3 36.18,38.56C36.91,37.81 38.05,37.81 38.78,38.56L44.25,44.05C47.18,42.57 50.48,41.71 54,41.71C57.48,41.71 60.78,42.57 63.68,44.05L69.11,38.56C69.84,37.81 70.98,37.81 71.71,38.56C72.44,39.3 72.44,40.45 71.71,41.19L66.94,46.02ZM62.94,56.92C64.08,56.92 65,56.01 65,54.88C65,53.76 64.08,52.85 62.94,52.85C61.8,52.85 60.88,53.76 60.88,54.88C60.88,56.01 61.8,56.92 62.94,56.92ZM45.06,56.92C46.2,56.92 47.13,56.01 47.13,54.88C47.13,53.76 46.2,52.85 45.06,52.85C43.92,52.85 43,53.76 43,54.88C43,56.01 43.92,56.92 45.06,56.92Z"
+      android:strokeColor="#00000000"
+      android:strokeWidth="1"/>
+</vector>

android/samples/stream-test/app/src/main/res/drawable/ic_launcher_background.xml

@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="utf-8"?>
+<vector
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="108dp"
+    android:height="108dp"
+    android:viewportHeight="108"
+    android:viewportWidth="108">
+  <path
+      android:fillColor="#26A69A"
+      android:pathData="M0,0h108v108h-108z"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M9,0L9,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,0L19,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M29,0L29,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M39,0L39,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M49,0L49,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M59,0L59,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M69,0L69,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M79,0L79,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M89,0L89,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M99,0L99,108"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,9L108,9"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,19L108,19"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,29L108,29"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,39L108,39"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,49L108,49"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,59L108,59"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,69L108,69"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,79L108,79"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,89L108,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M0,99L108,99"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,29L89,29"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,39L89,39"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,49L89,49"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,59L89,59"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,69L89,69"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M19,79L89,79"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M29,19L29,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M39,19L39,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M49,19L49,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M59,19L59,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M69,19L69,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+  <path
+      android:fillColor="#00000000"
+      android:pathData="M79,19L79,89"
+      android:strokeColor="#33FFFFFF"
+      android:strokeWidth="0.8"/>
+</vector>

android/samples/stream-test/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+  <background android:drawable="@drawable/ic_launcher_background"/>
+  <foreground android:drawable="@drawable/ic_launcher_foreground"/>
+</adaptive-icon>

android/samples/stream-test/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+  <background android:drawable="@drawable/ic_launcher_background"/>
+  <foreground android:drawable="@drawable/ic_launcher_foreground"/>
+</adaptive-icon>

android/samples/stream-test/app/src/main/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+  <color name="colorPrimary">#3F51B5</color>
+  <color name="colorPrimaryDark">#303F9F</color>
+  <color name="colorAccent">#FF4081</color>
+</resources>

android/samples/stream-test/app/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+  <string name="app_name">Stream Test</string>
+</resources>

android/samples/stream-test/app/src/main/res/values/styles.xml

@@ -0,0 +1,8 @@
+<resources>
+
+  <!-- Base application theme. -->
+  <style name="AppTheme" parent="android:Theme.Material.Light.DarkActionBar">
+    <!-- Customize your theme here. -->
+  </style>
+
+</resources>

android/samples/stream-test/build.gradle

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+buildscript {
+    ext.kotlin_version = '1.3.50'
+    repositories {
+        google()
+        jcenter()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.5.0'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

android/samples/stream-test/gradle.properties

@@ -0,0 +1,13 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx1536m
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true

android/samples/stream-test/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Mon Jan 14 11:08:15 PST 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip

android/samples/stream-test/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

android/samples/stream-test/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android/samples/stream-test/settings.gradle

@@ -0,0 +1,3 @@
+includeBuild '../../filament-android'
+
+include ':app'

build.sh

@@ -197,7 +197,7 @@ function build_webgl_with_target {
         cmake \
             -G "$BUILD_GENERATOR" \
             -DIMPORT_EXECUTABLES_DIR=out \
-            -DCMAKE_TOOLCHAIN_FILE=${EMSCRIPTEN}/cmake/Modules/Platform/Emscripten.cmake \
+            -DCMAKE_TOOLCHAIN_FILE=${EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
             -DCMAKE_BUILD_TYPE=$1 \
             -DCMAKE_INSTALL_PREFIX=../webgl-${lc_target}/filament \
             -DWEBGL=1 \

build/common/test_list.txt

@@ -1,4 +1,4 @@
-filament/test/test_filament --gtest_filter=-FilamentTest.FroxelData:FilamentExposureWithEngineTest.SetExposure:FilamentExposureWithEngineTest.ComputeEV100
+filament/test/test_filament --gtest_filter=-FilamentTest.FroxelData:FilamentExposureWithEngineTest.SetExposure:FilamentExposureWithEngineTest.ComputeEV100:RenderingTest.*
 libs/math/test_math
 libs/image/test_image compare libs/image/tests/reference/
 libs/utils/test_utils

build/common/upload-assets.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from github import Github
+import os, sys
+
+def print_usage():
+    print('Upload assets to a Filament GitHub release.')
+    print()
+    print('Usage:')
+    print('  upload-assets.py <tag> <asset>...')
+    print()
+    print('Notes:')
+    print('  The GitHub release must already be created prior to running this script. This is typically done')
+    print('  through the GitHub web UI.')
+    print()
+    print('  <tag> is the Git tag for the desired release to attach assets to, for example, "v1.4.2".')
+    print()
+    print('  <asset> is a path to the asset file to upload. The file name will be used as the name of the')
+    print('  asset.')
+    print()
+    print('  The GITHUB_API_KEY environment variable must be set to a valid authentication token for a')
+    print('  collaborator account of the Filament repository.')
+
+# The first argument is the path to this script.
+if len(sys.argv) < 3:
+    print_usage()
+    sys.exit(1)
+
+tag = sys.argv[1]
+assets = sys.argv[2:]
+
+authentication_token = os.environ.get('GITHUB_API_KEY')
+if not authentication_token:
+    sys.stderr.write('Error: the GITHUB_API_KEY is not set.\n')
+    sys.exit(1)
+
+g = Github(authentication_token)
+
+FILAMENT_REPO = "google/filament"
+filament = g.get_repo(FILAMENT_REPO)
+
+def find_release_from_tag(repo, tag):
+    """ Find a release in the repo for the given Git tag string. """
+    releases = repo.get_releases()
+    for r in releases:
+        if r.tag_name == tag:
+            return r
+    return None
+
+release = find_release_from_tag(filament, tag)
+if not release:
+    sys.stderr.write(f"Error: Could not find release with tag '{tag}'.\n")
+    sys.exit(1)
+
+print(f"Found release with tag '{tag}'.")
+
+for asset_path in assets:
+    sys.stdout.write(f'Uploding asset: {asset_path}... ')
+    asset_name = os.path.basename(asset_path)
+    asset = release.upload_asset(asset_path, name=asset_name)
+    if asset:
+        sys.stdout.write('Success!\n')

build/web/ci-common.sh

@@ -9,10 +9,11 @@ export PATH="$PWD:$PATH"
 # npm install -g typescript
 
 # Install emscripten.
-curl -L https://github.com/emscripten-core/emsdk/archive/a77638d.zip > emsdk.zip
+curl -L https://github.com/emscripten-core/emsdk/archive/1b1f08f.zip > emsdk.zip
 unzip emsdk.zip ; mv emsdk-* emsdk ; cd emsdk
-python emsdk install latest
-python emsdk activate latest
+./emsdk install latest
+./emsdk activate latest
+source ./emsdk_env.sh
 
 export EMSDK="$PWD"
 cd ..

build/windows/build-github.bat

@@ -1,9 +1,43 @@
 @echo off
 
+setlocal
+
 if "%GITHUB_WORKFLOW%" == "" (set RUNNING_LOCALLY=1)
 
+if "%TARGET%" == "" (
+    if "%1" == "" (
+        set TARGET=release
+    ) else (
+        set TARGET=%1
+    )
+)
+
+set BUILD_DEBUG=
+set BUILD_RELEASE=
+set INSTALL=
+set BUILD_RELEASE_VARIANTS=
+
+if "%TARGET%" == "presubmit" (
+    set BUILD_RELEASE=1
+)
+
+if "%TARGET%" == "continuous" (
+    set BUILD_DEBUG=1
+    set BUILD_RELEASE=1
+)
+
+if "%TARGET%" == "release" (
+    set BUILD_DEBUG=1
+    set BUILD_RELEASE=1
+    set INSTALL=--target install
+    set BUILD_RELEASE_VARIANTS=1
+)
+
 set VISUAL_STUDIO_VERSION="Enterprise"
-if "%RUNNING_LOCALLY%" == "1" (set VISUAL_STUDIO_VERSION="Professional")
+if "%RUNNING_LOCALLY%" == "1" (
+    set VISUAL_STUDIO_VERSION="Professional"
+    set "PATH=%PATH%;C:\Program Files\7-Zip"
+)
 
 call "C:\Program Files (x86)\Microsoft Visual Studio\2019\%VISUAL_STUDIO_VERSION%\VC\Auxiliary\Build\vcvars64.bat"
 if errorlevel 1 exit /b %errorlevel%
@@ -11,10 +45,60 @@ if errorlevel 1 exit /b %errorlevel%
 msbuild /version
 cmake --version
 
-mkdir out\cmake-release
-cd out\cmake-release
+if "%BUILD_RELEASE%" == "1" (
+    :: /MT
+    call :BuildVariant mt "-DUSE_STATIC_CRT=ON" Release || exit /b
+
+    if "%BUILD_RELEASE_VARIANTS%" == "1" (
+        :: /MD
+        call :BuildVariant md "-DUSE_STATIC_CRT=OFF" Release || exit /b
+    )
+)
+
+if "%BUILD_DEBUG%" == "1" (
+    :: MTd
+    call :BuildVariant mtd "-DUSE_STATIC_CRT=ON" Debug || exit /b
+
+    if "%BUILD_RELEASE_VARIANTS%" == "1" (
+        :: MDd
+        call :BuildVariant mdd "-DUSE_STATIC_CRT=OFF" Debug || exit /b
+    )
+)
+
+if "%BUILD_RELEASE_VARIANTS%" == "1" (
+    :: Use the /MT version as the "base" install.
+    move out\mt out\install
+    mkdir out\install\lib\x86_64\mt\
+    move out\install\lib\x86_64\*.lib out\install\lib\x86_64\mt\
+
+    xcopy out\md\lib\x86_64\*.lib out\install\lib\x86_64\md\
+    xcopy out\mtd\lib\x86_64\*.lib out\install\lib\x86_64\mtd\
+    xcopy out\mdd\lib\x86_64\*.lib out\install\lib\x86_64\mdd\
+)
+
+:: Create an archive.
+if "%BUILD_RELEASE_VARIANTS%" == "1" (
+    cd out\install
+    7z a -ttar -so ..\..\filament-release.tar * | 7z a -si ..\filament-windows.tgz
+)
+
+exit /b 0
+
+:BuildVariant
+set variant=%~1
+set flag=%~2
+set config=%~3
+
+mkdir out\cmake-%variant%
+cd out\cmake-%variant%
 if errorlevel 1 exit /b %errorlevel%
 
-cmake ..\.. -G "Visual Studio 16 2019" -A x64 || exit /b
+cmake ..\.. -G "Visual Studio 16 2019" -A x64 %flag% -DCMAKE_INSTALL_PREFIX=..\%variant% || exit /b
+cmake --build . %INSTALL% --config %config% -- /m || exit /b
 
-msbuild TNT.sln /m /p:configuration=Release
+cd ..\..
+
+:: Delete the cmake build folder, otherwise we run out of disk space on CI when
+:: building multiple variants.
+rd /s /q out\cmake-%variant%
+exit /b 0

docs/Materials.md.html

@@ -1620,6 +1620,7 @@ The following APIs are only available from the fragment block:
 **inverseTonemap(float3)**          | float3   |  Applies the inverse tone mapping operator to the specified linear sRGB color and returns a linear sRGB color. This operation may be an approximation
 **inverseTonemapSRGB(float3)**      | float3   |  Applies the inverse tone mapping operator to the specified non-linear sRGB color and returns a linear sRGB color. This operation may be an approximation
 **luminance(float3)**               | float    |  Computes the luminance of the specified linear sRGB color
+**ycbcrToRgb(float, float2)**       | float3   |  Converts a luminance and CbCr pair to a sRGB color
 **uvToRenderTargetUV(float2)**      | float2   |  Transforms a UV coordinate to allow sampling from a `RenderTarget` attachment
 
 !!! TIP: world space

filament/CMakeLists.txt

@@ -137,11 +137,11 @@ set(PRIVATE_HDRS
 
 set(MATERIAL_SRCS
         src/materials/defaultMaterial.mat
+        src/materials/blit.mat
         src/materials/blur.mat
         src/materials/mipmapDepth.mat
         src/materials/skybox.mat
         src/materials/sao.mat
-        src/materials/ssao.mat
         src/materials/tonemapping.mat
         src/materials/fxaa.mat
 )

filament/backend/CMakeLists.txt

@@ -34,6 +34,7 @@ set(SRCS
 )
 
 set(PRIVATE_HDRS
+        include/private/backend/AcquiredImage.h
         include/private/backend/CircularBuffer.h
         include/private/backend/CommandBufferQueue.h
         include/private/backend/CommandStream.h
@@ -248,6 +249,11 @@ else()
     set(FILAMENT_WARNINGS /W0)
 endif()
 
+if (APPLE)
+    # Turn on Automatic Reference Counting.
+    target_compile_options(${TARGET} PRIVATE "-fobjc-arc")
+endif()
+
 # clang-cl maps -Wall to -Weverything, which clutters logs with benign -Wc++98-compat warnings
 # Use /W4 instead for clang-cl builds
 if (CLANG_CL)
@@ -285,7 +291,8 @@ if (APPLE)
         test/ShaderGenerator.cpp
         test/TrianglePrimitive.cpp
         test/Arguments.cpp
-        test/test_MissingRequiredAttributes.cpp)
+        test/test_MissingRequiredAttributes.cpp
+        test/test_ReadPixels.cpp)
 
     target_link_libraries(backend_test PRIVATE
         backend

filament/backend/include/backend/BufferDescriptor.h

@@ -39,7 +39,12 @@ class UTILS_PUBLIC BufferDescriptor {
 public:
     /**
      * Callback used to destroy the buffer data.
-     * It is guaranteed to be called on the main filament thread.
+     * Guarantees:
+     *      Called on the main filament thread.
+     *
+     * Limitations:
+     *      Must be lightweight.
+     *      Must not call filament APIs.
      */
     using Callback = void(*)(void* buffer, size_t size, void* user);
 

filament/backend/include/backend/DriverEnums.h

@@ -624,6 +624,16 @@ enum class BlendFunction : uint8_t {
     SRC_ALPHA_SATURATE      //!< f(src, dst) = (1,1,1) * min(src.a, 1 - dst.a), 1
 };
 
+//! Stream for external textures
+enum class StreamType {
+    NATIVE,     //!< Not synchronized but copy-free. Good for video.
+    TEXTURE_ID, //!< Synchronized, but GL-only and incurs copies. Good for AR on devices before API 26.
+    ACQUIRED,   //!< Synchronized, copy-free, and take a release callback. Good for AR but requires API 26+.
+};
+
+//! Releases an ACQUIRED external texture, guaranteed to be called on the application thread.
+using StreamCallback = void(*)(void* image, void* user);
+
 //! Vertex attribute descriptor
 struct Attribute {
     //! attribute is normalized (remapped between 0 and 1)

filament/backend/include/private/backend/AcquiredImage.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_ACQUIRED_IMAGE_H
+#define TNT_FILAMENT_BACKEND_ACQUIRED_IMAGE_H
+
+#include <backend/DriverEnums.h>
+
+namespace filament {
+namespace backend {
+
+// This lightweight POD allows us to bundle the state required to process an ACQUIRED stream.
+// Since these types of external images need to be moved around and queued up, an encapsulation is
+// very useful.
+
+struct AcquiredImage {
+    void* image = nullptr;
+    backend::StreamCallback callback = nullptr;
+    void* userData = nullptr;
+};
+
+} // namespace backend
+} // namespace filament
+
+#endif // TNT_FILAMENT_BACKEND_ACQUIRED_IMAGE_H

filament/backend/include/private/backend/BackendUtils.h

@@ -22,9 +22,16 @@ namespace filament {
 namespace backend {
 
 /**
- * Returns the number of bytes per pixel for the given format.
+ * Returns the number of bytes per pixel for the given format. For compressed texture formats,
+ * returns the number of bytes per block.
  */
 size_t getFormatSize(TextureFormat format) noexcept;
 
+/**
+ * For compressed texture formats, returns the number of horizontal pixels per block. Otherwise
+ * returns 0.
+ */
+size_t getBlockWidth(TextureFormat format) noexcept;
+
 } // namespace backend
 } // namespace filament

filament/backend/include/private/backend/CommandStream.h

@@ -111,14 +111,21 @@ private:
 
 // ------------------------------------------------------------------------------------------------
 
+template<typename T, typename Type, typename D, typename ... ARGS>
+constexpr decltype(auto) invoke(Type T::* m, D&& d, ARGS&& ... args) {
+    static_assert(std::is_base_of<T, std::decay_t<D>>::value,
+            "member function and object not related");
+    return (std::forward<D>(d).*m)(std::forward<ARGS>(args)...);
+}
+
 template<typename M, typename D, typename T, std::size_t... I>
-constexpr void trampoline(M&& m, D&& d, T&& t, std::index_sequence<I...>) {
-    (d.*m)(std::move(std::get<I>(std::forward<T>(t)))...);
+constexpr decltype(auto) trampoline(M&& m, D&& d, T&& t, std::index_sequence<I...>) {
+    return invoke(std::forward<M>(m), std::forward<D>(d), std::get<I>(std::forward<T>(t))...);
 }
 
 template<typename M, typename D, typename T>
-constexpr void apply(M&& m, D&& d, T&& t) {
-    trampoline(std::forward<M>(m), std::forward<D>(d), std::forward<T>(t),
+constexpr decltype(auto) apply(M&& m, D&& d, T&& t) {
+    return trampoline(std::forward<M>(m), std::forward<D>(d), std::forward<T>(t),
             std::make_index_sequence< std::tuple_size<std::remove_reference_t<T>>::value >{});
 }
 
@@ -158,7 +165,7 @@ struct CommandType<void (Driver::*)(ARGS...)> {
                 // must call this before invoking the method
                 self->log();
 #endif
-            apply(std::forward<M>(method), std::forward<D>(driver), self->mArgs);
+            apply(std::forward<M>(method), std::forward<D>(driver), std::move(self->mArgs));
             self->~Command();
         }
 
@@ -227,7 +234,7 @@ public:
 #define DECL_DRIVER_API_SYNCHRONOUS(RetType, methodName, paramsDecl, params)                    \
     inline RetType methodName(paramsDecl) {                                                     \
         DEBUG_COMMAND(methodName, params);                                                      \
-        return mDriver->methodName(params);                                                     \
+        return apply(&Driver::methodName, *mDriver, std::forward_as_tuple(params));             \
     }
 
 #define DECL_DRIVER_API_RETURN(RetType, methodName, paramsDecl, params)                         \

filament/backend/include/private/backend/Driver.h

@@ -33,6 +33,8 @@
 #include <utils/compiler.h>
 #include <utils/Log.h>
 
+#include <functional>
+
 #include <stdint.h>
 
 namespace filament {
@@ -56,6 +58,12 @@ public:
 
     virtual Dispatcher& getDispatcher() noexcept = 0;
 
+    // called from CommandStream::execute on the render-thread
+    // the fn function will execute a batch of driver commands
+    // this gives the driver a chance to wrap their execution in a meaningful manner
+    // the default implementation simply calls fn
+    virtual void execute(std::function<void(void)> fn) noexcept;
+
 #ifndef NDEBUG
     virtual void debugCommand(const char* methodName) {}
 #endif

filament/backend/include/private/backend/DriverAPI.inc

@@ -124,6 +124,9 @@ DECL_DRIVER_API_N(endFrame,
 // can start rendering. e.g. correspond to glFlush() for a GLES driver.
 DECL_DRIVER_API_0(flush)
 
+// flush and wait for the effects to be done
+DECL_DRIVER_API_0(finish)
+
 /*
  * Creating driver objects
  * -----------------------
@@ -180,6 +183,11 @@ DECL_DRIVER_API_R_N(backend::SwapChainHandle, createSwapChain,
         void*, nativeWindow,
         uint64_t, flags)
 
+DECL_DRIVER_API_R_N(backend::SwapChainHandle, createSwapChainHeadless,
+        uint32_t, width,
+        uint32_t, height,
+        uint64_t, flags)
+
 DECL_DRIVER_API_R_N(backend::StreamHandle, createStreamFromTextureId,
         intptr_t, externalTextureId,
         uint32_t, width,
@@ -207,7 +215,9 @@ DECL_DRIVER_API_N(destroyStream,          backend::StreamHandle, sh)
  */
 
 DECL_DRIVER_API_SYNCHRONOUS_0(void, terminate)
-DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStream, void*, stream)
+DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamNative, void*, stream)
+DECL_DRIVER_API_SYNCHRONOUS_0(backend::StreamHandle, createStreamAcquired)
+DECL_DRIVER_API_SYNCHRONOUS_N(void, setAcquiredImage, backend::StreamHandle, stream, void*, image, backend::StreamCallback, cb, void*, userData)
 DECL_DRIVER_API_SYNCHRONOUS_N(void, setStreamDimensions, backend::StreamHandle, stream, uint32_t, width, uint32_t, height)
 DECL_DRIVER_API_SYNCHRONOUS_N(int64_t, getStreamTimestamp, backend::StreamHandle, stream)
 DECL_DRIVER_API_SYNCHRONOUS_N(void, updateStreams, backend::DriverApi*, driver)
@@ -267,6 +277,11 @@ DECL_DRIVER_API_N(setExternalImage,
         backend::TextureHandle, th,
         void*, image)
 
+DECL_DRIVER_API_N(setExternalImagePlane,
+        backend::TextureHandle, th,
+        void*, image,
+        size_t, plane)
+
 DECL_DRIVER_API_N(setExternalStream,
         backend::TextureHandle, th,
         backend::StreamHandle, sh)

filament/backend/include/private/backend/OpenGLPlatform.h

@@ -19,6 +19,8 @@
 
 #include <backend/Platform.h>
 
+#include "private/backend/AcquiredImage.h"
+
 namespace filament {
 namespace backend {
 
@@ -41,6 +43,10 @@ public:
     virtual void terminate() noexcept = 0;
 
     virtual SwapChain* createSwapChain(void* nativeWindow, uint64_t& flags) noexcept = 0;
+
+    // headless swapchain
+    virtual SwapChain* createSwapChain(uint32_t width, uint32_t height, uint64_t& flags) noexcept = 0;
+
     virtual void destroySwapChain(SwapChain* swapChain) noexcept = 0;
 
     virtual void createDefaultRenderTarget(uint32_t& framebuffer, uint32_t& colorbuffer,
@@ -85,6 +91,10 @@ public:
 
     virtual void destroyExternalTextureStorage(ExternalTexture* ets) noexcept = 0;
 
+    // The method allows platforms to convert a user-supplied external image object into a new type
+    // (e.g. HardwareBuffer => EGLImage). It makes sense for the default implementation to do nothing.
+    virtual AcquiredImage transformAcquiredImage(AcquiredImage source) noexcept { return source; }
+
     // called to bind the platform-specific externalImage to a texture
     // texture points to a OpenGLDriver::GLTexture
     virtual bool setExternalImage(void* externalImage, void* texture) noexcept {
@@ -102,7 +112,6 @@ public:
 
     // called once before a SAMPLER_EXTERNAL texture is destroyed.
     virtual void destroyExternalImage(void* texture) noexcept {}
-
 };
 
 } // namespace backend

filament/backend/src/BackendUtils.cpp

@@ -99,10 +99,130 @@ size_t getFormatSize(TextureFormat format) noexcept {
         case TextureFormat::RGBA32I:
             return 16;
 
+        // Compressed formats ---------------------------------------------------------------------
+
+        case TextureFormat::EAC_RG11:
+        case TextureFormat::EAC_RG11_SIGNED:
+        case TextureFormat::ETC2_EAC_RGBA8:
+        case TextureFormat::ETC2_EAC_SRGBA8:
+            return 16;
+
+        case TextureFormat::EAC_R11:
+        case TextureFormat::EAC_R11_SIGNED:
+        case TextureFormat::ETC2_RGB8:
+        case TextureFormat::ETC2_SRGB8:
+        case TextureFormat::ETC2_RGB8_A1:
+        case TextureFormat::ETC2_SRGB8_A1:
+            return 8;
+
+        case TextureFormat::DXT1_RGB:
+            return 8;
+
+        case TextureFormat::DXT3_RGBA:
+        case TextureFormat::DXT5_RGBA:
+            return 16;
+
+        // The block size for ASTC compression is always 16 bytes.
+        case TextureFormat::RGBA_ASTC_4x4:
+        case TextureFormat::RGBA_ASTC_5x4:
+        case TextureFormat::RGBA_ASTC_5x5:
+        case TextureFormat::RGBA_ASTC_6x5:
+        case TextureFormat::RGBA_ASTC_6x6:
+        case TextureFormat::RGBA_ASTC_8x5:
+        case TextureFormat::RGBA_ASTC_8x6:
+        case TextureFormat::RGBA_ASTC_8x8:
+        case TextureFormat::RGBA_ASTC_10x5:
+        case TextureFormat::RGBA_ASTC_10x6:
+        case TextureFormat::RGBA_ASTC_10x8:
+        case TextureFormat::RGBA_ASTC_10x10:
+        case TextureFormat::RGBA_ASTC_12x10:
+        case TextureFormat::RGBA_ASTC_12x12:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_4x4:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_5x4:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_5x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_6x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_6x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x8:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x8:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x10:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_12x10:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_12x12:
+            return 16;
+
         default:
             return 0;
     }
 }
 
+size_t getBlockWidth(TextureFormat format) noexcept {
+    switch (format) {
+        case TextureFormat::EAC_RG11:
+        case TextureFormat::EAC_RG11_SIGNED:
+        case TextureFormat::ETC2_EAC_RGBA8:
+        case TextureFormat::ETC2_EAC_SRGBA8:
+        case TextureFormat::EAC_R11:
+        case TextureFormat::EAC_R11_SIGNED:
+        case TextureFormat::ETC2_RGB8:
+        case TextureFormat::ETC2_SRGB8:
+        case TextureFormat::ETC2_RGB8_A1:
+        case TextureFormat::ETC2_SRGB8_A1:
+            return 4;
+
+        case TextureFormat::DXT1_RGB:
+        case TextureFormat::DXT3_RGBA:
+        case TextureFormat::DXT5_RGBA:
+            return 4;
+
+        case TextureFormat::RGBA_ASTC_4x4:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_4x4:
+            return 4;
+
+        case TextureFormat::RGBA_ASTC_5x4:
+        case TextureFormat::RGBA_ASTC_5x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_5x4:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_5x5:
+            return 5;
+            return 5;
+
+        case TextureFormat::RGBA_ASTC_6x5:
+        case TextureFormat::RGBA_ASTC_6x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_6x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_6x6:
+            return 6;
+
+        case TextureFormat::RGBA_ASTC_8x5:
+        case TextureFormat::RGBA_ASTC_8x6:
+        case TextureFormat::RGBA_ASTC_8x8:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_8x8:
+            return 8;
+
+        case TextureFormat::RGBA_ASTC_10x5:
+        case TextureFormat::RGBA_ASTC_10x6:
+        case TextureFormat::RGBA_ASTC_10x8:
+        case TextureFormat::RGBA_ASTC_10x10:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x5:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x6:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x8:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_10x10:
+            return 10;
+
+        case TextureFormat::RGBA_ASTC_12x10:
+        case TextureFormat::RGBA_ASTC_12x12:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_12x10:
+        case TextureFormat::SRGB8_ALPHA8_ASTC_12x12:
+            return 12;
+
+        default:
+            return 0;
+    }
+}
+
+
 } // namespace backend
 } // namespace filament

filament/backend/src/CommandStream.cpp

@@ -73,11 +73,13 @@ void CommandStream::execute(void* buffer) {
         profiler.start();
     }
 
-    Driver& UTILS_RESTRICT driver = *mDriver;
-    CommandBase* UTILS_RESTRICT base = static_cast<CommandBase*>(buffer);
-    while (UTILS_LIKELY(base)) {
-        base = base->execute(driver);
-    }
+    mDriver->execute([this, buffer]() {
+        Driver& UTILS_RESTRICT driver = *mDriver;
+        CommandBase* UTILS_RESTRICT base = static_cast<CommandBase*>(buffer);
+        while (UTILS_LIKELY(base)) {
+            base = base->execute(driver);
+        }
+    });
 
     if (SYSTRACE_TAG) {
         // we want to remove all this when tracing is completely disabled

filament/backend/src/Driver.cpp

@@ -43,9 +43,15 @@ DriverBase::~DriverBase() noexcept {
 
 void DriverBase::purge() noexcept {
     std::vector<BufferDescriptor> buffersToPurge;
+    std::vector<AcquiredImage> imagesToPurge;
     std::unique_lock<std::mutex> lock(mPurgeLock);
     std::swap(buffersToPurge, mBufferToPurge);
+    std::swap(imagesToPurge, mImagesToPurge);
     lock.unlock(); // don't remove this, it ensures mBufferToPurge is destroyed without lock held
+    for (auto& image : imagesToPurge) {
+        image.callback(image.image, image.userData);
+    }
+    // When the BufferDescriptors go out of scope, their destructors invoke their callbacks.
 }
 
 void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept {
@@ -53,10 +59,21 @@ void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept {
     mBufferToPurge.push_back(std::move(buffer));
 }
 
+// This is called from an async driver method so it's in the GL thread, but purge is called
+// on the user thread. This is typically called 0 or 1 times per frame.
+void DriverBase::scheduleRelease(AcquiredImage&& image) noexcept {
+    std::lock_guard<std::mutex> lock(mPurgeLock);
+    mImagesToPurge.push_back(std::move(image));
+}
+
 // ------------------------------------------------------------------------------------------------
 
 Driver::~Driver() noexcept = default;
 
+void Driver::execute(std::function<void(void)> fn) noexcept {
+    fn();
+}
+
 size_t Driver::getElementTypeSize(ElementType type) noexcept {
     switch (type) {
         case ElementType::BYTE:     return sizeof(int8_t);

filament/backend/src/DriverBase.h

@@ -24,6 +24,7 @@
 
 #include <backend/DriverEnums.h>
 
+#include "private/backend/AcquiredImage.h"
 #include "private/backend/Driver.h"
 #include "private/backend/SamplerGroup.h"
 
@@ -133,8 +134,9 @@ struct HwSwapChain : public HwBase {
 
 struct HwStream : public HwBase {
     HwStream() = default;
-    explicit HwStream(Platform::Stream* stream) : stream(stream) { }
+    explicit HwStream(Platform::Stream* stream) : stream(stream), streamType(StreamType::NATIVE) { }
     Platform::Stream* stream = nullptr;
+    StreamType streamType = StreamType::ACQUIRED;
     uint32_t width = 0;
     uint32_t height = 0;
 };
@@ -168,9 +170,12 @@ protected:
 
     void scheduleDestroySlow(BufferDescriptor&& buffer) noexcept;
 
+    void scheduleRelease(AcquiredImage&& image) noexcept;
+
 private:
     std::mutex mPurgeLock;
     std::vector<BufferDescriptor> mBufferToPurge;
+    std::vector<AcquiredImage> mImagesToPurge;
 };
 
 

filament/backend/src/metal/MetalBlitter.mm

@@ -289,11 +289,7 @@ void MetalBlitter::blitFastPath(bool& blitColor, bool& blitDepth, const BlitArgs
 }
 
 void MetalBlitter::shutdown() noexcept {
-    for (auto it = mBlitFunctions.begin(); it != mBlitFunctions.end(); ++it) {
-        [it.value() release];
-    }
     mBlitFunctions.clear();
-    [mVertexFunction release];
     mVertexFunction = nil;
 }
 
@@ -348,8 +344,6 @@ id<MTLFunction> MetalBlitter::compileFragmentFunction(BlitFunctionKey key) {
                                                               error:&error];
     id<MTLFunction> function = [library newFunctionWithName:@"blitterFrag"];
     NSERROR_CHECK("Unable to compile shading library for MetalBlitter.");
-    [options release];
-    [library release];
 
     return function;
 }
@@ -367,7 +361,6 @@ id<MTLFunction> MetalBlitter::getBlitVertexFunction() {
                                                              error:&error];
     id<MTLFunction> function = [library newFunctionWithName:@"blitterVertex"];
     NSERROR_CHECK("Unable to compile shading library for MetalBlitter.");
-    [library release];
 
     mVertexFunction = function;
 

filament/backend/src/metal/MetalBufferPool.mm

@@ -89,7 +89,6 @@ void MetalBufferPool::gc() noexcept {
     const uint64_t evictionTime = mCurrentFrame - TIME_BEFORE_EVICTION;
     for (auto pair : stages) {
         if (pair.second->lastAccessed < evictionTime) {
-            [pair.second->buffer release];
             delete pair.second;
         } else {
             mFreeStages.insert(pair);
@@ -102,7 +101,6 @@ void MetalBufferPool::reset() noexcept {
 
     assert(mUsedStages.empty());
     for (auto pair : mFreeStages) {
-        [pair.second->buffer release];
         delete pair.second;
     }
     mFreeStages.clear();

filament/backend/src/metal/MetalContext.h

@@ -42,13 +42,6 @@ struct MetalContext {
     id<MTLDevice> device = nullptr;
     id<MTLCommandQueue> commandQueue = nullptr;
 
-    // A pool for autoreleased objects throughout the lifetime of the Metal driver.
-    NSAutoreleasePool* driverPool = nil;
-
-    // A pool for autoreleased objects allocated during the execution of a frame.
-    // The pool is created in beginFrame() and drained in endFrame().
-    NSAutoreleasePool* framePool = nil;
-
     // Single use, re-created each frame.
     id<MTLCommandBuffer> currentCommandBuffer = nullptr;
     id<MTLRenderCommandEncoder> currentCommandEncoder = nullptr;
@@ -85,13 +78,18 @@ struct MetalContext {
 
     // Surface-related properties.
     MetalSwapChain* currentSurface = nullptr;
-    id<CAMetalDrawable> currentDrawable = nullptr;
+    id<CAMetalDrawable> currentDrawable = nil;
+    id<MTLTexture> currentDepthTexture = nil;
+    id<MTLTexture> headlessDrawable = nil;
     MTLPixelFormat currentSurfacePixelFormat = MTLPixelFormatInvalid;
     MTLPixelFormat currentDepthPixelFormat = MTLPixelFormatInvalid;
 
     // External textures.
     CVMetalTextureCacheRef textureCache = nullptr;
 
+    // Empty texture used to prevent GPU errors when a sampler has been bound without a texture.
+    id<MTLTexture> emptyTexture = nil;
+
     MetalBlitter* blitter = nullptr;
 
     // Fences.
@@ -102,12 +100,17 @@ struct MetalContext {
 };
 
 // Acquire the current surface's CAMetalDrawable for the current frame if it has not already been
-// acquired.
-// This method returns the drawable and stores it in the context's currentDrawable field.
-id<CAMetalDrawable> acquireDrawable(MetalContext* context);
+// acquired. This method stores it in the context's currentDrawable field and returns the
+// drawable's texture.
+// For headless swapchains a new texture is created.
+id<MTLTexture> acquireDrawable(MetalContext* context);
+
+id<MTLTexture> acquireDepthTexture(MetalContext* context);
 
 id<MTLCommandBuffer> acquireCommandBuffer(MetalContext* context);
 
+id<MTLTexture> getOrCreateEmptyTexture(MetalContext* context);
+
 } // namespace metal
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalContext.mm

@@ -25,34 +25,89 @@ namespace backend {
 namespace metal {
 
 void presentDrawable(bool presentFrame, void* user) {
-    id<CAMetalDrawable> drawable = (id<CAMetalDrawable>) user;
+    // CFBridgingRelease here is used to balance the CFBridgingRetain inside of acquireDrawable.
+    id<CAMetalDrawable> drawable = (id<CAMetalDrawable>) CFBridgingRelease(user);
     if (presentFrame) {
         [drawable present];
     }
-    [drawable release];
+    // The drawable will be released here when the "drawable" variable goes out of scope.
 }
 
-id<CAMetalDrawable> acquireDrawable(MetalContext* context) {
-    if (!context->currentDrawable) {
-        // The drawable is retained here and will be released either:
-        // 1. in MetalDriver::commit
-        // 2. in the presentDrawable function, when the client calls the PresentCallable
-        context->currentDrawable = [[context->currentSurface->layer nextDrawable] retain];
-
-        if (context->frameFinishedCallback) {
-            id<CAMetalDrawable> drawable = context->currentDrawable;
-            backend::FrameFinishedCallback callback = context->frameFinishedCallback;
-            void* userData = context->frameFinishedUserData;
-            [context->currentCommandBuffer addScheduledHandler:^(id<MTLCommandBuffer> cb) {
-                PresentCallable callable(presentDrawable, (void*) drawable);
-                dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-                    callback(callable, userData);
-                });
-            }];
+id<MTLTexture> acquireDrawable(MetalContext* context) {
+    if (context->currentDrawable) {
+        return context->currentDrawable.texture;
+    }
+    if (context->currentSurface->isHeadless()) {
+        if (context->headlessDrawable) {
+            return context->headlessDrawable;
         }
+        // For headless surfaces we construct a "fake" drawable, which is simply a renderable
+        // texture.
+        MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor new];
+        textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
+        textureDescriptor.width = context->currentSurface->surfaceWidth;
+        textureDescriptor.height = context->currentSurface->surfaceHeight;
+        textureDescriptor.usage = MTLTextureUsageRenderTarget;
+#if defined(IOS)
+        textureDescriptor.storageMode = MTLStorageModeShared;
+#else
+        textureDescriptor.storageMode = MTLStorageModeManaged;
+#endif
+        context->headlessDrawable = [context->device newTextureWithDescriptor:textureDescriptor];
+        return context->headlessDrawable;
+    }
+
+    context->currentDrawable = [context->currentSurface->layer nextDrawable];
+
+    if (context->frameFinishedCallback) {
+        id<CAMetalDrawable> drawable = context->currentDrawable;
+        backend::FrameFinishedCallback callback = context->frameFinishedCallback;
+        void* userData = context->frameFinishedUserData;
+        // This block strongly captures drawable to keep it alive until the handler executes.
+        [context->currentCommandBuffer addScheduledHandler:^(id<MTLCommandBuffer> cb) {
+            // CFBridgingRetain is used here to give the drawable a +1 retain count before
+            // casting it to a void*.
+            PresentCallable callable(presentDrawable, (void*) CFBridgingRetain(drawable));
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                callback(callable, userData);
+            });
+        }];
     }
     ASSERT_POSTCONDITION(context->currentDrawable != nil, "Could not obtain drawable.");
-    return context->currentDrawable;
+    return context->currentDrawable.texture;
+}
+
+id<MTLTexture> acquireDepthTexture(MetalContext* context) {
+    if (context->currentDepthTexture) {
+        // If the surface size has changed, we'll need to allocate a new depth texture.
+        if (context->currentDepthTexture.width != context->currentSurface->surfaceWidth ||
+            context->currentDepthTexture.height != context->currentSurface->surfaceHeight) {
+            context->currentDepthTexture = nil;
+        } else {
+            return context->currentDepthTexture;
+        }
+    }
+
+    const MTLPixelFormat depthFormat =
+#if defined(IOS)
+            MTLPixelFormatDepth32Float;
+#else
+            MTLPixelFormatDepth24Unorm_Stencil8;
+#endif
+
+    const NSUInteger width = context->currentSurface->surfaceWidth;
+    const NSUInteger height = context->currentSurface->surfaceHeight;
+    MTLTextureDescriptor* descriptor =
+            [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:depthFormat
+                                                               width:width
+                                                              height:height
+                                                           mipmapped:NO];
+    descriptor.usage = MTLTextureUsageRenderTarget;
+    descriptor.resourceOptions = MTLResourceStorageModePrivate;
+
+    context->currentDepthTexture = [context->device newTextureWithDescriptor:descriptor];
+
+    return context->currentDepthTexture;
 }
 
 id<MTLCommandBuffer> acquireCommandBuffer(MetalContext* context) {
@@ -62,6 +117,29 @@ id<MTLCommandBuffer> acquireCommandBuffer(MetalContext* context) {
     return commandBuffer;
 }
 
+id<MTLTexture> getOrCreateEmptyTexture(MetalContext* context) {
+    if (context->emptyTexture) {
+        return context->emptyTexture;
+    }
+
+    MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor new];
+    textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
+    textureDescriptor.width = 1;
+    textureDescriptor.height = 1;
+    id<MTLTexture> texture = [context->device newTextureWithDescriptor:textureDescriptor];
+
+    MTLRegion region = {
+        { 0, 0, 0 },    // MTLOrigin
+        { 1, 1, 1 }     // MTLSize
+    };
+    uint8_t imageData[4] = {0, 0, 0, 0};
+    [texture replaceRegion:region mipmapLevel:0 withBytes:imageData bytesPerRow:4];
+
+    context->emptyTexture = texture;
+
+    return context->emptyTexture;
+}
+
 } // namespace metal
 } // namespace backend
 } // namespace filament

filament/backend/src/metal/MetalDriver.h

@@ -58,6 +58,9 @@ private:
 
     ShaderModel getShaderModel() const noexcept final;
 
+    // Overrides the default implementation by wrapping the call to fn in an @autoreleasepool block.
+    void execute(std::function<void(void)> fn) noexcept final;
+
     /*
      * Driver interface
      */

filament/backend/src/metal/MetalDriver.mm

@@ -52,7 +52,6 @@ MetalDriver::MetalDriver(backend::MetalPlatform* platform) noexcept
         : DriverBase(new ConcreteDispatcher<MetalDriver>()),
         mPlatform(*platform),
         mContext(new MetalContext) {
-    mContext->driverPool = [[NSAutoreleasePool alloc] init];
     mContext->device = MTLCreateSystemDefaultDevice();
     mContext->commandQueue = [mContext->device newCommandQueue];
     mContext->commandQueue.label = @"Filament";
@@ -73,7 +72,8 @@ MetalDriver::MetalDriver(backend::MetalPlatform* platform) noexcept
 }
 
 MetalDriver::~MetalDriver() noexcept {
-    [mContext->device release];
+    mContext->device = nil;
+    mContext->emptyTexture = nil;
     CFRelease(mContext->textureCache);
     delete mContext->bufferPool;
     delete mContext->blitter;
@@ -91,11 +91,9 @@ void MetalDriver::debugCommand(const char *methodName) {
 
 void MetalDriver::beginFrame(int64_t monotonic_clock_ns, uint32_t frameId,
         backend::FrameFinishedCallback callback, void* user) {
-    mContext->framePool = [[NSAutoreleasePool alloc] init];
-
     id<MTLCommandBuffer> commandBuffer = acquireCommandBuffer(mContext);
     [commandBuffer addCompletedHandler:^(id <MTLCommandBuffer> buffer) {
-        mContext->resourceTracker.clearResources(buffer);
+        mContext->resourceTracker.clearResources((__bridge void*) buffer);
     }];
 
     // If a callback was specified, then the client is responsible for presenting the frame.
@@ -103,6 +101,12 @@ void MetalDriver::beginFrame(int64_t monotonic_clock_ns, uint32_t frameId,
     mContext->frameFinishedUserData = user;
 }
 
+void MetalDriver::execute(std::function<void(void)> fn) noexcept {
+    @autoreleasepool {
+        fn();
+    }
+}
+
 void MetalDriver::setPresentationTime(int64_t monotonic_clock_ns) {
 
 }
@@ -116,21 +120,24 @@ void MetalDriver::endFrame(uint32_t frameId) {
         mContext->currentCommandBuffer = nil;
     }
 
-    // Release resources created during frame execution- like commandBuffer and currentDrawable.
-    [mContext->framePool drain];
     mContext->bufferPool->gc();
 
-    if (!mContext->frameFinishedCallback) {
-        // If we're responsible for presenting the frame, then by this point we've already done so
-        // and it's safe to release the drawable.
-        [mContext->currentDrawable release];
-    }
+    // If we acquired a drawable for this frame, ensure that we release it here.
+    mContext->currentDrawable = nil;
+    mContext->headlessDrawable = nil;
 
     CVMetalTextureCacheFlush(mContext->textureCache, 0);
 }
 
-void MetalDriver::flush(int dummy) {
+void MetalDriver::flush(int) {
+    // TODO: implement flush, equivalent of glFlush() (needed for performance)
+}
 
+void MetalDriver::finish(int) {
+    // Wait for all frames to finish by submitting and waiting on a dummy command buffer.
+    id<MTLCommandBuffer> oneOffBuffer = [mContext->commandQueue commandBuffer];
+    [oneOffBuffer commit];
+    [oneOffBuffer waitUntilCompleted];
 }
 
 void MetalDriver::createVertexBufferR(Handle<HwVertexBuffer> vbh, uint8_t bufferCount,
@@ -183,6 +190,8 @@ void MetalDriver::createRenderTargetR(Handle<HwRenderTarget> rth,
     auto getColorTexture = [&]() -> id<MTLTexture> {
         if (color.handle) {
             auto colorTexture = handle_cast<MetalTexture>(mHandleMap, color.handle);
+            ASSERT_PRECONDITION(colorTexture->texture,
+                    "Color texture passed to render target has no texture allocation");
             return colorTexture->texture;
         } else if (any(targetBufferFlags & TargetBufferFlags::COLOR)) {
             ASSERT_POSTCONDITION(false, "The COLOR flag was specified, but no color texture provided.");
@@ -193,6 +202,8 @@ void MetalDriver::createRenderTargetR(Handle<HwRenderTarget> rth,
     auto getDepthTexture = [&]() -> id<MTLTexture> {
         if (depth.handle) {
             auto depthTexture = handle_cast<MetalTexture>(mHandleMap, depth.handle);
+            ASSERT_PRECONDITION(depthTexture->texture,
+                    "Depth texture passed to render target has no texture allocation.");
             return depthTexture->texture;
         } else if (any(targetBufferFlags & TargetBufferFlags::DEPTH)) {
             ASSERT_POSTCONDITION(false, "The DEPTH flag was specified, but no depth texture provided.");
@@ -215,10 +226,15 @@ void MetalDriver::createFenceR(Handle<HwFence> fh, int dummy) {
 }
 
 void MetalDriver::createSwapChainR(Handle<HwSwapChain> sch, void* nativeWindow, uint64_t flags) {
-    auto* metalLayer = (CAMetalLayer*) nativeWindow;
+    auto* metalLayer = (__bridge CAMetalLayer*) nativeWindow;
     construct_handle<MetalSwapChain>(mHandleMap, sch, mContext->device, metalLayer);
 }
 
+void MetalDriver::createSwapChainHeadlessR(Handle<HwSwapChain> sch,
+        uint32_t width, uint32_t height, uint64_t flags) {
+    construct_handle<MetalSwapChain>(mHandleMap, sch, width, height);
+}
+
 void MetalDriver::createStreamFromTextureIdR(Handle<HwStream>, intptr_t externalTextureId,
         uint32_t width, uint32_t height) {
 
@@ -268,6 +284,10 @@ Handle<HwSwapChain> MetalDriver::createSwapChainS() noexcept {
     return alloc_handle<MetalSwapChain, HwSwapChain>();
 }
 
+Handle<HwSwapChain> MetalDriver::createSwapChainHeadlessS() noexcept {
+    return alloc_handle<MetalSwapChain, HwSwapChain>();
+}
+
 Handle<HwStream> MetalDriver::createStreamFromTextureIdS() noexcept {
     return {};
 }
@@ -366,8 +386,7 @@ void MetalDriver::terminate() {
     [oneOffBuffer waitUntilCompleted];
 
     mContext->bufferPool->reset();
-    [mContext->commandQueue release];
-    [mContext->driverPool drain];
+    mContext->commandQueue = nil;
 
     MetalExternalImage::shutdown();
     mContext->blitter->shutdown();
@@ -381,10 +400,18 @@ ShaderModel MetalDriver::getShaderModel() const noexcept {
 #endif
 }
 
-Handle<HwStream> MetalDriver::createStream(void* stream) {
+Handle<HwStream> MetalDriver::createStreamNative(void* stream) {
     return {};
 }
 
+Handle<HwStream> MetalDriver::createStreamAcquired() {
+    return {};
+}
+
+void MetalDriver::setAcquiredImage(Handle<HwStream> sh, void* image, backend::StreamCallback cb,
+        void* userData) {
+}
+
 void MetalDriver::setStreamDimensions(Handle<HwStream> stream, uint32_t width,
         uint32_t height) {
 
@@ -525,24 +552,25 @@ void MetalDriver::setExternalImage(Handle<HwTexture> th, void* image) {
     texture->externalImage.set((CVPixelBufferRef) image);
 }
 
+void MetalDriver::setExternalImagePlane(Handle<HwTexture> th, void* image, size_t plane) {
+    auto texture = handle_cast<MetalTexture>(mHandleMap, th);
+    texture->externalImage.set((CVPixelBufferRef) image, plane);
+}
+
 void MetalDriver::setExternalStream(Handle<HwTexture> th, Handle<HwStream> sh) {
 
 }
 
 void MetalDriver::generateMipmaps(Handle<HwTexture> th) {
-    // @autoreleasepool is used to release the one-off command buffer and encoder in case this work
-    // is done outside a frame.
-    @autoreleasepool {
-        auto tex = handle_cast<MetalTexture>(mHandleMap, th);
-        // Create a one-off command buffer to execute the blit command. Technically, we could re-use
-        // this command buffer for later rendering commands, but we'll just commit it here for
-        // simplicity.
-        id <MTLCommandBuffer> commandBuffer = [mContext->commandQueue commandBuffer];
-        id <MTLBlitCommandEncoder> blitEncoder = [commandBuffer blitCommandEncoder];
-        [blitEncoder generateMipmapsForTexture:tex->texture];
-        [blitEncoder endEncoding];
-        [commandBuffer commit];
-    }
+    auto tex = handle_cast<MetalTexture>(mHandleMap, th);
+    // Create a one-off command buffer to execute the blit command. Technically, we could re-use
+    // this command buffer for later rendering commands, but we'll just commit it here for
+    // simplicity.
+    id <MTLCommandBuffer> commandBuffer = [mContext->commandQueue commandBuffer];
+    id <MTLBlitCommandEncoder> blitEncoder = [commandBuffer blitCommandEncoder];
+    [blitEncoder generateMipmapsForTexture:tex->texture];
+    [blitEncoder endEncoding];
+    [commandBuffer commit];
 }
 
 bool MetalDriver::canGenerateMipmaps() {
@@ -638,8 +666,9 @@ void MetalDriver::beginRenderPass(Handle<HwRenderTarget> rth,
 void MetalDriver::endRenderPass(int dummy) {
     [mContext->currentCommandEncoder endEncoding];
 
-    // Command encoders are one time use. Set it to nullptr so we don't accidentally use it again.
-    mContext->currentCommandEncoder = nullptr;
+    // Command encoders are one time use. Set it to nil to release the encoder and ensure we don't
+    // accidentally use it again.
+    mContext->currentCommandEncoder = nil;
 }
 
 void MetalDriver::discardSubRenderTargetBuffers(Handle<HwRenderTarget> rth,
@@ -677,11 +706,11 @@ void MetalDriver::makeCurrent(Handle<HwSwapChain> schDraw, Handle<HwSwapChain> s
 void MetalDriver::commit(Handle<HwSwapChain> sch) {
     if (mContext->currentDrawable != nil && !mContext->frameFinishedCallback) {
         [mContext->currentCommandBuffer presentDrawable:mContext->currentDrawable];
-        [mContext->currentDrawable release];
     }
     [mContext->currentCommandBuffer commit];
     mContext->currentCommandBuffer = nil;
     mContext->currentDrawable = nil;
+    mContext->headlessDrawable = nil;
 }
 
 void MetalDriver::bindUniformBuffer(size_t index, Handle<HwUniformBuffer> ubh) {
@@ -726,10 +755,84 @@ void MetalDriver::stopCapture(int) {
     [[MTLCaptureManager sharedCaptureManager] stopCapture];
 }
 
-
 void MetalDriver::readPixels(Handle<HwRenderTarget> src, uint32_t x, uint32_t y, uint32_t width,
         uint32_t height, PixelBufferDescriptor&& data) {
+    ASSERT_PRECONDITION(mContext->currentCommandBuffer != nil &&
+                        mContext->currentCommandEncoder == nil,
+                        "readPixels must be called during a frame, but outside of a render pass.");
 
+    auto srcTarget = handle_cast<MetalRenderTarget>(mHandleMap, src);
+    id<MTLTexture> srcTexture = srcTarget->getColor();
+    size_t miplevel = srcTarget->getColorLevel();
+
+    auto chooseMetalPixelFormat = [] (PixelDataFormat format, PixelDataType type) {
+        // TODO: Add support for UINT and INT
+        if (format == PixelDataFormat::RGBA && type == PixelDataType::UBYTE) {
+                return MTLPixelFormatRGBA8Unorm;
+        }
+
+        if (format == PixelDataFormat::RGBA && type == PixelDataType::FLOAT) {
+                return MTLPixelFormatRGBA32Float;
+        }
+
+        return MTLPixelFormatInvalid;
+    };
+
+    const MTLPixelFormat format = chooseMetalPixelFormat(data.format, data.type);
+    ASSERT_PRECONDITION(format != MTLPixelFormatInvalid,
+            "The chosen combination of PixelDataFormat and PixelDataType is not supported for "
+            "readPixels.");
+    MTLTextureDescriptor* textureDescriptor =
+            [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format
+                                                               width:(srcTexture.width >> miplevel)
+                                                              height:(srcTexture.height >> miplevel)
+                                                           mipmapped:NO];
+#if defined(IOS)
+    textureDescriptor.storageMode = MTLStorageModeShared;
+#else
+    textureDescriptor.storageMode = MTLStorageModeManaged;
+#endif
+    textureDescriptor.usage = MTLTextureUsageShaderWrite | MTLTextureUsageRenderTarget;
+    id<MTLTexture> readPixelsTexture = [mContext->device newTextureWithDescriptor:textureDescriptor];
+
+    MetalBlitter::BlitArgs args;
+    args.filter = SamplerMagFilter::NEAREST;
+    args.source.level = miplevel;
+    args.source.region = MTLRegionMake2D(0, 0, srcTexture.width >> miplevel, srcTexture.height >> miplevel);
+    args.destination.level = 0;
+    args.destination.region = MTLRegionMake2D(0, 0, readPixelsTexture.width, readPixelsTexture.height);
+    args.source.color = srcTexture;
+    args.destination.color = readPixelsTexture;
+
+    mContext->blitter->blit(args);
+
+#if !defined(IOS)
+    // Managed textures on macOS require explicit synchronization between GPU / CPU.
+    id <MTLBlitCommandEncoder> blitEncoder = [mContext->currentCommandBuffer blitCommandEncoder];
+    [blitEncoder synchronizeResource:readPixelsTexture];
+    [blitEncoder endEncoding];
+#endif
+
+    // TODO: right now, every Filament frame gets its own command buffer. We should adopt a more
+    // granular approach to command buffer allocation, as this command buffer won't start executing
+    // until the entire frame has been encoded.
+
+    PixelBufferDescriptor* p = new PixelBufferDescriptor(std::move(data));
+    [mContext->currentCommandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+        size_t stride = p->stride ? p->stride : width;
+        size_t bpp = PixelBufferDescriptor::computeDataSize(p->format, p->type, 1, 1, 1);
+        size_t bpr = PixelBufferDescriptor::computeDataSize(p->format, p->type, stride, 1, p->alignment);
+        // Metal's texture coordinates have (0, 0) at the top-left of the texture, but readPixels
+        // assumes (0, 0) at bottom-left.
+        MTLRegion srcRegion = MTLRegionMake2D(x, readPixelsTexture.height - y - height, width, height);
+        const uint8_t* bufferStart = (const uint8_t*) p->buffer + (p->left * bpp) +
+                                                                  (p->top * bpr);
+        [readPixelsTexture getBytes:(void*) bufferStart
+                       bytesPerRow:bpr
+                        fromRegion:srcRegion
+                       mipmapLevel:0];
+        scheduleDestroy(std::move(*p));
+    }];
 }
 
 void MetalDriver::readStreamPixels(Handle<HwStream> sh, uint32_t x, uint32_t y, uint32_t width,
@@ -915,6 +1018,12 @@ void MetalDriver::draw(backend::PipelineState ps, Handle<HwRenderPrimitive> rph)
             texturesToBind[binding] = metalTexture->externalImage.getMetalTextureForDraw();
         }
 
+        if (!texturesToBind[binding]) {
+            utils::slog.w << "Warning: no texture bound at binding point " << (size_t) binding
+                    << "." << utils::io::endl;
+            texturesToBind[binding] = getOrCreateEmptyTexture(mContext);
+        }
+
         id <MTLSamplerState> samplerState = mContext->samplerStateCache.getOrCreateState(sampler->s);
         samplersToBind[binding] = samplerState;
     });

filament/backend/src/metal/MetalEnums.h

@@ -215,22 +215,29 @@ constexpr inline MTLPixelFormat getMetalFormat(TextureFormat format) noexcept {
         case TextureFormat::RGBA32UI: return MTLPixelFormatRGBA32Uint;
         case TextureFormat::RGBA32I: return MTLPixelFormatRGBA32Sint;
 
-        // TODO: add compressed formats
-        case TextureFormat::EAC_R11:
-        case TextureFormat::EAC_R11_SIGNED:
-        case TextureFormat::EAC_RG11:
-        case TextureFormat::EAC_RG11_SIGNED:
-        case TextureFormat::ETC2_RGB8:
-        case TextureFormat::ETC2_SRGB8:
-        case TextureFormat::ETC2_RGB8_A1:
-        case TextureFormat::ETC2_SRGB8_A1:
-        case TextureFormat::ETC2_EAC_RGBA8:
-        case TextureFormat::ETC2_EAC_SRGBA8:
+#if defined(IOS)
+        // EAC / ETC2 formats are only available on iPhone.
+        case TextureFormat::EAC_R11: return MTLPixelFormatEAC_R11Unorm;
+        case TextureFormat::EAC_R11_SIGNED: return MTLPixelFormatEAC_R11Snorm;
+        case TextureFormat::EAC_RG11: return MTLPixelFormatEAC_RG11Unorm;
+        case TextureFormat::EAC_RG11_SIGNED: return MTLPixelFormatEAC_RG11Snorm;
+        case TextureFormat::ETC2_RGB8: return MTLPixelFormatETC2_RGB8;
+        case TextureFormat::ETC2_SRGB8: return MTLPixelFormatETC2_RGB8_sRGB;
+        case TextureFormat::ETC2_RGB8_A1: return MTLPixelFormatETC2_RGB8A1;
+        case TextureFormat::ETC2_SRGB8_A1: return MTLPixelFormatETC2_RGB8A1_sRGB;
+        case TextureFormat::ETC2_EAC_RGBA8: return MTLPixelFormatEAC_RGBA8;
+        case TextureFormat::ETC2_EAC_SRGBA8: return MTLPixelFormatEAC_RGBA8_sRGB;
+#endif
 
-        case TextureFormat::DXT1_RGB:
-        case TextureFormat::DXT1_RGBA:
-        case TextureFormat::DXT3_RGBA:
-        case TextureFormat::DXT5_RGBA:
+#if !defined(IOS)
+        // DXT (BC) formats are only available on macOS desktkop.
+        // See https://en.wikipedia.org/wiki/S3_Texture_Compression#S3TC_format_comparison
+        case TextureFormat::DXT1_RGBA: return MTLPixelFormatBC1_RGBA;
+        case TextureFormat::DXT3_RGBA: return MTLPixelFormatBC2_RGBA;
+        case TextureFormat::DXT5_RGBA: return MTLPixelFormatBC3_RGBA;
+
+        case TextureFormat::DXT1_RGB: return MTLPixelFormatInvalid;
+#endif
 
         case TextureFormat::RGBA_ASTC_4x4:
         case TextureFormat::RGBA_ASTC_5x4:

filament/backend/src/metal/MetalExternalImage.h

@@ -54,6 +54,14 @@ public:
      */
     void set(CVPixelBufferRef image) noexcept;
 
+    /**
+     * Set this external image to a specific plane of the passed-in CVPixelBuffer. Future calls to
+     * getMetalTextureForDraw will return a texture backed by a single plane of this CVPixelBuffer.
+     * Previous CVPixelBuffers and related resources will be released when all GPU work using them
+     * has finished.
+     */
+    void set(CVPixelBufferRef image, size_t plane) noexcept;
+
     /**
      * Get a Metal texture used to draw this image and denote that it is used for the current frame.
      * For future frames that use this external image, getMetalTextureForDraw must be called again.
@@ -68,6 +76,8 @@ public:
 
 private:
 
+    void unset();
+
     CVMetalTextureRef createTextureFromImage(CVPixelBufferRef image, MTLPixelFormat format,
             size_t plane);
     id<MTLTexture> createRgbTexture(size_t width, size_t height);

filament/backend/src/metal/MetalExternalImage.mm

@@ -78,21 +78,12 @@ bool MetalExternalImage::isValid() const noexcept {
 }
 
 void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
-    CVPixelBufferRelease(mImage);
-    CVBufferRelease(mTexture);
-    [mRgbTexture release];
-
-    mImage = nullptr;
-    mTexture = nullptr;
-    mRgbTexture = nil;
+    unset();
 
     if (!image) {
         return;
     }
 
-    // This pool is necessary because set is called outside of a frame.
-    NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
-
     OSType formatType = CVPixelBufferGetPixelFormatType(image);
     ASSERT_POSTCONDITION(formatType == kCVPixelFormatType_32BGRA ||
                          formatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
@@ -131,8 +122,37 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
 
         [commandBuffer commit];
     }
+}
 
-    [pool drain];
+void MetalExternalImage::set(CVPixelBufferRef image, size_t plane) noexcept {
+    unset();
+
+    if (!image) {
+        return;
+    }
+
+    const OSType formatType = CVPixelBufferGetPixelFormatType(image);
+    ASSERT_POSTCONDITION(formatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
+            "Metal planar external images must be in the 420f format.");
+
+    mImage = image;
+
+    auto getPlaneFormat = [] (size_t plane) {
+      // Right now Metal only supports kCVPixelFormatType_420YpCbCr8BiPlanarFullRange planar
+      // external images, so we can make the following assumptions about the format of each plane.
+      if (plane == 0) {
+        return MTLPixelFormatR8Unorm; // luminance
+      }
+      if (plane == 1) {
+        // CbCr
+        return MTLPixelFormatRG8Unorm; // CbCr
+      }
+      return MTLPixelFormatInvalid;
+    };
+
+    const MTLPixelFormat format = getPlaneFormat(plane);
+    assert(format != MTLPixelFormatInvalid);
+    mTexture = createTextureFromImage(image, format, plane);
 }
 
 id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
@@ -145,10 +165,10 @@ id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
     // lifetime is automatically managed by Metal.
     auto& tracker = mContext.resourceTracker;
     auto commandBuffer = mContext.currentCommandBuffer;
-    if (tracker.trackResource(commandBuffer, mImage, cvBufferDeleter)) {
+    if (tracker.trackResource((__bridge void*) commandBuffer, mImage, cvBufferDeleter)) {
         CVPixelBufferRetain(mImage);
     }
-    if (tracker.trackResource(commandBuffer, mTexture, cvBufferDeleter)) {
+    if (tracker.trackResource((__bridge void*) commandBuffer, mTexture, cvBufferDeleter)) {
         CVBufferRetain(mTexture);
     }
 
@@ -157,8 +177,8 @@ id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
 
 CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVPixelBufferRef image,
         MTLPixelFormat format, size_t plane) {
-    size_t width = CVPixelBufferGetWidthOfPlane(image, plane);
-    size_t height = CVPixelBufferGetHeightOfPlane(image, plane);
+    const size_t width = CVPixelBufferGetWidthOfPlane(image, plane);
+    const size_t height = CVPixelBufferGetHeightOfPlane(image, plane);
 
     CVMetalTextureRef texture;
     CVReturn result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
@@ -170,10 +190,18 @@ CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVPixelBufferRef im
 }
 
 void MetalExternalImage::shutdown() noexcept {
-    [gComputePipelineState release];
     gComputePipelineState = nil;
 }
 
+void MetalExternalImage::unset() {
+    CVPixelBufferRelease(mImage);
+    CVBufferRelease(mTexture);
+
+    mImage = nullptr;
+    mTexture = nullptr;
+    mRgbTexture = nil;
+}
+
 id<MTLTexture> MetalExternalImage::createRgbTexture(size_t width, size_t height) {
     MTLTextureDescriptor *descriptor =
             [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm
@@ -200,12 +228,10 @@ void MetalExternalImage::ensureComputePipelineState() {
     NSERROR_CHECK("Unable to compile Metal shading library.");
 
     id<MTLFunction> kernelFunction = [library newFunctionWithName:@"ycbcrToRgb"];
-    [library release];
 
     gComputePipelineState = [mContext.device newComputePipelineStateWithFunction:kernelFunction
                                                                            error:&error];
     NSERROR_CHECK("Unable to create Metal compute pipeline state.");
-    [kernelFunction release];
 }
 
 id<MTLCommandBuffer> MetalExternalImage::encodeColorConversionPass(id<MTLTexture> inYPlane,

filament/backend/src/metal/MetalHandles.h

@@ -43,21 +43,25 @@ namespace metal {
 struct MetalSwapChain : public HwSwapChain {
     MetalSwapChain(id<MTLDevice> device, CAMetalLayer* nativeWindow);
 
+    // Instantiate a headless SwapChain.
+    MetalSwapChain(int32_t width, int32_t height);
+
+    bool isHeadless() { return layer == nullptr; }
+
     CAMetalLayer* layer = nullptr;
+    NSUInteger surfaceWidth = 0;
     NSUInteger surfaceHeight = 0;
 };
 
 struct MetalVertexBuffer : public HwVertexBuffer {
     MetalVertexBuffer(id<MTLDevice> device, uint8_t bufferCount, uint8_t attributeCount,
             uint32_t vertexCount, AttributeArray const& attributes);
-    ~MetalVertexBuffer();
 
     std::vector<id<MTLBuffer>> buffers;
 };
 
 struct MetalIndexBuffer : public HwIndexBuffer {
     MetalIndexBuffer(id<MTLDevice> device, uint8_t elementSize, uint32_t indexCount);
-    ~MetalIndexBuffer();
 
     id<MTLBuffer> buffer;
 };
@@ -99,7 +103,7 @@ private:
 struct MetalRenderPrimitive : public HwRenderPrimitive {
     void setBuffers(MetalVertexBuffer* vertexBuffer, MetalIndexBuffer* indexBuffer,
             uint32_t enabledAttributes);
-    // The pointers to MetalVertexBuffer, MetalIndexBuffer, and id<MTLBuffer> are "weak".
+    // The pointers to MetalVertexBuffer and MetalIndexBuffer are "weak".
     // The MetalVertexBuffer and MetalIndexBuffer must outlive the MetalRenderPrimitive.
 
     MetalVertexBuffer* vertexBuffer = nullptr;
@@ -114,7 +118,6 @@ struct MetalRenderPrimitive : public HwRenderPrimitive {
 
 struct MetalProgram : public HwProgram {
     MetalProgram(id<MTLDevice> device, const Program& program) noexcept;
-    ~MetalProgram();
 
     id<MTLFunction> vertexFunction;
     id<MTLFunction> fragmentFunction;
@@ -132,10 +135,13 @@ struct MetalTexture : public HwTexture {
     void loadCubeImage(const PixelBufferDescriptor& data, const FaceOffsets& faceOffsets,
             int miplevel);
 
+    NSUInteger getBytesPerRow(PixelDataType type, NSUInteger width) const noexcept;
+
     MetalContext& context;
     MetalExternalImage externalImage;
     id<MTLTexture> texture = nil;
-    uint8_t bytesPerPixel;
+    uint8_t bytesPerElement; // The number of bytes per pixel, or block (for compressed texture formats).
+    uint8_t blockWidth; // The number of horizontal pixels per block (only for compressed texture formats).
     TextureReshaper reshaper;
 };
 
@@ -149,7 +155,6 @@ public:
             id<MTLTexture> color, id<MTLTexture> depth, uint8_t colorLevel, uint8_t depthLevel);
     explicit MetalRenderTarget(MetalContext* context)
             : HwRenderTarget(0, 0), context(context), defaultRenderTarget(true) {}
-    ~MetalRenderTarget();
 
     bool isDefaultRenderTarget() const { return defaultRenderTarget; }
     uint8_t getSamples() const { return samples; }
@@ -185,7 +190,6 @@ class MetalFence : public HwFence {
 public:
 
     MetalFence(MetalContext& context);
-    ~MetalFence();
 
     FenceStatus wait(uint64_t timeoutNs);
 

filament/backend/src/metal/MetalHandles.mm

@@ -23,6 +23,8 @@
 #include <utils/Panic.h>
 #include <utils/trap.h>
 
+#include <math.h>
+
 namespace filament {
 namespace backend {
 namespace metal {
@@ -60,9 +62,13 @@ MetalSwapChain::MetalSwapChain(id<MTLDevice> device, CAMetalLayer* nativeWindow)
         : layer(nativeWindow) {
     layer.device = device;
     CGSize size = layer.drawableSize;
-    surfaceHeight = (NSUInteger)(size.height);
+    surfaceHeight = (NSUInteger) (size.height);
+    surfaceWidth = (NSUInteger) (size.width);
 }
 
+MetalSwapChain::MetalSwapChain(int32_t width, int32_t height) : surfaceWidth(width),
+        surfaceHeight(height) { }
+
 MetalVertexBuffer::MetalVertexBuffer(id<MTLDevice> device, uint8_t bufferCount, uint8_t attributeCount,
             uint32_t vertexCount, AttributeArray const& attributes)
     : HwVertexBuffer(bufferCount, attributeCount, vertexCount, attributes) {
@@ -87,22 +93,12 @@ MetalVertexBuffer::MetalVertexBuffer(id<MTLDevice> device, uint8_t bufferCount,
     }
 }
 
-MetalVertexBuffer::~MetalVertexBuffer() {
-    for (auto buffer : buffers) {
-        [buffer release];
-    }
-}
-
 MetalIndexBuffer::MetalIndexBuffer(id<MTLDevice> device, uint8_t elementSize, uint32_t indexCount)
     : HwIndexBuffer(elementSize, indexCount) {
     buffer = [device newBufferWithLength:(elementSize * indexCount)
                                  options:MTLResourceStorageModeShared];
 }
 
-MetalIndexBuffer::~MetalIndexBuffer() {
-    [buffer release];
-}
-
 MetalUniformBuffer::MetalUniformBuffer(MetalContext& context, size_t size) : HwUniformBuffer(),
         uniformSize(size), context(context) {
     ASSERT_PRECONDITION(size > 0, "Cannot create Metal uniform with size %d.", size);
@@ -168,7 +164,8 @@ id<MTLBuffer> MetalUniformBuffer::getGpuBufferForDraw() {
         bufferPool->releaseBuffer((const MetalBufferPoolEntry*) resource);
     };
     id<MTLCommandBuffer> commandBuffer = context.currentCommandBuffer;
-    if (context.resourceTracker.trackResource(commandBuffer, bufferPoolEntry, uniformDeleter)) {
+    if (context.resourceTracker.trackResource((__bridge void*) commandBuffer, bufferPoolEntry,
+            uniformDeleter)) {
         // We only want to retain the buffer once per command buffer- trackResource will return
         // true if this is the first time tracking this uniform for this command buffer.
         context.bufferPool->retainBuffer(bufferPoolEntry);
@@ -238,7 +235,7 @@ void MetalRenderPrimitive::setBuffers(MetalVertexBuffer* vertexBuffer, MetalInde
 MetalProgram::MetalProgram(id<MTLDevice> device, const Program& program) noexcept
     : HwProgram(program.getName()) {
 
-    using MetalFunctionPtr = id<MTLFunction>*;
+    using MetalFunctionPtr = __strong id<MTLFunction>*;
 
     static_assert(Program::SHADER_TYPE_COUNT == 2, "Only vertex and fragment shaders expected.");
     MetalFunctionPtr shaderFunctions[2] = { &vertexFunction, &fragmentFunction };
@@ -260,8 +257,6 @@ MetalProgram::MetalProgram(id<MTLDevice> device, const Program& program) noexcep
         id<MTLLibrary> library = [device newLibraryWithSource:objcSource
                                                       options:nil
                                                         error:&error];
-        [objcSource release];
-        [options release];
         if (library == nil) {
             if (error) {
                 auto description =
@@ -272,18 +267,11 @@ MetalProgram::MetalProgram(id<MTLDevice> device, const Program& program) noexcep
         }
 
         *shaderFunctions[i] = [library newFunctionWithName:@"main0"];
-
-        [library release];
     }
 
     samplerGroupInfo = program.getSamplerGroupInfo();
 }
 
-MetalProgram::~MetalProgram() {
-    [vertexFunction release];
-    [fragmentFunction release];
-}
-
 static MTLPixelFormat decidePixelFormat(id<MTLDevice> device, TextureFormat format) {
     const MTLPixelFormat metalFormat = getMetalFormat(format);
 #if !defined(IOS)
@@ -307,7 +295,9 @@ MetalTexture::MetalTexture(MetalContext& context, backend::SamplerType target, u
     const TextureFormat reshapedFormat = reshaper.getReshapedFormat();
     const MTLPixelFormat pixelFormat = decidePixelFormat(context.device, reshapedFormat);
 
-    bytesPerPixel = static_cast<uint8_t>(getFormatSize(reshapedFormat));
+    bytesPerElement = static_cast<uint8_t>(getFormatSize(reshapedFormat));
+    assert(bytesPerElement > 0);
+    blockWidth = static_cast<uint8_t>(getBlockWidth(reshapedFormat));
 
     ASSERT_POSTCONDITION(pixelFormat != MTLPixelFormatInvalid, "Pixel format not supported.");
 
@@ -348,7 +338,6 @@ MetalTexture::MetalTexture(MetalContext& context, backend::SamplerType target, u
 }
 
 MetalTexture::~MetalTexture() {
-    [texture release];
     externalImage.set(nullptr);
 }
 
@@ -368,7 +357,7 @@ void MetalTexture::load2DImage(uint32_t level, uint32_t xoffset, uint32_t yoffse
             .depth = 1
         }
     };
-    NSUInteger bytesPerRow = bytesPerPixel * width;
+    const NSUInteger bytesPerRow = getBytesPerRow(data.type, width);
     [texture replaceRegion:region
                mipmapLevel:level
                      slice:0
@@ -381,8 +370,9 @@ void MetalTexture::load2DImage(uint32_t level, uint32_t xoffset, uint32_t yoffse
 
 void MetalTexture::loadCubeImage(const PixelBufferDescriptor& data, const FaceOffsets& faceOffsets,
         int miplevel) {
-    NSUInteger faceWidth = width >> miplevel;
-    NSUInteger bytesPerRow = bytesPerPixel * faceWidth;
+    const NSUInteger faceWidth = width >> miplevel;
+    const NSUInteger bytesPerRow = getBytesPerRow(data.type, faceWidth);
+
     MTLRegion region = MTLRegionMake2D(0, 0, faceWidth, faceWidth);
     for (NSUInteger slice = 0; slice < 6; slice++) {
         FaceOffsets::size_type faceOffset = faceOffsets.offsets[slice];
@@ -395,15 +385,26 @@ void MetalTexture::loadCubeImage(const PixelBufferDescriptor& data, const FaceOf
     }
 }
 
+NSUInteger MetalTexture::getBytesPerRow(PixelDataType type, NSUInteger width) const noexcept {
+    // From https://developer.apple.com/documentation/metal/mtltexture/1515464-replaceregion:
+    // For an ordinary or packed pixel format, the stride, in bytes, between rows of source data.
+    // For a compressed pixel format, the stride is the number of bytes from the beginning of one
+    // row of blocks to the beginning of the next.
+    if (type == PixelDataType::COMPRESSED) {
+        assert(blockWidth > 0);
+        const NSUInteger blocksPerRow = std::ceil(width / (float) blockWidth);
+        return bytesPerElement * blocksPerRow;
+    } else {
+        return bytesPerElement * width;
+    }
+}
+
 MetalRenderTarget::MetalRenderTarget(MetalContext* context, uint32_t width, uint32_t height,
         uint8_t samples, id<MTLTexture> color, id<MTLTexture> depth, uint8_t colorLevel,
         uint8_t depthLevel) : HwRenderTarget(width, height), context(context), samples(samples),
         colorLevel(colorLevel), depthLevel(depthLevel) {
     ASSERT_PRECONDITION(color || depth, "Must provide either a color or depth texture.");
 
-    [color retain];
-    [depth retain];
-
     if (color) {
         if (color.textureType == MTLTextureType2DMultisample) {
             this->multisampledColor = color;
@@ -442,7 +443,7 @@ MetalRenderTarget::MetalRenderTarget(MetalContext* context, uint32_t width, uint
 
 id<MTLTexture> MetalRenderTarget::getColor() {
     if (defaultRenderTarget) {
-        return acquireDrawable(context).texture;
+        return acquireDrawable(context);
     }
     if (multisampledColor) {
         return multisampledColor;
@@ -451,6 +452,9 @@ id<MTLTexture> MetalRenderTarget::getColor() {
 }
 
 id<MTLTexture> MetalRenderTarget::getDepth() {
+    if (defaultRenderTarget) {
+        return acquireDepthTexture(context);
+    }
     if (multisampledDepth) {
         return multisampledDepth;
     }
@@ -511,13 +515,6 @@ MTLStoreAction MetalRenderTarget::getStoreAction(const RenderPassParams& params,
     return MTLStoreActionStore;
 }
 
-MetalRenderTarget::~MetalRenderTarget() {
-    [color release];
-    [depth release];
-    [multisampledColor release];
-    [multisampledDepth release];
-}
-
 id<MTLTexture> MetalRenderTarget::createMultisampledTexture(id<MTLDevice> device,
         MTLPixelFormat format, uint32_t width, uint32_t height, uint8_t samples) {
     MTLTextureDescriptor* descriptor =
@@ -537,7 +534,6 @@ id<MTLTexture> MetalRenderTarget::createMultisampledTexture(id<MTLDevice> device
     return [device newTextureWithDescriptor:descriptor];
 }
 
-
 MetalFence::MetalFence(MetalContext& context) {
 #if METAL_FENCES_SUPPORTED
     cv = std::make_shared<std::condition_variable>();
@@ -556,12 +552,6 @@ MetalFence::MetalFence(MetalContext& context) {
 #endif
 }
 
-MetalFence::~MetalFence() {
-#if METAL_FENCES_SUPPORTED
-    [event release];
-#endif
-}
-
 FenceStatus MetalFence::wait(uint64_t timeoutNs) {
 #if METAL_FENCES_SUPPORTED
     std::unique_lock<std::mutex> guard(mutex);

filament/backend/src/metal/MetalState.h

@@ -128,9 +128,7 @@ static_assert(sizeof(BlendState) == 56, "BlendState is unexpected size.");
 
 // StateCache caches Metal state objects using StateType as a key.
 // MetalType is the corresponding Metal API type.
-// StateCreator is a functor that creates a new state of type MetalType. It is assumed that this
-// type is created with a positive reference count (i.e., a new* method, per Apple convention), and
-// thus each cached object is released in StateCache's destructor.
+// StateCreator is a functor that creates a new state of type MetalType.
 template<typename StateType,
          typename MetalType,
          typename StateCreator>
@@ -143,12 +141,6 @@ public:
     StateCache(const StateCache&) = delete;
     StateCache& operator=(const StateCache&) = delete;
 
-    ~StateCache() {
-        for (auto it = mStateCache.begin(); it != mStateCache.end(); ++it) {
-            [it.value() release];
-        }
-    }
-
     void setDevice(id<MTLDevice> device) noexcept { mDevice = device; }
 
     MetalType getOrCreateState(const StateType& state) noexcept {

filament/backend/src/metal/MetalState.mm

@@ -84,8 +84,6 @@ id<MTLRenderPipelineState> PipelineStateCreator::operator()(id<MTLDevice> device
     }
     ASSERT_POSTCONDITION(error == nil, "Could not create Metal pipeline state.");
 
-    [descriptor release];
-
     return pipeline;
 }
 
@@ -94,15 +92,12 @@ id<MTLDepthStencilState> DepthStateCreator::operator()(id<MTLDevice> device,
     MTLDepthStencilDescriptor* depthStencilDescriptor = [MTLDepthStencilDescriptor new];
     depthStencilDescriptor.depthCompareFunction = state.compareFunction;
     depthStencilDescriptor.depthWriteEnabled = state.depthWriteEnabled;
-    id<MTLDepthStencilState> depthStencilState =
-            [device newDepthStencilStateWithDescriptor:depthStencilDescriptor];
-    [depthStencilDescriptor release];
-    return depthStencilState;
+    return [device newDepthStencilStateWithDescriptor:depthStencilDescriptor];
 }
 
 id<MTLSamplerState> SamplerStateCreator::operator()(id<MTLDevice> device,
         const backend::SamplerParams& state) noexcept {
-    MTLSamplerDescriptor* samplerDescriptor = [[MTLSamplerDescriptor new] autorelease];
+    MTLSamplerDescriptor* samplerDescriptor = [MTLSamplerDescriptor new];
     samplerDescriptor.minFilter = getFilter(state.filterMin);
     samplerDescriptor.magFilter = getFilter(state.filterMag);
     samplerDescriptor.mipFilter = getMipFilter(state.filterMin);

filament/backend/src/opengl/OpenGLContext.cpp

@@ -36,11 +36,7 @@ OpenGLContext::OpenGLContext() noexcept {
     UTILS_UNUSED char const* const shader   = (char const*) glGetString(GL_SHADING_LANGUAGE_VERSION);
 
 #ifndef NDEBUG
-    slog.i
-        << vendor << io::endl
-        << renderer << io::endl
-        << version << io::endl
-        << shader << io::endl;
+    slog.i << vendor << ", " << renderer << ", " << version << ", " << shader << io::endl;
 #endif
 
     // OpenGL (ES) version
@@ -51,7 +47,8 @@ OpenGLContext::OpenGLContext() noexcept {
     glGetIntegerv(GL_MAX_UNIFORM_BLOCK_SIZE, &gets.max_uniform_block_size);
     glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &gets.uniform_buffer_offset_alignment);
 
-#ifndef NDEBUG
+#if 0
+    // this is useful for development, but too verbose even for debug builds
     slog.i
         << "GL_MAX_RENDERBUFFER_SIZE = " << gets.max_renderbuffer_size << io::endl
         << "GL_MAX_UNIFORM_BLOCK_SIZE = " << gets.max_uniform_block_size << io::endl

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -163,6 +163,15 @@ OpenGLDriver::~OpenGLDriver() noexcept {
 // ------------------------------------------------------------------------------------------------
 
 void OpenGLDriver::terminate() {
+    // wait for the GPU to finish executing all commands
+    glFinish();
+
+    // and make sure to execute all the GpuCommandCompleteOps callbacks
+    executeGpuCommandsCompleteOps();
+
+    // because we called glFinish(), all callbacks should have been executed
+    assert(!mGpuCommandCompleteOps.size());
+
     for (auto& item : mSamplerMap) {
         mContext.unbindSampler(item.second);
         glDeleteSamplers(1, &item.second);
@@ -328,7 +337,7 @@ void OpenGLDriver::setRasterStateSlow(RasterState rs) noexcept {
 // For reference on a 64-bits machine:
 //    GLFence                   :  8
 //    GLIndexBuffer             : 12        moderate
-//    GLSamplerGroup           : 16        moderate
+//    GLSamplerGroup            : 16        moderate
 // -- less than 16 bytes
 
 //    GLRenderPrimitive         : 40        many
@@ -351,7 +360,8 @@ OpenGLDriver::HandleAllocator::HandleAllocator(const utils::HeapArea& area)
           mPool2( pointermath::add(area.begin(), (6 * area.getSize()) / 16),
                   area.end()) {
 
-#ifndef NDEBUG
+#if 0
+    // this is useful for development, but too verbose even for debug builds
     slog.d << "HwFence: " << sizeof(HwFence) << io::endl;
     slog.d << "GLIndexBuffer: " << sizeof(GLIndexBuffer) << io::endl;
     slog.d << "GLSamplerGroup: " << sizeof(GLSamplerGroup) << io::endl;
@@ -464,6 +474,10 @@ Handle<HwSwapChain> OpenGLDriver::createSwapChainS() noexcept {
     return Handle<HwSwapChain>( allocateHandle(sizeof(HwSwapChain)) );
 }
 
+Handle<HwSwapChain> OpenGLDriver::createSwapChainHeadlessS() noexcept {
+    return Handle<HwSwapChain>( allocateHandle(sizeof(HwSwapChain)) );
+}
+
 Handle<HwStream> OpenGLDriver::createStreamFromTextureIdS() noexcept {
     return Handle<HwStream>( allocateHandle(sizeof(GLStream)) );
 }
@@ -983,6 +997,14 @@ void OpenGLDriver::createSwapChainR(Handle<HwSwapChain> sch, void* nativeWindow,
     sc->swapChain = mPlatform.createSwapChain(nativeWindow, flags);
 }
 
+void OpenGLDriver::createSwapChainHeadlessR(Handle<HwSwapChain> sch,
+        uint32_t width, uint32_t height, uint64_t flags) {
+    DEBUG_MARKER()
+
+    HwSwapChain* sc = construct<HwSwapChain>(sch);
+    sc->swapChain = mPlatform.createSwapChain(width, height, flags);
+}
+
 void OpenGLDriver::createStreamFromTextureIdR(Handle<HwStream> sh,
         intptr_t externalTextureId, uint32_t width, uint32_t height) {
     DEBUG_MARKER()
@@ -993,6 +1015,7 @@ void OpenGLDriver::createStreamFromTextureIdR(Handle<HwStream> sh,
     s->width = width;
     s->height = height;
     s->gl.externalTextureId = static_cast<GLuint>(externalTextureId);
+    s->streamType = StreamType::TEXTURE_ID;
     glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read);
     glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write);
     for (auto& info : s->user_thread.infos) {
@@ -1139,9 +1162,9 @@ void OpenGLDriver::destroyStream(Handle<HwStream> sh) {
         if (pos != externalStreams.end()) {
             detachStream(*pos);
         }
-        if (s->isNativeStream()) {
+        if (s->streamType == StreamType::NATIVE) {
             mPlatform.destroyStream(s->stream);
-        } else {
+        } else if (s->streamType == StreamType::TEXTURE_ID) {
             glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read);
             glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write);
             if (s->gl.fbo) {
@@ -1160,13 +1183,35 @@ void OpenGLDriver::destroyStream(Handle<HwStream> sh) {
 // These are called on the application's thread
 // ------------------------------------------------------------------------------------------------
 
-Handle<HwStream> OpenGLDriver::createStream(void* nativeStream) {
+Handle<HwStream> OpenGLDriver::createStreamNative(void* nativeStream) {
     Handle<HwStream> sh( allocateHandle(sizeof(GLStream)) );
     Platform::Stream* stream = mPlatform.createStream(nativeStream);
     construct<GLStream>(sh, stream);
     return sh;
 }
 
+Handle<HwStream> OpenGLDriver::createStreamAcquired() {
+    Handle<HwStream> sh(allocateHandle(sizeof(GLStream)));
+    construct<GLStream>(sh);
+    return sh;
+}
+
+// Stashes an acquired external image and a release callback. The image is not bound to OpenGL until
+// the subsequent call to beginFrame (see updateStreamAcquired).
+//
+// setAcquiredImage should be called by the user outside of beginFrame / endFrame, and should be
+// called only once per frame. If the user pushes images to the same stream multiple times in a
+// single frame, we emit a warning and honor only the final image, but still invoke all callbacks.
+void OpenGLDriver::setAcquiredImage(Handle<HwStream> sh, void* hwbuffer,
+        backend::StreamCallback cb, void* userData) {
+    GLStream* glstream = handle_cast<GLStream*>(sh);
+    if (glstream->user_thread.pending.image) {
+        scheduleRelease(std::move(glstream->user_thread.pending));
+        slog.w << "Acquired image is set more than once per frame." << io::endl;
+    }
+    glstream->user_thread.pending = mPlatform.transformAcquiredImage({hwbuffer, cb, userData});
+}
+
 void OpenGLDriver::updateStreams(DriverApi* driver) {
     if (UTILS_UNLIKELY(!mExternalStreams.empty())) {
         OpenGLBlitter::State state;
@@ -1176,13 +1221,17 @@ void OpenGLDriver::updateStreams(DriverApi* driver) {
             GLStream* s = static_cast<GLStream*>(t->hwStream);
             if (UTILS_UNLIKELY(s == nullptr)) {
                 // this can happen because we're called synchronously and the setExternalStream()
-                // call may bot have been processed yet.
+                // call may not have been processed yet.
                 continue;
             }
 
-            if (!s->isNativeStream()) {
+            if (s->streamType == StreamType::TEXTURE_ID) {
                 state.setup();
-                updateStream(t, driver);
+                updateStreamTexId(t, driver);
+            }
+
+            if (s->streamType == StreamType::ACQUIRED) {
+                updateStreamAcquired(t, driver);
             }
         }
     }
@@ -1696,10 +1745,16 @@ void OpenGLDriver::cancelExternalImage(void* image) {
 }
 
 void OpenGLDriver::setExternalImage(Handle<HwTexture> th, void* image) {
-    GLTexture* t = handle_cast<GLTexture*>(th);
-    auto& gl = mContext;
+    mPlatform.setExternalImage(image, handle_cast<GLTexture*>(th));
+    setExternalTexture(handle_cast<GLTexture*>(th), image);
+}
 
-    mPlatform.setExternalImage(image, t);
+void OpenGLDriver::setExternalImagePlane(Handle<HwTexture> th, void* image, size_t plane) {
+
+}
+
+void OpenGLDriver::setExternalTexture(GLTexture* t, void* image) {
+    auto& gl = mContext;
 
     // TODO: move this logic to PlatformEGL.
     if (gl.ext.OES_EGL_image_external_essl3) {
@@ -1726,7 +1781,7 @@ void OpenGLDriver::setExternalStream(Handle<HwTexture> th, Handle<HwStream> sh)
         if (UTILS_LIKELY(sh)) {
             GLStream* s = handle_cast<GLStream*>(sh);
             if (UTILS_LIKELY(!t->hwStream)) {
-                // we're not attached alread
+                // we're not attached already
                 attachStream(t, s);
             } else {
                 if (s->stream != t->hwStream->stream) {
@@ -1746,20 +1801,26 @@ void OpenGLDriver::attachStream(GLTexture* t, GLStream* hwStream) noexcept {
     auto& gl = mContext;
     mExternalStreams.push_back(t);
 
-    if (hwStream->isNativeStream()) {
-        mPlatform.attach(hwStream->stream, t->gl.id);
-    } else {
-        assert(t->target == SamplerType::SAMPLER_EXTERNAL);
-        // The texture doesn't need a texture name anymore, get rid of it
-        gl.unbindTexture(t->gl.target, t->gl.id);
-        glDeleteTextures(1, &t->gl.id);
-        t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur];
+    switch (hwStream->streamType) {
+        case StreamType::NATIVE:
+            mPlatform.attach(hwStream->stream, t->gl.id);
+            break;
+        case StreamType::TEXTURE_ID:
+            assert(t->target == SamplerType::SAMPLER_EXTERNAL);
+            // The texture doesn't need a texture name anymore, get rid of it
+            gl.unbindTexture(t->gl.target, t->gl.id);
+            glDeleteTextures(1, &t->gl.id);
+            t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur];
+            break;
+        case StreamType::ACQUIRED:
+            break;
     }
     t->hwStream = hwStream;
 }
 
 UTILS_NOINLINE
 void OpenGLDriver::detachStream(GLTexture* t) noexcept {
+    auto& gl = mContext;
     auto& streams = mExternalStreams;
     auto pos = std::find(streams.begin(), streams.end(), t);
     if (pos != streams.end()) {
@@ -1767,30 +1828,57 @@ void OpenGLDriver::detachStream(GLTexture* t) noexcept {
     }
 
     GLStream* s = static_cast<GLStream*>(t->hwStream);
-    if (s->isNativeStream()) {
-        mPlatform.detach(t->hwStream->stream);
-        // this deletes the texture id
+    switch (s->streamType) {
+        case StreamType::NATIVE:
+            mPlatform.detach(t->hwStream->stream);
+            // ^ this deletes the texture id
+            break;
+        case StreamType::TEXTURE_ID:
+            break;
+        case StreamType::ACQUIRED:
+            gl.unbindTexture(t->gl.target, t->gl.id);
+            glDeleteTextures(1, &t->gl.id);
+            break;
     }
+
     glGenTextures(1, &t->gl.id);
+
     t->hwStream = nullptr;
 }
 
 UTILS_NOINLINE
-void OpenGLDriver::replaceStream(GLTexture* t, GLStream* hwStream) noexcept {
-    GLStream* s = static_cast<GLStream*>(t->hwStream);
-    if (s->isNativeStream()) {
-        mPlatform.detach(t->hwStream->stream);
-        // this deletes the texture id
+void OpenGLDriver::replaceStream(GLTexture* texture, GLStream* newStream) noexcept {
+    assert(newStream && "Do not use replaceStream to detach a stream.");
+
+    // This could be implemented via detachStream + attachStream but inlining allows
+    // a few small optimizations, like not touching the mExternalStreams list.
+
+    GLStream* oldStream = static_cast<GLStream*>(texture->hwStream);
+    switch (oldStream->streamType) {
+        case StreamType::NATIVE:
+            mPlatform.detach(texture->hwStream->stream);
+            // ^ this deletes the texture id
+            break;
+        case StreamType::TEXTURE_ID:
+        case StreamType::ACQUIRED:
+            break;
     }
 
-    if (hwStream->isNativeStream()) {
-        glGenTextures(1, &t->gl.id);
-        mPlatform.attach(hwStream->stream, t->gl.id);
-    } else {
-        assert(t->target == SamplerType::SAMPLER_EXTERNAL);
-        t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur];
+    switch (newStream->streamType) {
+        case StreamType::NATIVE:
+            glGenTextures(1, &texture->gl.id);
+            mPlatform.attach(newStream->stream, texture->gl.id);
+            break;
+        case StreamType::TEXTURE_ID:
+            assert(texture->target == SamplerType::SAMPLER_EXTERNAL);
+            texture->gl.id = newStream->user_thread.read[newStream->user_thread.cur];
+            break;
+        case StreamType::ACQUIRED:
+            // Just re-use the old texture id.
+            break;
     }
-    t->hwStream = hwStream;
+
+    texture->hwStream = newStream;
 }
 
 void OpenGLDriver::beginRenderPass(Handle<HwRenderTarget> rth,
@@ -1805,20 +1893,18 @@ void OpenGLDriver::beginRenderPass(Handle<HwRenderTarget> rth,
     TargetBufferFlags discardFlags = params.flags.discardStart;
 
     GLRenderTarget* rt = handle_cast<GLRenderTarget*>(rth);
-    if (UTILS_UNLIKELY(gl.getDrawFbo() != rt->gl.fbo)) {
-        gl.bindFramebuffer(GL_FRAMEBUFFER, rt->gl.fbo);
+    gl.bindFramebuffer(GL_FRAMEBUFFER, rt->gl.fbo);
 
-        // glInvalidateFramebuffer appeared on GLES 3.0 and GL4.3, for simplicity we just
-        // ignore it on GL (rather than having to do a runtime check).
-        if (GLES30_HEADERS) {
-            if (!gl.bugs.disable_invalidate_framebuffer) {
-                std::array<GLenum, 3> attachments; // NOLINT
-                GLsizei attachmentCount = getAttachments(attachments, rt, discardFlags);
-                if (attachmentCount) {
-                    glInvalidateFramebuffer(GL_FRAMEBUFFER, attachmentCount, attachments.data());
-                }
-                CHECK_GL_ERROR(utils::slog.e)
+    // glInvalidateFramebuffer appeared on GLES 3.0 and GL4.3, for simplicity we just
+    // ignore it on GL (rather than having to do a runtime check).
+    if (GLES30_HEADERS) {
+        if (!gl.bugs.disable_invalidate_framebuffer) {
+            std::array<GLenum, 3> attachments; // NOLINT
+            GLsizei attachmentCount = getAttachments(attachments, rt, discardFlags);
+            if (attachmentCount) {
+                glInvalidateFramebuffer(GL_FRAMEBUFFER, attachmentCount, attachments.data());
             }
+            CHECK_GL_ERROR(utils::slog.e)
         }
     }
 
@@ -2092,19 +2178,49 @@ void OpenGLDriver::setViewportScissor(Viewport const& viewportScissor) noexcept
     gl.enable(GL_SCISSOR_TEST);
 }
 
-/*
- * This is called in the user thread
- */
+// Binds the external image stashed in the associated stream.
+//
+// updateStreamAcquired() and setAcquiredImage() are both called from on the application's thread
+// and therefore do not require synchronization. The former is always called immediately before
+// beginFrame, the latter is called by the user from anywhere outside beginFrame / endFrame.
+void OpenGLDriver::updateStreamAcquired(GLTexture* gltexture, DriverApi* driver) noexcept {
+    SYSTRACE_CALL();
+
+    GLStream* glstream = static_cast<GLStream*>(gltexture->hwStream);
+    assert(glstream);
+    assert(glstream->streamType == StreamType::ACQUIRED);
+
+    // If there's no pending image, do nothing. Note that GL_OES_EGL_image does not let you pass
+    // NULL to glEGLImageTargetTexture2DOES, and there is no concept of "detaching" an EGLimage from
+    // a texture.
+    if (glstream->user_thread.pending.image == nullptr) {
+        return;
+    }
+
+    AcquiredImage previousImage = glstream->user_thread.acquired;
+    glstream->user_thread.acquired = glstream->user_thread.pending;
+    glstream->user_thread.pending = {0};
+
+    // Bind the stashed EGLImage to its corresponding GL texture as soon as we start making the GL
+    // calls for the upcoming frame.
+    void* image = glstream->user_thread.acquired.image;
+    driver->queueCommand([this, gltexture, image, previousImage]() {
+        setExternalTexture(gltexture, image);
+        if (previousImage.image) {
+            scheduleRelease(AcquiredImage(previousImage));
+        }
+    });
+}
 
 #define DEBUG_NO_EXTERNAL_STREAM_COPY false
 
-void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept {
+void OpenGLDriver::updateStreamTexId(GLTexture* t, DriverApi* driver) noexcept {
     SYSTRACE_CALL();
     auto& gl = mContext;
 
     GLStream* s = static_cast<GLStream*>(t->hwStream);
     assert(s);
-    assert(!s->isNativeStream());
+    assert(s->streamType == StreamType::TEXTURE_ID);
 
     // round-robin to the next texture name
     if (UTILS_UNLIKELY(DEBUG_NO_EXTERNAL_STREAM_COPY ||
@@ -2114,7 +2230,7 @@ void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept {
             // also make sure that this texture is still associated with the same stream
             auto& streams = mExternalStreams;
             if (UTILS_LIKELY(std::find(streams.begin(), streams.end(), t) != streams.end()) &&
-                (t->hwStream == s)) {
+                    (t->hwStream == s)) {
                 t->gl.id = s->gl.externalTextureId;
             }
         });
@@ -2199,7 +2315,13 @@ void OpenGLDriver::readStreamPixels(Handle<HwStream> sh,
     auto& gl = mContext;
 
     GLStream* s = handle_cast<GLStream*>(sh);
-    if (UTILS_LIKELY(!s->isNativeStream())) {
+
+    if (UTILS_UNLIKELY(s->streamType == StreamType::ACQUIRED)) {
+        PANIC_LOG("readStreamPixels with ACQUIRED streams is not yet implemented.");
+        return;
+    }
+
+    if (UTILS_LIKELY(s->streamType == StreamType::TEXTURE_ID)) {
         GLuint tid = s->gl.externalTexture2DId;
         if (tid == 0) {
             return;
@@ -2406,26 +2528,75 @@ void OpenGLDriver::readPixels(Handle<HwRenderTarget> src,
     gl.bindFramebuffer(GL_READ_FRAMEBUFFER, s->gl.fbo);
 
     // TODO: we could use a PBO to make this asynchronous
-    glReadPixels(GLint(x), GLint(y), GLint(width), GLint(height), glFormat, glType, p.buffer);
+    //glReadPixels(GLint(x), GLint(y), GLint(width), GLint(height), glFormat, glType, p.buffer);
 
-    // now we need to flip the buffer vertically to match our API
-    size_t stride = p.stride ? p.stride : width;
-    size_t bpp = PixelBufferDescriptor::computeDataSize(p.format, p.type, 1, 1, 1);
-    size_t bpr = PixelBufferDescriptor::computeDataSize(p.format, p.type, stride, 1, p.alignment);
-    char* head = (char*)p.buffer + p.left * bpp + bpr * p.top;
-    char* tail = (char*)p.buffer + p.left * bpp + bpr * (p.top + height - 1);
-    // clang vectorizes this loop
-    while (head < tail) {
-        std::swap_ranges(head, head + bpp * width, tail);
-        head += bpr;
-        tail -= bpr;
-    }
+    GLuint pbo;
+    glGenBuffers(1, &pbo);
+    gl.bindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
+    glBufferData(GL_PIXEL_PACK_BUFFER, p.size, nullptr, GL_STATIC_DRAW);
+    glReadPixels(GLint(x), GLint(y), GLint(width), GLint(height), glFormat, glType, nullptr);
+    gl.bindBuffer(GL_PIXEL_PACK_BUFFER, 0);
 
-    scheduleDestroy(std::move(p));
+    // we're forced to make a copy on the heap because otherwise it deletes std::function<> copy
+    // constructor.
+    auto* pUserBuffer = new PixelBufferDescriptor(std::move(p));
+    whenGpuCommandsComplete([this, width, height, pbo, pUserBuffer]() mutable {
+        PixelBufferDescriptor& p = *pUserBuffer;
+        auto& gl = mContext;
+        gl.bindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
+        void* vaddr = glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0,  p.size, GL_MAP_READ_BIT);
+        if (vaddr) {
+            // now we need to flip the buffer vertically to match our API
+            size_t stride = p.stride ? p.stride : width;
+            size_t bpp = PixelBufferDescriptor::computeDataSize(
+                    p.format, p.type, 1, 1, 1);
+            size_t bpr = PixelBufferDescriptor::computeDataSize(
+                    p.format, p.type, stride, 1, p.alignment);
+            char const* head = (char const*)vaddr + p.left * bpp + bpr * p.top;
+            char* tail = (char*)p.buffer + p.left * bpp + bpr * (p.top + height - 1);
+            for (size_t i = 0; i < height; ++i) {
+                memcpy(tail, head, bpp * width);
+                head += bpr;
+                tail -= bpr;
+            }
+            glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
+        }
+        gl.bindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+        glDeleteBuffers(1, &pbo);
+        scheduleDestroy(std::move(p));
+        delete pUserBuffer;
+        CHECK_GL_ERROR(utils::slog.e)
+    });
 
     CHECK_GL_ERROR(utils::slog.e)
 }
 
+void OpenGLDriver::whenGpuCommandsComplete(std::function<void(void)> fn) noexcept {
+    GLsync sync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
+    mGpuCommandCompleteOps.emplace_back(sync, std::move(fn));
+    CHECK_GL_ERROR(utils::slog.e)
+}
+
+void OpenGLDriver::executeGpuCommandsCompleteOps() noexcept {
+    auto& v = mGpuCommandCompleteOps;
+    auto it = v.begin();
+    while (it != v.end()) {
+        GLenum status = glClientWaitSync(it->first, 0, 0);
+        if (status == GL_ALREADY_SIGNALED || status == GL_CONDITION_SATISFIED) {
+            it->second();
+            glDeleteSync(it->first);
+            it = v.erase(it);
+        } else if (UTILS_UNLIKELY(status == GL_WAIT_FAILED)) {
+            // This should never happen, but is very problematic if it does, as we might leak
+            // some data depending on what the callback does. However, we clean-up our own state.
+            glDeleteSync(it->first);
+            it = v.erase(it);
+        } else {
+            ++it;
+        }
+    }
+}
+
 // ------------------------------------------------------------------------------------------------
 // Rendering ops
 // ------------------------------------------------------------------------------------------------
@@ -2434,11 +2605,12 @@ void OpenGLDriver::beginFrame(int64_t monotonic_clock_ns, uint32_t frameId,
         backend::FrameFinishedCallback, void*) {
     auto& gl = mContext;
     insertEventMarker("beginFrame");
+    executeGpuCommandsCompleteOps();
     if (UTILS_UNLIKELY(!mExternalStreams.empty())) {
         OpenGLPlatform& platform = mPlatform;
         for (GLTexture const* t : mExternalStreams) {
             assert(t && t->hwStream);
-            if (static_cast<GLStream*>(t->hwStream)->isNativeStream()) {
+            if (t->hwStream->streamType == StreamType::NATIVE) {
                 assert(t->hwStream->stream);
                 platform.updateTexImage(t->hwStream->stream,
                         &static_cast<GLStream*>(t->hwStream)->user_thread.timestamp);
@@ -2456,6 +2628,7 @@ void OpenGLDriver::setPresentationTime(int64_t monotonic_clock_ns) {
 void OpenGLDriver::endFrame(uint32_t frameId) {
     //SYSTRACE_NAME("glFinish");
     //glFinish();
+    //executeGpuCommandsCompleteOps();
     insertEventMarker("endFrame");
 }
 
@@ -2467,6 +2640,12 @@ void OpenGLDriver::flush(int) {
     }
 }
 
+void OpenGLDriver::finish(int) {
+    DEBUG_MARKER()
+    glFinish();
+    executeGpuCommandsCompleteOps();
+}
+
 UTILS_NOINLINE
 void OpenGLDriver::clearWithRasterPipe(
         bool clearColor, float4 const& linearColor,

filament/backend/src/opengl/OpenGLDriver.h

@@ -124,7 +124,6 @@ public:
     struct GLStream : public backend::HwStream {
         static constexpr size_t ROUND_ROBIN_TEXTURE_COUNT = 3;      // 3 maximum
         using HwStream::HwStream;
-        bool isNativeStream() const { return gl.externalTextureId == 0; }
         struct Info {
             // storage for the read/write textures below
             backend::Platform::ExternalTexture* ets = nullptr;
@@ -143,8 +142,9 @@ public:
             GLuint fbo = 0;
         } gl;   // 20 bytes
 
+
         /*
-         * The fields below are access from the main application thread
+         * The fields below are accessed from the main application thread
          * (not the GL thread)
          */
         struct {
@@ -155,6 +155,8 @@ public:
             Info infos[ROUND_ROBIN_TEXTURE_COUNT];      // 48 bytes
             int64_t timestamp = 0;
             uint8_t cur = 0;
+            backend::AcquiredImage acquired;
+            backend::AcquiredImage pending;
         } user_thread;
     };
 
@@ -366,7 +368,6 @@ private:
     mutable tsl::robin_map<uint32_t, GLuint> mSamplerMap;
     mutable std::vector<GLTexture*> mExternalStreams;
 
-
     void attachStream(GLTexture* t, GLStream* stream) noexcept;
     void detachStream(GLTexture* t) noexcept;
     void replaceStream(GLTexture* t, GLStream* stream) noexcept;
@@ -374,9 +375,16 @@ private:
     backend::OpenGLPlatform& mPlatform;
 
     OpenGLBlitter* mOpenGLBlitter = nullptr;
-    void updateStream(GLTexture* t, backend::DriverApi* driver) noexcept;
+    void updateStreamTexId(GLTexture* t, backend::DriverApi* driver) noexcept;
+    void updateStreamAcquired(GLTexture* t, backend::DriverApi* driver) noexcept;
     void updateBuffer(GLenum target, GLBuffer* buffer, backend::BufferDescriptor const& p, uint32_t alignment = 16) noexcept;
     void updateTextureLodRange(GLTexture* texture, int8_t targetLevel) noexcept;
+
+    void setExternalTexture(GLTexture* t, void* image);
+
+    void whenGpuCommandsComplete(std::function<void()> fn) noexcept;
+    void executeGpuCommandsCompleteOps() noexcept;
+    std::vector<std::pair<GLsync, std::function<void(void)>>> mGpuCommandCompleteOps;
 };
 
 // ------------------------------------------------------------------------------------------------

filament/backend/src/opengl/PlatformCocoaGL.h

@@ -36,6 +36,7 @@ public:
     void terminate() noexcept final;
 
     SwapChain* createSwapChain(void* nativewindow, uint64_t& flags) noexcept final;
+    SwapChain* createSwapChain(uint32_t width, uint32_t height, uint64_t& flags) noexcept final;
     void destroySwapChain(SwapChain* swapChain) noexcept final;
     void makeCurrent(SwapChain* drawSwapChain, SwapChain* readSwapChain) noexcept final;
     void commit(SwapChain* swapChain) noexcept final;