Add external format to cache prewarming (#9558)

Provides logic to load and check for external format ids, and
build fake pipelines against them when relevant, to prevent
hitching when using external formats.

For now, we're going to use upto five likely types of YCbCr
conversions, which seem to cover all usecases encountered. This
means we might compile pipelines with external samplers a total of
4 additional times on top of the baseline (5 in total), or 3 additional
times on the devices we're currently testing (4 in total).

* Add base pipeline prewarm call for ext samplers

This is necessary because in some cases, a material that supports
external samplers will use RGB inputs instead of YCbCr inputs, and will
miss the cache.

README.md

@@ -31,7 +31,7 @@ repositories {
 }
 
 dependencies {
-    implementation 'com.google.android.filament:filament-android:1.68.3'
+    implementation 'com.google.android.filament:filament-android:1.68.4'
 }
 ```
 
@@ -51,7 +51,7 @@ Here are all the libraries available in the group `com.google.android.filament`:
 iOS projects can use CocoaPods to install the latest release:
 
 ```shell
-pod 'Filament', '~> 1.68.3'
+pod 'Filament', '~> 1.68.4'
 ```
 
 ## Documentation

android/gradle.properties

@@ -1,5 +1,5 @@
 GROUP=com.google.android.filament
-VERSION_NAME=1.68.3
+VERSION_NAME=1.68.4
 
 POM_DESCRIPTION=Real-time physically based rendering engine for Android.
 

filament/backend/include/backend/platforms/VulkanPlatform.h

@@ -56,6 +56,16 @@ struct VulkanCmdFence;
  */
 class VulkanPlatform : public Platform, utils::PrivateImplementation<VulkanPlatformPrivate> {
 public:
+    /**
+     * Encapsulates information required to instantiate a known external format,
+     * typically for the purpose of preloading a pipeline cache for materials using
+     * external formats for samplers.
+     */
+    struct ExternalYcbcrFormat {
+        uint64_t externalFormat;
+        VkSamplerYcbcrModelConversion ycbcrModelConversion;
+        VkSamplerYcbcrRange ycbcrRange;
+    };
 
     struct ExtensionHashFn {
         std::size_t operator()(utils::CString const& s) const noexcept {
@@ -501,6 +511,17 @@ protected:
      */
     bool isTransientAttachmentSupported() const noexcept;
 
+    /**
+     * For pipeline cache prewarming, if external samplers are present, we need to build
+     * the fake pipeline using the proper formats specified. Since there's no way to
+     * get these at material build time, we allow the app to register them before
+     * creating materials.
+     *
+     * @param format The format, containing the external format value which should be
+     *               extracted from an AHardwareBuffer.
+     */
+    void registerPipelineCachePrewarmExternalFormat(const ExternalYcbcrFormat& format) noexcept;
+
 private:
     /**
      * Contains information about features that should be requested

filament/backend/include/backend/platforms/VulkanPlatformAndroid.h

@@ -81,7 +81,6 @@ public:
     bool queryFrameTimestamps(SwapChain const* swapchain, uint64_t frameId,
             FrameTimestamps* outFrameTimestamps) const noexcept override;
 
-
 protected:
     ExtensionSet getSwapchainInstanceExtensions() const override;
 

filament/backend/src/vulkan/VulkanContext.h

@@ -22,8 +22,9 @@
 
 #include "vulkan/memory/ResourcePointer.h"
 
+#include <vector>
+
 #include <utils/bitset.h>
-#include <utils/FixedCapacityVector.h>
 #include <utils/Mutex.h>
 #include <utils/Slice.h>
 
@@ -90,6 +91,19 @@ public:
         return selectMemoryType(mMemoryProperties, types, reqs);
     }
 
+    /**
+     * For pipeline cache prewarming, if external samplers are present, we need to build
+     * the fake pipeline using the proper formats specified. Since there's no way to
+     * get these at material build time, we allow the app to register them before
+     * creating materials.
+     *
+     * @param format The format, containing the external format value which should be
+     *               extracted from an AHardwareBuffer.
+     */
+    inline void addPipelineCachePrewarmExternalFormat(const VulkanPlatform::ExternalYcbcrFormat& format) {
+        mPipelineCachePrewarmExternalFormats.push_back(format);
+    }
+
     inline fvkutils::VkFormatList const& getAttachmentDepthStencilFormats() const {
         return mDepthStencilFormats;
     }
@@ -106,6 +120,18 @@ public:
         return mPhysicalDeviceProperties.properties.vendorID;
     }
 
+    /**
+     * Fetches a list of pre-registered external formats for prewarming the Vulkan
+     * pipeline cache.
+     *
+     * @return A list containing an external format number, YCbCr color model conversion,
+     *         and YCbCr color range.
+     */
+    inline const std::vector<VulkanPlatform::ExternalYcbcrFormat>&
+    getPipelineCachePrewarmExternalFormats() const noexcept {
+        return mPipelineCachePrewarmExternalFormats;
+    }
+
     inline VkExternalFenceHandleTypeFlags getFenceExportFlags() const noexcept {
         return mFenceExportFlags;
     }
@@ -223,6 +249,8 @@ private:
     fvkutils::VkFormatList mDepthStencilFormats;
     fvkutils::VkFormatList mBlittableDepthStencilFormats;
 
+    std::vector<VulkanPlatform::ExternalYcbcrFormat> mPipelineCachePrewarmExternalFormats;
+
     // For convenience so that VulkanPlatform can initialize the private fields.
     friend class VulkanPlatform;
 };

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -151,6 +151,28 @@ static CallbackHandler::Callback syncCallbackWrapper = [](void* userData) {
     cbData->cb(cbData->sync, cbData->userData);
 };
 
+/**
+ * Shorthand for converting a description for an external YCbCr format used for
+ * pipeline cache prewarming to the params for the actual YCbCr conversion.
+ */
+inline VulkanYcbcrConversionCache::Params getYcbcrConversionParams(const VulkanPlatform::ExternalYcbcrFormat& format) {
+    return VulkanYcbcrConversionCache::Params {
+        .conversion = {
+            .ycbcrModel = fvkutils::getYcbcrModelConversionFilament(format.ycbcrModelConversion),
+            .r = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_R, 0),
+            .g = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_G, 1),
+            .b = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_B, 2),
+            .a = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_A, 3),
+            .ycbcrRange = fvkutils::getYcbcrRangeFilament(format.ycbcrRange),
+            .xChromaOffset = fvkutils::getChromaLocationFilament(VK_CHROMA_LOCATION_MIDPOINT),
+            .yChromaOffset = fvkutils::getChromaLocationFilament(VK_CHROMA_LOCATION_MIDPOINT),
+            .chromaFilter = SamplerMagFilter::NEAREST,
+        },
+        .format = VK_FORMAT_UNDEFINED,
+        .externalFormat = format.externalFormat,
+    };
+}
+
 }// anonymous namespace
 
 #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS)
@@ -803,12 +825,18 @@ void VulkanDriver::createProgramR(Handle<HwProgram> ph, Program&& program, utils
     }
 
     // If async prewarming is enabled, let's find the proper layout and build the pipeline.
+    std::array<resource_ptr<VulkanDescriptorSetLayout>, MAX_DESCRIPTOR_SET_COUNT> layouts {};
     VulkanDescriptorSetLayout::DescriptorSetLayoutArray vkLayouts {};
+    bool hasExternalSamplers = false;
     for (const auto& layoutBinding : program.getDescriptorSetLayouts()) {
         DescriptorSetLayout layoutDescription = layoutBinding.layout;
         auto layoutHandle = mResourceManager.allocHandle<VulkanDescriptorSetLayout>();
         auto layout = mDescriptorSetLayoutCache.createLayout(layoutHandle, std::move(layoutDescription));
+        layouts[layoutBinding.set] = layout;
         vkLayouts[layoutBinding.set] = layout->getVkLayout();
+        if (layout->bitmask.externalSampler.count() > 0) {
+            hasExternalSamplers = true;
+        }
     }
 
     StereoscopicType stereoscopicType = mStereoscopicType;
@@ -816,13 +844,51 @@ void VulkanDriver::createProgramR(Handle<HwProgram> ph, Program&& program, utils
         stereoscopicType = StereoscopicType::NONE;
     }
 
-    VkPipelineLayout layout = mPipelineLayoutCache.getLayout(vkLayouts, vprogram);
+    // Base case - build the pipeline without any external samplers.
     mPipelineCache.asyncPrewarmCache(
         *vprogram.get(),
-        layout,
+        mPipelineLayoutCache.getLayout(vkLayouts, vprogram),
         stereoscopicType,
         mStereoscopicEyeCount,
         program.getPriorityQueue());
+
+    if (!hasExternalSamplers) {
+        return;
+    }
+
+    // If we have external samplers, let's do this again with the external samplers
+    // specified.
+    for (const auto& format : mContext.getPipelineCachePrewarmExternalFormats()) {
+        // The values that seem to matter in terms of cache hits here are the model conversion,
+        // and model range. We need some value for externalFormat that is known to support that
+        // pair of values, but we need not find every possible externalFormat. As we test on
+        // more devices, this may change.
+        VkSamplerYcbcrConversion vkConversion = mYcbcrConversionCache.getConversion(
+            getYcbcrConversionParams(format));
+        VkSampler externalSampler = mSamplerCache.getSampler({.sampler = {}, .conversion = vkConversion});
+
+        // Update all layouts to use the external samplers.
+        for (size_t i = 0; i < MAX_DESCRIPTOR_SET_COUNT; ++i) {
+            if (!layouts[i]) {
+                continue;
+            }
+
+            // For cache prewarming, we don't need every single possible combination of external sampler
+            // formats. It seems to be enough, in practicce, to simply run through a list of the types of
+            // samplers that *might* appear. As long as the real pipeline is close enough to something that
+            // the driver has seen before, we are able to get a cache hit.
+            utils::FixedCapacityVector<VkSampler> externalSamplers (layouts[i]->bitmask.externalSampler.count(), externalSampler);
+            vkLayouts[i] = mDescriptorSetLayoutCache.getVkLayout(
+                layouts[i]->bitmask, layouts[i]->bitmask.externalSampler, externalSamplers);
+        }
+
+        mPipelineCache.asyncPrewarmCache(
+            *vprogram.get(),
+            mPipelineLayoutCache.getLayout(vkLayouts, vprogram),
+            stereoscopicType,
+            mStereoscopicEyeCount,
+            program.getPriorityQueue());
+    }
 }
 
 void VulkanDriver::destroyProgram(Handle<HwProgram> ph) {

filament/backend/src/vulkan/VulkanPipelineCache.cpp

@@ -180,7 +180,12 @@ void VulkanPipelineCache::asyncPrewarmCache(const VulkanProgram& program,
         mCallbackManager.put(cmh);
         // We don't actually need this pipeline, we just wanted to force the driver to cache
         // the pipeline's information.
-        vkDestroyPipeline(mDevice, pipeline, VKALLOC);
+        if (pipeline != VK_NULL_HANDLE) {
+            vkDestroyPipeline(mDevice, pipeline, VKALLOC);
+        } else {
+            FVK_LOGW << "Failed to create a pipeline during prewarming, draw-time pipeline "
+                        "creation may fail.";
+        }
     });
 }
 

filament/backend/src/vulkan/platform/VulkanPlatform.cpp

@@ -798,6 +798,11 @@ bool VulkanPlatform::isTransientAttachmentSupported() const noexcept {
     return mImpl->mContext.isLazilyAllocatedMemorySupported();
 }
 
+void VulkanPlatform::registerPipelineCachePrewarmExternalFormat(
+        const ExternalYcbcrFormat& format) noexcept {
+    mImpl->mContext.addPipelineCachePrewarmExternalFormat(format);
+}
+
 VkInstance VulkanPlatform::createVkInstance(const VkInstanceCreateInfo& createInfo) noexcept {
     VkInstance instance = VK_NULL_HANDLE;
     VkResult result = vkCreateInstance(&createInfo, VKALLOC, &instance);

ios/CocoaPods/Filament.podspec

@@ -1,12 +1,12 @@
 Pod::Spec.new do |spec|
   spec.name = "Filament"
-  spec.version = "1.68.3"
+  spec.version = "1.68.4"
   spec.license = { :type => "Apache 2.0", :file => "LICENSE" }
   spec.homepage = "https://google.github.io/filament"
   spec.authors = "Google LLC."
   spec.summary = "Filament is a real-time physically based rendering engine for Android, iOS, Windows, Linux, macOS, and WASM/WebGL."
   spec.platform = :ios, "11.0"
-  spec.source = { :http => "https://github.com/google/filament/releases/download/v1.68.3/filament-v1.68.3-ios.tgz" }
+  spec.source = { :http => "https://github.com/google/filament/releases/download/v1.68.4/filament-v1.68.4-ios.tgz" }
 
   # Fix linking error with Xcode 12; we do not yet support the simulator on Apple silicon.
   spec.pod_target_xcconfig = {

web/filament-js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "filament",
-  "version": "1.68.3",
+  "version": "1.68.4",
   "description": "Real-time physically based rendering engine",
   "main": "filament.js",
   "module": "filament.js",