From f2bca03360df0ae2b27456ea6697c12220a85aeb Mon Sep 17 00:00:00 2001 From: Anish Goyal Date: Mon, 12 Jan 2026 14:32:01 -0500 Subject: [PATCH] Add external format to cache prewarming (#9558) Provides logic to load and check for external format ids, and build fake pipelines against them when relevant, to prevent hitching when using external formats. For now, we're going to use upto five likely types of YCbCr conversions, which seem to cover all usecases encountered. This means we might compile pipelines with external samplers a total of 4 additional times on top of the baseline (5 in total), or 3 additional times on the devices we're currently testing (4 in total). * Add base pipeline prewarm call for ext samplers This is necessary because in some cases, a material that supports external samplers will use RGB inputs instead of YCbCr inputs, and will miss the cache. --- .../backend/platforms/VulkanPlatform.h | 21 ++++++ .../backend/platforms/VulkanPlatformAndroid.h | 1 - filament/backend/src/vulkan/VulkanContext.h | 30 +++++++- filament/backend/src/vulkan/VulkanDriver.cpp | 70 ++++++++++++++++++- .../src/vulkan/VulkanPipelineCache.cpp | 7 +- .../src/vulkan/platform/VulkanPlatform.cpp | 5 ++ 6 files changed, 129 insertions(+), 5 deletions(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index d56d2d83b6..2bba69e2d0 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -56,6 +56,16 @@ struct VulkanCmdFence; */ class VulkanPlatform : public Platform, utils::PrivateImplementation { public: + /** + * Encapsulates information required to instantiate a known external format, + * typically for the purpose of preloading a pipeline cache for materials using + * external formats for samplers. + */ + struct ExternalYcbcrFormat { + uint64_t externalFormat; + VkSamplerYcbcrModelConversion ycbcrModelConversion; + VkSamplerYcbcrRange ycbcrRange; + }; struct ExtensionHashFn { std::size_t operator()(utils::CString const& s) const noexcept { @@ -501,6 +511,17 @@ protected: */ bool isTransientAttachmentSupported() const noexcept; + /** + * For pipeline cache prewarming, if external samplers are present, we need to build + * the fake pipeline using the proper formats specified. Since there's no way to + * get these at material build time, we allow the app to register them before + * creating materials. + * + * @param format The format, containing the external format value which should be + * extracted from an AHardwareBuffer. + */ + void registerPipelineCachePrewarmExternalFormat(const ExternalYcbcrFormat& format) noexcept; + private: /** * Contains information about features that should be requested diff --git a/filament/backend/include/backend/platforms/VulkanPlatformAndroid.h b/filament/backend/include/backend/platforms/VulkanPlatformAndroid.h index ba82db819e..dd56c25358 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatformAndroid.h +++ b/filament/backend/include/backend/platforms/VulkanPlatformAndroid.h @@ -81,7 +81,6 @@ public: bool queryFrameTimestamps(SwapChain const* swapchain, uint64_t frameId, FrameTimestamps* outFrameTimestamps) const noexcept override; - protected: ExtensionSet getSwapchainInstanceExtensions() const override; diff --git a/filament/backend/src/vulkan/VulkanContext.h b/filament/backend/src/vulkan/VulkanContext.h index ecaef4832a..dfb5775cb3 100644 --- a/filament/backend/src/vulkan/VulkanContext.h +++ b/filament/backend/src/vulkan/VulkanContext.h @@ -22,8 +22,9 @@ #include "vulkan/memory/ResourcePointer.h" +#include + #include -#include #include #include @@ -90,6 +91,19 @@ public: return selectMemoryType(mMemoryProperties, types, reqs); } + /** + * For pipeline cache prewarming, if external samplers are present, we need to build + * the fake pipeline using the proper formats specified. Since there's no way to + * get these at material build time, we allow the app to register them before + * creating materials. + * + * @param format The format, containing the external format value which should be + * extracted from an AHardwareBuffer. + */ + inline void addPipelineCachePrewarmExternalFormat(const VulkanPlatform::ExternalYcbcrFormat& format) { + mPipelineCachePrewarmExternalFormats.push_back(format); + } + inline fvkutils::VkFormatList const& getAttachmentDepthStencilFormats() const { return mDepthStencilFormats; } @@ -106,6 +120,18 @@ public: return mPhysicalDeviceProperties.properties.vendorID; } + /** + * Fetches a list of pre-registered external formats for prewarming the Vulkan + * pipeline cache. + * + * @return A list containing an external format number, YCbCr color model conversion, + * and YCbCr color range. + */ + inline const std::vector& + getPipelineCachePrewarmExternalFormats() const noexcept { + return mPipelineCachePrewarmExternalFormats; + } + inline VkExternalFenceHandleTypeFlags getFenceExportFlags() const noexcept { return mFenceExportFlags; } @@ -223,6 +249,8 @@ private: fvkutils::VkFormatList mDepthStencilFormats; fvkutils::VkFormatList mBlittableDepthStencilFormats; + std::vector mPipelineCachePrewarmExternalFormats; + // For convenience so that VulkanPlatform can initialize the private fields. friend class VulkanPlatform; }; diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 68c0eb43c6..0a021f4eca 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -151,6 +151,28 @@ static CallbackHandler::Callback syncCallbackWrapper = [](void* userData) { cbData->cb(cbData->sync, cbData->userData); }; +/** + * Shorthand for converting a description for an external YCbCr format used for + * pipeline cache prewarming to the params for the actual YCbCr conversion. + */ +inline VulkanYcbcrConversionCache::Params getYcbcrConversionParams(const VulkanPlatform::ExternalYcbcrFormat& format) { + return VulkanYcbcrConversionCache::Params { + .conversion = { + .ycbcrModel = fvkutils::getYcbcrModelConversionFilament(format.ycbcrModelConversion), + .r = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_R, 0), + .g = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_G, 1), + .b = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_B, 2), + .a = fvkutils::getSwizzleFilament(VK_COMPONENT_SWIZZLE_A, 3), + .ycbcrRange = fvkutils::getYcbcrRangeFilament(format.ycbcrRange), + .xChromaOffset = fvkutils::getChromaLocationFilament(VK_CHROMA_LOCATION_MIDPOINT), + .yChromaOffset = fvkutils::getChromaLocationFilament(VK_CHROMA_LOCATION_MIDPOINT), + .chromaFilter = SamplerMagFilter::NEAREST, + }, + .format = VK_FORMAT_UNDEFINED, + .externalFormat = format.externalFormat, + }; +} + }// anonymous namespace #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS) @@ -803,12 +825,18 @@ void VulkanDriver::createProgramR(Handle ph, Program&& program, utils } // If async prewarming is enabled, let's find the proper layout and build the pipeline. + std::array, MAX_DESCRIPTOR_SET_COUNT> layouts {}; VulkanDescriptorSetLayout::DescriptorSetLayoutArray vkLayouts {}; + bool hasExternalSamplers = false; for (const auto& layoutBinding : program.getDescriptorSetLayouts()) { DescriptorSetLayout layoutDescription = layoutBinding.layout; auto layoutHandle = mResourceManager.allocHandle(); auto layout = mDescriptorSetLayoutCache.createLayout(layoutHandle, std::move(layoutDescription)); + layouts[layoutBinding.set] = layout; vkLayouts[layoutBinding.set] = layout->getVkLayout(); + if (layout->bitmask.externalSampler.count() > 0) { + hasExternalSamplers = true; + } } StereoscopicType stereoscopicType = mStereoscopicType; @@ -816,13 +844,51 @@ void VulkanDriver::createProgramR(Handle ph, Program&& program, utils stereoscopicType = StereoscopicType::NONE; } - VkPipelineLayout layout = mPipelineLayoutCache.getLayout(vkLayouts, vprogram); + // Base case - build the pipeline without any external samplers. mPipelineCache.asyncPrewarmCache( *vprogram.get(), - layout, + mPipelineLayoutCache.getLayout(vkLayouts, vprogram), stereoscopicType, mStereoscopicEyeCount, program.getPriorityQueue()); + + if (!hasExternalSamplers) { + return; + } + + // If we have external samplers, let's do this again with the external samplers + // specified. + for (const auto& format : mContext.getPipelineCachePrewarmExternalFormats()) { + // The values that seem to matter in terms of cache hits here are the model conversion, + // and model range. We need some value for externalFormat that is known to support that + // pair of values, but we need not find every possible externalFormat. As we test on + // more devices, this may change. + VkSamplerYcbcrConversion vkConversion = mYcbcrConversionCache.getConversion( + getYcbcrConversionParams(format)); + VkSampler externalSampler = mSamplerCache.getSampler({.sampler = {}, .conversion = vkConversion}); + + // Update all layouts to use the external samplers. + for (size_t i = 0; i < MAX_DESCRIPTOR_SET_COUNT; ++i) { + if (!layouts[i]) { + continue; + } + + // For cache prewarming, we don't need every single possible combination of external sampler + // formats. It seems to be enough, in practicce, to simply run through a list of the types of + // samplers that *might* appear. As long as the real pipeline is close enough to something that + // the driver has seen before, we are able to get a cache hit. + utils::FixedCapacityVector externalSamplers (layouts[i]->bitmask.externalSampler.count(), externalSampler); + vkLayouts[i] = mDescriptorSetLayoutCache.getVkLayout( + layouts[i]->bitmask, layouts[i]->bitmask.externalSampler, externalSamplers); + } + + mPipelineCache.asyncPrewarmCache( + *vprogram.get(), + mPipelineLayoutCache.getLayout(vkLayouts, vprogram), + stereoscopicType, + mStereoscopicEyeCount, + program.getPriorityQueue()); + } } void VulkanDriver::destroyProgram(Handle ph) { diff --git a/filament/backend/src/vulkan/VulkanPipelineCache.cpp b/filament/backend/src/vulkan/VulkanPipelineCache.cpp index e3373537a8..784b23b16c 100644 --- a/filament/backend/src/vulkan/VulkanPipelineCache.cpp +++ b/filament/backend/src/vulkan/VulkanPipelineCache.cpp @@ -180,7 +180,12 @@ void VulkanPipelineCache::asyncPrewarmCache(const VulkanProgram& program, mCallbackManager.put(cmh); // We don't actually need this pipeline, we just wanted to force the driver to cache // the pipeline's information. - vkDestroyPipeline(mDevice, pipeline, VKALLOC); + if (pipeline != VK_NULL_HANDLE) { + vkDestroyPipeline(mDevice, pipeline, VKALLOC); + } else { + FVK_LOGW << "Failed to create a pipeline during prewarming, draw-time pipeline " + "creation may fail."; + } }); } diff --git a/filament/backend/src/vulkan/platform/VulkanPlatform.cpp b/filament/backend/src/vulkan/platform/VulkanPlatform.cpp index f234068cd7..4a01026216 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatform.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatform.cpp @@ -798,6 +798,11 @@ bool VulkanPlatform::isTransientAttachmentSupported() const noexcept { return mImpl->mContext.isLazilyAllocatedMemorySupported(); } +void VulkanPlatform::registerPipelineCachePrewarmExternalFormat( + const ExternalYcbcrFormat& format) noexcept { + mImpl->mContext.addPipelineCachePrewarmExternalFormat(format); +} + VkInstance VulkanPlatform::createVkInstance(const VkInstanceCreateInfo& createInfo) noexcept { VkInstance instance = VK_NULL_HANDLE; VkResult result = vkCreateInstance(&createInfo, VKALLOC, &instance);