From ff1f2fcac65cc63653a803113271dcad35e84875 Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Thu, 27 Feb 2025 11:08:17 -0800 Subject: [PATCH] vk: timer query refactor (#8459) - Put QueryManager into its own .h/.cpp - Do not assert when endTime <= beginTime. This could happen if they were set on different command buffers. To be resolved later. - Better logic for iterating through a bitset. --- filament/backend/CMakeLists.txt | 3 +- .../backend/src/vulkan/VulkanAsyncHandles.cpp | 43 ------- .../backend/src/vulkan/VulkanAsyncHandles.h | 27 +++-- filament/backend/src/vulkan/VulkanContext.cpp | 70 ----------- filament/backend/src/vulkan/VulkanContext.h | 29 +---- filament/backend/src/vulkan/VulkanDriver.cpp | 36 +++--- filament/backend/src/vulkan/VulkanDriver.h | 3 +- filament/backend/src/vulkan/VulkanHandles.h | 1 - .../backend/src/vulkan/VulkanQueryManager.cpp | 110 ++++++++++++++++++ .../backend/src/vulkan/VulkanQueryManager.h | 63 ++++++++++ 10 files changed, 214 insertions(+), 171 deletions(-) delete mode 100644 filament/backend/src/vulkan/VulkanAsyncHandles.cpp create mode 100644 filament/backend/src/vulkan/VulkanQueryManager.cpp create mode 100644 filament/backend/src/vulkan/VulkanQueryManager.h diff --git a/filament/backend/CMakeLists.txt b/filament/backend/CMakeLists.txt index 112c3e2249..9242f29513 100644 --- a/filament/backend/CMakeLists.txt +++ b/filament/backend/CMakeLists.txt @@ -194,7 +194,6 @@ if (FILAMENT_SUPPORTS_VULKAN) src/vulkan/utils/Spirv.h src/vulkan/utils/Spirv.cpp src/vulkan/utils/StaticVector.h - src/vulkan/VulkanAsyncHandles.cpp src/vulkan/VulkanAsyncHandles.h src/vulkan/VulkanBlitter.cpp src/vulkan/VulkanBlitter.h @@ -216,6 +215,8 @@ if (FILAMENT_SUPPORTS_VULKAN) src/vulkan/VulkanMemory.cpp src/vulkan/VulkanPipelineCache.cpp src/vulkan/VulkanPipelineCache.h + src/vulkan/VulkanQueryManager.cpp + src/vulkan/VulkanQueryManager.h src/vulkan/VulkanSamplerCache.cpp src/vulkan/VulkanSamplerCache.h src/vulkan/VulkanStagePool.cpp diff --git a/filament/backend/src/vulkan/VulkanAsyncHandles.cpp b/filament/backend/src/vulkan/VulkanAsyncHandles.cpp deleted file mode 100644 index 82963b05c4..0000000000 --- a/filament/backend/src/vulkan/VulkanAsyncHandles.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "VulkanAsyncHandles.h" - -namespace filament::backend { - -VulkanTimerQuery::VulkanTimerQuery(std::tuple indices) - : mStartingQueryIndex(std::get<0>(indices)), - mStoppingQueryIndex(std::get<1>(indices)) {} - -void VulkanTimerQuery::setFence(std::shared_ptr fence) noexcept { - std::unique_lock lock(mFenceMutex); - mFence = fence; -} - -bool VulkanTimerQuery::isCompleted() noexcept { - std::unique_lock lock(mFenceMutex); - // QueryValue is a synchronous call and might occur before beginTimerQuery has written anything - // into the command buffer, which is an error according to the validation layer that ships in - // the Android NDK. Even when AVAILABILITY_BIT is set, validation seems to require that the - // timestamp has at least been written into a processed command buffer. - - // This fence indicates that the corresponding buffer has been completed. - return mFence && mFence->getStatus() == VK_SUCCESS; -} - -VulkanTimerQuery::~VulkanTimerQuery() = default; - -} // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanAsyncHandles.h b/filament/backend/src/vulkan/VulkanAsyncHandles.h index eb626871ae..c70fabc0b6 100644 --- a/filament/backend/src/vulkan/VulkanAsyncHandles.h +++ b/filament/backend/src/vulkan/VulkanAsyncHandles.h @@ -53,17 +53,28 @@ struct VulkanFence : public HwFence, fvkmemory::ThreadSafeResource { }; struct VulkanTimerQuery : public HwTimerQuery, fvkmemory::ThreadSafeResource { - explicit VulkanTimerQuery(std::tuple indices); - ~VulkanTimerQuery(); + VulkanTimerQuery(uint32_t startingIndex, uint32_t stoppingIndex) + : mStartingQueryIndex(startingIndex), + mStoppingQueryIndex(stoppingIndex) {} - void setFence(std::shared_ptr fence) noexcept; - - bool isCompleted() noexcept; - - uint32_t getStartingQueryIndex() const { - return mStartingQueryIndex; + void setFence(std::shared_ptr fence) noexcept { + std::unique_lock lock(mFenceMutex); + mFence = fence; } + bool isCompleted() noexcept { + std::unique_lock lock(mFenceMutex); + // QueryValue is a synchronous call and might occur before beginTimerQuery has written + // anything into the command buffer, which is an error according to the validation layer + // that ships in the Android NDK. Even when AVAILABILITY_BIT is set, validation seems to + // require that the timestamp has at least been written into a processed command buffer. + + // This fence indicates that the corresponding buffer has been completed. + return mFence && mFence->getStatus() == VK_SUCCESS; + } + + uint32_t getStartingQueryIndex() const { return mStartingQueryIndex; } + uint32_t getStoppingQueryIndex() const { return mStoppingQueryIndex; } diff --git a/filament/backend/src/vulkan/VulkanContext.cpp b/filament/backend/src/vulkan/VulkanContext.cpp index 0345e69cf9..5f149a85d7 100644 --- a/filament/backend/src/vulkan/VulkanContext.cpp +++ b/filament/backend/src/vulkan/VulkanContext.cpp @@ -76,75 +76,5 @@ VkImageSubresourceRange VulkanAttachment::getSubresourceRange() const { }; } -VulkanTimestamps::VulkanTimestamps(VkDevice device) : mDevice(device) { - // Create a timestamp pool large enough to hold a pair of queries for each timer. - VkQueryPoolCreateInfo tqpCreateInfo = { - .sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, - .queryType = VK_QUERY_TYPE_TIMESTAMP, - }; - std::unique_lock lock(mMutex); - tqpCreateInfo.queryCount = mUsed.size() * 2; - VkResult result = vkCreateQueryPool(mDevice, &tqpCreateInfo, VKALLOC, &mPool); - FILAMENT_CHECK_POSTCONDITION(result == VK_SUCCESS) << "vkCreateQueryPool failed." - << " error=" << static_cast(result); - mUsed.reset(); -} - -std::tuple VulkanTimestamps::getNextQuery() { - std::unique_lock lock(mMutex); - size_t const maxTimers = mUsed.size(); - assert_invariant(mUsed.count() < maxTimers); - for (size_t timerIndex = 0; timerIndex < maxTimers; ++timerIndex) { - if (!mUsed.test(timerIndex)) { - mUsed.set(timerIndex); - return std::make_tuple(timerIndex * 2, timerIndex * 2 + 1); - } - } - FVK_LOGE << "More than " << maxTimers << " timers are not supported." << utils::io::endl; - return std::make_tuple(0, 1); -} - -void VulkanTimestamps::clearQuery(uint32_t queryIndex) { - mUsed.unset(queryIndex / 2); -} - -void VulkanTimestamps::beginQuery(VulkanCommandBuffer const* commands, - fvkmemory::resource_ptr query) { - uint32_t const index = query->getStartingQueryIndex(); - - auto const cmdbuffer = commands->buffer(); - vkCmdResetQueryPool(cmdbuffer, mPool, index, 2); - vkCmdWriteTimestamp(cmdbuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, mPool, index); - - // We stash this because getResult might come before the query is actually processed. - query->setFence(commands->getFenceStatus()); -} - -void VulkanTimestamps::endQuery(VulkanCommandBuffer const* commands, - fvkmemory::resource_ptr query) { - uint32_t const index = query->getStoppingQueryIndex(); - vkCmdWriteTimestamp(commands->buffer(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, mPool, index); -} - -VulkanTimestamps::QueryResult VulkanTimestamps::getResult( - fvkmemory::resource_ptr query) { - uint32_t const index = query->getStartingQueryIndex(); - QueryResult result; - size_t const dataSize = result.size() * sizeof(uint64_t); - VkDeviceSize const stride = sizeof(uint64_t) * 2; - VkResult vkresult = - vkGetQueryPoolResults(mDevice, mPool, index, 2, dataSize, (void*) result.data(), stride, - VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); - FILAMENT_CHECK_POSTCONDITION(vkresult == VK_SUCCESS || vkresult == VK_NOT_READY) - << "vkGetQueryPoolResults error=" << static_cast(vkresult); - if (vkresult == VK_NOT_READY) { - return {0, 0, 0, 0}; - } - return result; -} - -VulkanTimestamps::~VulkanTimestamps() { - vkDestroyQueryPool(mDevice, mPool, VKALLOC); -} } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanContext.h b/filament/backend/src/vulkan/VulkanContext.h index 30969142de..06adc7791c 100644 --- a/filament/backend/src/vulkan/VulkanContext.h +++ b/filament/backend/src/vulkan/VulkanContext.h @@ -37,12 +37,12 @@ VK_DEFINE_HANDLE(VmaPool) namespace filament::backend { +struct VulkanCommandBuffer; struct VulkanRenderTarget; struct VulkanSwapChain; struct VulkanTexture; class VulkanStagePool; struct VulkanTimerQuery; -struct VulkanCommandBuffer; struct VulkanAttachment { fvkmemory::resource_ptr texture; @@ -60,33 +60,6 @@ struct VulkanAttachment { VkImageSubresourceRange getSubresourceRange() const; }; -class VulkanTimestamps { -public: - using QueryResult = std::array; - - VulkanTimestamps(VkDevice device); - ~VulkanTimestamps(); - - // Not copy-able. - VulkanTimestamps(VulkanTimestamps const&) = delete; - VulkanTimestamps& operator=(VulkanTimestamps const&) = delete; - - std::tuple getNextQuery(); - void clearQuery(uint32_t queryIndex); - - void beginQuery(VulkanCommandBuffer const* commands, - fvkmemory::resource_ptr query); - void endQuery(VulkanCommandBuffer const* commands, - fvkmemory::resource_ptr query); - QueryResult getResult(fvkmemory::resource_ptr query); - -private: - VkDevice mDevice; - VkQueryPool mPool; - utils::bitset32 mUsed; - utils::Mutex mMutex; -}; - struct VulkanRenderPass { // Between the begin and end command render pass we cache the command buffer VulkanCommandBuffer* commandBuffer; diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index c72ecde6cb..577c025757 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -217,6 +217,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext const& contex mBlitter(mPlatform->getPhysicalDevice(), &mCommands), mReadPixels(mPlatform->getDevice()), mDescriptorSetManager(mPlatform->getDevice(), &mResourceManager), + mQueryManager(mPlatform->getDevice()), mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported), mStereoscopicType(driverConfig.stereoscopicType) { @@ -241,8 +242,6 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext const& contex << " error=" << static_cast(result); } #endif - - mTimestamps = std::make_unique(mPlatform->getDevice()); } VulkanDriver::~VulkanDriver() noexcept = default; @@ -313,7 +312,7 @@ void VulkanDriver::terminate() { mDefaultRenderTarget = {}; mBoundPipeline = {}; - mTimestamps.reset(); + mQueryManager.terminate(); mBlitter.terminate(); mReadPixels.terminate(); @@ -851,8 +850,7 @@ Handle VulkanDriver::createSwapChainHeadlessS() noexcept { Handle VulkanDriver::createTimerQueryS() noexcept { // The handle must be constructed here, as a synchronous call to getTimerQueryValue might happen // before createTimerQueryR is executed. - auto query = resource_ptr::construct(&mResourceManager, - mTimestamps->getNextQuery()); + auto query = mQueryManager.getNextQuery(&mResourceManager); query.inc(); return Handle(query.id()); } @@ -884,6 +882,7 @@ void VulkanDriver::destroyTimerQuery(Handle tqh) { return; } auto vtq = resource_ptr::cast(&mResourceManager, tqh); + mQueryManager.clearQuery(vtq); vtq.dec(); } @@ -1196,26 +1195,25 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle tqh, uint return TimerQueryResult::NOT_READY; } - auto results = mTimestamps->getResult(vtq); - uint64_t timestamp0 = results[0]; - uint64_t available0 = results[1]; - uint64_t timestamp1 = results[2]; - uint64_t available1 = results[3]; - - if (available0 == 0 || available1 == 0) { + auto results = mQueryManager.getResult(vtq); + if (results.beginAvailable == 0 || results.endAvailable == 0) { return TimerQueryResult::NOT_READY; } - FILAMENT_CHECK_POSTCONDITION(timestamp1 >= timestamp0) - << "Timestamps are not monotonically increasing."; + uint64_t const begin = results.beginTime; + uint64_t const end = results.endTime; + if (begin >= end) { + // TODO: queries might have ran on different command buffers. + FVK_LOGW << "Timestamps are not monotonically increasing. " << utils::io::endl; + *elapsedTime = 0; + return TimerQueryResult::ERROR; + } // NOTE: MoltenVK currently writes system time so the following delta will always be zero. // However there are plans for implementing this properly. See the following GitHub ticket. // https://github.com/KhronosGroup/MoltenVK/issues/773 - float const period = mContext.getPhysicalDeviceLimits().timestampPeriod; - uint64_t delta = uint64_t(float(timestamp1 - timestamp0) * period); - *elapsedTime = delta; + *elapsedTime = uint64_t(float(end - begin) * period); return TimerQueryResult::AVAILABLE; } @@ -1856,12 +1854,12 @@ void VulkanDriver::scissor(Viewport scissorBox) { void VulkanDriver::beginTimerQuery(Handle tqh) { auto vtq = resource_ptr::cast(&mResourceManager, tqh); - mTimestamps->beginQuery(&(mCommands.get()), vtq); + mQueryManager.beginQuery(&(mCommands.get()), vtq); } void VulkanDriver::endTimerQuery(Handle tqh) { auto vtq = resource_ptr::cast(&mResourceManager, tqh); - mTimestamps->endQuery(&(mCommands.get()), vtq); + mQueryManager.endQuery(&(mCommands.get()), vtq); } void VulkanDriver::debugCommandBegin(CommandStream* cmds, bool synchronous, const char* methodName) noexcept { diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 1621dc1106..f488200abe 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -26,6 +26,7 @@ #include "VulkanReadPixels.h" #include "VulkanSamplerCache.h" #include "VulkanStagePool.h" +#include "VulkanQueryManager.h" #include "vulkan/caching/VulkanDescriptorSetManager.h" #include "vulkan/caching/VulkanPipelineLayoutCache.h" #include "vulkan/memory/ResourceManager.h" @@ -119,7 +120,6 @@ private: VulkanPlatform* mPlatform = nullptr; fvkmemory::ResourceManager mResourceManager; - std::unique_ptr mTimestamps; resource_ptr mCurrentSwapChain; resource_ptr mDefaultRenderTarget; @@ -138,6 +138,7 @@ private: VulkanBlitter mBlitter; VulkanReadPixels mReadPixels; VulkanDescriptorSetManager mDescriptorSetManager; + VulkanQueryManager mQueryManager; // This is necessary for us to write to push constants after binding a pipeline. struct { diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index ae5708c424..7eb24b82e1 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -55,7 +55,6 @@ inline uint8_t collapsedCount(Bitmask const& mask) { } // anonymous namespace -class VulkanTimestamps; struct VulkanBufferObject; struct VulkanDescriptorSetLayout : public HwDescriptorSetLayout, fvkmemory::Resource { diff --git a/filament/backend/src/vulkan/VulkanQueryManager.cpp b/filament/backend/src/vulkan/VulkanQueryManager.cpp new file mode 100644 index 0000000000..302e367953 --- /dev/null +++ b/filament/backend/src/vulkan/VulkanQueryManager.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VulkanQueryManager.h" + +#include "VulkanAsyncHandles.h" +#include "VulkanCommands.h" +#include "VulkanConstants.h" + +#include + +namespace filament::backend { + +using namespace bluevk; + +VulkanQueryManager::VulkanQueryManager(VkDevice device) : mDevice(device) { + // Create a timestamp pool large enough to hold a pair of queries for each timer. + VkQueryPoolCreateInfo tqpCreateInfo = { + .sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = (uint32_t) mUsed.size() * 2, + }; + VkResult result = vkCreateQueryPool(mDevice, &tqpCreateInfo, VKALLOC, &mPool); + FILAMENT_CHECK_POSTCONDITION(result == VK_SUCCESS) << "vkCreateQueryPool failed." + << " error=" << static_cast(result); +} + +fvkmemory::resource_ptr VulkanQueryManager::getNextQuery( + fvkmemory::ResourceManager* resourceManager) { + auto unused = ~mUsed; + if (unused.empty()) { + FVK_LOGE << "More than " << mUsed.size() << " timers are not supported." << utils::io::endl; + return {}; + } + + bool found = false; + std::pair queryIndices; + unused.forEachSetBit([&](size_t index) { + if (found) { + return; + } + std::unique_lock lock(mMutex); + mUsed.set(index); + found = true; + queryIndices = std::make_pair(index * 2, index * 2 + 1); + }); + return fvkmemory::resource_ptr::construct(resourceManager, queryIndices.first, + queryIndices.second); +} + +void VulkanQueryManager::clearQuery(fvkmemory::resource_ptr query) { + std::unique_lock lock(mMutex); + uint32_t const startingIndex = query->getStartingQueryIndex(); + mUsed.unset(startingIndex / 2); +} + +void VulkanQueryManager::beginQuery(VulkanCommandBuffer const* commands, + fvkmemory::resource_ptr query) { + uint32_t const index = query->getStartingQueryIndex(); + + auto const cmdbuffer = commands->buffer(); + vkCmdResetQueryPool(cmdbuffer, mPool, index, 2); + vkCmdWriteTimestamp(cmdbuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, mPool, index); + + // We stash this because getResult might come before the query is actually processed. + query->setFence(commands->getFenceStatus()); +} + +void VulkanQueryManager::endQuery(VulkanCommandBuffer const* commands, + fvkmemory::resource_ptr query) { + uint32_t const index = query->getStoppingQueryIndex(); + vkCmdWriteTimestamp(commands->buffer(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, mPool, index); +} + +VulkanQueryManager::QueryResult VulkanQueryManager::getResult( + fvkmemory::resource_ptr query) { + uint32_t const index = query->getStartingQueryIndex(); + QueryResult result; + size_t const dataSize = sizeof(result); + VkDeviceSize const stride = sizeof(uint64_t) * 2; + VkResult vkresult = + vkGetQueryPoolResults(mDevice, mPool, index, 2, dataSize, (void*) &result, stride, + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + FILAMENT_CHECK_POSTCONDITION(vkresult == VK_SUCCESS || vkresult == VK_NOT_READY) + << "vkGetQueryPoolResults error=" << static_cast(vkresult); + if (vkresult == VK_NOT_READY) { + return {}; + } + return result; +} + +void VulkanQueryManager::terminate() noexcept { + vkDestroyQueryPool(mDevice, mPool, VKALLOC); + mPool = VK_NULL_HANDLE; +} + +} // filament::backend diff --git a/filament/backend/src/vulkan/VulkanQueryManager.h b/filament/backend/src/vulkan/VulkanQueryManager.h new file mode 100644 index 0000000000..1580c52fd7 --- /dev/null +++ b/filament/backend/src/vulkan/VulkanQueryManager.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_BACKEND_VULKANQUERYMANAGER_H +#define TNT_FILAMENT_BACKEND_VULKANQUERYMANAGER_H + +#include "vulkan/memory/ResourcePointer.h" + +namespace filament::backend { + +struct VulkanCommandBuffer; +struct VulkanTimerQuery; + +class VulkanQueryManager { +public: + struct QueryResult { + uint64_t beginTime = 0; + uint64_t beginAvailable = 0; + uint64_t endTime = 0; + uint64_t endAvailable = 0; + }; + + VulkanQueryManager(VkDevice device); + ~VulkanQueryManager() = default; + + // Not copy-able. + VulkanQueryManager(VulkanQueryManager const&) = delete; + VulkanQueryManager& operator=(VulkanQueryManager const&) = delete; + + fvkmemory::resource_ptr + getNextQuery(fvkmemory::ResourceManager* mResourceManager); + void clearQuery(fvkmemory::resource_ptr query); + void beginQuery(VulkanCommandBuffer const* commands, + fvkmemory::resource_ptr query); + void endQuery(VulkanCommandBuffer const* commands, + fvkmemory::resource_ptr query); + QueryResult getResult(fvkmemory::resource_ptr query); + + void terminate() noexcept; + +private: + VkDevice mDevice; + VkQueryPool mPool; + utils::bitset32 mUsed; + utils::Mutex mMutex; +}; + +} // filament::backend + +#endif // TNT_FILAMENT_BACKEND_VULKANQUERYMANAGER_H