diff --git a/filament/backend/src/vulkan/VulkanContext.cpp b/filament/backend/src/vulkan/VulkanContext.cpp index 28cbf871c3..99f61c8907 100644 --- a/filament/backend/src/vulkan/VulkanContext.cpp +++ b/filament/backend/src/vulkan/VulkanContext.cpp @@ -22,6 +22,9 @@ #pragma clang diagnostic ignored "-Wunused-variable" #pragma clang diagnostic ignored "-Wmissing-field-initializers" #pragma clang diagnostic ignored "-Wtautological-compare" +#pragma clang diagnostic ignored "-Wnullability-completeness" +#pragma clang diagnostic ignored "-Wweak-vtables" +#pragma clang diagnostic ignored "-Wimplicit-fallthrough" #define VMA_IMPLEMENTATION #include #include "vk_mem_alloc.h" diff --git a/filament/backend/src/vulkan/VulkanContext.h b/filament/backend/src/vulkan/VulkanContext.h index 58a8a94bce..164fd61458 100644 --- a/filament/backend/src/vulkan/VulkanContext.h +++ b/filament/backend/src/vulkan/VulkanContext.h @@ -24,9 +24,10 @@ #include -#include - +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wundef" #include "vk_mem_alloc.h" +#pragma clang diagnostic pop #include #include diff --git a/third_party/vkmemalloc/.travis.yml b/third_party/vkmemalloc/.travis.yml index 41f671a621..2a68740864 100644 --- a/third_party/vkmemalloc/.travis.yml +++ b/third_party/vkmemalloc/.travis.yml @@ -15,14 +15,14 @@ before_script: - sudo apt-get install install: - - wget https://github.com/premake/premake-core/releases/download/v5.0.0-alpha11/premake-5.0.0-alpha11-linux.tar.gz + - wget https://github.com/premake/premake-core/releases/download/v5.0.0-alpha11/premake-5.0.0-alpha11-linux.tar.gz?Human=true -O premake-5.0.0-alpha11-linux.tar.gz - tar -xzvf premake-5.0.0-alpha11-linux.tar.gz - sudo apt-get -qq update - sudo apt-get install -y libassimp-dev libglm-dev graphviz libxcb-dri3-0 libxcb-present0 libpciaccess0 cmake libpng-dev libxcb-dri3-dev libx11-dev libx11-xcb-dev libmirclient-dev libwayland-dev libxrandr-dev - - wget -O vulkansdk-linux-x86_64-1.0.61.1.run https://sdk.lunarg.com/sdk/download/1.0.61.1/linux/vulkansdk-linux-x86_64-1.0.61.1.run - - chmod ugo+x vulkansdk-linux-x86_64-1.0.61.1.run - - ./vulkansdk-linux-x86_64-1.0.61.1.run - - export VULKAN_SDK=$TRAVIS_BUILD_DIR/VulkanSDK/1.0.61.1/x86_64 + - export VK_VERSION=1.2.131.1 + - wget -O vulkansdk-linux-x86_64-$VK_VERSION.tar.gz https://sdk.lunarg.com/sdk/download/$VK_VERSION/linux/vulkansdk-linux-x86_64-$VK_VERSION.tar.gz + - tar zxf vulkansdk-linux-x86_64-$VK_VERSION.tar.gz + - export VULKAN_SDK=$TRAVIS_BUILD_DIR/$VK_VERSION/x86_64 script: - cd premake diff --git a/third_party/vkmemalloc/CHANGELOG.md b/third_party/vkmemalloc/CHANGELOG.md index be4cb6e66e..87ad08da88 100644 --- a/third_party/vkmemalloc/CHANGELOG.md +++ b/third_party/vkmemalloc/CHANGELOG.md @@ -1,3 +1,105 @@ +# 2.3.0 (2019-12-04) + +Major release after a year of development in "master" branch and feature branches. Notable new features: supporting Vulkan 1.1, supporting query for memory budget. + +Major changes: + +- Added support for Vulkan 1.1. + - Added member `VmaAllocatorCreateInfo::vulkanApiVersion`. + - When Vulkan 1.1 is used, there is no need to enable VK_KHR_dedicated_allocation or VK_KHR_bind_memory2 extensions, as they are promoted to Vulkan itself. +- Added support for query for memory budget and staying within the budget. + - Added function `vmaGetBudget`, structure `VmaBudget`. This can also serve as simple statistics, more efficient than `vmaCalculateStats`. + - By default the budget it is estimated based on memory heap sizes. It may be queried from the system using VK_EXT_memory_budget extension if you use `VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT` flag and `VmaAllocatorCreateInfo::instance` member. + - Added flag `VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT` that fails an allocation if it would exceed the budget. +- Added new memory usage options: + - `VMA_MEMORY_USAGE_CPU_COPY` for memory that is preferably not `DEVICE_LOCAL` but not guaranteed to be `HOST_VISIBLE`. + - `VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED` for memory that is `LAZILY_ALLOCATED`. +- Added support for VK_KHR_bind_memory2 extension: + - Added `VMA_ALLOCATION_CREATE_DONT_BIND_BIT` flag that lets you create both buffer/image and allocation, but don't bind them together. + - Added flag `VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT`, functions `vmaBindBufferMemory2`, `vmaBindImageMemory2` that let you specify additional local offset and `pNext` pointer while binding. +- Added functions `vmaSetPoolName`, `vmaGetPoolName` that let you assign string names to custom pools. JSON dump file format and VmaDumpVis tool is updated to show these names. +- Defragmentation is legal only on buffers and images in `VK_IMAGE_TILING_LINEAR`. This is due to the way it is currently implemented in the library and the restrictions of the Vulkan specification. Clarified documentation in this regard. See discussion in #59. + +Minor changes: + +- Made `vmaResizeAllocation` function deprecated, always returning failure. +- Made changes in the internal algorithm for the choice of memory type. Be careful! You may now get a type that is not `HOST_VISIBLE` or `HOST_COHERENT` if it's not stated as always ensured by some `VMA_MEMORY_USAGE_*` flag. +- Extended VmaReplay application with more detailed statistics printed at the end. +- Added macros `VMA_CALL_PRE`, `VMA_CALL_POST` that let you decorate declarations of all library functions if you want to e.g. export/import them as dynamically linked library. +- Optimized `VmaAllocation` objects to be allocated out of an internal free-list allocator. This makes allocation and deallocation causing 0 dynamic CPU heap allocations on average. +- Updated recording CSV file format version to 1.8, to support new functions. +- Many additions and fixes in documentation. Many compatibility fixes for various compilers and platforms. Other internal bugfixes, optimizations, updates, refactoring... + +# 2.2.0 (2018-12-13) + +Major release after many months of development in "master" branch and feature branches. Notable new features: defragmentation of GPU memory, buddy algorithm, convenience functions for sparse binding. + +Major changes: + +- New, more powerful defragmentation: + - Added structure `VmaDefragmentationInfo2`, functions `vmaDefragmentationBegin`, `vmaDefragmentationEnd`. + - Added support for defragmentation of GPU memory. + - Defragmentation of CPU memory now uses `memmove`, so it can move data to overlapping regions. + - Defragmentation of CPU memory is now available for memory types that are `HOST_VISIBLE` but not `HOST_COHERENT`. + - Added structure member `VmaVulkanFunctions::vkCmdCopyBuffer`. + - Major internal changes in defragmentation algorithm. + - VmaReplay: added parameters: `--DefragmentAfterLine`, `--DefragmentationFlags`. + - Old interface (structure `VmaDefragmentationInfo`, function `vmaDefragment`) is now deprecated. +- Added buddy algorithm, available for custom pools - flag `VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT`. +- Added convenience functions for multiple allocations and deallocations at once, intended for sparse binding resources - functions `vmaAllocateMemoryPages`, `vmaFreeMemoryPages`. +- Added function that tries to resize existing allocation in place: `vmaResizeAllocation`. +- Added flags for allocation strategy: `VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT`, `VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT`, `VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT`, and their aliases: `VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT`, `VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT`, `VMA_ALLOCATION_CREATE_STRATEGY_MIN_FRAGMENTATION_BIT`. + +Minor changes: + +- Changed behavior of allocation functions to return `VK_ERROR_VALIDATION_FAILED_EXT` when trying to allocate memory of size 0, create buffer with size 0, or image with one of the dimensions 0. +- VmaReplay: Added support for Windows end of lines. +- Updated recording CSV file format version to 1.5, to support new functions. +- Internal optimization: using read-write mutex on some platforms. +- Many additions and fixes in documentation. Many compatibility fixes for various compilers. Other internal bugfixes, optimizations, refactoring, added more internal validation... + +# 2.1.0 (2018-09-10) + +Minor bugfixes. + +# 2.1.0-beta.1 (2018-08-27) + +Major release after many months of development in "development" branch and features branches. Many new features added, some bugs fixed. API stays backward-compatible. + +Major changes: + +- Added linear allocation algorithm, accessible for custom pools, that can be used as free-at-once, stack, double stack, or ring buffer. See "Linear allocation algorithm" documentation chapter. + - Added `VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT`, `VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT`. +- Added feature to record sequence of calls to the library to a file and replay it using dedicated application. See documentation chapter "Record and replay". + - Recording: added `VmaAllocatorCreateInfo::pRecordSettings`. + - Replaying: added VmaReplay project. + - Recording file format: added document "docs/Recording file format.md". +- Improved support for non-coherent memory. + - Added functions: `vmaFlushAllocation`, `vmaInvalidateAllocation`. + - `nonCoherentAtomSize` is now respected automatically. + - Added `VmaVulkanFunctions::vkFlushMappedMemoryRanges`, `vkInvalidateMappedMemoryRanges`. +- Improved debug features related to detecting incorrect mapped memory usage. See documentation chapter "Debugging incorrect memory usage". + - Added debug macro `VMA_DEBUG_DETECT_CORRUPTION`, functions `vmaCheckCorruption`, `vmaCheckPoolCorruption`. + - Added debug macro `VMA_DEBUG_INITIALIZE_ALLOCATIONS` to initialize contents of allocations with a bit pattern. + - Changed behavior of `VMA_DEBUG_MARGIN` macro - it now adds margin also before first and after last allocation in a block. +- Changed format of JSON dump returned by `vmaBuildStatsString` (not backward compatible!). + - Custom pools and memory blocks now have IDs that don't change after sorting. + - Added properties: "CreationFrameIndex", "LastUseFrameIndex", "Usage". + - Changed VmaDumpVis tool to use these new properties for better coloring. + - Changed behavior of `vmaGetAllocationInfo` and `vmaTouchAllocation` to update `allocation.lastUseFrameIndex` even if allocation cannot become lost. + +Minor changes: + +- Changes in custom pools: + - Added new structure member `VmaPoolStats::blockCount`. + - Changed behavior of `VmaPoolCreateInfo::blockSize` = 0 (default) - it now means that pool may use variable block sizes, just like default pools do. +- Improved logic of `vmaFindMemoryTypeIndex` for some cases, especially integrated GPUs. +- VulkanSample application: Removed dependency on external library MathFu. Added own vector and matrix structures. +- Changes that improve compatibility with various platforms, including: Visual Studio 2012, 32-bit code, C compilers. + - Changed usage of "VK_KHR_dedicated_allocation" extension in the code to be optional, driven by macro `VMA_DEDICATED_ALLOCATION`, for compatibility with Android. +- Many additions and fixes in documentation, including description of new features, as well as "Validation layer warnings". +- Other bugfixes. + # 2.0.0 (2018-03-19) A major release with many compatibility-breaking changes. diff --git a/third_party/vkmemalloc/LICENSE.txt b/third_party/vkmemalloc/LICENSE.txt index dbfe253391..bee6af703f 100644 --- a/third_party/vkmemalloc/LICENSE.txt +++ b/third_party/vkmemalloc/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/third_party/vkmemalloc/README.md b/third_party/vkmemalloc/README.md index 6815d7a6ac..3c778a2057 100644 --- a/third_party/vkmemalloc/README.md +++ b/third_party/vkmemalloc/README.md @@ -20,7 +20,7 @@ Easy to integrate Vulkan memory allocation library. Memory allocation and resource (buffer and image) creation in Vulkan is difficult (comparing to older graphics API-s, like D3D11 or OpenGL) for several reasons: - It requires a lot of boilerplate code, just like everything else in Vulkan, because it is a low-level and high-performance API. -- There is additional level of indirection: `VkDeviceMemory` is allocated separately from creating `VkBuffer`/`VkImage` and they must be bound together. The binding cannot be changed later - resource must be recreated. +- There is additional level of indirection: `VkDeviceMemory` is allocated separately from creating `VkBuffer`/`VkImage` and they must be bound together. - Driver must be queried for supported memory heaps and memory types. Different IHVs provide different types of it. - It is recommended practice to allocate bigger chunks of memory and assign parts of them to particular resources. @@ -41,10 +41,18 @@ Additional features: - Configuration: Fill optional members of CreateInfo structure to provide custom CPU memory allocator, pointers to Vulkan functions and other parameters. - Customization: Predefine appropriate macros to provide your own implementation of all external facilities used by the library, from assert, mutex, and atomic, to vector and linked list. - Support for memory mapping, reference-counted internally. Support for persistently mapped memory: Just allocate with appropriate flag and you get access to mapped pointer. -- Support for non-coherent memory. Functions that flush/invalidate memory. nonCoherentAtomSize is respected automatically. +- Support for non-coherent memory. Functions that flush/invalidate memory. `nonCoherentAtomSize` is respected automatically. +- Support for resource aliasing (overlap). +- Support for sparse binding and sparse residency: Convenience functions that allocate or free multiple memory pages at once. - Custom memory pools: Create a pool with desired parameters (e.g. fixed or limited maximum size) and allocate memory out of it. -- Support for VK_KHR_dedicated_allocation extension: Just enable it and it will be used automatically by the library. -- Defragmentation: Call one function and let the library move data around to free some memory blocks and make your allocations better compacted. +- Linear allocator: Create a pool with linear algorithm and use it for much faster allocations and deallocations in free-at-once, stack, double stack, or ring buffer fashion. +- Support for Vulkan 1.0, 1.1, 1.2. +- Support for extensions (and equivalent functionality included in new Vulkan versions): + - VK_EXT_memory_budget: Used internally if available to query for current usage and budget. If not available, it falls back to an estimation based on memory heap sizes. + - VK_KHR_dedicated_allocation: Just enable it and it will be used automatically by the library. + - VK_AMD_device_coherent_memory + - VK_KHR_buffer_device_address +- Defragmentation of GPU and CPU memory: Let the library move data around to free some memory blocks and make your allocations better compacted. - Lost allocations: Allocate memory with appropriate flags and let the library remove allocations that are not used for many frames to make room for new ones. - Statistics: Obtain detailed statistics about the amount of memory used, unused, number of allocated blocks, number of allocations etc. - globally, per memory heap, and per memory type. - Debug annotations: Associate string with name or opaque pointer to your own data with every allocation. @@ -55,11 +63,11 @@ Additional features: # Prequisites -- Self-contained C++ library in single header file. No external dependencies other than standard C and C++ library and of course Vulkan. +- Self-contained C++ library in single header file. No external dependencies other than standard C and C++ library and of course Vulkan. STL containers are not used by default. - Public interface in C, in same convention as Vulkan API. Implementation in C++. - Error handling implemented by returning `VkResult` error codes - same way as in Vulkan. - Interface documented using Doxygen-style comments. -- Platform-independent, but developed and tested on Windows using Visual Studio. Continuous integration setup for Windows and Linux. Tested also on Android and MacOS. +- Platform-independent, but developed and tested on Windows using Visual Studio. Continuous integration setup for Windows and Linux. Used also on Android, MacOS, and other platforms. # Example @@ -86,19 +94,39 @@ With this one function call: `VmaAllocation` is an object that represents memory assigned to this buffer. It can be queried for parameters like Vulkan memory handle and offset. +# Binaries + +The release comes with precompiled binary executables for "VulkanSample" application which contains test suite and "VmaReplay" tool. They are compiled using Visual Studio 2017, so they require appropriate libraries to work, including "vcruntime140.dll" and "msvcp140.dll". If their launch fails with error message telling about those files missing, please download and install [Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads), "x64" version. + # Read more See **[Documentation](https://gpuopen-librariesandsdks.github.io/VulkanMemoryAllocator/html/)**. # Software using this library +- **[Vulkan Samples](https://github.com/LunarG/VulkanSamples)** - official Khronos Vulkan samples. License: Apache-style. - **[Anvil](https://github.com/GPUOpen-LibrariesAndSDKs/Anvil)** - cross-platform framework for Vulkan. License: MIT. -- **[vkDOOM3](https://github.com/DustinHLand/vkDOOM3)** - Vulkan port of GPL DOOM 3 BFG Edition. License: GNU GPL. +- **[Filament](https://github.com/google/filament)** - physically based rendering engine for Android, Windows, Linux and macOS, from Google. Apache License 2.0. +- **[Atypical Games - proprietary game engine](https://developer.samsung.com/galaxy-gamedev/gamedev-blog/infinitejet.html)** +- **[Flax Engine](https://flaxengine.com/)** - **[Lightweight Java Game Library (LWJGL)](https://www.lwjgl.org/)** - includes binding of the library for Java. License: BSD. +- **[PowerVR SDK](https://github.com/powervr-graphics/Native_SDK)** - C++ cross-platform 3D graphics SDK, from Imagination. License: MIT. +- **[Skia](https://github.com/google/skia)** - complete 2D graphic library for drawing Text, Geometries, and Images, from Google. - **[The Forge](https://github.com/ConfettiFX/The-Forge)** - cross-platform rendering framework. Apache License 2.0. +- **[VK9](https://github.com/disks86/VK9)** - Direct3D 9 compatibility layer using Vulkan. Zlib lincese. +- **[vkDOOM3](https://github.com/DustinHLand/vkDOOM3)** - Vulkan port of GPL DOOM 3 BFG Edition. License: GNU GPL. +- **[vkQuake2](https://github.com/kondrak/vkQuake2)** - vanilla Quake 2 with Vulkan support. License: GNU GPL. +- **[Vulkan Best Practice for Mobile Developers](https://github.com/ARM-software/vulkan_best_practice_for_mobile_developers)** from ARM. License: MIT. + +[Many other projects on GitHub](https://github.com/search?q=AMD_VULKAN_MEMORY_ALLOCATOR_H&type=Code) and some game development studios that use Vulkan in their games. # See also +- **[D3D12 Memory Allocator](https://github.com/GPUOpen-LibrariesAndSDKs/D3D12MemoryAllocator)** - equivalent library for Direct3D 12. License: MIT. - **[Awesome Vulkan](https://github.com/vinjn/awesome-vulkan)** - a curated list of awesome Vulkan libraries, debuggers and resources. +- **[VulkanMemoryAllocator-Hpp](https://github.com/malte-v/VulkanMemoryAllocator-Hpp)** - C++ binding for this library. License: CC0-1.0. - **[PyVMA](https://github.com/realitix/pyvma)** - Python wrapper for this library. Author: Jean-Sébastien B. (@realitix). License: Apache 2.0. +- **[vk-mem](https://github.com/gwihlidal/vk-mem-rs)** - Rust binding for this library. Author: Graham Wihlidal. License: Apache 2.0 or MIT. +- **[Haskell bindings](https://hackage.haskell.org/package/VulkanMemoryAllocator)**, **[github](https://github.com/expipiplus1/vulkan/tree/master/VulkanMemoryAllocator)** - Haskell bindings for this library. Author: Joe Hermaszewski (@expipiplus1). License BSD-3-Clause. +- **[vma_sample_sdl](https://github.com/rextimmy/vma_sample_sdl)** - SDL port of the sample app of this library (with the goal of running it on multiple platforms, including MacOS). Author: @rextimmy. License: MIT. - **[vulkan-malloc](https://github.com/dylanede/vulkan-malloc)** - Vulkan memory allocation library for Rust. Based on version 1 of this library. Author: Dylan Ede (@dylanede). License: MIT / Apache 2.0. diff --git a/third_party/vkmemalloc/src/.editorconfig b/third_party/vkmemalloc/src/.editorconfig new file mode 100644 index 0000000000..8abf0efc44 --- /dev/null +++ b/third_party/vkmemalloc/src/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[**.{cpp,h}] +indent_style = space +indent_size = 4 diff --git a/third_party/vkmemalloc/src/Common.cpp b/third_party/vkmemalloc/src/Common.cpp index ea7b9a1830..246acaa6b3 100644 --- a/third_party/vkmemalloc/src/Common.cpp +++ b/third_party/vkmemalloc/src/Common.cpp @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #include "Common.h" #ifdef _WIN32 @@ -23,7 +45,7 @@ void SetConsoleColor(CONSOLE_COLOR color) switch(color) { case CONSOLE_COLOR::INFO: - attr = FOREGROUND_INTENSITY;; + attr = FOREGROUND_INTENSITY; break; case CONSOLE_COLOR::NORMAL: attr = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; diff --git a/third_party/vkmemalloc/src/Common.h b/third_party/vkmemalloc/src/Common.h index d034196045..19c0297a4e 100644 --- a/third_party/vkmemalloc/src/Common.h +++ b/third_party/vkmemalloc/src/Common.h @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #ifndef COMMON_H_ #define COMMON_H_ @@ -16,6 +38,7 @@ #include #include #include +#include #include #include @@ -25,12 +48,30 @@ typedef std::chrono::high_resolution_clock::time_point time_point; typedef std::chrono::high_resolution_clock::duration duration; -#define ERR_GUARD_VULKAN(Expr) do { VkResult res__ = (Expr); if (res__ < 0) assert(0); } while(0) +#ifdef _DEBUG + #define TEST(expr) do { \ + if(!(expr)) { \ + assert(0 && #expr); \ + } \ + } while(0) +#else + #define TEST(expr) do { \ + if(!(expr)) { \ + throw std::runtime_error("TEST FAILED: " #expr); \ + } \ + } while(0) +#endif +#define ERR_GUARD_VULKAN(expr) TEST((expr) >= 0) + +extern VkInstance g_hVulkanInstance; extern VkPhysicalDevice g_hPhysicalDevice; extern VkDevice g_hDevice; +extern VkInstance g_hVulkanInstance; extern VmaAllocator g_hAllocator; -extern bool g_MemoryAliasingWarningEnabled; +extern bool VK_AMD_device_coherent_memory_enabled; + +void SetAllocatorCreateInfo(VmaAllocatorCreateInfo& outInfo); inline float ToFloatSeconds(duration d) { @@ -42,6 +83,11 @@ inline T ceil_div(T x, T y) { return (x+y-1) / y; } +template +inline T round_div(T x, T y) +{ + return (x+y/(T)2) / y; +} template static inline T align_up(T val, T align) @@ -51,6 +97,29 @@ static inline T align_up(T val, T align) static const float PI = 3.14159265358979323846264338327950288419716939937510582f; +template +inline void PnextChainPushFront(MainT* mainStruct, NewT* newStruct) +{ + newStruct->pNext = mainStruct->pNext; + mainStruct->pNext = newStruct; +} +template +inline void PnextChainPushBack(MainT* mainStruct, NewT* newStruct) +{ + struct VkAnyStruct + { + VkStructureType sType; + void* pNext; + }; + VkAnyStruct* lastStruct = (VkAnyStruct*)mainStruct; + while(lastStruct->pNext != nullptr) + { + lastStruct = (VkAnyStruct*)lastStruct->pNext; + } + newStruct->pNext = nullptr; + lastStruct->pNext = newStruct; +} + struct vec3 { float x, y, z; @@ -206,6 +275,19 @@ private: uint32_t GenerateFast() { return m_Value = (m_Value * 196314165 + 907633515); } }; +// Wrapper for RandomNumberGenerator compatible with STL "UniformRandomNumberGenerator" idea. +struct MyUniformRandomNumberGenerator +{ + typedef uint32_t result_type; + MyUniformRandomNumberGenerator(RandomNumberGenerator& gen) : m_Gen(gen) { } + static uint32_t min() { return 0; } + static uint32_t max() { return UINT32_MAX; } + uint32_t operator()() { return m_Gen.Generate(); } + +private: + RandomNumberGenerator& m_Gen; +}; + void ReadFile(std::vector& out, const char* fileName); enum class CONSOLE_COLOR diff --git a/third_party/vkmemalloc/src/Doxyfile b/third_party/vkmemalloc/src/Doxyfile index 6d71733299..b880bb2c59 100644 --- a/third_party/vkmemalloc/src/Doxyfile +++ b/third_party/vkmemalloc/src/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.8.13 +# Doxyfile 1.8.16 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -17,11 +17,11 @@ # Project related configuration options #--------------------------------------------------------------------------- -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv -# for the list of possible encodings. +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 @@ -93,6 +93,14 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. + +OUTPUT_TEXT_DIRECTION = None + # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -189,6 +197,16 @@ SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus @@ -236,7 +254,12 @@ TAB_SIZE = 4 # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) ALIASES = @@ -274,17 +297,26 @@ OPTIMIZE_FOR_FORTRAN = NO OPTIMIZE_OUTPUT_VHDL = NO +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat +# .inc files as Fortran files (default is PHP), and .f files as C (default is +# Fortran), use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # @@ -295,7 +327,7 @@ EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. +# documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. @@ -307,7 +339,7 @@ MARKDOWN_SUPPORT = YES # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. +# Minimum value: 0, maximum value: 99, default value: 5. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. TOC_INCLUDE_HEADINGS = 0 @@ -337,7 +369,7 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. @@ -443,6 +475,12 @@ EXTRACT_ALL = YES EXTRACT_PRIVATE = NO +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. @@ -521,7 +559,7 @@ INTERNAL_DOCS = NO # names in lower-case letters. If set to YES, upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. +# (including Cygwin) ands Mac users are advised to set this option to NO. # The default value is: system dependent. CASE_SENSE_NAMES = NO @@ -708,7 +746,7 @@ LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. @@ -753,7 +791,8 @@ WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. # The default value is: NO. WARN_NO_PARAMDOC = NO @@ -795,7 +834,7 @@ INPUT = vk_mem_alloc.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# documentation (see: https://www.gnu.org/software/libiconv/) for the list of # possible encodings. # The default value is: UTF-8. @@ -813,7 +852,7 @@ INPUT_ENCODING = UTF-8 # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, # *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.cc \ @@ -1011,7 +1050,7 @@ INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. +# entity all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO @@ -1043,12 +1082,12 @@ SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version +# (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # @@ -1076,7 +1115,7 @@ VERBATIM_HEADERS = YES # rich C++ code for which doxygen's built-in parser lacks the necessary type # information. # Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse-libclang=ON option for CMake. +# generated with the -Duse_libclang=ON option for CMake. # The default value is: NO. CLANG_ASSISTED_PARSING = NO @@ -1089,6 +1128,16 @@ CLANG_ASSISTED_PARSING = NO CLANG_OPTIONS = +# If clang assisted parsing is enabled you can provide the clang parser with the +# path to the compilation database (see: +# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files +# were built. This is equivalent to specifying the "-p" option to a clang tool, +# such as clang-check. These options will then be passed to the parser. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse_libclang=ON option for CMake. + +CLANG_DATABASE_PATH = + #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- @@ -1207,7 +1256,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. @@ -1243,6 +1292,17 @@ HTML_COLORSTYLE_GAMMA = 80 HTML_TIMESTAMP = NO +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via Javascript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have Javascript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. @@ -1266,13 +1326,13 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# environment (see: https://developer.apple.com/xcode/), introduced with OSX +# 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1311,7 +1371,7 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output @@ -1387,7 +1447,7 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1395,7 +1455,7 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1404,7 +1464,7 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1412,7 +1472,7 @@ QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1420,7 +1480,7 @@ QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = @@ -1513,7 +1573,7 @@ EXT_LINKS_IN_WINDOW = NO FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # @@ -1525,7 +1585,7 @@ FORMULA_FONTSIZE = 10 FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering +# https://www.mathjax.org) which uses client side Javascript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path @@ -1552,8 +1612,8 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest @@ -1614,7 +1674,7 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). +# Xapian (see: https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1627,7 +1687,7 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). See the section "External Indexing and +# Xapian (see: https://xapian.org/). See the section "External Indexing and # Searching" for details. # This tag requires that the tag SEARCHENGINE is set to YES. @@ -1679,21 +1739,35 @@ LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. # -# Note that when enabling USE_PDFLATEX this option is only used for generating -# bitmaps for formulas in the HTML output, but not in the Makefile that is -# written to the output directory. -# The default file is: latex. +# Note that when not enabling USE_PDFLATEX the default is latex when enabling +# USE_PDFLATEX the default is pdflatex and when in the later case latex is +# chosen this is overwritten by pdflatex. For specific output languages the +# default can have been set differently, this depends on the implementation of +# the output language. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate # index for LaTeX. +# Note: This tag is used in the Makefile / make.bat. +# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file +# (.tex). # The default file is: makeindex. # This tag requires that the tag GENERATE_LATEX is set to YES. MAKEINDEX_CMD_NAME = makeindex +# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to +# generate index for LaTeX. In case there is no backslash (\) as first character +# it will be automatically added in the LaTeX code. +# Note: This tag is used in the generated output file (.tex). +# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat. +# The default value is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_MAKEINDEX_CMD = makeindex + # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some # trees in general. @@ -1814,7 +1888,7 @@ LATEX_SOURCE_CODE = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See -# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# https://en.wikipedia.org/wiki/BibTeX and \cite for more info. # The default value is: plain. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1828,6 +1902,14 @@ LATEX_BIB_STYLE = plain LATEX_TIMESTAMP = NO +# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) +# path from which the emoji images will be read. If a relative path is entered, +# it will be relative to the LATEX_OUTPUT directory. If left blank the +# LATEX_OUTPUT directory will be used. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EMOJI_DIRECTORY = + #--------------------------------------------------------------------------- # Configuration options related to the RTF output #--------------------------------------------------------------------------- @@ -1867,9 +1949,9 @@ COMPACT_RTF = NO RTF_HYPERLINKS = NO -# Load stylesheet definitions from file. Syntax is similar to doxygen's config -# file, i.e. a series of assignments. You only have to provide replacements, -# missing definitions are set to their default value. +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# configuration file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. # # See also section "Doxygen usage" for information on how to generate the # default style sheet that doxygen normally uses. @@ -1878,8 +1960,8 @@ RTF_HYPERLINKS = NO RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an RTF document. Syntax is -# similar to doxygen's config file. A template extensions file can be generated -# using doxygen -e rtf extensionFile. +# similar to doxygen's configuration file. A template extensions file can be +# generated using doxygen -e rtf extensionFile. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_EXTENSIONS_FILE = @@ -1965,6 +2047,13 @@ XML_OUTPUT = xml XML_PROGRAMLISTING = YES +# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include +# namespace members in file scope as well, matching the HTML output. +# The default value is: NO. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_NS_MEMB_FILE_SCOPE = NO + #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- @@ -1997,9 +2086,9 @@ DOCBOOK_PROGRAMLISTING = NO #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sf.net) file that captures the -# structure of the code including all documentation. Note that this feature is -# still experimental and incomplete at the moment. +# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# the structure of the code including all documentation. Note that this feature +# is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO @@ -2059,7 +2148,7 @@ ENABLE_PREPROCESSING = YES # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -MACRO_EXPANSION = NO +MACRO_EXPANSION = YES # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then # the macro expansion is limited to the macros specified with the PREDEFINED and @@ -2067,7 +2156,7 @@ MACRO_EXPANSION = NO # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -EXPAND_ONLY_PREDEF = NO +EXPAND_ONLY_PREDEF = YES # If the SEARCH_INCLUDES tag is set to YES, the include files in the # INCLUDE_PATH will be searched if a #include is found. @@ -2099,7 +2188,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = +PREDEFINED = VMA_CALL_PRE= VMA_CALL_POST= VMA_NOT_NULL= VMA_NULLABLE= VMA_LEN_IF_NOT_NULL(len)= VMA_NOT_NULL_NON_DISPATCHABLE= VMA_NULLABLE_NON_DISPATCHABLE= # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The @@ -2166,12 +2255,6 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of 'which perl'). -# The default file (with absolute path) is: /usr/bin/perl. - -PERL_PATH = /usr/bin/perl - #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- @@ -2185,15 +2268,6 @@ PERL_PATH = /usr/bin/perl CLASS_DIAGRAMS = YES -# You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see: -# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the -# documentation. The MSCGEN_PATH tag allows you to specify the directory where -# the mscgen tool resides. If left empty the tool is assumed to be found in the -# default search path. - -MSCGEN_PATH = - # You can include diagrams made with dia in doxygen documentation. Doxygen will # then run dia to produce the diagram and insert it in the documentation. The # DIA_PATH tag allows you to specify the directory where the dia binary resides. diff --git a/third_party/vkmemalloc/src/Shaders/CompileShaders.bat b/third_party/vkmemalloc/src/Shaders/CompileShaders.bat index b0f4a65a53..5d9d815039 100644 --- a/third_party/vkmemalloc/src/Shaders/CompileShaders.bat +++ b/third_party/vkmemalloc/src/Shaders/CompileShaders.bat @@ -1,3 +1,4 @@ %VULKAN_SDK%/Bin32/glslangValidator.exe -V -o ../../bin/Shader.vert.spv Shader.vert %VULKAN_SDK%/Bin32/glslangValidator.exe -V -o ../../bin/Shader.frag.spv Shader.frag +%VULKAN_SDK%/Bin32/glslangValidator.exe -V -o ../../bin/SparseBindingTest.comp.spv SparseBindingTest.comp pause diff --git a/third_party/vkmemalloc/src/Shaders/Shader.frag b/third_party/vkmemalloc/src/Shaders/Shader.frag index 6f1f9d32ec..bbe754b000 100644 --- a/third_party/vkmemalloc/src/Shaders/Shader.frag +++ b/third_party/vkmemalloc/src/Shaders/Shader.frag @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/third_party/vkmemalloc/src/Shaders/Shader.vert b/third_party/vkmemalloc/src/Shaders/Shader.vert index c6e6cab37d..e62544f428 100644 --- a/third_party/vkmemalloc/src/Shaders/Shader.vert +++ b/third_party/vkmemalloc/src/Shaders/Shader.vert @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/third_party/vkmemalloc/src/Shaders/SparseBindingTest.comp b/third_party/vkmemalloc/src/Shaders/SparseBindingTest.comp new file mode 100644 index 0000000000..f615278513 --- /dev/null +++ b/third_party/vkmemalloc/src/Shaders/SparseBindingTest.comp @@ -0,0 +1,44 @@ +// +// Copyright (c) 2018-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#version 450 +#extension GL_ARB_separate_shader_objects : enable + +layout(local_size_x=1, local_size_y=1, local_size_z=1) in; + +layout(binding=0) uniform sampler2D img; +layout(binding=1) buffer buf +{ + uint bufValues[]; +}; + +void main() +{ + ivec2 xy = ivec2(bufValues[gl_GlobalInvocationID.x * 3], + bufValues[gl_GlobalInvocationID.x * 3 + 1]); + vec4 color = texture(img, xy); + bufValues[gl_GlobalInvocationID.x * 3 + 2] = + uint(color.r * 255.0) << 24 | + uint(color.g * 255.0) << 16 | + uint(color.b * 255.0) << 8 | + uint(color.a * 255.0); +} diff --git a/third_party/vkmemalloc/src/SparseBindingTest.cpp b/third_party/vkmemalloc/src/SparseBindingTest.cpp new file mode 100644 index 0000000000..3c636418ed --- /dev/null +++ b/third_party/vkmemalloc/src/SparseBindingTest.cpp @@ -0,0 +1,593 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#include "Common.h" +#include "SparseBindingTest.h" + +#ifdef _WIN32 + +//////////////////////////////////////////////////////////////////////////////// +// External imports + +extern VkDevice g_hDevice; +extern VmaAllocator g_hAllocator; +extern uint32_t g_FrameIndex; +extern bool g_SparseBindingEnabled; +extern VkQueue g_hSparseBindingQueue; +extern VkFence g_ImmediateFence; +extern VkCommandBuffer g_hTemporaryCommandBuffer; + +void BeginSingleTimeCommands(); +void EndSingleTimeCommands(); +void SaveAllocatorStatsToFile(const wchar_t* filePath); +void LoadShader(std::vector& out, const char* fileName); + +//////////////////////////////////////////////////////////////////////////////// +// Class definitions + +static uint32_t CalculateMipMapCount(uint32_t width, uint32_t height, uint32_t depth) +{ + uint32_t mipMapCount = 1; + while(width > 1 || height > 1 || depth > 1) + { + ++mipMapCount; + width /= 2; + height /= 2; + depth /= 2; + } + return mipMapCount; +} + +class BaseImage +{ +public: + virtual void Init(RandomNumberGenerator& rand) = 0; + virtual ~BaseImage(); + + const VkImageCreateInfo& GetCreateInfo() const { return m_CreateInfo; } + + void TestContent(RandomNumberGenerator& rand); + +protected: + VkImageCreateInfo m_CreateInfo = {}; + VkImage m_Image = VK_NULL_HANDLE; + + void FillImageCreateInfo(RandomNumberGenerator& rand); + void UploadContent(); + void ValidateContent(RandomNumberGenerator& rand); +}; + +class TraditionalImage : public BaseImage +{ +public: + virtual void Init(RandomNumberGenerator& rand); + virtual ~TraditionalImage(); + +private: + VmaAllocation m_Allocation = VK_NULL_HANDLE; +}; + +class SparseBindingImage : public BaseImage +{ +public: + virtual void Init(RandomNumberGenerator& rand); + virtual ~SparseBindingImage(); + +private: + std::vector m_Allocations; +}; + +//////////////////////////////////////////////////////////////////////////////// +// class BaseImage + +BaseImage::~BaseImage() +{ + if(m_Image) + { + vkDestroyImage(g_hDevice, m_Image, nullptr); + } +} + +void BaseImage::TestContent(RandomNumberGenerator& rand) +{ + printf("Validating content of %u x %u texture...\n", + m_CreateInfo.extent.width, m_CreateInfo.extent.height); + UploadContent(); + ValidateContent(rand); +} + +void BaseImage::FillImageCreateInfo(RandomNumberGenerator& rand) +{ + constexpr uint32_t imageSizeMin = 8; + constexpr uint32_t imageSizeMax = 2048; + + const bool useMipMaps = rand.Generate() % 2 != 0; + + ZeroMemory(&m_CreateInfo, sizeof(m_CreateInfo)); + m_CreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + m_CreateInfo.imageType = VK_IMAGE_TYPE_2D; + m_CreateInfo.extent.width = rand.Generate() % (imageSizeMax - imageSizeMin) + imageSizeMin; + m_CreateInfo.extent.height = rand.Generate() % (imageSizeMax - imageSizeMin) + imageSizeMin; + m_CreateInfo.extent.depth = 1; + m_CreateInfo.mipLevels = useMipMaps ? + CalculateMipMapCount(m_CreateInfo.extent.width, m_CreateInfo.extent.height, m_CreateInfo.extent.depth) : 1; + m_CreateInfo.arrayLayers = 1; + m_CreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + m_CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + m_CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + m_CreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + m_CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + m_CreateInfo.flags = 0; +} + +void BaseImage::UploadContent() +{ + VkBufferCreateInfo srcBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + srcBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + srcBufCreateInfo.size = 4 * m_CreateInfo.extent.width * m_CreateInfo.extent.height; + + VmaAllocationCreateInfo srcBufAllocCreateInfo = {}; + srcBufAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + srcBufAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + + VkBuffer srcBuf = nullptr; + VmaAllocation srcBufAlloc = nullptr; + VmaAllocationInfo srcAllocInfo = {}; + TEST( vmaCreateBuffer(g_hAllocator, &srcBufCreateInfo, &srcBufAllocCreateInfo, &srcBuf, &srcBufAlloc, &srcAllocInfo) == VK_SUCCESS ); + + // Fill texels with: r = x % 255, g = u % 255, b = 13, a = 25 + uint32_t* srcBufPtr = (uint32_t*)srcAllocInfo.pMappedData; + for(uint32_t y = 0, sizeY = m_CreateInfo.extent.height; y < sizeY; ++y) + { + for(uint32_t x = 0, sizeX = m_CreateInfo.extent.width; x < sizeX; ++x, ++srcBufPtr) + { + const uint8_t r = (uint8_t)x; + const uint8_t g = (uint8_t)y; + const uint8_t b = 13; + const uint8_t a = 25; + *srcBufPtr = (uint32_t)r << 24 | (uint32_t)g << 16 | + (uint32_t)b << 8 | (uint32_t)a; + } + } + + BeginSingleTimeCommands(); + + // Barrier undefined to transfer dst. + { + VkImageMemoryBarrier barrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + barrier.srcAccessMask = 0; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = m_Image; + barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.layerCount = 1; + barrier.subresourceRange.levelCount = 1; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // srcStageMask + VK_PIPELINE_STAGE_TRANSFER_BIT, // dstStageMask + 0, // dependencyFlags + 0, nullptr, // memoryBarriers + 0, nullptr, // bufferMemoryBarriers + 1, &barrier); // imageMemoryBarriers + } + + // CopyBufferToImage + { + VkBufferImageCopy region = {}; + region.bufferOffset = 0; + region.bufferRowLength = 0; // Zeros mean tightly packed. + region.bufferImageHeight = 0; // Zeros mean tightly packed. + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.mipLevel = 0; + region.imageSubresource.baseArrayLayer = 0; + region.imageSubresource.layerCount = 1; + region.imageOffset = { 0, 0, 0 }; + region.imageExtent = m_CreateInfo.extent; + vkCmdCopyBufferToImage(g_hTemporaryCommandBuffer, srcBuf, m_Image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + } + + // Barrier transfer dst to fragment shader read only. + { + VkImageMemoryBarrier barrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = m_Image; + barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.layerCount = 1; + barrier.subresourceRange.levelCount = 1; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, + VK_PIPELINE_STAGE_TRANSFER_BIT, // srcStageMask + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, // dstStageMask + 0, // dependencyFlags + 0, nullptr, // memoryBarriers + 0, nullptr, // bufferMemoryBarriers + 1, &barrier); // imageMemoryBarriers + } + + EndSingleTimeCommands(); + + vmaDestroyBuffer(g_hAllocator, srcBuf, srcBufAlloc); +} + +void BaseImage::ValidateContent(RandomNumberGenerator& rand) +{ + /* + dstBuf has following layout: + For each of texels to be sampled, [0..valueCount): + struct { + in uint32_t pixelX; + in uint32_t pixelY; + out uint32_t pixelColor; + } + */ + + const uint32_t valueCount = 128; + + VkBufferCreateInfo dstBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + dstBufCreateInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + dstBufCreateInfo.size = valueCount * sizeof(uint32_t) * 3; + + VmaAllocationCreateInfo dstBufAllocCreateInfo = {}; + dstBufAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + dstBufAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_TO_CPU; + + VkBuffer dstBuf = nullptr; + VmaAllocation dstBufAlloc = nullptr; + VmaAllocationInfo dstBufAllocInfo = {}; + TEST( vmaCreateBuffer(g_hAllocator, &dstBufCreateInfo, &dstBufAllocCreateInfo, &dstBuf, &dstBufAlloc, &dstBufAllocInfo) == VK_SUCCESS ); + + // Fill dstBuf input data. + { + uint32_t* dstBufContent = (uint32_t*)dstBufAllocInfo.pMappedData; + for(uint32_t i = 0; i < valueCount; ++i) + { + const uint32_t x = rand.Generate() % m_CreateInfo.extent.width; + const uint32_t y = rand.Generate() % m_CreateInfo.extent.height; + dstBufContent[i * 3 ] = x; + dstBufContent[i * 3 + 1] = y; + dstBufContent[i * 3 + 2] = 0; + } + } + + VkSamplerCreateInfo samplerCreateInfo = { VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO }; + samplerCreateInfo.magFilter = VK_FILTER_NEAREST; + samplerCreateInfo.minFilter = VK_FILTER_NEAREST; + samplerCreateInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + samplerCreateInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerCreateInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerCreateInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerCreateInfo.unnormalizedCoordinates = VK_TRUE; + + VkSampler sampler = nullptr; + TEST( vkCreateSampler( g_hDevice, &samplerCreateInfo, nullptr, &sampler) == VK_SUCCESS ); + + VkDescriptorSetLayoutBinding bindings[2] = {}; + bindings[0].binding = 0; + bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + bindings[0].descriptorCount = 1; + bindings[0].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + bindings[0].pImmutableSamplers = &sampler; + bindings[1].binding = 1; + bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + bindings[1].descriptorCount = 1; + bindings[1].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + + VkDescriptorSetLayoutCreateInfo descSetLayoutCreateInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO }; + descSetLayoutCreateInfo.bindingCount = 2; + descSetLayoutCreateInfo.pBindings = bindings; + + VkDescriptorSetLayout descSetLayout = nullptr; + TEST( vkCreateDescriptorSetLayout(g_hDevice, &descSetLayoutCreateInfo, nullptr, &descSetLayout) == VK_SUCCESS ); + + VkPipelineLayoutCreateInfo pipelineLayoutCreateInfo = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO }; + pipelineLayoutCreateInfo.setLayoutCount = 1; + pipelineLayoutCreateInfo.pSetLayouts = &descSetLayout; + + VkPipelineLayout pipelineLayout = nullptr; + TEST( vkCreatePipelineLayout(g_hDevice, &pipelineLayoutCreateInfo, nullptr, &pipelineLayout) == VK_SUCCESS ); + + std::vector shaderCode; + LoadShader(shaderCode, "SparseBindingTest.comp.spv"); + + VkShaderModuleCreateInfo shaderModuleCreateInfo = { VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO }; + shaderModuleCreateInfo.codeSize = shaderCode.size(); + shaderModuleCreateInfo.pCode = (const uint32_t*)shaderCode.data(); + + VkShaderModule shaderModule = nullptr; + TEST( vkCreateShaderModule(g_hDevice, &shaderModuleCreateInfo, nullptr, &shaderModule) == VK_SUCCESS ); + + VkComputePipelineCreateInfo pipelineCreateInfo = { VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO }; + pipelineCreateInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pipelineCreateInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineCreateInfo.stage.module = shaderModule; + pipelineCreateInfo.stage.pName = "main"; + pipelineCreateInfo.layout = pipelineLayout; + + VkPipeline pipeline = nullptr; + TEST( vkCreateComputePipelines(g_hDevice, nullptr, 1, &pipelineCreateInfo, nullptr, &pipeline) == VK_SUCCESS ); + + VkDescriptorPoolSize poolSizes[2] = {}; + poolSizes[0].type = bindings[0].descriptorType; + poolSizes[0].descriptorCount = bindings[0].descriptorCount; + poolSizes[1].type = bindings[1].descriptorType; + poolSizes[1].descriptorCount = bindings[1].descriptorCount; + + VkDescriptorPoolCreateInfo descPoolCreateInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO }; + descPoolCreateInfo.maxSets = 1; + descPoolCreateInfo.poolSizeCount = 2; + descPoolCreateInfo.pPoolSizes = poolSizes; + + VkDescriptorPool descPool = nullptr; + TEST( vkCreateDescriptorPool(g_hDevice, &descPoolCreateInfo, nullptr, &descPool) == VK_SUCCESS ); + + VkDescriptorSetAllocateInfo descSetAllocInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO }; + descSetAllocInfo.descriptorPool = descPool; + descSetAllocInfo.descriptorSetCount = 1; + descSetAllocInfo.pSetLayouts = &descSetLayout; + + VkDescriptorSet descSet = nullptr; + TEST( vkAllocateDescriptorSets(g_hDevice, &descSetAllocInfo, &descSet) == VK_SUCCESS ); + + VkImageViewCreateInfo imageViewCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; + imageViewCreateInfo.image = m_Image; + imageViewCreateInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + imageViewCreateInfo.format = m_CreateInfo.format; + imageViewCreateInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageViewCreateInfo.subresourceRange.layerCount = 1; + imageViewCreateInfo.subresourceRange.levelCount = 1; + + VkImageView imageView = nullptr; + TEST( vkCreateImageView(g_hDevice, &imageViewCreateInfo, nullptr, &imageView) == VK_SUCCESS ); + + VkDescriptorImageInfo descImageInfo = {}; + descImageInfo.imageView = imageView; + descImageInfo.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + + VkDescriptorBufferInfo descBufferInfo = {}; + descBufferInfo.buffer = dstBuf; + descBufferInfo.offset = 0; + descBufferInfo.range = VK_WHOLE_SIZE; + + VkWriteDescriptorSet descWrites[2] = {}; + descWrites[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + descWrites[0].dstSet = descSet; + descWrites[0].dstBinding = bindings[0].binding; + descWrites[0].dstArrayElement = 0; + descWrites[0].descriptorCount = 1; + descWrites[0].descriptorType = bindings[0].descriptorType; + descWrites[0].pImageInfo = &descImageInfo; + descWrites[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + descWrites[1].dstSet = descSet; + descWrites[1].dstBinding = bindings[1].binding; + descWrites[1].dstArrayElement = 0; + descWrites[1].descriptorCount = 1; + descWrites[1].descriptorType = bindings[1].descriptorType; + descWrites[1].pBufferInfo = &descBufferInfo; + vkUpdateDescriptorSets(g_hDevice, 2, descWrites, 0, nullptr); + + BeginSingleTimeCommands(); + vkCmdBindPipeline(g_hTemporaryCommandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + vkCmdBindDescriptorSets(g_hTemporaryCommandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descSet, 0, nullptr); + vkCmdDispatch(g_hTemporaryCommandBuffer, valueCount, 1, 1); + EndSingleTimeCommands(); + + // Validate dstBuf output data. + { + const uint32_t* dstBufContent = (const uint32_t*)dstBufAllocInfo.pMappedData; + for(uint32_t i = 0; i < valueCount; ++i) + { + const uint32_t x = dstBufContent[i * 3 ]; + const uint32_t y = dstBufContent[i * 3 + 1]; + const uint32_t color = dstBufContent[i * 3 + 2]; + const uint8_t a = (uint8_t)(color >> 24); + const uint8_t b = (uint8_t)(color >> 16); + const uint8_t g = (uint8_t)(color >> 8); + const uint8_t r = (uint8_t)color; + TEST(r == (uint8_t)x && g == (uint8_t)y && b == 13 && a == 25); + } + } + + vkDestroyImageView(g_hDevice, imageView, nullptr); + vkDestroyDescriptorPool(g_hDevice, descPool, nullptr); + vmaDestroyBuffer(g_hAllocator, dstBuf, dstBufAlloc); + vkDestroyPipeline(g_hDevice, pipeline, nullptr); + vkDestroyShaderModule(g_hDevice, shaderModule, nullptr); + vkDestroyPipelineLayout(g_hDevice, pipelineLayout, nullptr); + vkDestroyDescriptorSetLayout(g_hDevice, descSetLayout, nullptr); + vkDestroySampler(g_hDevice, sampler, nullptr); +} + +//////////////////////////////////////////////////////////////////////////////// +// class TraditionalImage + +void TraditionalImage::Init(RandomNumberGenerator& rand) +{ + FillImageCreateInfo(rand); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + // Default BEST_FIT is clearly better. + //allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT; + + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &m_CreateInfo, &allocCreateInfo, + &m_Image, &m_Allocation, nullptr) ); +} + +TraditionalImage::~TraditionalImage() +{ + if(m_Allocation) + { + vmaFreeMemory(g_hAllocator, m_Allocation); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// class SparseBindingImage + +void SparseBindingImage::Init(RandomNumberGenerator& rand) +{ + assert(g_SparseBindingEnabled && g_hSparseBindingQueue); + + // Create image. + FillImageCreateInfo(rand); + m_CreateInfo.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT; + ERR_GUARD_VULKAN( vkCreateImage(g_hDevice, &m_CreateInfo, nullptr, &m_Image) ); + + // Get memory requirements. + VkMemoryRequirements imageMemReq; + vkGetImageMemoryRequirements(g_hDevice, m_Image, &imageMemReq); + + // This is just to silence validation layer warning. + // But it doesn't help. Looks like a bug in Vulkan validation layers. + // See: https://github.com/KhronosGroup/Vulkan-ValidationLayers/issues/364 + uint32_t sparseMemReqCount = 0; + vkGetImageSparseMemoryRequirements(g_hDevice, m_Image, &sparseMemReqCount, nullptr); + TEST(sparseMemReqCount <= 8); + VkSparseImageMemoryRequirements sparseMemReq[8]; + vkGetImageSparseMemoryRequirements(g_hDevice, m_Image, &sparseMemReqCount, sparseMemReq); + + // According to Vulkan specification, for sparse resources memReq.alignment is also page size. + const VkDeviceSize pageSize = imageMemReq.alignment; + const uint32_t pageCount = (uint32_t)ceil_div(imageMemReq.size, pageSize); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VkMemoryRequirements pageMemReq = imageMemReq; + pageMemReq.size = pageSize; + + // Allocate and bind memory pages. + m_Allocations.resize(pageCount); + std::fill(m_Allocations.begin(), m_Allocations.end(), nullptr); + std::vector binds{pageCount}; + std::vector allocInfo{pageCount}; + ERR_GUARD_VULKAN( vmaAllocateMemoryPages(g_hAllocator, &pageMemReq, &allocCreateInfo, pageCount, m_Allocations.data(), allocInfo.data()) ); + + for(uint32_t i = 0; i < pageCount; ++i) + { + binds[i] = {}; + binds[i].resourceOffset = pageSize * i; + binds[i].size = pageSize; + binds[i].memory = allocInfo[i].deviceMemory; + binds[i].memoryOffset = allocInfo[i].offset; + } + + VkSparseImageOpaqueMemoryBindInfo imageBindInfo; + imageBindInfo.image = m_Image; + imageBindInfo.bindCount = pageCount; + imageBindInfo.pBinds = binds.data(); + + VkBindSparseInfo bindSparseInfo = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO }; + bindSparseInfo.pImageOpaqueBinds = &imageBindInfo; + bindSparseInfo.imageOpaqueBindCount = 1; + + ERR_GUARD_VULKAN( vkResetFences(g_hDevice, 1, &g_ImmediateFence) ); + ERR_GUARD_VULKAN( vkQueueBindSparse(g_hSparseBindingQueue, 1, &bindSparseInfo, g_ImmediateFence) ); + ERR_GUARD_VULKAN( vkWaitForFences(g_hDevice, 1, &g_ImmediateFence, VK_TRUE, UINT64_MAX) ); +} + +SparseBindingImage::~SparseBindingImage() +{ + vmaFreeMemoryPages(g_hAllocator, m_Allocations.size(), m_Allocations.data()); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private functions + +//////////////////////////////////////////////////////////////////////////////// +// Public functions + +void TestSparseBinding() +{ + struct ImageInfo + { + std::unique_ptr image; + uint32_t endFrame; + }; + std::vector images; + + constexpr uint32_t frameCount = 1000; + constexpr uint32_t imageLifeFramesMin = 1; + constexpr uint32_t imageLifeFramesMax = 400; + + RandomNumberGenerator rand(4652467); + + for(uint32_t i = 0; i < frameCount; ++i) + { + // Bump frame index. + ++g_FrameIndex; + vmaSetCurrentFrameIndex(g_hAllocator, g_FrameIndex); + + // Create one new, random image. + ImageInfo imageInfo; + //imageInfo.image = std::make_unique(); + imageInfo.image = std::make_unique(); + imageInfo.image->Init(rand); + imageInfo.endFrame = g_FrameIndex + rand.Generate() % (imageLifeFramesMax - imageLifeFramesMin) + imageLifeFramesMin; + images.push_back(std::move(imageInfo)); + + // Delete all images that expired. + for(size_t i = images.size(); i--; ) + { + if(g_FrameIndex >= images[i].endFrame) + { + images.erase(images.begin() + i); + } + } + } + + SaveAllocatorStatsToFile(L"SparseBindingTest.json"); + + // Choose biggest image. Test uploading and sampling. + BaseImage* biggestImage = nullptr; + for(size_t i = 0, count = images.size(); i < count; ++i) + { + if(!biggestImage || + images[i].image->GetCreateInfo().extent.width * images[i].image->GetCreateInfo().extent.height > + biggestImage->GetCreateInfo().extent.width * biggestImage->GetCreateInfo().extent.height) + { + biggestImage = images[i].image.get(); + } + } + assert(biggestImage); + + biggestImage->TestContent(rand); + + // Free remaining images. + images.clear(); +} + +#endif // #ifdef _WIN32 diff --git a/third_party/vkmemalloc/src/SparseBindingTest.h b/third_party/vkmemalloc/src/SparseBindingTest.h new file mode 100644 index 0000000000..3b7a6e183c --- /dev/null +++ b/third_party/vkmemalloc/src/SparseBindingTest.h @@ -0,0 +1,29 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#pragma once + +#ifdef _WIN32 + +void TestSparseBinding(); + +#endif // #ifdef _WIN32 diff --git a/third_party/vkmemalloc/src/Tests.cpp b/third_party/vkmemalloc/src/Tests.cpp index c408b0f8d6..55bd67cc1d 100644 --- a/third_party/vkmemalloc/src/Tests.cpp +++ b/third_party/vkmemalloc/src/Tests.cpp @@ -1,14 +1,85 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #include "Tests.h" #include "VmaUsage.h" #include "Common.h" #include #include #include +#include #ifdef _WIN32 +static const char* CODE_DESCRIPTION = "Foo"; + +extern VkCommandBuffer g_hTemporaryCommandBuffer; +extern const VkAllocationCallbacks* g_Allocs; +extern bool g_BufferDeviceAddressEnabled; +extern PFN_vkGetBufferDeviceAddressEXT g_vkGetBufferDeviceAddressEXT; +void BeginSingleTimeCommands(); +void EndSingleTimeCommands(); + +#ifndef VMA_DEBUG_MARGIN + #define VMA_DEBUG_MARGIN 0 +#endif + +enum CONFIG_TYPE { + CONFIG_TYPE_MINIMUM, + CONFIG_TYPE_SMALL, + CONFIG_TYPE_AVERAGE, + CONFIG_TYPE_LARGE, + CONFIG_TYPE_MAXIMUM, + CONFIG_TYPE_COUNT +}; + +static constexpr CONFIG_TYPE ConfigType = CONFIG_TYPE_SMALL; +//static constexpr CONFIG_TYPE ConfigType = CONFIG_TYPE_LARGE; + enum class FREE_ORDER { FORWARD, BACKWARD, RANDOM, COUNT }; +static const char* FREE_ORDER_NAMES[] = { + "FORWARD", + "BACKWARD", + "RANDOM", +}; + +// Copy of internal VmaAlgorithmToStr. +static const char* AlgorithmToStr(uint32_t algorithm) +{ + switch(algorithm) + { + case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT: + return "Linear"; + case VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT: + return "Buddy"; + case 0: + return "Default"; + default: + assert(0); + return ""; + } +} + struct AllocationSize { uint32_t Probability; @@ -27,6 +98,7 @@ struct Config uint32_t ThreadCount; uint32_t ThreadsUsingCommonAllocationsProbabilityPercent; FREE_ORDER FreeOrder; + VmaAllocationCreateFlags AllocationStrategy; // For VMA_ALLOCATION_CREATE_STRATEGY_* }; struct Result @@ -101,12 +173,48 @@ struct PoolTestResult static const uint32_t IMAGE_BYTES_PER_PIXEL = 1; +uint32_t g_FrameIndex = 0; + struct BufferInfo { VkBuffer Buffer = VK_NULL_HANDLE; VmaAllocation Allocation = VK_NULL_HANDLE; }; +static uint32_t MemoryTypeToHeap(uint32_t memoryTypeIndex) +{ + const VkPhysicalDeviceMemoryProperties* props; + vmaGetMemoryProperties(g_hAllocator, &props); + return props->memoryTypes[memoryTypeIndex].heapIndex; +} + +static uint32_t GetAllocationStrategyCount() +{ + uint32_t strategyCount = 0; + switch(ConfigType) + { + case CONFIG_TYPE_MINIMUM: strategyCount = 1; break; + case CONFIG_TYPE_SMALL: strategyCount = 1; break; + case CONFIG_TYPE_AVERAGE: strategyCount = 2; break; + case CONFIG_TYPE_LARGE: strategyCount = 2; break; + case CONFIG_TYPE_MAXIMUM: strategyCount = 3; break; + default: assert(0); + } + return strategyCount; +} + +static const char* GetAllocationStrategyName(VmaAllocationCreateFlags allocStrategy) +{ + switch(allocStrategy) + { + case VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT: return "BEST_FIT"; break; + case VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT: return "WORST_FIT"; break; + case VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT: return "FIRST_FIT"; break; + case 0: return "Default"; break; + default: assert(0); return ""; + } +} + static void InitResult(Result& outResult) { outResult.TotalTime = duration::zero(); @@ -192,6 +300,15 @@ public: } }; +static void CurrentTimeToStr(std::string& out) +{ + time_t rawTime; time(&rawTime); + struct tm timeInfo; localtime_s(&timeInfo, &rawTime); + char timeStr[128]; + strftime(timeStr, _countof(timeStr), "%c", &timeInfo); + out = timeStr; +} + VkResult MainTest(Result& outResult, const Config& config) { assert(config.ThreadCount > 0); @@ -244,6 +361,7 @@ VkResult MainTest(Result& outResult, const Config& config) VmaAllocationCreateInfo memReq = {}; memReq.usage = (VmaMemoryUsage)(VMA_MEMORY_USAGE_GPU_ONLY + memUsageIndex); + memReq.flags |= config.AllocationStrategy; Allocation allocation = {}; VmaAllocationInfo allocationInfo; @@ -324,7 +442,7 @@ VkResult MainTest(Result& outResult, const Config& config) } else { - assert(0); + TEST(0); } return res; }; @@ -558,8 +676,9 @@ VkResult MainTest(Result& outResult, const Config& config) return res; } -static void SaveAllocatorStatsToFile(const wchar_t* filePath) +void SaveAllocatorStatsToFile(const wchar_t* filePath) { + wprintf(L"Saving JSON dump to file \"%s\"\n", filePath); char* stats; vmaBuildStatsString(g_hAllocator, &stats, VK_TRUE); SaveFile(filePath, stats, strlen(stats)); @@ -568,17 +687,445 @@ static void SaveAllocatorStatsToFile(const wchar_t* filePath) struct AllocInfo { - VmaAllocation m_Allocation; - VkBuffer m_Buffer; - VkImage m_Image; - uint32_t m_StartValue; + VmaAllocation m_Allocation = VK_NULL_HANDLE; + VkBuffer m_Buffer = VK_NULL_HANDLE; + VkImage m_Image = VK_NULL_HANDLE; + VkImageLayout m_ImageLayout = VK_IMAGE_LAYOUT_UNDEFINED; + uint32_t m_StartValue = 0; union { VkBufferCreateInfo m_BufferInfo; VkImageCreateInfo m_ImageInfo; }; + + // After defragmentation. + VkBuffer m_NewBuffer = VK_NULL_HANDLE; + VkImage m_NewImage = VK_NULL_HANDLE; + + void CreateBuffer( + const VkBufferCreateInfo& bufCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo); + void CreateImage( + const VkImageCreateInfo& imageCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VkImageLayout layout); + void Destroy(); }; +void AllocInfo::CreateBuffer( + const VkBufferCreateInfo& bufCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo) +{ + m_BufferInfo = bufCreateInfo; + VkResult res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &m_Buffer, &m_Allocation, nullptr); + TEST(res == VK_SUCCESS); +} +void AllocInfo::CreateImage( + const VkImageCreateInfo& imageCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VkImageLayout layout) +{ + m_ImageInfo = imageCreateInfo; + m_ImageLayout = layout; + VkResult res = vmaCreateImage(g_hAllocator, &imageCreateInfo, &allocCreateInfo, &m_Image, &m_Allocation, nullptr); + TEST(res == VK_SUCCESS); +} + +void AllocInfo::Destroy() +{ + if(m_Image) + { + assert(!m_Buffer); + vkDestroyImage(g_hDevice, m_Image, g_Allocs); + m_Image = VK_NULL_HANDLE; + } + if(m_Buffer) + { + assert(!m_Image); + vkDestroyBuffer(g_hDevice, m_Buffer, g_Allocs); + m_Buffer = VK_NULL_HANDLE; + } + if(m_Allocation) + { + vmaFreeMemory(g_hAllocator, m_Allocation); + m_Allocation = VK_NULL_HANDLE; + } +} + +class StagingBufferCollection +{ +public: + StagingBufferCollection() { } + ~StagingBufferCollection(); + // Returns false if maximum total size of buffers would be exceeded. + bool AcquireBuffer(VkDeviceSize size, VkBuffer& outBuffer, void*& outMappedPtr); + void ReleaseAllBuffers(); + +private: + static const VkDeviceSize MAX_TOTAL_SIZE = 256ull * 1024 * 1024; + struct BufInfo + { + VmaAllocation Allocation = VK_NULL_HANDLE; + VkBuffer Buffer = VK_NULL_HANDLE; + VkDeviceSize Size = VK_WHOLE_SIZE; + void* MappedPtr = nullptr; + bool Used = false; + }; + std::vector m_Bufs; + // Including both used and unused. + VkDeviceSize m_TotalSize = 0; +}; + +StagingBufferCollection::~StagingBufferCollection() +{ + for(size_t i = m_Bufs.size(); i--; ) + { + vmaDestroyBuffer(g_hAllocator, m_Bufs[i].Buffer, m_Bufs[i].Allocation); + } +} + +bool StagingBufferCollection::AcquireBuffer(VkDeviceSize size, VkBuffer& outBuffer, void*& outMappedPtr) +{ + assert(size <= MAX_TOTAL_SIZE); + + // Try to find existing unused buffer with best size. + size_t bestIndex = SIZE_MAX; + for(size_t i = 0, count = m_Bufs.size(); i < count; ++i) + { + BufInfo& currBufInfo = m_Bufs[i]; + if(!currBufInfo.Used && currBufInfo.Size >= size && + (bestIndex == SIZE_MAX || currBufInfo.Size < m_Bufs[bestIndex].Size)) + { + bestIndex = i; + } + } + + if(bestIndex != SIZE_MAX) + { + m_Bufs[bestIndex].Used = true; + outBuffer = m_Bufs[bestIndex].Buffer; + outMappedPtr = m_Bufs[bestIndex].MappedPtr; + return true; + } + + // Allocate new buffer with requested size. + if(m_TotalSize + size <= MAX_TOTAL_SIZE) + { + BufInfo bufInfo; + bufInfo.Size = size; + bufInfo.Used = true; + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = size; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + + VmaAllocationInfo allocInfo; + VkResult res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &bufInfo.Buffer, &bufInfo.Allocation, &allocInfo); + bufInfo.MappedPtr = allocInfo.pMappedData; + TEST(res == VK_SUCCESS && bufInfo.MappedPtr); + + outBuffer = bufInfo.Buffer; + outMappedPtr = bufInfo.MappedPtr; + + m_Bufs.push_back(std::move(bufInfo)); + + m_TotalSize += size; + + return true; + } + + // There are some unused but smaller buffers: Free them and try again. + bool hasUnused = false; + for(size_t i = 0, count = m_Bufs.size(); i < count; ++i) + { + if(!m_Bufs[i].Used) + { + hasUnused = true; + break; + } + } + if(hasUnused) + { + for(size_t i = m_Bufs.size(); i--; ) + { + if(!m_Bufs[i].Used) + { + m_TotalSize -= m_Bufs[i].Size; + vmaDestroyBuffer(g_hAllocator, m_Bufs[i].Buffer, m_Bufs[i].Allocation); + m_Bufs.erase(m_Bufs.begin() + i); + } + } + + return AcquireBuffer(size, outBuffer, outMappedPtr); + } + + return false; +} + +void StagingBufferCollection::ReleaseAllBuffers() +{ + for(size_t i = 0, count = m_Bufs.size(); i < count; ++i) + { + m_Bufs[i].Used = false; + } +} + +static void UploadGpuData(const AllocInfo* allocInfo, size_t allocInfoCount) +{ + StagingBufferCollection stagingBufs; + + bool cmdBufferStarted = false; + for(size_t allocInfoIndex = 0; allocInfoIndex < allocInfoCount; ++allocInfoIndex) + { + const AllocInfo& currAllocInfo = allocInfo[allocInfoIndex]; + if(currAllocInfo.m_Buffer) + { + const VkDeviceSize size = currAllocInfo.m_BufferInfo.size; + + VkBuffer stagingBuf = VK_NULL_HANDLE; + void* stagingBufMappedPtr = nullptr; + if(!stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr)) + { + TEST(cmdBufferStarted); + EndSingleTimeCommands(); + stagingBufs.ReleaseAllBuffers(); + cmdBufferStarted = false; + + bool ok = stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr); + TEST(ok); + } + + // Fill staging buffer. + { + assert(size % sizeof(uint32_t) == 0); + uint32_t* stagingValPtr = (uint32_t*)stagingBufMappedPtr; + uint32_t val = currAllocInfo.m_StartValue; + for(size_t i = 0; i < size / sizeof(uint32_t); ++i) + { + *stagingValPtr = val; + ++stagingValPtr; + ++val; + } + } + + // Issue copy command from staging buffer to destination buffer. + if(!cmdBufferStarted) + { + cmdBufferStarted = true; + BeginSingleTimeCommands(); + } + + VkBufferCopy copy = {}; + copy.srcOffset = 0; + copy.dstOffset = 0; + copy.size = size; + vkCmdCopyBuffer(g_hTemporaryCommandBuffer, stagingBuf, currAllocInfo.m_Buffer, 1, ©); + } + else + { + TEST(currAllocInfo.m_ImageInfo.format == VK_FORMAT_R8G8B8A8_UNORM && "Only RGBA8 images are currently supported."); + TEST(currAllocInfo.m_ImageInfo.mipLevels == 1 && "Only single mip images are currently supported."); + + const VkDeviceSize size = (VkDeviceSize)currAllocInfo.m_ImageInfo.extent.width * currAllocInfo.m_ImageInfo.extent.height * sizeof(uint32_t); + + VkBuffer stagingBuf = VK_NULL_HANDLE; + void* stagingBufMappedPtr = nullptr; + if(!stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr)) + { + TEST(cmdBufferStarted); + EndSingleTimeCommands(); + stagingBufs.ReleaseAllBuffers(); + cmdBufferStarted = false; + + bool ok = stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr); + TEST(ok); + } + + // Fill staging buffer. + { + assert(size % sizeof(uint32_t) == 0); + uint32_t *stagingValPtr = (uint32_t *)stagingBufMappedPtr; + uint32_t val = currAllocInfo.m_StartValue; + for(size_t i = 0; i < size / sizeof(uint32_t); ++i) + { + *stagingValPtr = val; + ++stagingValPtr; + ++val; + } + } + + // Issue copy command from staging buffer to destination buffer. + if(!cmdBufferStarted) + { + cmdBufferStarted = true; + BeginSingleTimeCommands(); + } + + + // Transfer to transfer dst layout + VkImageSubresourceRange subresourceRange = { + VK_IMAGE_ASPECT_COLOR_BIT, + 0, VK_REMAINING_MIP_LEVELS, + 0, VK_REMAINING_ARRAY_LAYERS + }; + + VkImageMemoryBarrier barrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + barrier.srcAccessMask = 0; + barrier.dstAccessMask = 0; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = currAllocInfo.m_Image; + barrier.subresourceRange = subresourceRange; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, + 0, nullptr, + 0, nullptr, + 1, &barrier); + + // Copy image date + VkBufferImageCopy copy = {}; + copy.bufferOffset = 0; + copy.bufferRowLength = 0; + copy.bufferImageHeight = 0; + copy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copy.imageSubresource.layerCount = 1; + copy.imageExtent = currAllocInfo.m_ImageInfo.extent; + + vkCmdCopyBufferToImage(g_hTemporaryCommandBuffer, stagingBuf, currAllocInfo.m_Image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + + // Transfer to desired layout + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = currAllocInfo.m_ImageLayout; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, + 0, nullptr, + 0, nullptr, + 1, &barrier); + } + } + + if(cmdBufferStarted) + { + EndSingleTimeCommands(); + stagingBufs.ReleaseAllBuffers(); + } +} + +static void ValidateGpuData(const AllocInfo* allocInfo, size_t allocInfoCount) +{ + StagingBufferCollection stagingBufs; + + bool cmdBufferStarted = false; + size_t validateAllocIndexOffset = 0; + std::vector validateStagingBuffers; + for(size_t allocInfoIndex = 0; allocInfoIndex < allocInfoCount; ++allocInfoIndex) + { + const AllocInfo& currAllocInfo = allocInfo[allocInfoIndex]; + if(currAllocInfo.m_Buffer) + { + const VkDeviceSize size = currAllocInfo.m_BufferInfo.size; + + VkBuffer stagingBuf = VK_NULL_HANDLE; + void* stagingBufMappedPtr = nullptr; + if(!stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr)) + { + TEST(cmdBufferStarted); + EndSingleTimeCommands(); + cmdBufferStarted = false; + + for(size_t validateIndex = 0; + validateIndex < validateStagingBuffers.size(); + ++validateIndex) + { + const size_t validateAllocIndex = validateIndex + validateAllocIndexOffset; + const VkDeviceSize validateSize = allocInfo[validateAllocIndex].m_BufferInfo.size; + TEST(validateSize % sizeof(uint32_t) == 0); + const uint32_t* stagingValPtr = (const uint32_t*)validateStagingBuffers[validateIndex]; + uint32_t val = allocInfo[validateAllocIndex].m_StartValue; + bool valid = true; + for(size_t i = 0; i < validateSize / sizeof(uint32_t); ++i) + { + if(*stagingValPtr != val) + { + valid = false; + break; + } + ++stagingValPtr; + ++val; + } + TEST(valid); + } + + stagingBufs.ReleaseAllBuffers(); + + validateAllocIndexOffset = allocInfoIndex; + validateStagingBuffers.clear(); + + bool ok = stagingBufs.AcquireBuffer(size, stagingBuf, stagingBufMappedPtr); + TEST(ok); + } + + // Issue copy command from staging buffer to destination buffer. + if(!cmdBufferStarted) + { + cmdBufferStarted = true; + BeginSingleTimeCommands(); + } + + VkBufferCopy copy = {}; + copy.srcOffset = 0; + copy.dstOffset = 0; + copy.size = size; + vkCmdCopyBuffer(g_hTemporaryCommandBuffer, currAllocInfo.m_Buffer, stagingBuf, 1, ©); + + // Sava mapped pointer for later validation. + validateStagingBuffers.push_back(stagingBufMappedPtr); + } + else + { + TEST(0 && "Images not currently supported."); + } + } + + if(cmdBufferStarted) + { + EndSingleTimeCommands(); + + for(size_t validateIndex = 0; + validateIndex < validateStagingBuffers.size(); + ++validateIndex) + { + const size_t validateAllocIndex = validateIndex + validateAllocIndexOffset; + const VkDeviceSize validateSize = allocInfo[validateAllocIndex].m_BufferInfo.size; + TEST(validateSize % sizeof(uint32_t) == 0); + const uint32_t* stagingValPtr = (const uint32_t*)validateStagingBuffers[validateIndex]; + uint32_t val = allocInfo[validateAllocIndex].m_StartValue; + bool valid = true; + for(size_t i = 0; i < validateSize / sizeof(uint32_t); ++i) + { + if(*stagingValPtr != val) + { + valid = false; + break; + } + ++stagingValPtr; + ++val; + } + TEST(valid); + } + + stagingBufs.ReleaseAllBuffers(); + } +} + static void GetMemReq(VmaAllocationCreateInfo& outMemReq) { outMemReq = {}; @@ -607,14 +1154,14 @@ static void CreateBuffer( { outAllocInfo.m_StartValue = (uint32_t)rand(); uint32_t* data = (uint32_t*)vmaAllocInfo.pMappedData; - assert((data != nullptr) == persistentlyMapped); + TEST((data != nullptr) == persistentlyMapped); if(!persistentlyMapped) { ERR_GUARD_VULKAN( vmaMapMemory(g_hAllocator, outAllocInfo.m_Allocation, (void**)&data) ); } uint32_t value = outAllocInfo.m_StartValue; - assert(bufCreateInfo.size % 4 == 0); + TEST(bufCreateInfo.size % 4 == 0); for(size_t i = 0; i < bufCreateInfo.size / sizeof(uint32_t); ++i) data[i] = value++; @@ -623,7 +1170,7 @@ static void CreateBuffer( } } -static void CreateAllocation(AllocInfo& outAllocation, VmaAllocator allocator) +static void CreateAllocation(AllocInfo& outAllocation) { outAllocation.m_Allocation = nullptr; outAllocation.m_Buffer = nullptr; @@ -647,9 +1194,9 @@ static void CreateAllocation(AllocInfo& outAllocation, VmaAllocator allocator) bufferInfo.size = bufferSize; bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; - VkResult res = vmaCreateBuffer(allocator, &bufferInfo, &vmaMemReq, &outAllocation.m_Buffer, &outAllocation.m_Allocation, &allocInfo); + VkResult res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &vmaMemReq, &outAllocation.m_Buffer, &outAllocation.m_Allocation, &allocInfo); outAllocation.m_BufferInfo = bufferInfo; - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } else { @@ -673,25 +1220,25 @@ static void CreateAllocation(AllocInfo& outAllocation, VmaAllocator allocator) imageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT; - VkResult res = vmaCreateImage(allocator, &imageInfo, &vmaMemReq, &outAllocation.m_Image, &outAllocation.m_Allocation, &allocInfo); + VkResult res = vmaCreateImage(g_hAllocator, &imageInfo, &vmaMemReq, &outAllocation.m_Image, &outAllocation.m_Allocation, &allocInfo); outAllocation.m_ImageInfo = imageInfo; - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } uint32_t* data = (uint32_t*)allocInfo.pMappedData; if(allocInfo.pMappedData == nullptr) { - VkResult res = vmaMapMemory(allocator, outAllocation.m_Allocation, (void**)&data); - assert(res == VK_SUCCESS); + VkResult res = vmaMapMemory(g_hAllocator, outAllocation.m_Allocation, (void**)&data); + TEST(res == VK_SUCCESS); } uint32_t value = outAllocation.m_StartValue; - assert(allocInfo.size % 4 == 0); + TEST(allocInfo.size % 4 == 0); for(size_t i = 0; i < allocInfo.size / sizeof(uint32_t); ++i) data[i] = value++; if(allocInfo.pMappedData == nullptr) - vmaUnmapMemory(allocator, outAllocation.m_Allocation); + vmaUnmapMemory(g_hAllocator, outAllocation.m_Allocation); } static void DestroyAllocation(const AllocInfo& allocation) @@ -718,13 +1265,13 @@ static void ValidateAllocationData(const AllocInfo& allocation) if(allocInfo.pMappedData == nullptr) { VkResult res = vmaMapMemory(g_hAllocator, allocation.m_Allocation, (void**)&data); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } uint32_t value = allocation.m_StartValue; bool ok = true; size_t i; - assert(allocInfo.size % 4 == 0); + TEST(allocInfo.size % 4 == 0); for(i = 0; i < allocInfo.size / sizeof(uint32_t); ++i) { if(data[i] != value++) @@ -733,7 +1280,7 @@ static void ValidateAllocationData(const AllocInfo& allocation) break; } } - assert(ok); + TEST(ok); if(allocInfo.pMappedData == nullptr) vmaUnmapMemory(g_hAllocator, allocation.m_Allocation); @@ -746,32 +1293,32 @@ static void RecreateAllocationResource(AllocInfo& allocation) if(allocation.m_Buffer) { - vkDestroyBuffer(g_hDevice, allocation.m_Buffer, nullptr); + vkDestroyBuffer(g_hDevice, allocation.m_Buffer, g_Allocs); - VkResult res = vkCreateBuffer(g_hDevice, &allocation.m_BufferInfo, nullptr, &allocation.m_Buffer); - assert(res == VK_SUCCESS); + VkResult res = vkCreateBuffer(g_hDevice, &allocation.m_BufferInfo, g_Allocs, &allocation.m_Buffer); + TEST(res == VK_SUCCESS); // Just to silence validation layer warnings. VkMemoryRequirements vkMemReq; vkGetBufferMemoryRequirements(g_hDevice, allocation.m_Buffer, &vkMemReq); - assert(vkMemReq.size == allocation.m_BufferInfo.size); + TEST(vkMemReq.size >= allocation.m_BufferInfo.size); - res = vkBindBufferMemory(g_hDevice, allocation.m_Buffer, allocInfo.deviceMemory, allocInfo.offset); - assert(res == VK_SUCCESS); + res = vmaBindBufferMemory(g_hAllocator, allocation.m_Allocation, allocation.m_Buffer); + TEST(res == VK_SUCCESS); } else { - vkDestroyImage(g_hDevice, allocation.m_Image, nullptr); + vkDestroyImage(g_hDevice, allocation.m_Image, g_Allocs); - VkResult res = vkCreateImage(g_hDevice, &allocation.m_ImageInfo, nullptr, &allocation.m_Image); - assert(res == VK_SUCCESS); + VkResult res = vkCreateImage(g_hDevice, &allocation.m_ImageInfo, g_Allocs, &allocation.m_Image); + TEST(res == VK_SUCCESS); // Just to silence validation layer warnings. VkMemoryRequirements vkMemReq; vkGetImageMemoryRequirements(g_hDevice, allocation.m_Image, &vkMemReq); - res = vkBindImageMemory(g_hDevice, allocation.m_Image, allocInfo.deviceMemory, allocInfo.offset); - assert(res == VK_SUCCESS); + res = vmaBindImageMemory(g_hAllocator, allocation.m_Allocation, allocation.m_Image); + TEST(res == VK_SUCCESS); } } @@ -836,6 +1383,23 @@ void TestDefragmentationSimple() VmaPool pool; ERR_GUARD_VULKAN( vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool) ); + // Defragmentation of empty pool. + { + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.maxCpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.maxCpuAllocationsToMove = UINT32_MAX; + defragInfo.poolCount = 1; + defragInfo.pPools = &pool; + + VmaDefragmentationStats defragStats = {}; + VmaDefragmentationContext defragCtx = nullptr; + VkResult res = vmaDefragmentationBegin(g_hAllocator, &defragInfo, &defragStats, &defragCtx); + TEST(res >= VK_SUCCESS); + vmaDefragmentationEnd(g_hAllocator, defragCtx); + TEST(defragStats.allocationsMoved == 0 && defragStats.bytesFreed == 0 && + defragStats.bytesMoved == 0 && defragStats.deviceMemoryBlocksFreed == 0); + } + std::vector allocations; // persistentlyMappedOption = 0 - not persistently mapped. @@ -865,8 +1429,8 @@ void TestDefragmentationSimple() VmaDefragmentationStats defragStats; Defragment(allocations.data(), allocations.size(), nullptr, &defragStats); - assert(defragStats.allocationsMoved > 0 && defragStats.bytesMoved > 0); - assert(defragStats.deviceMemoryBlocksFreed >= 1); + TEST(defragStats.allocationsMoved > 0 && defragStats.bytesMoved > 0); + TEST(defragStats.deviceMemoryBlocksFreed >= 1); ValidateAllocationsData(allocations.data(), allocations.size()); @@ -899,7 +1463,7 @@ void TestDefragmentationSimple() { VmaDefragmentationStats defragStats; Defragment(allocations.data(), allocations.size(), &defragInfo, &defragStats); - assert(defragStats.allocationsMoved > 0 && defragStats.bytesMoved > 0); + TEST(defragStats.allocationsMoved > 0 && defragStats.bytesMoved > 0); } ValidateAllocationsData(allocations.data(), allocations.size()); @@ -954,9 +1518,121 @@ void TestDefragmentationSimple() } } + /* + Allocation that must be move to an overlapping place using memmove(). + Create 2 buffers, second slightly bigger than the first. Delete first. Then defragment. + */ + if(VMA_DEBUG_MARGIN == 0) // FAST algorithm works only when DEBUG_MARGIN disabled. + { + AllocInfo allocInfo[2]; + + bufCreateInfo.size = BUF_SIZE; + CreateBuffer(pool, bufCreateInfo, false, allocInfo[0]); + const VkDeviceSize biggerBufSize = BUF_SIZE + BUF_SIZE / 256; + bufCreateInfo.size = biggerBufSize; + CreateBuffer(pool, bufCreateInfo, false, allocInfo[1]); + + DestroyAllocation(allocInfo[0]); + + VmaDefragmentationStats defragStats; + Defragment(&allocInfo[1], 1, nullptr, &defragStats); + // If this fails, it means we couldn't do memmove with overlapping regions. + TEST(defragStats.allocationsMoved == 1 && defragStats.bytesMoved > 0); + + ValidateAllocationsData(&allocInfo[1], 1); + DestroyAllocation(allocInfo[1]); + } + vmaDestroyPool(g_hAllocator, pool); } +void TestDefragmentationWholePool() +{ + wprintf(L"Test defragmentation whole pool\n"); + + RandomNumberGenerator rand(668); + + const VkDeviceSize BUF_SIZE = 0x10000; + const VkDeviceSize BLOCK_SIZE = BUF_SIZE * 8; + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = BUF_SIZE; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VmaAllocationCreateInfo exampleAllocCreateInfo = {}; + exampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + + uint32_t memTypeIndex = UINT32_MAX; + vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &bufCreateInfo, &exampleAllocCreateInfo, &memTypeIndex); + + VmaPoolCreateInfo poolCreateInfo = {}; + poolCreateInfo.blockSize = BLOCK_SIZE; + poolCreateInfo.memoryTypeIndex = memTypeIndex; + + VmaDefragmentationStats defragStats[2]; + for(size_t caseIndex = 0; caseIndex < 2; ++caseIndex) + { + VmaPool pool; + ERR_GUARD_VULKAN( vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool) ); + + std::vector allocations; + + // Buffers of fixed size. + // Fill 2 blocks. Remove odd buffers. Defragment all of them. + for(size_t i = 0; i < BLOCK_SIZE / BUF_SIZE * 2; ++i) + { + AllocInfo allocInfo; + CreateBuffer(pool, bufCreateInfo, false, allocInfo); + allocations.push_back(allocInfo); + } + + for(size_t i = 1; i < allocations.size(); ++i) + { + DestroyAllocation(allocations[i]); + allocations.erase(allocations.begin() + i); + } + + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.maxCpuAllocationsToMove = UINT32_MAX; + defragInfo.maxCpuBytesToMove = VK_WHOLE_SIZE; + std::vector allocationsToDefrag; + if(caseIndex == 0) + { + defragInfo.poolCount = 1; + defragInfo.pPools = &pool; + } + else + { + const size_t allocCount = allocations.size(); + allocationsToDefrag.resize(allocCount); + std::transform( + allocations.begin(), allocations.end(), + allocationsToDefrag.begin(), + [](const AllocInfo& allocInfo) { return allocInfo.m_Allocation; }); + defragInfo.allocationCount = (uint32_t)allocCount; + defragInfo.pAllocations = allocationsToDefrag.data(); + } + + VmaDefragmentationContext defragCtx = VK_NULL_HANDLE; + VkResult res = vmaDefragmentationBegin(g_hAllocator, &defragInfo, &defragStats[caseIndex], &defragCtx); + TEST(res >= VK_SUCCESS); + vmaDefragmentationEnd(g_hAllocator, defragCtx); + + TEST(defragStats[caseIndex].allocationsMoved > 0 && defragStats[caseIndex].bytesMoved > 0); + + ValidateAllocationsData(allocations.data(), allocations.size()); + + DestroyAllAllocations(allocations); + + vmaDestroyPool(g_hAllocator, pool); + } + + TEST(defragStats[0].bytesMoved == defragStats[1].bytesMoved); + TEST(defragStats[0].allocationsMoved == defragStats[1].allocationsMoved); + TEST(defragStats[0].bytesFreed == defragStats[1].bytesFreed); + TEST(defragStats[0].deviceMemoryBlocksFreed == defragStats[1].deviceMemoryBlocksFreed); +} + void TestDefragmentationFull() { std::vector allocations; @@ -965,7 +1641,7 @@ void TestDefragmentationFull() for(size_t i = 0; i < 400; ++i) { AllocInfo allocation; - CreateAllocation(allocation, g_hAllocator); + CreateAllocation(allocation); allocations.push_back(allocation); } @@ -982,7 +1658,7 @@ void TestDefragmentationFull() for(size_t i = 0; i < allocations.size(); ++i) ValidateAllocationData(allocations[i]); - SaveAllocatorStatsToFile(L"Before.csv"); + //SaveAllocatorStatsToFile(L"Before.csv"); { std::vector vmaAllocations(allocations.size()); @@ -1012,7 +1688,7 @@ void TestDefragmentationFull() VmaDefragmentationStats stats; VkResult res = vmaDefragment(g_hAllocator, vmaAllocations.data(), vmaAllocations.size(), allocationsChanged.data(), &defragmentationInfo, &stats); - assert(res >= 0); + TEST(res >= 0); float defragmentDuration = ToFloatSeconds(std::chrono::high_resolution_clock::now() - begTime); @@ -1031,9 +1707,9 @@ void TestDefragmentationFull() for(size_t i = 0; i < allocations.size(); ++i) ValidateAllocationData(allocations[i]); - wchar_t fileName[MAX_PATH]; - swprintf(fileName, MAX_PATH, L"After_%02u.csv", defragIndex); - SaveAllocatorStatsToFile(fileName); + //wchar_t fileName[MAX_PATH]; + //swprintf(fileName, MAX_PATH, L"After_%02u.csv", defragIndex); + //SaveAllocatorStatsToFile(fileName); } } @@ -1041,6 +1717,723 @@ void TestDefragmentationFull() DestroyAllAllocations(allocations); } +static void TestDefragmentationGpu() +{ + wprintf(L"Test defragmentation GPU\n"); + + std::vector allocations; + + // Create that many allocations to surely fill 3 new blocks of 256 MB. + const VkDeviceSize bufSizeMin = 5ull * 1024 * 1024; + const VkDeviceSize bufSizeMax = 10ull * 1024 * 1024; + const VkDeviceSize totalSize = 3ull * 256 * 1024 * 1024; + const size_t bufCount = (size_t)(totalSize / bufSizeMin); + const size_t percentToLeave = 30; + const size_t percentNonMovable = 3; + RandomNumberGenerator rand = { 234522 }; + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + allocCreateInfo.flags = 0; + + // Create all intended buffers. + for(size_t i = 0; i < bufCount; ++i) + { + bufCreateInfo.size = align_up(rand.Generate() % (bufSizeMax - bufSizeMin) + bufSizeMin, 32ull); + + if(rand.Generate() % 100 < percentNonMovable) + { + bufCreateInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + allocCreateInfo.pUserData = (void*)(uintptr_t)2; + } + else + { + // Different usage just to see different color in output from VmaDumpVis. + bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + // And in JSON dump. + allocCreateInfo.pUserData = (void*)(uintptr_t)1; + } + + AllocInfo alloc; + alloc.CreateBuffer(bufCreateInfo, allocCreateInfo); + alloc.m_StartValue = rand.Generate(); + allocations.push_back(alloc); + } + + // Destroy some percentage of them. + { + const size_t buffersToDestroy = round_div(bufCount * (100 - percentToLeave), 100); + for(size_t i = 0; i < buffersToDestroy; ++i) + { + const size_t index = rand.Generate() % allocations.size(); + allocations[index].Destroy(); + allocations.erase(allocations.begin() + index); + } + } + + // Fill them with meaningful data. + UploadGpuData(allocations.data(), allocations.size()); + + wchar_t fileName[MAX_PATH]; + swprintf_s(fileName, L"GPU_defragmentation_A_before.json"); + SaveAllocatorStatsToFile(fileName); + + // Defragment using GPU only. + { + const size_t allocCount = allocations.size(); + + std::vector allocationPtrs; + std::vector allocationChanged; + std::vector allocationOriginalIndex; + + for(size_t i = 0; i < allocCount; ++i) + { + VmaAllocationInfo allocInfo = {}; + vmaGetAllocationInfo(g_hAllocator, allocations[i].m_Allocation, &allocInfo); + if((uintptr_t)allocInfo.pUserData == 1) // Movable + { + allocationPtrs.push_back(allocations[i].m_Allocation); + allocationChanged.push_back(VK_FALSE); + allocationOriginalIndex.push_back(i); + } + } + + const size_t movableAllocCount = allocationPtrs.size(); + + BeginSingleTimeCommands(); + + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.flags = 0; + defragInfo.allocationCount = (uint32_t)movableAllocCount; + defragInfo.pAllocations = allocationPtrs.data(); + defragInfo.pAllocationsChanged = allocationChanged.data(); + defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.maxGpuAllocationsToMove = UINT32_MAX; + defragInfo.commandBuffer = g_hTemporaryCommandBuffer; + + VmaDefragmentationStats stats = {}; + VmaDefragmentationContext ctx = VK_NULL_HANDLE; + VkResult res = vmaDefragmentationBegin(g_hAllocator, &defragInfo, &stats, &ctx); + TEST(res >= VK_SUCCESS); + + EndSingleTimeCommands(); + + vmaDefragmentationEnd(g_hAllocator, ctx); + + for(size_t i = 0; i < movableAllocCount; ++i) + { + if(allocationChanged[i]) + { + const size_t origAllocIndex = allocationOriginalIndex[i]; + RecreateAllocationResource(allocations[origAllocIndex]); + } + } + + // If corruption detection is enabled, GPU defragmentation may not work on + // memory types that have this detection active, e.g. on Intel. + #if !defined(VMA_DEBUG_DETECT_CORRUPTION) || VMA_DEBUG_DETECT_CORRUPTION == 0 + TEST(stats.allocationsMoved > 0 && stats.bytesMoved > 0); + TEST(stats.deviceMemoryBlocksFreed > 0 && stats.bytesFreed > 0); + #endif + } + + ValidateGpuData(allocations.data(), allocations.size()); + + swprintf_s(fileName, L"GPU_defragmentation_B_after.json"); + SaveAllocatorStatsToFile(fileName); + + // Destroy all remaining buffers. + for(size_t i = allocations.size(); i--; ) + { + allocations[i].Destroy(); + } +} + +static void ProcessDefragmentationStepInfo(VmaDefragmentationPassInfo &stepInfo) +{ + std::vector beginImageBarriers; + std::vector finalizeImageBarriers; + + VkPipelineStageFlags beginSrcStageMask = 0; + VkPipelineStageFlags beginDstStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT; + + VkPipelineStageFlags finalizeSrcStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT; + VkPipelineStageFlags finalizeDstStageMask = 0; + + bool wantsMemoryBarrier = false; + + VkMemoryBarrier beginMemoryBarrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER }; + VkMemoryBarrier finalizeMemoryBarrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER }; + + for(uint32_t i = 0; i < stepInfo.moveCount; ++i) + { + VmaAllocationInfo info; + vmaGetAllocationInfo(g_hAllocator, stepInfo.pMoves[i].allocation, &info); + + AllocInfo *allocInfo = (AllocInfo *)info.pUserData; + + if(allocInfo->m_Image) + { + VkImage newImage; + + const VkResult result = vkCreateImage(g_hDevice, &allocInfo->m_ImageInfo, g_Allocs, &newImage); + TEST(result >= VK_SUCCESS); + + vkBindImageMemory(g_hDevice, newImage, stepInfo.pMoves[i].memory, stepInfo.pMoves[i].offset); + allocInfo->m_NewImage = newImage; + + // Keep track of our pipeline stages that we need to wait/signal on + beginSrcStageMask |= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + finalizeDstStageMask |= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + + // We need one pipeline barrier and two image layout transitions here + // First we'll have to turn our newly created image into VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL + // And the second one is turning the old image into VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL + + VkImageSubresourceRange subresourceRange = { + VK_IMAGE_ASPECT_COLOR_BIT, + 0, VK_REMAINING_MIP_LEVELS, + 0, VK_REMAINING_ARRAY_LAYERS + }; + + VkImageMemoryBarrier barrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + barrier.srcAccessMask = 0; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = newImage; + barrier.subresourceRange = subresourceRange; + + beginImageBarriers.push_back(barrier); + + // Second barrier to convert the existing image. This one actually needs a real barrier + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = allocInfo->m_ImageLayout; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.image = allocInfo->m_Image; + + beginImageBarriers.push_back(barrier); + + // And lastly we need a barrier that turns our new image into the layout of the old one + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = allocInfo->m_ImageLayout; + barrier.image = newImage; + + finalizeImageBarriers.push_back(barrier); + } + else if(allocInfo->m_Buffer) + { + VkBuffer newBuffer; + + const VkResult result = vkCreateBuffer(g_hDevice, &allocInfo->m_BufferInfo, g_Allocs, &newBuffer); + TEST(result >= VK_SUCCESS); + + vkBindBufferMemory(g_hDevice, newBuffer, stepInfo.pMoves[i].memory, stepInfo.pMoves[i].offset); + allocInfo->m_NewBuffer = newBuffer; + + // Keep track of our pipeline stages that we need to wait/signal on + beginSrcStageMask |= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + finalizeDstStageMask |= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + + beginMemoryBarrier.srcAccessMask |= VK_ACCESS_MEMORY_WRITE_BIT; + beginMemoryBarrier.dstAccessMask |= VK_ACCESS_TRANSFER_READ_BIT; + + finalizeMemoryBarrier.srcAccessMask |= VK_ACCESS_TRANSFER_WRITE_BIT; + finalizeMemoryBarrier.dstAccessMask |= VK_ACCESS_MEMORY_READ_BIT; + + wantsMemoryBarrier = true; + } + } + + if(!beginImageBarriers.empty() || wantsMemoryBarrier) + { + const uint32_t memoryBarrierCount = wantsMemoryBarrier ? 1 : 0; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, beginSrcStageMask, beginDstStageMask, 0, + memoryBarrierCount, &beginMemoryBarrier, + 0, nullptr, + (uint32_t)beginImageBarriers.size(), beginImageBarriers.data()); + } + + for(uint32_t i = 0; i < stepInfo.moveCount; ++ i) + { + VmaAllocationInfo info; + vmaGetAllocationInfo(g_hAllocator, stepInfo.pMoves[i].allocation, &info); + + AllocInfo *allocInfo = (AllocInfo *)info.pUserData; + + if(allocInfo->m_Image) + { + std::vector imageCopies; + + // Copy all mips of the source image into the target image + VkOffset3D offset = { 0, 0, 0 }; + VkExtent3D extent = allocInfo->m_ImageInfo.extent; + + VkImageSubresourceLayers subresourceLayers = { + VK_IMAGE_ASPECT_COLOR_BIT, + 0, + 0, 1 + }; + + for(uint32_t mip = 0; mip < allocInfo->m_ImageInfo.mipLevels; ++ mip) + { + subresourceLayers.mipLevel = mip; + + VkImageCopy imageCopy{ + subresourceLayers, + offset, + subresourceLayers, + offset, + extent + }; + + imageCopies.push_back(imageCopy); + + extent.width = std::max(uint32_t(1), extent.width >> 1); + extent.height = std::max(uint32_t(1), extent.height >> 1); + extent.depth = std::max(uint32_t(1), extent.depth >> 1); + } + + vkCmdCopyImage( + g_hTemporaryCommandBuffer, + allocInfo->m_Image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + allocInfo->m_NewImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + (uint32_t)imageCopies.size(), imageCopies.data()); + } + else if(allocInfo->m_Buffer) + { + VkBufferCopy region = { + 0, + 0, + allocInfo->m_BufferInfo.size }; + + vkCmdCopyBuffer(g_hTemporaryCommandBuffer, + allocInfo->m_Buffer, allocInfo->m_NewBuffer, + 1, ®ion); + } + } + + if(!finalizeImageBarriers.empty() || wantsMemoryBarrier) + { + const uint32_t memoryBarrierCount = wantsMemoryBarrier ? 1 : 0; + + vkCmdPipelineBarrier(g_hTemporaryCommandBuffer, finalizeSrcStageMask, finalizeDstStageMask, 0, + memoryBarrierCount, &finalizeMemoryBarrier, + 0, nullptr, + (uint32_t)finalizeImageBarriers.size(), finalizeImageBarriers.data()); + } +} + + +static void TestDefragmentationIncrementalBasic() +{ + wprintf(L"Test defragmentation incremental basic\n"); + + std::vector allocations; + + // Create that many allocations to surely fill 3 new blocks of 256 MB. + const std::array imageSizes = { 256, 512, 1024 }; + const VkDeviceSize bufSizeMin = 5ull * 1024 * 1024; + const VkDeviceSize bufSizeMax = 10ull * 1024 * 1024; + const VkDeviceSize totalSize = 3ull * 256 * 1024 * 1024; + const size_t imageCount = totalSize / ((size_t)imageSizes[0] * imageSizes[0] * 4) / 2; + const size_t bufCount = (size_t)(totalSize / bufSizeMin) / 2; + const size_t percentToLeave = 30; + RandomNumberGenerator rand = { 234522 }; + + VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.extent.depth = 1; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; + imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + allocCreateInfo.flags = 0; + + // Create all intended images. + for(size_t i = 0; i < imageCount; ++i) + { + const uint32_t size = imageSizes[rand.Generate() % 3]; + + imageInfo.extent.width = size; + imageInfo.extent.height = size; + + AllocInfo alloc; + alloc.CreateImage(imageInfo, allocCreateInfo, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + alloc.m_StartValue = 0; + + allocations.push_back(alloc); + } + + // And all buffers + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + + for(size_t i = 0; i < bufCount; ++i) + { + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + AllocInfo alloc; + alloc.CreateBuffer(bufCreateInfo, allocCreateInfo); + alloc.m_StartValue = 0; + + allocations.push_back(alloc); + } + + // Destroy some percentage of them. + { + const size_t allocationsToDestroy = round_div((imageCount + bufCount) * (100 - percentToLeave), 100); + for(size_t i = 0; i < allocationsToDestroy; ++i) + { + const size_t index = rand.Generate() % allocations.size(); + allocations[index].Destroy(); + allocations.erase(allocations.begin() + index); + } + } + + { + // Set our user data pointers. A real application should probably be more clever here + const size_t allocationCount = allocations.size(); + for(size_t i = 0; i < allocationCount; ++i) + { + AllocInfo &alloc = allocations[i]; + vmaSetAllocationUserData(g_hAllocator, alloc.m_Allocation, &alloc); + } + } + + // Fill them with meaningful data. + UploadGpuData(allocations.data(), allocations.size()); + + wchar_t fileName[MAX_PATH]; + swprintf_s(fileName, L"GPU_defragmentation_incremental_basic_A_before.json"); + SaveAllocatorStatsToFile(fileName); + + // Defragment using GPU only. + { + const size_t allocCount = allocations.size(); + + std::vector allocationPtrs; + + for(size_t i = 0; i < allocCount; ++i) + { + allocationPtrs.push_back(allocations[i].m_Allocation); + } + + const size_t movableAllocCount = allocationPtrs.size(); + + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.flags = VMA_DEFRAGMENTATION_FLAG_INCREMENTAL; + defragInfo.allocationCount = (uint32_t)movableAllocCount; + defragInfo.pAllocations = allocationPtrs.data(); + defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.maxGpuAllocationsToMove = UINT32_MAX; + + VmaDefragmentationStats stats = {}; + VmaDefragmentationContext ctx = VK_NULL_HANDLE; + VkResult res = vmaDefragmentationBegin(g_hAllocator, &defragInfo, &stats, &ctx); + TEST(res >= VK_SUCCESS); + + res = VK_NOT_READY; + + std::vector moveInfo; + moveInfo.resize(movableAllocCount); + + while(res == VK_NOT_READY) + { + VmaDefragmentationPassInfo stepInfo = {}; + stepInfo.pMoves = moveInfo.data(); + stepInfo.moveCount = (uint32_t)moveInfo.size(); + + res = vmaBeginDefragmentationPass(g_hAllocator, ctx, &stepInfo); + TEST(res >= VK_SUCCESS); + + BeginSingleTimeCommands(); + std::vector newHandles; + ProcessDefragmentationStepInfo(stepInfo); + EndSingleTimeCommands(); + + res = vmaEndDefragmentationPass(g_hAllocator, ctx); + + // Destroy old buffers/images and replace them with new handles. + for(size_t i = 0; i < stepInfo.moveCount; ++i) + { + VmaAllocation const alloc = stepInfo.pMoves[i].allocation; + VmaAllocationInfo vmaAllocInfo; + vmaGetAllocationInfo(g_hAllocator, alloc, &vmaAllocInfo); + AllocInfo* allocInfo = (AllocInfo*)vmaAllocInfo.pUserData; + if(allocInfo->m_Buffer) + { + assert(allocInfo->m_NewBuffer && !allocInfo->m_Image && !allocInfo->m_NewImage); + vkDestroyBuffer(g_hDevice, allocInfo->m_Buffer, g_Allocs); + allocInfo->m_Buffer = allocInfo->m_NewBuffer; + allocInfo->m_NewBuffer = VK_NULL_HANDLE; + } + else if(allocInfo->m_Image) + { + assert(allocInfo->m_NewImage && !allocInfo->m_Buffer && !allocInfo->m_NewBuffer); + vkDestroyImage(g_hDevice, allocInfo->m_Image, g_Allocs); + allocInfo->m_Image = allocInfo->m_NewImage; + allocInfo->m_NewImage = VK_NULL_HANDLE; + } + else + assert(0); + } + } + + TEST(res >= VK_SUCCESS); + vmaDefragmentationEnd(g_hAllocator, ctx); + + // If corruption detection is enabled, GPU defragmentation may not work on + // memory types that have this detection active, e.g. on Intel. +#if !defined(VMA_DEBUG_DETECT_CORRUPTION) || VMA_DEBUG_DETECT_CORRUPTION == 0 + TEST(stats.allocationsMoved > 0 && stats.bytesMoved > 0); + TEST(stats.deviceMemoryBlocksFreed > 0 && stats.bytesFreed > 0); +#endif + } + + //ValidateGpuData(allocations.data(), allocations.size()); + + swprintf_s(fileName, L"GPU_defragmentation_incremental_basic_B_after.json"); + SaveAllocatorStatsToFile(fileName); + + // Destroy all remaining buffers and images. + for(size_t i = allocations.size(); i--; ) + { + allocations[i].Destroy(); + } +} + +void TestDefragmentationIncrementalComplex() +{ + wprintf(L"Test defragmentation incremental complex\n"); + + std::vector allocations; + + // Create that many allocations to surely fill 3 new blocks of 256 MB. + const std::array imageSizes = { 256, 512, 1024 }; + const VkDeviceSize bufSizeMin = 5ull * 1024 * 1024; + const VkDeviceSize bufSizeMax = 10ull * 1024 * 1024; + const VkDeviceSize totalSize = 3ull * 256 * 1024 * 1024; + const size_t imageCount = (size_t)(totalSize / (imageSizes[0] * imageSizes[0] * 4)) / 2; + const size_t bufCount = (size_t)(totalSize / bufSizeMin) / 2; + const size_t percentToLeave = 30; + RandomNumberGenerator rand = { 234522 }; + + VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.extent.depth = 1; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; + imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + allocCreateInfo.flags = 0; + + // Create all intended images. + for(size_t i = 0; i < imageCount; ++i) + { + const uint32_t size = imageSizes[rand.Generate() % 3]; + + imageInfo.extent.width = size; + imageInfo.extent.height = size; + + AllocInfo alloc; + alloc.CreateImage(imageInfo, allocCreateInfo, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + alloc.m_StartValue = 0; + + allocations.push_back(alloc); + } + + // And all buffers + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + + for(size_t i = 0; i < bufCount; ++i) + { + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + AllocInfo alloc; + alloc.CreateBuffer(bufCreateInfo, allocCreateInfo); + alloc.m_StartValue = 0; + + allocations.push_back(alloc); + } + + // Destroy some percentage of them. + { + const size_t allocationsToDestroy = round_div((imageCount + bufCount) * (100 - percentToLeave), 100); + for(size_t i = 0; i < allocationsToDestroy; ++i) + { + const size_t index = rand.Generate() % allocations.size(); + allocations[index].Destroy(); + allocations.erase(allocations.begin() + index); + } + } + + { + // Set our user data pointers. A real application should probably be more clever here + const size_t allocationCount = allocations.size(); + for(size_t i = 0; i < allocationCount; ++i) + { + AllocInfo &alloc = allocations[i]; + vmaSetAllocationUserData(g_hAllocator, alloc.m_Allocation, &alloc); + } + } + + // Fill them with meaningful data. + UploadGpuData(allocations.data(), allocations.size()); + + wchar_t fileName[MAX_PATH]; + swprintf_s(fileName, L"GPU_defragmentation_incremental_complex_A_before.json"); + SaveAllocatorStatsToFile(fileName); + + std::vector additionalAllocations; + +#define MakeAdditionalAllocation() \ + do { \ + { \ + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); \ + bufCreateInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; \ + \ + AllocInfo alloc; \ + alloc.CreateBuffer(bufCreateInfo, allocCreateInfo); \ + \ + additionalAllocations.push_back(alloc); \ + } \ + } while(0) + + // Defragment using GPU only. + { + const size_t allocCount = allocations.size(); + + std::vector allocationPtrs; + + for(size_t i = 0; i < allocCount; ++i) + { + VmaAllocationInfo allocInfo = {}; + vmaGetAllocationInfo(g_hAllocator, allocations[i].m_Allocation, &allocInfo); + + allocationPtrs.push_back(allocations[i].m_Allocation); + } + + const size_t movableAllocCount = allocationPtrs.size(); + + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.flags = VMA_DEFRAGMENTATION_FLAG_INCREMENTAL; + defragInfo.allocationCount = (uint32_t)movableAllocCount; + defragInfo.pAllocations = allocationPtrs.data(); + defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.maxGpuAllocationsToMove = UINT32_MAX; + + VmaDefragmentationStats stats = {}; + VmaDefragmentationContext ctx = VK_NULL_HANDLE; + VkResult res = vmaDefragmentationBegin(g_hAllocator, &defragInfo, &stats, &ctx); + TEST(res >= VK_SUCCESS); + + res = VK_NOT_READY; + + std::vector moveInfo; + moveInfo.resize(movableAllocCount); + + MakeAdditionalAllocation(); + + while(res == VK_NOT_READY) + { + VmaDefragmentationPassInfo stepInfo = {}; + stepInfo.pMoves = moveInfo.data(); + stepInfo.moveCount = (uint32_t)moveInfo.size(); + + res = vmaBeginDefragmentationPass(g_hAllocator, ctx, &stepInfo); + TEST(res >= VK_SUCCESS); + + MakeAdditionalAllocation(); + + BeginSingleTimeCommands(); + ProcessDefragmentationStepInfo(stepInfo); + EndSingleTimeCommands(); + + res = vmaEndDefragmentationPass(g_hAllocator, ctx); + + // Destroy old buffers/images and replace them with new handles. + for(size_t i = 0; i < stepInfo.moveCount; ++i) + { + VmaAllocation const alloc = stepInfo.pMoves[i].allocation; + VmaAllocationInfo vmaAllocInfo; + vmaGetAllocationInfo(g_hAllocator, alloc, &vmaAllocInfo); + AllocInfo* allocInfo = (AllocInfo*)vmaAllocInfo.pUserData; + if(allocInfo->m_Buffer) + { + assert(allocInfo->m_NewBuffer && !allocInfo->m_Image && !allocInfo->m_NewImage); + vkDestroyBuffer(g_hDevice, allocInfo->m_Buffer, g_Allocs); + allocInfo->m_Buffer = allocInfo->m_NewBuffer; + allocInfo->m_NewBuffer = VK_NULL_HANDLE; + } + else if(allocInfo->m_Image) + { + assert(allocInfo->m_NewImage && !allocInfo->m_Buffer && !allocInfo->m_NewBuffer); + vkDestroyImage(g_hDevice, allocInfo->m_Image, g_Allocs); + allocInfo->m_Image = allocInfo->m_NewImage; + allocInfo->m_NewImage = VK_NULL_HANDLE; + } + else + assert(0); + } + + MakeAdditionalAllocation(); + } + + TEST(res >= VK_SUCCESS); + vmaDefragmentationEnd(g_hAllocator, ctx); + + // If corruption detection is enabled, GPU defragmentation may not work on + // memory types that have this detection active, e.g. on Intel. +#if !defined(VMA_DEBUG_DETECT_CORRUPTION) || VMA_DEBUG_DETECT_CORRUPTION == 0 + TEST(stats.allocationsMoved > 0 && stats.bytesMoved > 0); + TEST(stats.deviceMemoryBlocksFreed > 0 && stats.bytesFreed > 0); +#endif + } + + //ValidateGpuData(allocations.data(), allocations.size()); + + swprintf_s(fileName, L"GPU_defragmentation_incremental_complex_B_after.json"); + SaveAllocatorStatsToFile(fileName); + + // Destroy all remaining buffers. + for(size_t i = allocations.size(); i--; ) + { + allocations[i].Destroy(); + } + + for(size_t i = additionalAllocations.size(); i--; ) + { + additionalAllocations[i].Destroy(); + } +} + + static void TestUserData() { VkResult res; @@ -1065,15 +2458,15 @@ static void TestUserData() VkBuffer buf; VmaAllocation alloc; VmaAllocationInfo allocInfo; res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); - assert(allocInfo.pUserData = numberAsPointer); + TEST(res == VK_SUCCESS); + TEST(allocInfo.pUserData = numberAsPointer); vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(allocInfo.pUserData == numberAsPointer); + TEST(allocInfo.pUserData == numberAsPointer); vmaSetAllocationUserData(g_hAllocator, alloc, pointerToSomething); vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(allocInfo.pUserData == pointerToSomething); + TEST(allocInfo.pUserData == pointerToSomething); vmaDestroyBuffer(g_hAllocator, buf, alloc); } @@ -1096,28 +2489,78 @@ static void TestUserData() VkBuffer buf; VmaAllocation alloc; VmaAllocationInfo allocInfo; res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); - assert(allocInfo.pUserData != nullptr && allocInfo.pUserData != name1Buf); - assert(strcmp(name1, (const char*)allocInfo.pUserData) == 0); + TEST(res == VK_SUCCESS); + TEST(allocInfo.pUserData != nullptr && allocInfo.pUserData != name1Buf); + TEST(strcmp(name1, (const char*)allocInfo.pUserData) == 0); delete[] name1Buf; vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(strcmp(name1, (const char*)allocInfo.pUserData) == 0); + TEST(strcmp(name1, (const char*)allocInfo.pUserData) == 0); vmaSetAllocationUserData(g_hAllocator, alloc, (void*)name2); vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(strcmp(name2, (const char*)allocInfo.pUserData) == 0); + TEST(strcmp(name2, (const char*)allocInfo.pUserData) == 0); vmaSetAllocationUserData(g_hAllocator, alloc, nullptr); vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(allocInfo.pUserData == nullptr); + TEST(allocInfo.pUserData == nullptr); vmaDestroyBuffer(g_hAllocator, buf, alloc); } } } +static void TestInvalidAllocations() +{ + VkResult res; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + + // Try to allocate 0 bytes. + { + VkMemoryRequirements memReq = {}; + memReq.size = 0; // !!! + memReq.alignment = 4; + memReq.memoryTypeBits = UINT32_MAX; + VmaAllocation alloc = VK_NULL_HANDLE; + res = vmaAllocateMemory(g_hAllocator, &memReq, &allocCreateInfo, &alloc, nullptr); + TEST(res == VK_ERROR_VALIDATION_FAILED_EXT && alloc == VK_NULL_HANDLE); + } + + // Try to create buffer with size = 0. + { + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + bufCreateInfo.size = 0; // !!! + VkBuffer buf = VK_NULL_HANDLE; + VmaAllocation alloc = VK_NULL_HANDLE; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, nullptr); + TEST(res == VK_ERROR_VALIDATION_FAILED_EXT && buf == VK_NULL_HANDLE && alloc == VK_NULL_HANDLE); + } + + // Try to create image with one dimension = 0. + { + VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + imageCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imageCreateInfo.format = VK_FORMAT_B8G8R8A8_UNORM; + imageCreateInfo.extent.width = 128; + imageCreateInfo.extent.height = 0; // !!! + imageCreateInfo.extent.depth = 1; + imageCreateInfo.mipLevels = 1; + imageCreateInfo.arrayLayers = 1; + imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageCreateInfo.tiling = VK_IMAGE_TILING_LINEAR; + imageCreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + imageCreateInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; + VkImage image = VK_NULL_HANDLE; + VmaAllocation alloc = VK_NULL_HANDLE; + res = vmaCreateImage(g_hAllocator, &imageCreateInfo, &allocCreateInfo, &image, &alloc, nullptr); + TEST(res == VK_ERROR_VALIDATION_FAILED_EXT && image == VK_NULL_HANDLE && alloc == VK_NULL_HANDLE); + } +} + static void TestMemoryRequirements() { VkResult res; @@ -1136,7 +2579,7 @@ static void TestMemoryRequirements() // No requirements. res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); vmaDestroyBuffer(g_hAllocator, buf, alloc); // Usage. @@ -1146,8 +2589,8 @@ static void TestMemoryRequirements() allocCreateInfo.memoryTypeBits = UINT32_MAX; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); - assert(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + TEST(res == VK_SUCCESS); + TEST(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); vmaDestroyBuffer(g_hAllocator, buf, alloc); // Required flags, preferred flags. @@ -1157,9 +2600,9 @@ static void TestMemoryRequirements() allocCreateInfo.memoryTypeBits = 0; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); - assert(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); - assert(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + TEST(res == VK_SUCCESS); + TEST(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + TEST(memProps->memoryTypes[allocInfo.memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); vmaDestroyBuffer(g_hAllocator, buf, alloc); // memoryTypeBits. @@ -1170,28 +2613,43 @@ static void TestMemoryRequirements() allocCreateInfo.memoryTypeBits = 1u << memType; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); - assert(allocInfo.memoryType == memType); + TEST(res == VK_SUCCESS); + TEST(allocInfo.memoryType == memType); vmaDestroyBuffer(g_hAllocator, buf, alloc); } +static void TestGetAllocatorInfo() +{ + wprintf(L"Test vnaGetAllocatorInfo\n"); + + VmaAllocatorInfo allocInfo = {}; + vmaGetAllocatorInfo(g_hAllocator, &allocInfo); + TEST(allocInfo.instance == g_hVulkanInstance); + TEST(allocInfo.physicalDevice == g_hPhysicalDevice); + TEST(allocInfo.device == g_hDevice); +} + static void TestBasics() { + wprintf(L"Test basics\n"); + VkResult res; + TestGetAllocatorInfo(); + TestMemoryRequirements(); // Lost allocation { VmaAllocation alloc = VK_NULL_HANDLE; vmaCreateLostAllocation(g_hAllocator, &alloc); - assert(alloc != VK_NULL_HANDLE); + TEST(alloc != VK_NULL_HANDLE); VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, alloc, &allocInfo); - assert(allocInfo.deviceMemory == VK_NULL_HANDLE); - assert(allocInfo.size == 0); + TEST(allocInfo.deviceMemory == VK_NULL_HANDLE); + TEST(allocInfo.size == 0); vmaFreeMemory(g_hAllocator, alloc); } @@ -1208,7 +2666,7 @@ static void TestBasics() VkBuffer buf; VmaAllocation alloc; VmaAllocationInfo allocInfo; res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); vmaDestroyBuffer(g_hAllocator, buf, alloc); @@ -1216,18 +2674,129 @@ static void TestBasics() allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); vmaDestroyBuffer(g_hAllocator, buf, alloc); } TestUserData(); + + TestInvalidAllocations(); +} + +static void TestAllocationVersusResourceSize() +{ + wprintf(L"Test allocation versus resource size\n"); + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 22921; // Prime number + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + + for(uint32_t i = 0; i < 2; ++i) + { + allocCreateInfo.flags = (i == 1) ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; + + AllocInfo info; + info.CreateBuffer(bufCreateInfo, allocCreateInfo); + + VmaAllocationInfo allocInfo = {}; + vmaGetAllocationInfo(g_hAllocator, info.m_Allocation, &allocInfo); + //wprintf(L" Buffer size = %llu, allocation size = %llu\n", bufCreateInfo.size, allocInfo.size); + + // Map and test accessing entire area of the allocation, not only the buffer. + void* mappedPtr = nullptr; + VkResult res = vmaMapMemory(g_hAllocator, info.m_Allocation, &mappedPtr); + TEST(res == VK_SUCCESS); + + memset(mappedPtr, 0xCC, (size_t)allocInfo.size); + + vmaUnmapMemory(g_hAllocator, info.m_Allocation); + + info.Destroy(); + } +} + +static void TestPool_MinBlockCount() +{ +#if defined(VMA_DEBUG_MARGIN) && VMA_DEBUG_MARGIN > 0 + return; +#endif + + wprintf(L"Test Pool MinBlockCount\n"); + VkResult res; + + static const VkDeviceSize ALLOC_SIZE = 512ull * 1024; + static const VkDeviceSize BLOCK_SIZE = ALLOC_SIZE * 2; // Each block can fit 2 allocations. + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_COPY; + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + bufCreateInfo.size = ALLOC_SIZE; + + VmaPoolCreateInfo poolCreateInfo = {}; + poolCreateInfo.blockSize = BLOCK_SIZE; + poolCreateInfo.minBlockCount = 2; // At least 2 blocks always present. + res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + VmaPool pool = VK_NULL_HANDLE; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS && pool != VK_NULL_HANDLE); + + // Check that there are 2 blocks preallocated as requested. + VmaPoolStats begPoolStats = {}; + vmaGetPoolStats(g_hAllocator, pool, &begPoolStats); + TEST(begPoolStats.blockCount == 2 && begPoolStats.allocationCount == 0 && begPoolStats.size == BLOCK_SIZE * 2); + + // Allocate 5 buffers to create 3 blocks. + static const uint32_t BUF_COUNT = 5; + allocCreateInfo.pool = pool; + std::vector allocs(BUF_COUNT); + for(uint32_t i = 0; i < BUF_COUNT; ++i) + { + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &allocs[i].m_Buffer, &allocs[i].m_Allocation, nullptr); + TEST(res == VK_SUCCESS && allocs[i].m_Buffer != VK_NULL_HANDLE && allocs[i].m_Allocation != VK_NULL_HANDLE); + } + + // Check that there are really 3 blocks. + VmaPoolStats poolStats2 = {}; + vmaGetPoolStats(g_hAllocator, pool, &poolStats2); + TEST(poolStats2.blockCount == 3 && poolStats2.allocationCount == BUF_COUNT && poolStats2.size == BLOCK_SIZE * 3); + + // Free two first allocations to make one block empty. + allocs[0].Destroy(); + allocs[1].Destroy(); + + // Check that there are still 3 blocks due to hysteresis. + VmaPoolStats poolStats3 = {}; + vmaGetPoolStats(g_hAllocator, pool, &poolStats3); + TEST(poolStats3.blockCount == 3 && poolStats3.allocationCount == BUF_COUNT - 2 && poolStats2.size == BLOCK_SIZE * 3); + + // Free the last allocation to make second block empty. + allocs[BUF_COUNT - 1].Destroy(); + + // Check that there are now 2 blocks only. + VmaPoolStats poolStats4 = {}; + vmaGetPoolStats(g_hAllocator, pool, &poolStats4); + TEST(poolStats4.blockCount == 2 && poolStats4.allocationCount == BUF_COUNT - 3 && poolStats4.size == BLOCK_SIZE * 2); + + // Cleanup. + for(size_t i = allocs.size(); i--; ) + { + allocs[i].Destroy(); + } + vmaDestroyPool(g_hAllocator, pool); } void TestHeapSizeLimit() { - const VkDeviceSize HEAP_SIZE_LIMIT = 1ull * 1024 * 1024 * 1024; // 1 GB - const VkDeviceSize BLOCK_SIZE = 128ull * 1024 * 1024; // 128 MB + const VkDeviceSize HEAP_SIZE_LIMIT = 100ull * 1024 * 1024; // 100 MB + const VkDeviceSize BLOCK_SIZE = 10ull * 1024 * 1024; // 10 MB VkDeviceSize heapSizeLimit[VK_MAX_MEMORY_HEAPS]; for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i) @@ -1238,11 +2807,12 @@ void TestHeapSizeLimit() VmaAllocatorCreateInfo allocatorCreateInfo = {}; allocatorCreateInfo.physicalDevice = g_hPhysicalDevice; allocatorCreateInfo.device = g_hDevice; + allocatorCreateInfo.instance = g_hVulkanInstance; allocatorCreateInfo.pHeapSizeLimit = heapSizeLimit; VmaAllocator hAllocator; VkResult res = vmaCreateAllocator(&allocatorCreateInfo, &hAllocator); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); struct Item { @@ -1254,8 +2824,8 @@ void TestHeapSizeLimit() VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; bufCreateInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; - // 1. Allocate two blocks of Own Memory, half the size of BLOCK_SIZE. - VmaAllocationInfo ownAllocInfo; + // 1. Allocate two blocks of dedicated memory, half the size of BLOCK_SIZE. + VmaAllocationInfo dedicatedAllocInfo; { VmaAllocationCreateInfo allocCreateInfo = {}; allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; @@ -1266,20 +2836,20 @@ void TestHeapSizeLimit() for(size_t i = 0; i < 2; ++i) { Item item; - res = vmaCreateBuffer(hAllocator, &bufCreateInfo, &allocCreateInfo, &item.hBuf, &item.hAlloc, &ownAllocInfo); - assert(res == VK_SUCCESS); + res = vmaCreateBuffer(hAllocator, &bufCreateInfo, &allocCreateInfo, &item.hBuf, &item.hAlloc, &dedicatedAllocInfo); + TEST(res == VK_SUCCESS); items.push_back(item); } } // Create pool to make sure allocations must be out of this memory type. VmaPoolCreateInfo poolCreateInfo = {}; - poolCreateInfo.memoryTypeIndex = ownAllocInfo.memoryType; + poolCreateInfo.memoryTypeIndex = dedicatedAllocInfo.memoryType; poolCreateInfo.blockSize = BLOCK_SIZE; VmaPool hPool; res = vmaCreatePool(hAllocator, &poolCreateInfo, &hPool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // 2. Allocate normal buffers from all the remaining memory. { @@ -1293,7 +2863,7 @@ void TestHeapSizeLimit() { Item item; res = vmaCreateBuffer(hAllocator, &bufCreateInfo, &allocCreateInfo, &item.hBuf, &item.hAlloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } } @@ -1308,7 +2878,7 @@ void TestHeapSizeLimit() VkBuffer hBuf; VmaAllocation hAlloc; res = vmaCreateBuffer(hAllocator, &bufCreateInfo, &allocCreateInfo, &hBuf, &hAlloc, nullptr); - assert(res == VK_ERROR_OUT_OF_DEVICE_MEMORY); + TEST(res == VK_ERROR_OUT_OF_DEVICE_MEMORY); } // Destroy everything. @@ -1347,14 +2917,14 @@ static void TestDebugMargin() allocCreateInfo.flags = (i == BUF_COUNT - 1) ? VMA_ALLOCATION_CREATE_MAPPED_BIT : 0; VkResult res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buffers[i].Buffer, &buffers[i].Allocation, &allocInfo[i]); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // Margin is preserved also at the beginning of a block. - assert(allocInfo[i].offset >= VMA_DEBUG_MARGIN); + TEST(allocInfo[i].offset >= VMA_DEBUG_MARGIN); if(i == BUF_COUNT - 1) { // Fill with data. - assert(allocInfo[i].pMappedData != nullptr); + TEST(allocInfo[i].pMappedData != nullptr); // Uncomment this "+ 1" to overwrite past end of allocation and check corruption detection. memset(allocInfo[i].pMappedData, 0xFF, bufInfo.size /* + 1 */); } @@ -1373,12 +2943,12 @@ static void TestDebugMargin() { if(allocInfo[i].deviceMemory == allocInfo[i - 1].deviceMemory) { - assert(allocInfo[i].offset >= allocInfo[i - 1].offset + VMA_DEBUG_MARGIN); + TEST(allocInfo[i].offset >= allocInfo[i - 1].offset + VMA_DEBUG_MARGIN); } } VkResult res = vmaCheckCorruption(g_hAllocator, UINT32_MAX); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // Destroy all buffers. for(size_t i = BUF_COUNT; i--; ) @@ -1388,6 +2958,960 @@ static void TestDebugMargin() } #endif +static void TestLinearAllocator() +{ + wprintf(L"Test linear allocator\n"); + + RandomNumberGenerator rand{645332}; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = 1024; // Whatever. + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + poolCreateInfo.blockSize = 1024 * 300; + poolCreateInfo.flags = VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT; + poolCreateInfo.minBlockCount = poolCreateInfo.maxBlockCount = 1; + + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + VkBufferCreateInfo bufCreateInfo = sampleBufCreateInfo; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + + constexpr size_t maxBufCount = 100; + std::vector bufInfo; + + constexpr VkDeviceSize bufSizeMin = 16; + constexpr VkDeviceSize bufSizeMax = 1024; + VmaAllocationInfo allocInfo; + VkDeviceSize prevOffset = 0; + + // Test one-time free. + for(size_t i = 0; i < 2; ++i) + { + // Allocate number of buffers of varying size that surely fit into this block. + VkDeviceSize bufSumSize = 0; + for(size_t i = 0; i < maxBufCount; ++i) + { + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + TEST(i == 0 || allocInfo.offset > prevOffset); + bufInfo.push_back(newBufInfo); + prevOffset = allocInfo.offset; + bufSumSize += bufCreateInfo.size; + } + + // Validate pool stats. + VmaPoolStats stats; + vmaGetPoolStats(g_hAllocator, pool, &stats); + TEST(stats.size == poolCreateInfo.blockSize); + TEST(stats.unusedSize = poolCreateInfo.blockSize - bufSumSize); + TEST(stats.allocationCount == bufInfo.size()); + + // Destroy the buffers in random order. + while(!bufInfo.empty()) + { + const size_t indexToDestroy = rand.Generate() % bufInfo.size(); + const BufferInfo& currBufInfo = bufInfo[indexToDestroy]; + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin() + indexToDestroy); + } + } + + // Test stack. + { + // Allocate number of buffers of varying size that surely fit into this block. + for(size_t i = 0; i < maxBufCount; ++i) + { + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + TEST(i == 0 || allocInfo.offset > prevOffset); + bufInfo.push_back(newBufInfo); + prevOffset = allocInfo.offset; + } + + // Destroy few buffers from top of the stack. + for(size_t i = 0; i < maxBufCount / 5; ++i) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + + // Create some more + for(size_t i = 0; i < maxBufCount / 5; ++i) + { + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + TEST(i == 0 || allocInfo.offset > prevOffset); + bufInfo.push_back(newBufInfo); + prevOffset = allocInfo.offset; + } + + // Destroy the buffers in reverse order. + while(!bufInfo.empty()) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + } + + // Test ring buffer. + { + // Allocate number of buffers that surely fit into this block. + bufCreateInfo.size = bufSizeMax; + for(size_t i = 0; i < maxBufCount; ++i) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + TEST(i == 0 || allocInfo.offset > prevOffset); + bufInfo.push_back(newBufInfo); + prevOffset = allocInfo.offset; + } + + // Free and allocate new buffers so many times that we make sure we wrap-around at least once. + const size_t buffersPerIter = maxBufCount / 10 - 1; + const size_t iterCount = poolCreateInfo.blockSize / bufCreateInfo.size / buffersPerIter * 2; + for(size_t iter = 0; iter < iterCount; ++iter) + { + for(size_t bufPerIter = 0; bufPerIter < buffersPerIter; ++bufPerIter) + { + const BufferInfo& currBufInfo = bufInfo.front(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin()); + } + for(size_t bufPerIter = 0; bufPerIter < buffersPerIter; ++bufPerIter) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + } + } + + // Allocate buffers until we reach out-of-memory. + uint32_t debugIndex = 0; + while(res == VK_SUCCESS) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + if(res == VK_SUCCESS) + { + bufInfo.push_back(newBufInfo); + } + else + { + TEST(res == VK_ERROR_OUT_OF_DEVICE_MEMORY); + } + ++debugIndex; + } + + // Destroy the buffers in random order. + while(!bufInfo.empty()) + { + const size_t indexToDestroy = rand.Generate() % bufInfo.size(); + const BufferInfo& currBufInfo = bufInfo[indexToDestroy]; + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin() + indexToDestroy); + } + } + + // Test double stack. + { + // Allocate number of buffers of varying size that surely fit into this block, alternate from bottom/top. + VkDeviceSize prevOffsetLower = 0; + VkDeviceSize prevOffsetUpper = poolCreateInfo.blockSize; + for(size_t i = 0; i < maxBufCount; ++i) + { + const bool upperAddress = (i % 2) != 0; + if(upperAddress) + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + else + allocCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + if(upperAddress) + { + TEST(allocInfo.offset < prevOffsetUpper); + prevOffsetUpper = allocInfo.offset; + } + else + { + TEST(allocInfo.offset >= prevOffsetLower); + prevOffsetLower = allocInfo.offset; + } + TEST(prevOffsetLower < prevOffsetUpper); + bufInfo.push_back(newBufInfo); + } + + // Destroy few buffers from top of the stack. + for(size_t i = 0; i < maxBufCount / 5; ++i) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + + // Create some more + for(size_t i = 0; i < maxBufCount / 5; ++i) + { + const bool upperAddress = (i % 2) != 0; + if(upperAddress) + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + else + allocCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + } + + // Destroy the buffers in reverse order. + while(!bufInfo.empty()) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + + // Create buffers on both sides until we reach out of memory. + prevOffsetLower = 0; + prevOffsetUpper = poolCreateInfo.blockSize; + res = VK_SUCCESS; + for(size_t i = 0; res == VK_SUCCESS; ++i) + { + const bool upperAddress = (i % 2) != 0; + if(upperAddress) + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + else + allocCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + if(res == VK_SUCCESS) + { + if(upperAddress) + { + TEST(allocInfo.offset < prevOffsetUpper); + prevOffsetUpper = allocInfo.offset; + } + else + { + TEST(allocInfo.offset >= prevOffsetLower); + prevOffsetLower = allocInfo.offset; + } + TEST(prevOffsetLower < prevOffsetUpper); + bufInfo.push_back(newBufInfo); + } + } + + // Destroy the buffers in random order. + while(!bufInfo.empty()) + { + const size_t indexToDestroy = rand.Generate() % bufInfo.size(); + const BufferInfo& currBufInfo = bufInfo[indexToDestroy]; + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin() + indexToDestroy); + } + + // Create buffers on upper side only, constant size, until we reach out of memory. + prevOffsetUpper = poolCreateInfo.blockSize; + res = VK_SUCCESS; + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + bufCreateInfo.size = bufSizeMax; + for(size_t i = 0; res == VK_SUCCESS; ++i) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + if(res == VK_SUCCESS) + { + TEST(allocInfo.offset < prevOffsetUpper); + prevOffsetUpper = allocInfo.offset; + bufInfo.push_back(newBufInfo); + } + } + + // Destroy the buffers in reverse order. + while(!bufInfo.empty()) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + } + + // Test ring buffer with lost allocations. + { + // Allocate number of buffers until pool is full. + // Notice CAN_BECOME_LOST flag and call to vmaSetCurrentFrameIndex. + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT; + res = VK_SUCCESS; + for(size_t i = 0; res == VK_SUCCESS; ++i) + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + if(res == VK_SUCCESS) + bufInfo.push_back(newBufInfo); + } + + // Free first half of it. + { + const size_t buffersToDelete = bufInfo.size() / 2; + for(size_t i = 0; i < buffersToDelete; ++i) + { + vmaDestroyBuffer(g_hAllocator, bufInfo[i].Buffer, bufInfo[i].Allocation); + } + bufInfo.erase(bufInfo.begin(), bufInfo.begin() + buffersToDelete); + } + + // Allocate number of buffers until pool is full again. + // This way we make sure ring buffers wraps around, front in in the middle. + res = VK_SUCCESS; + for(size_t i = 0; res == VK_SUCCESS; ++i) + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + if(res == VK_SUCCESS) + bufInfo.push_back(newBufInfo); + } + + VkDeviceSize firstNewOffset; + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + // Allocate a large buffer with CAN_MAKE_OTHER_LOST. + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT; + bufCreateInfo.size = bufSizeMax; + + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + firstNewOffset = allocInfo.offset; + + // Make sure at least one buffer from the beginning became lost. + vmaGetAllocationInfo(g_hAllocator, bufInfo[0].Allocation, &allocInfo); + TEST(allocInfo.deviceMemory == VK_NULL_HANDLE); + } + +#if 0 // TODO Fix and uncomment. Failing on Intel. + // Allocate more buffers that CAN_MAKE_OTHER_LOST until we wrap-around with this. + size_t newCount = 1; + for(;;) + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + bufCreateInfo.size = align_up(bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin), 16); + + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + ++newCount; + if(allocInfo.offset < firstNewOffset) + break; + } +#endif + + // Delete buffers that are lost. + for(size_t i = bufInfo.size(); i--; ) + { + vmaGetAllocationInfo(g_hAllocator, bufInfo[i].Allocation, &allocInfo); + if(allocInfo.deviceMemory == VK_NULL_HANDLE) + { + vmaDestroyBuffer(g_hAllocator, bufInfo[i].Buffer, bufInfo[i].Allocation); + bufInfo.erase(bufInfo.begin() + i); + } + } + + // Test vmaMakePoolAllocationsLost + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + size_t lostAllocCount = 0; + vmaMakePoolAllocationsLost(g_hAllocator, pool, &lostAllocCount); + TEST(lostAllocCount > 0); + + size_t realLostAllocCount = 0; + for(size_t i = 0; i < bufInfo.size(); ++i) + { + vmaGetAllocationInfo(g_hAllocator, bufInfo[i].Allocation, &allocInfo); + if(allocInfo.deviceMemory == VK_NULL_HANDLE) + ++realLostAllocCount; + } + TEST(realLostAllocCount == lostAllocCount); + } + + // Destroy all the buffers in forward order. + for(size_t i = 0; i < bufInfo.size(); ++i) + vmaDestroyBuffer(g_hAllocator, bufInfo[i].Buffer, bufInfo[i].Allocation); + bufInfo.clear(); + } + + vmaDestroyPool(g_hAllocator, pool); +} + +static void TestLinearAllocatorMultiBlock() +{ + wprintf(L"Test linear allocator multi block\n"); + + RandomNumberGenerator rand{345673}; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = 1024 * 1024; + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + poolCreateInfo.flags = VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + VkBufferCreateInfo bufCreateInfo = sampleBufCreateInfo; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + + std::vector bufInfo; + VmaAllocationInfo allocInfo; + + // Test one-time free. + { + // Allocate buffers until we move to a second block. + VkDeviceMemory lastMem = VK_NULL_HANDLE; + for(uint32_t i = 0; ; ++i) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + if(lastMem && allocInfo.deviceMemory != lastMem) + { + break; + } + lastMem = allocInfo.deviceMemory; + } + + TEST(bufInfo.size() > 2); + + // Make sure that pool has now two blocks. + VmaPoolStats poolStats = {}; + vmaGetPoolStats(g_hAllocator, pool, &poolStats); + TEST(poolStats.blockCount == 2); + + // Destroy all the buffers in random order. + while(!bufInfo.empty()) + { + const size_t indexToDestroy = rand.Generate() % bufInfo.size(); + const BufferInfo& currBufInfo = bufInfo[indexToDestroy]; + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin() + indexToDestroy); + } + + // Make sure that pool has now at most one block. + vmaGetPoolStats(g_hAllocator, pool, &poolStats); + TEST(poolStats.blockCount <= 1); + } + + // Test stack. + { + // Allocate buffers until we move to a second block. + VkDeviceMemory lastMem = VK_NULL_HANDLE; + for(uint32_t i = 0; ; ++i) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + if(lastMem && allocInfo.deviceMemory != lastMem) + { + break; + } + lastMem = allocInfo.deviceMemory; + } + + TEST(bufInfo.size() > 2); + + // Add few more buffers. + for(uint32_t i = 0; i < 5; ++i) + { + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + } + + // Make sure that pool has now two blocks. + VmaPoolStats poolStats = {}; + vmaGetPoolStats(g_hAllocator, pool, &poolStats); + TEST(poolStats.blockCount == 2); + + // Delete half of buffers, LIFO. + for(size_t i = 0, countToDelete = bufInfo.size() / 2; i < countToDelete; ++i) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + + // Add one more buffer. + BufferInfo newBufInfo; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + // Make sure that pool has now one block. + vmaGetPoolStats(g_hAllocator, pool, &poolStats); + TEST(poolStats.blockCount == 1); + + // Delete all the remaining buffers, LIFO. + while(!bufInfo.empty()) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + } + + vmaDestroyPool(g_hAllocator, pool); +} + +static void ManuallyTestLinearAllocator() +{ + VmaStats origStats; + vmaCalculateStats(g_hAllocator, &origStats); + + wprintf(L"Manually test linear allocator\n"); + + RandomNumberGenerator rand{645332}; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = 1024; // Whatever. + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + poolCreateInfo.blockSize = 10 * 1024; + poolCreateInfo.flags = VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT; + poolCreateInfo.minBlockCount = poolCreateInfo.maxBlockCount = 1; + + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + VkBufferCreateInfo bufCreateInfo = sampleBufCreateInfo; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + + std::vector bufInfo; + VmaAllocationInfo allocInfo; + BufferInfo newBufInfo; + + // Test double stack. + { + /* + Lower: Buffer 32 B, Buffer 1024 B, Buffer 32 B + Upper: Buffer 16 B, Buffer 1024 B, Buffer 128 B + + Totally: + 1 block allocated + 10240 Vulkan bytes + 6 new allocations + 2256 bytes in allocations + */ + + bufCreateInfo.size = 32; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 1024; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 32; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT; + + bufCreateInfo.size = 128; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 1024; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 16; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + VmaStats currStats; + vmaCalculateStats(g_hAllocator, &currStats); + VmaPoolStats poolStats; + vmaGetPoolStats(g_hAllocator, pool, &poolStats); + + char* statsStr = nullptr; + vmaBuildStatsString(g_hAllocator, &statsStr, VK_TRUE); + + // PUT BREAKPOINT HERE TO CHECK. + // Inspect: currStats versus origStats, poolStats, statsStr. + int I = 0; + + vmaFreeStatsString(g_hAllocator, statsStr); + + // Destroy the buffers in reverse order. + while(!bufInfo.empty()) + { + const BufferInfo& currBufInfo = bufInfo.back(); + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.pop_back(); + } + } + + vmaDestroyPool(g_hAllocator, pool); +} + +static void BenchmarkAlgorithmsCase(FILE* file, + uint32_t algorithm, + bool empty, + VmaAllocationCreateFlags allocStrategy, + FREE_ORDER freeOrder) +{ + RandomNumberGenerator rand{16223}; + + const VkDeviceSize bufSizeMin = 32; + const VkDeviceSize bufSizeMax = 1024; + const size_t maxBufCapacity = 10000; + const uint32_t iterationCount = 10; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = bufSizeMax; + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + poolCreateInfo.blockSize = bufSizeMax * maxBufCapacity; + poolCreateInfo.flags |= algorithm; + poolCreateInfo.minBlockCount = poolCreateInfo.maxBlockCount = 1; + + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + // Buffer created just to get memory requirements. Never bound to any memory. + VkBuffer dummyBuffer = VK_NULL_HANDLE; + res = vkCreateBuffer(g_hDevice, &sampleBufCreateInfo, g_Allocs, &dummyBuffer); + TEST(res == VK_SUCCESS && dummyBuffer); + + VkMemoryRequirements memReq = {}; + vkGetBufferMemoryRequirements(g_hDevice, dummyBuffer, &memReq); + + vkDestroyBuffer(g_hDevice, dummyBuffer, g_Allocs); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + allocCreateInfo.flags = allocStrategy; + + VmaAllocation alloc; + std::vector baseAllocations; + + if(!empty) + { + // Make allocations up to 1/3 of pool size. + VkDeviceSize totalSize = 0; + while(totalSize < poolCreateInfo.blockSize / 3) + { + // This test intentionally allows sizes that are aligned to 4 or 16 bytes. + // This is theoretically allowed and already uncovered one bug. + memReq.size = bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin); + res = vmaAllocateMemory(g_hAllocator, &memReq, &allocCreateInfo, &alloc, nullptr); + TEST(res == VK_SUCCESS); + baseAllocations.push_back(alloc); + totalSize += memReq.size; + } + + // Delete half of them, choose randomly. + size_t allocsToDelete = baseAllocations.size() / 2; + for(size_t i = 0; i < allocsToDelete; ++i) + { + const size_t index = (size_t)rand.Generate() % baseAllocations.size(); + vmaFreeMemory(g_hAllocator, baseAllocations[index]); + baseAllocations.erase(baseAllocations.begin() + index); + } + } + + // BENCHMARK + const size_t allocCount = maxBufCapacity / 3; + std::vector testAllocations; + testAllocations.reserve(allocCount); + duration allocTotalDuration = duration::zero(); + duration freeTotalDuration = duration::zero(); + for(uint32_t iterationIndex = 0; iterationIndex < iterationCount; ++iterationIndex) + { + // Allocations + time_point allocTimeBeg = std::chrono::high_resolution_clock::now(); + for(size_t i = 0; i < allocCount; ++i) + { + memReq.size = bufSizeMin + rand.Generate() % (bufSizeMax - bufSizeMin); + res = vmaAllocateMemory(g_hAllocator, &memReq, &allocCreateInfo, &alloc, nullptr); + TEST(res == VK_SUCCESS); + testAllocations.push_back(alloc); + } + allocTotalDuration += std::chrono::high_resolution_clock::now() - allocTimeBeg; + + // Deallocations + switch(freeOrder) + { + case FREE_ORDER::FORWARD: + // Leave testAllocations unchanged. + break; + case FREE_ORDER::BACKWARD: + std::reverse(testAllocations.begin(), testAllocations.end()); + break; + case FREE_ORDER::RANDOM: + std::shuffle(testAllocations.begin(), testAllocations.end(), MyUniformRandomNumberGenerator(rand)); + break; + default: assert(0); + } + + time_point freeTimeBeg = std::chrono::high_resolution_clock::now(); + for(size_t i = 0; i < allocCount; ++i) + vmaFreeMemory(g_hAllocator, testAllocations[i]); + freeTotalDuration += std::chrono::high_resolution_clock::now() - freeTimeBeg; + + testAllocations.clear(); + } + + // Delete baseAllocations + while(!baseAllocations.empty()) + { + vmaFreeMemory(g_hAllocator, baseAllocations.back()); + baseAllocations.pop_back(); + } + + vmaDestroyPool(g_hAllocator, pool); + + const float allocTotalSeconds = ToFloatSeconds(allocTotalDuration); + const float freeTotalSeconds = ToFloatSeconds(freeTotalDuration); + + printf(" Algorithm=%s %s Allocation=%s FreeOrder=%s: allocations %g s, free %g s\n", + AlgorithmToStr(algorithm), + empty ? "Empty" : "Not empty", + GetAllocationStrategyName(allocStrategy), + FREE_ORDER_NAMES[(size_t)freeOrder], + allocTotalSeconds, + freeTotalSeconds); + + if(file) + { + std::string currTime; + CurrentTimeToStr(currTime); + + fprintf(file, "%s,%s,%s,%u,%s,%s,%g,%g\n", + CODE_DESCRIPTION, currTime.c_str(), + AlgorithmToStr(algorithm), + empty ? 1 : 0, + GetAllocationStrategyName(allocStrategy), + FREE_ORDER_NAMES[(uint32_t)freeOrder], + allocTotalSeconds, + freeTotalSeconds); + } +} + +static void TestBufferDeviceAddress() +{ + wprintf(L"Test buffer device address\n"); + + assert(g_BufferDeviceAddressEnabled); + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 0x10000; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; // !!! + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + for(uint32_t testIndex = 0; testIndex < 2; ++testIndex) + { + // 1st is placed, 2nd is dedicated. + if(testIndex == 1) + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + + BufferInfo bufInfo = {}; + VkResult res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &bufInfo.Buffer, &bufInfo.Allocation, nullptr); + TEST(res == VK_SUCCESS); + + VkBufferDeviceAddressInfoEXT bufferDeviceAddressInfo = { VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_EXT }; + bufferDeviceAddressInfo.buffer = bufInfo.Buffer; + //assert(g_vkGetBufferDeviceAddressEXT != nullptr); + if(g_vkGetBufferDeviceAddressEXT != nullptr) + { + VkDeviceAddress addr = g_vkGetBufferDeviceAddressEXT(g_hDevice, &bufferDeviceAddressInfo); + TEST(addr != 0); + } + + vmaDestroyBuffer(g_hAllocator, bufInfo.Buffer, bufInfo.Allocation); + } +} + +static void BenchmarkAlgorithms(FILE* file) +{ + wprintf(L"Benchmark algorithms\n"); + + if(file) + { + fprintf(file, + "Code,Time," + "Algorithm,Empty,Allocation strategy,Free order," + "Allocation time (s),Deallocation time (s)\n"); + } + + uint32_t freeOrderCount = 1; + if(ConfigType >= CONFIG_TYPE::CONFIG_TYPE_LARGE) + freeOrderCount = 3; + else if(ConfigType >= CONFIG_TYPE::CONFIG_TYPE_SMALL) + freeOrderCount = 2; + + const uint32_t emptyCount = ConfigType >= CONFIG_TYPE::CONFIG_TYPE_SMALL ? 2 : 1; + const uint32_t allocStrategyCount = GetAllocationStrategyCount(); + + for(uint32_t freeOrderIndex = 0; freeOrderIndex < freeOrderCount; ++freeOrderIndex) + { + FREE_ORDER freeOrder = FREE_ORDER::COUNT; + switch(freeOrderIndex) + { + case 0: freeOrder = FREE_ORDER::BACKWARD; break; + case 1: freeOrder = FREE_ORDER::FORWARD; break; + case 2: freeOrder = FREE_ORDER::RANDOM; break; + default: assert(0); + } + + for(uint32_t emptyIndex = 0; emptyIndex < emptyCount; ++emptyIndex) + { + for(uint32_t algorithmIndex = 0; algorithmIndex < 3; ++algorithmIndex) + { + uint32_t algorithm = 0; + switch(algorithmIndex) + { + case 0: + break; + case 1: + algorithm = VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT; + break; + case 2: + algorithm = VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT; + break; + default: + assert(0); + } + + uint32_t currAllocStrategyCount = algorithm != 0 ? 1 : allocStrategyCount; + for(uint32_t allocStrategyIndex = 0; allocStrategyIndex < currAllocStrategyCount; ++allocStrategyIndex) + { + VmaAllocatorCreateFlags strategy = 0; + if(currAllocStrategyCount > 1) + { + switch(allocStrategyIndex) + { + case 0: strategy = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT; break; + case 1: strategy = VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT; break; + case 2: strategy = VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT; break; + default: assert(0); + } + } + + BenchmarkAlgorithmsCase( + file, + algorithm, + (emptyIndex == 0), // empty + strategy, + freeOrder); // freeOrder + } + } + } + } +} + static void TestPool_SameSize() { const VkDeviceSize BUF_SIZE = 1024 * 1024; @@ -1403,14 +3927,14 @@ static void TestPool_SameSize() uint32_t memoryTypeBits = UINT32_MAX; { VkBuffer dummyBuffer; - res = vkCreateBuffer(g_hDevice, &bufferInfo, nullptr, &dummyBuffer); - assert(res == VK_SUCCESS); + res = vkCreateBuffer(g_hDevice, &bufferInfo, g_Allocs, &dummyBuffer); + TEST(res == VK_SUCCESS); VkMemoryRequirements memReq; vkGetBufferMemoryRequirements(g_hDevice, dummyBuffer, &memReq); memoryTypeBits = memReq.memoryTypeBits; - vkDestroyBuffer(g_hDevice, dummyBuffer, nullptr); + vkDestroyBuffer(g_hDevice, dummyBuffer, g_Allocs); } VmaAllocationCreateInfo poolAllocInfo = {}; @@ -1431,7 +3955,19 @@ static void TestPool_SameSize() VmaPool pool; res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); + + // Test pool name + { + static const char* const POOL_NAME = "Pool name"; + vmaSetPoolName(g_hAllocator, pool, POOL_NAME); + + const char* fetchedPoolName = nullptr; + vmaGetPoolName(g_hAllocator, pool, &fetchedPoolName); + TEST(strcmp(fetchedPoolName, POOL_NAME) == 0); + + vmaSetPoolName(g_hAllocator, pool, nullptr); + } vmaSetCurrentFrameIndex(g_hAllocator, 1); @@ -1452,7 +3988,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } @@ -1460,7 +3996,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_ERROR_OUT_OF_DEVICE_MEMORY); + TEST(res == VK_ERROR_OUT_OF_DEVICE_MEMORY); } // Validate that no buffer is lost. Also check that they are not mapped. @@ -1468,8 +4004,8 @@ static void TestPool_SameSize() { VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, items[i].Alloc, &allocInfo); - assert(allocInfo.deviceMemory != VK_NULL_HANDLE); - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.deviceMemory != VK_NULL_HANDLE); + TEST(allocInfo.pMappedData == nullptr); } // Free some percent of random items. @@ -1496,7 +4032,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } } @@ -1517,7 +4053,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } @@ -1526,7 +4062,7 @@ static void TestPool_SameSize() { VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, items[i].Alloc, &allocInfo); - assert(allocInfo.deviceMemory != VK_NULL_HANDLE); + TEST(allocInfo.deviceMemory != VK_NULL_HANDLE); } // Next frame. @@ -1537,7 +4073,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } @@ -1546,7 +4082,7 @@ static void TestPool_SameSize() { VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, items[i].Alloc, &allocInfo); - assert(allocInfo.deviceMemory == VK_NULL_HANDLE); + TEST(allocInfo.deviceMemory == VK_NULL_HANDLE); vmaDestroyBuffer(g_hAllocator, items[i].Buf, items[i].Alloc); } items.erase(items.begin(), items.begin() + BUF_COUNT); @@ -1556,7 +4092,7 @@ static void TestPool_SameSize() { VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, items[i].Alloc, &allocInfo); - assert(allocInfo.deviceMemory != VK_NULL_HANDLE); + TEST(allocInfo.deviceMemory != VK_NULL_HANDLE); } // Free one item. @@ -1567,11 +4103,11 @@ static void TestPool_SameSize() { VmaPoolStats poolStats = {}; vmaGetPoolStats(g_hAllocator, pool, &poolStats); - assert(poolStats.allocationCount == items.size()); - assert(poolStats.size = BUF_COUNT * BUF_SIZE); - assert(poolStats.unusedRangeCount == 1); - assert(poolStats.unusedRangeSizeMax == BUF_SIZE); - assert(poolStats.unusedSize == BUF_SIZE); + TEST(poolStats.allocationCount == items.size()); + TEST(poolStats.size = BUF_COUNT * BUF_SIZE); + TEST(poolStats.unusedRangeCount == 1); + TEST(poolStats.unusedRangeSizeMax == BUF_SIZE); + TEST(poolStats.unusedSize == BUF_SIZE); } // Free all remaining items. @@ -1584,7 +4120,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } @@ -1603,8 +4139,8 @@ static void TestPool_SameSize() VmaDefragmentationStats defragmentationStats; res = vmaDefragment(g_hAllocator, allocationsToDefragment.data(), items.size(), nullptr, nullptr, &defragmentationStats); - assert(res == VK_SUCCESS); - assert(defragmentationStats.deviceMemoryBlocksFreed == 2); + TEST(res == VK_SUCCESS); + TEST(defragmentationStats.deviceMemoryBlocksFreed == 2); } // Free all remaining items. @@ -1621,7 +4157,7 @@ static void TestPool_SameSize() { BufItem item; res = vmaCreateBuffer(g_hAllocator, &bufferInfo, &allocInfo, &item.Buf, &item.Alloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); items.push_back(item); } @@ -1636,11 +4172,11 @@ static void TestPool_SameSize() // vmaMakePoolAllocationsLost. Only remaining 2 should be lost. size_t lostCount = 0xDEADC0DE; vmaMakePoolAllocationsLost(g_hAllocator, pool, &lostCount); - assert(lostCount == 2); + TEST(lostCount == 2); // Make another call. Now 0 should be lost. vmaMakePoolAllocationsLost(g_hAllocator, pool, &lostCount); - assert(lostCount == 0); + TEST(lostCount == 0); // Make another call, with null count. Should not crash. vmaMakePoolAllocationsLost(g_hAllocator, pool, nullptr); @@ -1665,7 +4201,7 @@ static void TestPool_SameSize() VmaAllocation alloc = nullptr; res = vmaAllocateMemory(g_hAllocator, &memReq, &allocCreateInfo, &alloc, nullptr); - assert(res == VK_ERROR_OUT_OF_DEVICE_MEMORY && alloc == nullptr); + TEST(res == VK_ERROR_OUT_OF_DEVICE_MEMORY && alloc == nullptr); } vmaDestroyPool(g_hAllocator, pool); @@ -1704,11 +4240,11 @@ static void TestAllocationsInitialization() poolCreateInfo.minBlockCount = 1; // To keep memory alive while pool exists. poolCreateInfo.maxBlockCount = 1; res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &bufInfo, &dummyBufAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); VmaAllocationCreateInfo bufAllocCreateInfo = {}; res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &bufAllocCreateInfo.pool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // Create one persistently mapped buffer to keep memory of this block mapped, // so that pointer to mapped data will remain (more or less...) valid even @@ -1718,7 +4254,7 @@ static void TestAllocationsInitialization() VkBuffer firstBuf; VmaAllocation firstAlloc; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &bufAllocCreateInfo, &firstBuf, &firstAlloc, nullptr); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // Test buffers. @@ -1730,13 +4266,13 @@ static void TestAllocationsInitialization() VmaAllocation alloc; VmaAllocationInfo allocInfo; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &bufAllocCreateInfo, &buf, &alloc, &allocInfo); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); void* pMappedData; if(!persistentlyMapped) { res = vmaMapMemory(g_hAllocator, alloc, &pMappedData); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } else { @@ -1745,7 +4281,7 @@ static void TestAllocationsInitialization() // Validate initialized content bool valid = ValidatePattern(pMappedData, BUF_SIZE, 0xDC); - assert(valid); + TEST(valid); if(!persistentlyMapped) { @@ -1756,7 +4292,7 @@ static void TestAllocationsInitialization() // Validate freed content valid = ValidatePattern(pMappedData, BUF_SIZE, 0xEF); - assert(valid); + TEST(valid); } vmaDestroyBuffer(g_hAllocator, firstBuf, firstAlloc); @@ -1767,7 +4303,7 @@ static void TestPool_Benchmark( PoolTestResult& outResult, const PoolTestConfig& config) { - assert(config.ThreadCount > 0); + TEST(config.ThreadCount > 0); RandomNumberGenerator mainRand{config.RandSeed}; @@ -1799,27 +4335,27 @@ static void TestPool_Benchmark( uint32_t bufferMemoryTypeBits = UINT32_MAX; { VkBuffer dummyBuffer; - VkResult res = vkCreateBuffer(g_hDevice, &bufferInfo, nullptr, &dummyBuffer); - assert(res == VK_SUCCESS); + VkResult res = vkCreateBuffer(g_hDevice, &bufferInfo, g_Allocs, &dummyBuffer); + TEST(res == VK_SUCCESS); VkMemoryRequirements memReq; vkGetBufferMemoryRequirements(g_hDevice, dummyBuffer, &memReq); bufferMemoryTypeBits = memReq.memoryTypeBits; - vkDestroyBuffer(g_hDevice, dummyBuffer, nullptr); + vkDestroyBuffer(g_hDevice, dummyBuffer, g_Allocs); } uint32_t imageMemoryTypeBits = UINT32_MAX; { VkImage dummyImage; - VkResult res = vkCreateImage(g_hDevice, &imageInfo, nullptr, &dummyImage); - assert(res == VK_SUCCESS); + VkResult res = vkCreateImage(g_hDevice, &imageInfo, g_Allocs, &dummyImage); + TEST(res == VK_SUCCESS); VkMemoryRequirements memReq; vkGetImageMemoryRequirements(g_hDevice, dummyImage, &memReq); imageMemoryTypeBits = memReq.memoryTypeBits; - vkDestroyImage(g_hDevice, dummyImage, nullptr); + vkDestroyImage(g_hDevice, dummyImage, g_Allocs); } uint32_t memoryTypeBits = 0; @@ -1837,7 +4373,7 @@ static void TestPool_Benchmark( else if(config.UsesImages()) memoryTypeBits = imageMemoryTypeBits; else - assert(0); + TEST(0); VmaPoolCreateInfo poolCreateInfo = {}; poolCreateInfo.memoryTypeIndex = 0; @@ -1852,7 +4388,7 @@ static void TestPool_Benchmark( VmaPool pool; VkResult res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); // Start time measurement - after creating pool and initializing data structures. time_point timeBeg = std::chrono::high_resolution_clock::now(); @@ -1911,8 +4447,8 @@ static void TestPool_Benchmark( const AllocationSize& allocSize = config.AllocationSizes[allocSizeIndex]; if(allocSize.BufferSizeMax > 0) { - assert(allocSize.BufferSizeMin > 0); - assert(allocSize.ImageSizeMin == 0 && allocSize.ImageSizeMax == 0); + TEST(allocSize.BufferSizeMin > 0); + TEST(allocSize.ImageSizeMin == 0 && allocSize.ImageSizeMax == 0); if(allocSize.BufferSizeMax == allocSize.BufferSizeMin) item.BufferSize = allocSize.BufferSizeMin; else @@ -1923,7 +4459,7 @@ static void TestPool_Benchmark( } else { - assert(allocSize.ImageSizeMin > 0 && allocSize.ImageSizeMax > 0); + TEST(allocSize.ImageSizeMin > 0 && allocSize.ImageSizeMax > 0); if(allocSize.ImageSizeMax == allocSize.ImageSizeMin) item.ImageSize.width = item.ImageSize.height = allocSize.ImageSizeMax; else @@ -1951,7 +4487,7 @@ static void TestPool_Benchmark( } else { - assert(item.ImageSize.width && item.ImageSize.height); + TEST(item.ImageSize.width && item.ImageSize.height); imageInfo.extent.width = item.ImageSize.width; imageInfo.extent.height = item.ImageSize.height; @@ -1978,7 +4514,7 @@ static void TestPool_Benchmark( // Determine which bufs we want to use in this frame. const size_t usedBufCount = (threadRand.Generate() % (config.UsedItemCountMax - config.UsedItemCountMin) + config.UsedItemCountMin) / config.ThreadCount; - assert(usedBufCount < usedItems.size() + unusedItems.size()); + TEST(usedBufCount < usedItems.size() + unusedItems.size()); // Move some used to unused. while(usedBufCount < usedItems.size()) { @@ -2112,7 +4648,7 @@ static void TestPool_Benchmark( } // Execute frames. - assert(config.ThreadCount <= MAXIMUM_WAIT_OBJECTS); + TEST(config.ThreadCount <= MAXIMUM_WAIT_OBJECTS); for(uint32_t frameIndex = 0; frameIndex < config.FrameCount; ++frameIndex) { vmaSetCurrentFrameIndex(g_hAllocator, frameIndex); @@ -2179,6 +4715,459 @@ static inline bool MemoryRegionsOverlap(char* ptr1, size_t size1, char* ptr2, si return true; } +static void TestMemoryUsage() +{ + wprintf(L"Testing memory usage:\n"); + + static const VmaMemoryUsage lastUsage = VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED; + for(uint32_t usage = 0; usage <= lastUsage; ++usage) + { + switch(usage) + { + case VMA_MEMORY_USAGE_UNKNOWN: printf(" VMA_MEMORY_USAGE_UNKNOWN:\n"); break; + case VMA_MEMORY_USAGE_GPU_ONLY: printf(" VMA_MEMORY_USAGE_GPU_ONLY:\n"); break; + case VMA_MEMORY_USAGE_CPU_ONLY: printf(" VMA_MEMORY_USAGE_CPU_ONLY:\n"); break; + case VMA_MEMORY_USAGE_CPU_TO_GPU: printf(" VMA_MEMORY_USAGE_CPU_TO_GPU:\n"); break; + case VMA_MEMORY_USAGE_GPU_TO_CPU: printf(" VMA_MEMORY_USAGE_GPU_TO_CPU:\n"); break; + case VMA_MEMORY_USAGE_CPU_COPY: printf(" VMA_MEMORY_USAGE_CPU_COPY:\n"); break; + case VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED: printf(" VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED:\n"); break; + default: assert(0); + } + + auto printResult = [](const char* testName, VkResult res, uint32_t memoryTypeBits, uint32_t memoryTypeIndex) + { + if(res == VK_SUCCESS) + printf(" %s: memoryTypeBits=0x%X, memoryTypeIndex=%u\n", testName, memoryTypeBits, memoryTypeIndex); + else + printf(" %s: memoryTypeBits=0x%X, FAILED with res=%d\n", testName, memoryTypeBits, (int32_t)res); + }; + + // 1: Buffer for copy + { + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 65536; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VkBuffer buf = VK_NULL_HANDLE; + VkResult res = vkCreateBuffer(g_hDevice, &bufCreateInfo, g_Allocs, &buf); + TEST(res == VK_SUCCESS && buf != VK_NULL_HANDLE); + + VkMemoryRequirements memReq = {}; + vkGetBufferMemoryRequirements(g_hDevice, buf, &memReq); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = (VmaMemoryUsage)usage; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaAllocateMemoryForBuffer(g_hAllocator, buf, &allocCreateInfo, &alloc, &allocInfo); + if(res == VK_SUCCESS) + { + TEST((memReq.memoryTypeBits & (1u << allocInfo.memoryType)) != 0); + res = vkBindBufferMemory(g_hDevice, buf, allocInfo.deviceMemory, allocInfo.offset); + TEST(res == VK_SUCCESS); + } + printResult("Buffer TRANSFER_DST + TRANSFER_SRC", res, memReq.memoryTypeBits, allocInfo.memoryType); + vmaDestroyBuffer(g_hAllocator, buf, alloc); + } + + // 2: Vertex buffer + { + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 65536; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + + VkBuffer buf = VK_NULL_HANDLE; + VkResult res = vkCreateBuffer(g_hDevice, &bufCreateInfo, g_Allocs, &buf); + TEST(res == VK_SUCCESS && buf != VK_NULL_HANDLE); + + VkMemoryRequirements memReq = {}; + vkGetBufferMemoryRequirements(g_hDevice, buf, &memReq); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = (VmaMemoryUsage)usage; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaAllocateMemoryForBuffer(g_hAllocator, buf, &allocCreateInfo, &alloc, &allocInfo); + if(res == VK_SUCCESS) + { + TEST((memReq.memoryTypeBits & (1u << allocInfo.memoryType)) != 0); + res = vkBindBufferMemory(g_hDevice, buf, allocInfo.deviceMemory, allocInfo.offset); + TEST(res == VK_SUCCESS); + } + printResult("Buffer TRANSFER_DST + VERTEX_BUFFER", res, memReq.memoryTypeBits, allocInfo.memoryType); + vmaDestroyBuffer(g_hAllocator, buf, alloc); + } + + // 3: Image for copy, OPTIMAL + { + VkImageCreateInfo imgCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imgCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imgCreateInfo.extent.width = 256; + imgCreateInfo.extent.height = 256; + imgCreateInfo.extent.depth = 1; + imgCreateInfo.mipLevels = 1; + imgCreateInfo.arrayLayers = 1; + imgCreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imgCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imgCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imgCreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + imgCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VkImage img = VK_NULL_HANDLE; + VkResult res = vkCreateImage(g_hDevice, &imgCreateInfo, g_Allocs, &img); + TEST(res == VK_SUCCESS && img != VK_NULL_HANDLE); + + VkMemoryRequirements memReq = {}; + vkGetImageMemoryRequirements(g_hDevice, img, &memReq); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = (VmaMemoryUsage)usage; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaAllocateMemoryForImage(g_hAllocator, img, &allocCreateInfo, &alloc, &allocInfo); + if(res == VK_SUCCESS) + { + TEST((memReq.memoryTypeBits & (1u << allocInfo.memoryType)) != 0); + res = vkBindImageMemory(g_hDevice, img, allocInfo.deviceMemory, allocInfo.offset); + TEST(res == VK_SUCCESS); + } + printResult("Image OPTIMAL TRANSFER_DST + TRANSFER_SRC", res, memReq.memoryTypeBits, allocInfo.memoryType); + + vmaDestroyImage(g_hAllocator, img, alloc); + } + + // 4: Image SAMPLED, OPTIMAL + { + VkImageCreateInfo imgCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imgCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imgCreateInfo.extent.width = 256; + imgCreateInfo.extent.height = 256; + imgCreateInfo.extent.depth = 1; + imgCreateInfo.mipLevels = 1; + imgCreateInfo.arrayLayers = 1; + imgCreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imgCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imgCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imgCreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + imgCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VkImage img = VK_NULL_HANDLE; + VkResult res = vkCreateImage(g_hDevice, &imgCreateInfo, g_Allocs, &img); + TEST(res == VK_SUCCESS && img != VK_NULL_HANDLE); + + VkMemoryRequirements memReq = {}; + vkGetImageMemoryRequirements(g_hDevice, img, &memReq); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = (VmaMemoryUsage)usage; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaAllocateMemoryForImage(g_hAllocator, img, &allocCreateInfo, &alloc, &allocInfo); + if(res == VK_SUCCESS) + { + TEST((memReq.memoryTypeBits & (1u << allocInfo.memoryType)) != 0); + res = vkBindImageMemory(g_hDevice, img, allocInfo.deviceMemory, allocInfo.offset); + TEST(res == VK_SUCCESS); + } + printResult("Image OPTIMAL TRANSFER_DST + SAMPLED", res, memReq.memoryTypeBits, allocInfo.memoryType); + vmaDestroyImage(g_hAllocator, img, alloc); + } + + // 5: Image COLOR_ATTACHMENT, OPTIMAL + { + VkImageCreateInfo imgCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imgCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imgCreateInfo.extent.width = 256; + imgCreateInfo.extent.height = 256; + imgCreateInfo.extent.depth = 1; + imgCreateInfo.mipLevels = 1; + imgCreateInfo.arrayLayers = 1; + imgCreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imgCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imgCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imgCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + imgCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VkImage img = VK_NULL_HANDLE; + VkResult res = vkCreateImage(g_hDevice, &imgCreateInfo, g_Allocs, &img); + TEST(res == VK_SUCCESS && img != VK_NULL_HANDLE); + + VkMemoryRequirements memReq = {}; + vkGetImageMemoryRequirements(g_hDevice, img, &memReq); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = (VmaMemoryUsage)usage; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaAllocateMemoryForImage(g_hAllocator, img, &allocCreateInfo, &alloc, &allocInfo); + if(res == VK_SUCCESS) + { + TEST((memReq.memoryTypeBits & (1u << allocInfo.memoryType)) != 0); + res = vkBindImageMemory(g_hDevice, img, allocInfo.deviceMemory, allocInfo.offset); + TEST(res == VK_SUCCESS); + } + printResult("Image OPTIMAL SAMPLED + COLOR_ATTACHMENT", res, memReq.memoryTypeBits, allocInfo.memoryType); + vmaDestroyImage(g_hAllocator, img, alloc); + } + } +} + +static uint32_t FindDeviceCoherentMemoryTypeBits() +{ + VkPhysicalDeviceMemoryProperties memProps; + vkGetPhysicalDeviceMemoryProperties(g_hPhysicalDevice, &memProps); + + uint32_t memTypeBits = 0; + for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i) + { + if(memProps.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD) + memTypeBits |= 1u << i; + } + return memTypeBits; +} + +static void TestDeviceCoherentMemory() +{ + if(!VK_AMD_device_coherent_memory_enabled) + return; + + uint32_t deviceCoherentMemoryTypeBits = FindDeviceCoherentMemoryTypeBits(); + // Extension is enabled, feature is enabled, and the device still doesn't support any such memory type? + // OK then, so it's just fake! + if(deviceCoherentMemoryTypeBits == 0) + return; + + wprintf(L"Testing device coherent memory...\n"); + + // 1. Try to allocate buffer from a memory type that is DEVICE_COHERENT. + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 0x10000; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + allocCreateInfo.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD; + + AllocInfo alloc = {}; + VmaAllocationInfo allocInfo = {}; + VkResult res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &alloc.m_Buffer, &alloc.m_Allocation, &allocInfo); + + // Make sure it succeeded and was really created in such memory type. + TEST(res == VK_SUCCESS); + TEST((1u << allocInfo.memoryType) & deviceCoherentMemoryTypeBits); + + alloc.Destroy(); + + // 2. Try to create a pool in such memory type. + { + VmaPoolCreateInfo poolCreateInfo = {}; + + res = vmaFindMemoryTypeIndex(g_hAllocator, UINT32_MAX, &allocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + TEST((1u << poolCreateInfo.memoryTypeIndex) & deviceCoherentMemoryTypeBits); + + VmaPool pool = VK_NULL_HANDLE; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + vmaDestroyPool(g_hAllocator, pool); + } + + // 3. Try the same with a local allocator created without VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT. + + VmaAllocatorCreateInfo allocatorCreateInfo = {}; + SetAllocatorCreateInfo(allocatorCreateInfo); + allocatorCreateInfo.flags &= ~VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT; + + VmaAllocator localAllocator = VK_NULL_HANDLE; + res = vmaCreateAllocator(&allocatorCreateInfo, &localAllocator); + TEST(res == VK_SUCCESS && localAllocator); + + res = vmaCreateBuffer(localAllocator, &bufCreateInfo, &allocCreateInfo, &alloc.m_Buffer, &alloc.m_Allocation, &allocInfo); + + // Make sure it failed. + TEST(res != VK_SUCCESS && !alloc.m_Buffer && !alloc.m_Allocation); + + // 4. Try to find memory type. + { + uint32_t memTypeIndex = UINT_MAX; + res = vmaFindMemoryTypeIndex(localAllocator, UINT32_MAX, &allocCreateInfo, &memTypeIndex); + TEST(res != VK_SUCCESS); + } + + vmaDestroyAllocator(localAllocator); +} + +static void TestBudget() +{ + wprintf(L"Testing budget...\n"); + + static const VkDeviceSize BUF_SIZE = 10ull * 1024 * 1024; + static const uint32_t BUF_COUNT = 4; + + const VkPhysicalDeviceMemoryProperties* memProps = {}; + vmaGetMemoryProperties(g_hAllocator, &memProps); + + for(uint32_t testIndex = 0; testIndex < 2; ++testIndex) + { + vmaSetCurrentFrameIndex(g_hAllocator, ++g_FrameIndex); + + VmaBudget budgetBeg[VK_MAX_MEMORY_HEAPS] = {}; + vmaGetBudget(g_hAllocator, budgetBeg); + + for(uint32_t i = 0; i < memProps->memoryHeapCount; ++i) + { + TEST(budgetBeg[i].budget > 0); + TEST(budgetBeg[i].budget <= memProps->memoryHeaps[i].size); + TEST(budgetBeg[i].allocationBytes <= budgetBeg[i].blockBytes); + } + + VkBufferCreateInfo bufInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufInfo.size = BUF_SIZE; + bufInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + if(testIndex == 0) + { + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + } + + // CREATE BUFFERS + uint32_t heapIndex = 0; + BufferInfo bufInfos[BUF_COUNT] = {}; + for(uint32_t bufIndex = 0; bufIndex < BUF_COUNT; ++bufIndex) + { + VmaAllocationInfo allocInfo; + VkResult res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, + &bufInfos[bufIndex].Buffer, &bufInfos[bufIndex].Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + if(bufIndex == 0) + { + heapIndex = MemoryTypeToHeap(allocInfo.memoryType); + } + else + { + // All buffers need to fall into the same heap. + TEST(MemoryTypeToHeap(allocInfo.memoryType) == heapIndex); + } + } + + VmaBudget budgetWithBufs[VK_MAX_MEMORY_HEAPS] = {}; + vmaGetBudget(g_hAllocator, budgetWithBufs); + + // DESTROY BUFFERS + for(size_t bufIndex = BUF_COUNT; bufIndex--; ) + { + vmaDestroyBuffer(g_hAllocator, bufInfos[bufIndex].Buffer, bufInfos[bufIndex].Allocation); + } + + VmaBudget budgetEnd[VK_MAX_MEMORY_HEAPS] = {}; + vmaGetBudget(g_hAllocator, budgetEnd); + + // CHECK + for(uint32_t i = 0; i < memProps->memoryHeapCount; ++i) + { + TEST(budgetEnd[i].allocationBytes <= budgetEnd[i].blockBytes); + if(i == heapIndex) + { + TEST(budgetEnd[i].allocationBytes == budgetBeg[i].allocationBytes); + TEST(budgetWithBufs[i].allocationBytes == budgetBeg[i].allocationBytes + BUF_SIZE * BUF_COUNT); + TEST(budgetWithBufs[i].blockBytes >= budgetEnd[i].blockBytes); + } + else + { + TEST(budgetEnd[i].allocationBytes == budgetEnd[i].allocationBytes && + budgetEnd[i].allocationBytes == budgetWithBufs[i].allocationBytes); + TEST(budgetEnd[i].blockBytes == budgetEnd[i].blockBytes && + budgetEnd[i].blockBytes == budgetWithBufs[i].blockBytes); + } + } + } +} + +static void TestAliasing() +{ + wprintf(L"Testing aliasing...\n"); + + /* + This is just a simple test, more like a code sample to demonstrate it's possible. + */ + + // A 512x512 texture to be sampled. + VkImageCreateInfo img1CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + img1CreateInfo.imageType = VK_IMAGE_TYPE_2D; + img1CreateInfo.extent.width = 512; + img1CreateInfo.extent.height = 512; + img1CreateInfo.extent.depth = 1; + img1CreateInfo.mipLevels = 10; + img1CreateInfo.arrayLayers = 1; + img1CreateInfo.format = VK_FORMAT_R8G8B8A8_SRGB; + img1CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + img1CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + img1CreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + img1CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + // A full screen texture to be used as color attachment. + VkImageCreateInfo img2CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + img2CreateInfo.imageType = VK_IMAGE_TYPE_2D; + img2CreateInfo.extent.width = 1920; + img2CreateInfo.extent.height = 1080; + img2CreateInfo.extent.depth = 1; + img2CreateInfo.mipLevels = 1; + img2CreateInfo.arrayLayers = 1; + img2CreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + img2CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + img2CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + img2CreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + img2CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + + VkImage img1 = VK_NULL_HANDLE; + ERR_GUARD_VULKAN(vkCreateImage(g_hDevice, &img1CreateInfo, g_Allocs, &img1)); + VkImage img2 = VK_NULL_HANDLE; + ERR_GUARD_VULKAN(vkCreateImage(g_hDevice, &img2CreateInfo, g_Allocs, &img2)); + + VkMemoryRequirements img1MemReq = {}; + vkGetImageMemoryRequirements(g_hDevice, img1, &img1MemReq); + VkMemoryRequirements img2MemReq = {}; + vkGetImageMemoryRequirements(g_hDevice, img2, &img2MemReq); + + VkMemoryRequirements finalMemReq = {}; + finalMemReq.size = std::max(img1MemReq.size, img2MemReq.size); + finalMemReq.alignment = std::max(img1MemReq.alignment, img2MemReq.alignment); + finalMemReq.memoryTypeBits = img1MemReq.memoryTypeBits & img2MemReq.memoryTypeBits; + if(finalMemReq.memoryTypeBits != 0) + { + wprintf(L" size: max(%llu, %llu) = %llu\n", + img1MemReq.size, img2MemReq.size, finalMemReq.size); + wprintf(L" alignment: max(%llu, %llu) = %llu\n", + img1MemReq.alignment, img2MemReq.alignment, finalMemReq.alignment); + wprintf(L" memoryTypeBits: %u & %u = %u\n", + img1MemReq.memoryTypeBits, img2MemReq.memoryTypeBits, finalMemReq.memoryTypeBits); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VmaAllocation alloc = VK_NULL_HANDLE; + ERR_GUARD_VULKAN(vmaAllocateMemory(g_hAllocator, &finalMemReq, &allocCreateInfo, &alloc, nullptr)); + + ERR_GUARD_VULKAN(vmaBindImageMemory(g_hAllocator, alloc, img1)); + ERR_GUARD_VULKAN(vmaBindImageMemory(g_hAllocator, alloc, img2)); + + // You can use img1, img2 here, but not at the same time! + + vmaFreeMemory(g_hAllocator, alloc); + } + else + { + wprintf(L" Textures cannot alias!\n"); + } + + vkDestroyImage(g_hDevice, img2, g_Allocs); + vkDestroyImage(g_hDevice, img1, g_Allocs); +} + static void TestMapping() { wprintf(L"Testing mapping...\n"); @@ -2198,25 +5187,25 @@ static void TestMapping() VmaPool pool = nullptr; if(testIndex == TEST_POOL) { - assert(memTypeIndex != UINT32_MAX); + TEST(memTypeIndex != UINT32_MAX); VmaPoolCreateInfo poolInfo = {}; poolInfo.memoryTypeIndex = memTypeIndex; res = vmaCreatePool(g_hAllocator, &poolInfo, &pool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } VkBufferCreateInfo bufInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; bufInfo.size = 0x10000; bufInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; - + VmaAllocationCreateInfo allocCreateInfo = {}; allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; allocCreateInfo.pool = pool; if(testIndex == TEST_DEDICATED) allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; - + VmaAllocationInfo allocInfo; - + // Mapped manually // Create 2 buffers. @@ -2225,56 +5214,56 @@ static void TestMapping() { res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &bufferInfos[i].Buffer, &bufferInfos[i].Allocation, &allocInfo); - assert(res == VK_SUCCESS); - assert(allocInfo.pMappedData == nullptr); + TEST(res == VK_SUCCESS); + TEST(allocInfo.pMappedData == nullptr); memTypeIndex = allocInfo.memoryType; } - + // Map buffer 0. char* data00 = nullptr; res = vmaMapMemory(g_hAllocator, bufferInfos[0].Allocation, (void**)&data00); - assert(res == VK_SUCCESS && data00 != nullptr); + TEST(res == VK_SUCCESS && data00 != nullptr); data00[0xFFFF] = data00[0]; // Map buffer 0 second time. char* data01 = nullptr; res = vmaMapMemory(g_hAllocator, bufferInfos[0].Allocation, (void**)&data01); - assert(res == VK_SUCCESS && data01 == data00); + TEST(res == VK_SUCCESS && data01 == data00); // Map buffer 1. char* data1 = nullptr; res = vmaMapMemory(g_hAllocator, bufferInfos[1].Allocation, (void**)&data1); - assert(res == VK_SUCCESS && data1 != nullptr); - assert(!MemoryRegionsOverlap(data00, (size_t)bufInfo.size, data1, (size_t)bufInfo.size)); + TEST(res == VK_SUCCESS && data1 != nullptr); + TEST(!MemoryRegionsOverlap(data00, (size_t)bufInfo.size, data1, (size_t)bufInfo.size)); data1[0xFFFF] = data1[0]; // Unmap buffer 0 two times. vmaUnmapMemory(g_hAllocator, bufferInfos[0].Allocation); vmaUnmapMemory(g_hAllocator, bufferInfos[0].Allocation); vmaGetAllocationInfo(g_hAllocator, bufferInfos[0].Allocation, &allocInfo); - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.pMappedData == nullptr); // Unmap buffer 1. vmaUnmapMemory(g_hAllocator, bufferInfos[1].Allocation); vmaGetAllocationInfo(g_hAllocator, bufferInfos[1].Allocation, &allocInfo); - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.pMappedData == nullptr); // Create 3rd buffer - persistently mapped. allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_MAPPED_BIT; res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &bufferInfos[2].Buffer, &bufferInfos[2].Allocation, &allocInfo); - assert(res == VK_SUCCESS && allocInfo.pMappedData != nullptr); + TEST(res == VK_SUCCESS && allocInfo.pMappedData != nullptr); // Map buffer 2. char* data2 = nullptr; res = vmaMapMemory(g_hAllocator, bufferInfos[2].Allocation, (void**)&data2); - assert(res == VK_SUCCESS && data2 == allocInfo.pMappedData); + TEST(res == VK_SUCCESS && data2 == allocInfo.pMappedData); data2[0xFFFF] = data2[0]; // Unmap buffer 2. vmaUnmapMemory(g_hAllocator, bufferInfos[2].Allocation); vmaGetAllocationInfo(g_hAllocator, bufferInfos[2].Allocation, &allocInfo); - assert(allocInfo.pMappedData == data2); + TEST(allocInfo.pMappedData == data2); // Destroy all buffers. for(size_t i = 3; i--; ) @@ -2284,6 +5273,51 @@ static void TestMapping() } } +// Test CREATE_MAPPED with required DEVICE_LOCAL. There was a bug with it. +static void TestDeviceLocalMapped() +{ + VkResult res; + + for(uint32_t testIndex = 0; testIndex < 3; ++testIndex) + { + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + bufCreateInfo.size = 4096; + + VmaPool pool = VK_NULL_HANDLE; + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + if(testIndex == 2) + { + VmaPoolCreateInfo poolCreateInfo = {}; + res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + allocCreateInfo.pool = pool; + } + else if(testIndex == 1) + { + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT; + } + + VkBuffer buf = VK_NULL_HANDLE; + VmaAllocation alloc = VK_NULL_HANDLE; + VmaAllocationInfo allocInfo = {}; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); + TEST(res == VK_SUCCESS && alloc); + + VkMemoryPropertyFlags memTypeFlags = 0; + vmaGetMemoryTypeProperties(g_hAllocator, allocInfo.memoryType, &memTypeFlags); + const bool shouldBeMapped = (memTypeFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; + TEST((allocInfo.pMappedData != nullptr) == shouldBeMapped); + + vmaDestroyBuffer(g_hAllocator, buf, alloc); + vmaDestroyPool(g_hAllocator, pool); + } +} + static void TestMappingMultithreaded() { wprintf(L"Testing mapping multithreaded...\n"); @@ -2307,11 +5341,11 @@ static void TestMappingMultithreaded() VmaPool pool = nullptr; if(testIndex == TEST_POOL) { - assert(memTypeIndex != UINT32_MAX); + TEST(memTypeIndex != UINT32_MAX); VmaPoolCreateInfo poolInfo = {}; poolInfo.memoryTypeIndex = memTypeIndex; res = vmaCreatePool(g_hAllocator, &poolInfo, &pool); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); } VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; @@ -2362,7 +5396,7 @@ static void TestMappingMultithreaded() VmaAllocationInfo allocInfo; VkResult res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &localAllocCreateInfo, &bufInfo.Buffer, &bufInfo.Allocation, &allocInfo); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); if(memTypeIndex == UINT32_MAX) memTypeIndex = allocInfo.memoryType; @@ -2372,28 +5406,28 @@ static void TestMappingMultithreaded() if(mode == MODE::PERSISTENTLY_MAPPED) { data = (char*)allocInfo.pMappedData; - assert(data != nullptr); + TEST(data != nullptr); } else if(mode == MODE::MAP_FOR_MOMENT || mode == MODE::MAP_FOR_LONGER || mode == MODE::MAP_TWO_TIMES) { - assert(data == nullptr); + TEST(data == nullptr); res = vmaMapMemory(g_hAllocator, bufInfo.Allocation, (void**)&data); - assert(res == VK_SUCCESS && data != nullptr); + TEST(res == VK_SUCCESS && data != nullptr); if(mode == MODE::MAP_TWO_TIMES) { char* data2 = nullptr; res = vmaMapMemory(g_hAllocator, bufInfo.Allocation, (void**)&data2); - assert(res == VK_SUCCESS && data2 == data); + TEST(res == VK_SUCCESS && data2 == data); } } else if(mode == MODE::DONT_MAP) { - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.pMappedData == nullptr); } else - assert(0); + TEST(0); // Test if reading and writing from the beginning and end of mapped memory doesn't crash. if(data) @@ -2406,9 +5440,9 @@ static void TestMappingMultithreaded() VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, bufInfo.Allocation, &allocInfo); if(mode == MODE::MAP_FOR_MOMENT) - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.pMappedData == nullptr); else - assert(allocInfo.pMappedData == data); + TEST(allocInfo.pMappedData == data); } switch(rand.Generate() % 3) @@ -2432,7 +5466,7 @@ static void TestMappingMultithreaded() VmaAllocationInfo allocInfo; vmaGetAllocationInfo(g_hAllocator, bufInfos[bufferIndex].Allocation, &allocInfo); - assert(allocInfo.pMappedData == nullptr); + TEST(allocInfo.pMappedData == nullptr); } vmaDestroyBuffer(g_hAllocator, bufInfos[bufferIndex].Buffer, bufInfos[bufferIndex].Allocation); @@ -2450,8 +5484,8 @@ static void TestMappingMultithreaded() static void WriteMainTestResultHeader(FILE* file) { fprintf(file, - "Code,Test,Time," - "Config," + "Code,Time," + "Threads,Buffers and images,Sizes,Operations,Allocation strategy,Free order," "Total Time (us)," "Allocation Time Min (us)," "Allocation Time Avg (us)," @@ -2478,19 +5512,15 @@ static void WriteMainTestResult( float deallocationTimeAvgSeconds = ToFloatSeconds(result.DeallocationTimeAvg); float deallocationTimeMaxSeconds = ToFloatSeconds(result.DeallocationTimeMax); - time_t rawTime; time(&rawTime); - struct tm timeInfo; localtime_s(&timeInfo, &rawTime); - char timeStr[128]; - strftime(timeStr, _countof(timeStr), "%c", &timeInfo); + std::string currTime; + CurrentTimeToStr(currTime); fprintf(file, "%s,%s,%s," - "BeginBytesToAllocate=%I64u MaxBytesToAllocate=%I64u AdditionalOperationCount=%u ThreadCount=%u FreeOrder=%d," "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%I64u,%I64u,%I64u\n", codeDescription, + currTime.c_str(), testDescription, - timeStr, - config.BeginBytesToAllocate, config.MaxBytesToAllocate, config.AdditionalOperationCount, config.ThreadCount, (uint32_t)config.FreeOrder, totalTimeSeconds * 1e6f, allocationTimeMinSeconds * 1e6f, allocationTimeAvgSeconds * 1e6f, @@ -2536,10 +5566,8 @@ static void WritePoolTestResult( float deallocationTimeAvgSeconds = ToFloatSeconds(result.DeallocationTimeAvg); float deallocationTimeMaxSeconds = ToFloatSeconds(result.DeallocationTimeMax); - time_t rawTime; time(&rawTime); - struct tm timeInfo; localtime_s(&timeInfo, &rawTime); - char timeStr[128]; - strftime(timeStr, _countof(timeStr), "%c", &timeInfo); + std::string currTime; + CurrentTimeToStr(currTime); fprintf(file, "%s,%s,%s," @@ -2548,7 +5576,7 @@ static void WritePoolTestResult( // General codeDescription, testDescription, - timeStr, + currTime.c_str(), // Config config.ThreadCount, (unsigned long long)config.PoolSize, @@ -2581,6 +5609,7 @@ static void PerformCustomMainTest(FILE* file) config.FreeOrder = FREE_ORDER::FORWARD; config.ThreadCount = 16; config.ThreadsUsingCommonAllocationsProbabilityPercent = 50; + config.AllocationStrategy = 0; // Buffers //config.AllocationSizes.push_back({4, 16, 1024}); @@ -2595,7 +5624,7 @@ static void PerformCustomMainTest(FILE* file) Result result{}; VkResult res = MainTest(result, config); - assert(res == VK_SUCCESS); + TEST(res == VK_SUCCESS); WriteMainTestResult(file, "Foo", "CustomTest", config, result); } @@ -2626,27 +5655,12 @@ static void PerformCustomPoolTest(FILE* file) config.TotalItemCount = config.UsedItemCountMax * 10; config.UsedItemCountMin = config.UsedItemCountMax * 80 / 100; - g_MemoryAliasingWarningEnabled = false; PoolTestResult result = {}; TestPool_Benchmark(result, config); - g_MemoryAliasingWarningEnabled = true; WritePoolTestResult(file, "Code desc", "Test desc", config, result); } -enum CONFIG_TYPE { - CONFIG_TYPE_MINIMUM, - CONFIG_TYPE_SMALL, - CONFIG_TYPE_AVERAGE, - CONFIG_TYPE_LARGE, - CONFIG_TYPE_MAXIMUM, - CONFIG_TYPE_COUNT -}; - -static constexpr CONFIG_TYPE ConfigType = CONFIG_TYPE_SMALL; -//static constexpr CONFIG_TYPE ConfigType = CONFIG_TYPE_LARGE; -static const char* CODE_DESCRIPTION = "Foo"; - static void PerformMainTests(FILE* file) { uint32_t repeatCount = 1; @@ -2667,6 +5681,9 @@ static void PerformMainTests(FILE* file) case CONFIG_TYPE_MAXIMUM: threadCountCount = 7; break; default: assert(0); } + + const size_t strategyCount = GetAllocationStrategyCount(); + for(size_t threadCountIndex = 0; threadCountIndex < threadCountCount; ++threadCountIndex) { std::string desc1; @@ -2720,9 +5737,9 @@ static void PerformMainTests(FILE* file) std::string desc2 = desc1; switch(buffersVsImagesIndex) { - case 0: desc2 += " Buffers"; break; - case 1: desc2 += " Images"; break; - case 2: desc2 += " Buffers+Images"; break; + case 0: desc2 += ",Buffers"; break; + case 1: desc2 += ",Images"; break; + case 2: desc2 += ",Buffers+Images"; break; default: assert(0); } @@ -2734,9 +5751,9 @@ static void PerformMainTests(FILE* file) std::string desc3 = desc2; switch(smallVsLargeIndex) { - case 0: desc3 += " Small"; break; - case 1: desc3 += " Large"; break; - case 2: desc3 += " Small+Large"; break; + case 0: desc3 += ",Small"; break; + case 1: desc3 += ",Large"; break; + case 2: desc3 += ",Small+Large"; break; default: assert(0); } @@ -2840,22 +5857,22 @@ static void PerformMainTests(FILE* file) switch(beginBytesToAllocateIndex) { case 0: - desc5 += " Allocate_100%"; + desc5 += ",Allocate_100%"; config.BeginBytesToAllocate = config.MaxBytesToAllocate; config.AdditionalOperationCount = 0; break; case 1: - desc5 += " Allocate_50%+Operations"; + desc5 += ",Allocate_50%+Operations"; config.BeginBytesToAllocate = config.MaxBytesToAllocate * 50 / 100; config.AdditionalOperationCount = 1024; break; case 2: - desc5 += " Allocate_5%+Operations"; + desc5 += ",Allocate_5%+Operations"; config.BeginBytesToAllocate = config.MaxBytesToAllocate * 5 / 100; config.AdditionalOperationCount = 1024; break; case 3: - desc5 += " Allocate_95%+Operations"; + desc5 += ",Allocate_95%+Operations"; config.BeginBytesToAllocate = config.MaxBytesToAllocate * 95 / 100; config.AdditionalOperationCount = 1024; break; @@ -2863,16 +5880,44 @@ static void PerformMainTests(FILE* file) assert(0); } - const char* testDescription = desc5.c_str(); - - for(size_t repeat = 0; repeat < repeatCount; ++repeat) + for(size_t strategyIndex = 0; strategyIndex < strategyCount; ++strategyIndex) { - printf("%s Repeat %u\n", testDescription, (uint32_t)repeat); + std::string desc6 = desc5; + switch(strategyIndex) + { + case 0: + desc6 += ",BestFit"; + config.AllocationStrategy = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT; + break; + case 1: + desc6 += ",WorstFit"; + config.AllocationStrategy = VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT; + break; + case 2: + desc6 += ",FirstFit"; + config.AllocationStrategy = VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT; + break; + default: + assert(0); + } - Result result{}; - VkResult res = MainTest(result, config); - assert(res == VK_SUCCESS); - WriteMainTestResult(file, CODE_DESCRIPTION, testDescription, config, result); + desc6 += ','; + desc6 += FREE_ORDER_NAMES[(uint32_t)config.FreeOrder]; + + const char* testDescription = desc6.c_str(); + + for(size_t repeat = 0; repeat < repeatCount; ++repeat) + { + printf("%s #%u\n", testDescription, (uint32_t)repeat); + + Result result{}; + VkResult res = MainTest(result, config); + TEST(res == VK_SUCCESS); + if(file) + { + WriteMainTestResult(file, CODE_DESCRIPTION, testDescription, config, result); + } + } } } } @@ -3092,12 +6137,10 @@ static void PerformPoolTests(FILE* file) for(size_t repeat = 0; repeat < repeatCount; ++repeat) { - printf("%s Repeat %u\n", testDescription, (uint32_t)repeat); + printf("%s #%u\n", testDescription, (uint32_t)repeat); PoolTestResult result{}; - g_MemoryAliasingWarningEnabled = false; TestPool_Benchmark(result, config); - g_MemoryAliasingWarningEnabled = true; WritePoolTestResult(file, CODE_DESCRIPTION, testDescription, config, result); } } @@ -3107,28 +6150,300 @@ static void PerformPoolTests(FILE* file) } } +static void BasicTestBuddyAllocator() +{ + wprintf(L"Basic test buddy allocator\n"); + + RandomNumberGenerator rand{76543}; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = 1024; // Whatever. + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + // Deliberately adding 1023 to test usable size smaller than memory block size. + poolCreateInfo.blockSize = 1024 * 1024 + 1023; + poolCreateInfo.flags = VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT; + //poolCreateInfo.minBlockCount = poolCreateInfo.maxBlockCount = 1; + + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + VkBufferCreateInfo bufCreateInfo = sampleBufCreateInfo; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + + std::vector bufInfo; + BufferInfo newBufInfo; + VmaAllocationInfo allocInfo; + + bufCreateInfo.size = 1024 * 256; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 1024 * 512; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + bufCreateInfo.size = 1024 * 128; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + // Test very small allocation, smaller than minimum node size. + bufCreateInfo.size = 1; + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + + // Test some small allocation with alignment requirement. + { + VkMemoryRequirements memReq; + memReq.alignment = 256; + memReq.memoryTypeBits = UINT32_MAX; + memReq.size = 32; + + newBufInfo.Buffer = VK_NULL_HANDLE; + res = vmaAllocateMemory(g_hAllocator, &memReq, &allocCreateInfo, + &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + TEST(allocInfo.offset % memReq.alignment == 0); + bufInfo.push_back(newBufInfo); + } + + //SaveAllocatorStatsToFile(L"TEST.json"); + + VmaPoolStats stats = {}; + vmaGetPoolStats(g_hAllocator, pool, &stats); + int DBG = 0; // Set breakpoint here to inspect `stats`. + + // Allocate enough new buffers to surely fall into second block. + for(uint32_t i = 0; i < 32; ++i) + { + bufCreateInfo.size = 1024 * (rand.Generate() % 32 + 1); + res = vmaCreateBuffer(g_hAllocator, &bufCreateInfo, &allocCreateInfo, + &newBufInfo.Buffer, &newBufInfo.Allocation, &allocInfo); + TEST(res == VK_SUCCESS); + bufInfo.push_back(newBufInfo); + } + + SaveAllocatorStatsToFile(L"BuddyTest01.json"); + + // Destroy the buffers in random order. + while(!bufInfo.empty()) + { + const size_t indexToDestroy = rand.Generate() % bufInfo.size(); + const BufferInfo& currBufInfo = bufInfo[indexToDestroy]; + vmaDestroyBuffer(g_hAllocator, currBufInfo.Buffer, currBufInfo.Allocation); + bufInfo.erase(bufInfo.begin() + indexToDestroy); + } + + vmaDestroyPool(g_hAllocator, pool); +} + +static void BasicTestAllocatePages() +{ + wprintf(L"Basic test allocate pages\n"); + + RandomNumberGenerator rand{765461}; + + VkBufferCreateInfo sampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + sampleBufCreateInfo.size = 1024; // Whatever. + sampleBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VmaAllocationCreateInfo sampleAllocCreateInfo = {}; + sampleAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + + VmaPoolCreateInfo poolCreateInfo = {}; + VkResult res = vmaFindMemoryTypeIndexForBufferInfo(g_hAllocator, &sampleBufCreateInfo, &sampleAllocCreateInfo, &poolCreateInfo.memoryTypeIndex); + TEST(res == VK_SUCCESS); + + // 1 block of 1 MB. + poolCreateInfo.blockSize = 1024 * 1024; + poolCreateInfo.minBlockCount = poolCreateInfo.maxBlockCount = 1; + + // Create pool. + VmaPool pool = nullptr; + res = vmaCreatePool(g_hAllocator, &poolCreateInfo, &pool); + TEST(res == VK_SUCCESS); + + // Make 100 allocations of 4 KB - they should fit into the pool. + VkMemoryRequirements memReq; + memReq.memoryTypeBits = UINT32_MAX; + memReq.alignment = 4 * 1024; + memReq.size = 4 * 1024; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + allocCreateInfo.pool = pool; + + constexpr uint32_t allocCount = 100; + + std::vector alloc{allocCount}; + std::vector allocInfo{allocCount}; + res = vmaAllocateMemoryPages(g_hAllocator, &memReq, &allocCreateInfo, allocCount, alloc.data(), allocInfo.data()); + TEST(res == VK_SUCCESS); + for(uint32_t i = 0; i < allocCount; ++i) + { + TEST(alloc[i] != VK_NULL_HANDLE && + allocInfo[i].pMappedData != nullptr && + allocInfo[i].deviceMemory == allocInfo[0].deviceMemory && + allocInfo[i].memoryType == allocInfo[0].memoryType); + } + + // Free the allocations. + vmaFreeMemoryPages(g_hAllocator, allocCount, alloc.data()); + std::fill(alloc.begin(), alloc.end(), nullptr); + std::fill(allocInfo.begin(), allocInfo.end(), VmaAllocationInfo{}); + + // Try to make 100 allocations of 100 KB. This call should fail due to not enough memory. + // Also test optional allocationInfo = null. + memReq.size = 100 * 1024; + res = vmaAllocateMemoryPages(g_hAllocator, &memReq, &allocCreateInfo, allocCount, alloc.data(), nullptr); + TEST(res != VK_SUCCESS); + TEST(std::find_if(alloc.begin(), alloc.end(), [](VmaAllocation alloc){ return alloc != VK_NULL_HANDLE; }) == alloc.end()); + + // Make 100 allocations of 4 KB, but with required alignment of 128 KB. This should also fail. + memReq.size = 4 * 1024; + memReq.alignment = 128 * 1024; + res = vmaAllocateMemoryPages(g_hAllocator, &memReq, &allocCreateInfo, allocCount, alloc.data(), allocInfo.data()); + TEST(res != VK_SUCCESS); + + // Make 100 dedicated allocations of 4 KB. + memReq.alignment = 4 * 1024; + memReq.size = 4 * 1024; + + VmaAllocationCreateInfo dedicatedAllocCreateInfo = {}; + dedicatedAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + dedicatedAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + res = vmaAllocateMemoryPages(g_hAllocator, &memReq, &dedicatedAllocCreateInfo, allocCount, alloc.data(), allocInfo.data()); + TEST(res == VK_SUCCESS); + for(uint32_t i = 0; i < allocCount; ++i) + { + TEST(alloc[i] != VK_NULL_HANDLE && + allocInfo[i].pMappedData != nullptr && + allocInfo[i].memoryType == allocInfo[0].memoryType && + allocInfo[i].offset == 0); + if(i > 0) + { + TEST(allocInfo[i].deviceMemory != allocInfo[0].deviceMemory); + } + } + + // Free the allocations. + vmaFreeMemoryPages(g_hAllocator, allocCount, alloc.data()); + std::fill(alloc.begin(), alloc.end(), nullptr); + std::fill(allocInfo.begin(), allocInfo.end(), VmaAllocationInfo{}); + + vmaDestroyPool(g_hAllocator, pool); +} + +// Test the testing environment. +static void TestGpuData() +{ + RandomNumberGenerator rand = { 53434 }; + + std::vector allocInfo; + + for(size_t i = 0; i < 100; ++i) + { + AllocInfo info = {}; + + info.m_BufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + info.m_BufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + info.m_BufferInfo.size = 1024 * 1024 * (rand.Generate() % 9 + 1); + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + VkResult res = vmaCreateBuffer(g_hAllocator, &info.m_BufferInfo, &allocCreateInfo, &info.m_Buffer, &info.m_Allocation, nullptr); + TEST(res == VK_SUCCESS); + + info.m_StartValue = rand.Generate(); + + allocInfo.push_back(std::move(info)); + } + + UploadGpuData(allocInfo.data(), allocInfo.size()); + + ValidateGpuData(allocInfo.data(), allocInfo.size()); + + DestroyAllAllocations(allocInfo); +} + void Test() { wprintf(L"TESTING:\n"); - // TEMP tests + if(false) + { + //////////////////////////////////////////////////////////////////////////////// + // Temporarily insert custom tests here: + return; + } // # Simple tests TestBasics(); + TestAllocationVersusResourceSize(); + //TestGpuData(); // Not calling this because it's just testing the testing environment. #if VMA_DEBUG_MARGIN TestDebugMargin(); #else TestPool_SameSize(); + TestPool_MinBlockCount(); TestHeapSizeLimit(); #endif #if VMA_DEBUG_INITIALIZE_ALLOCATIONS TestAllocationsInitialization(); #endif + TestMemoryUsage(); + TestDeviceCoherentMemory(); + TestBudget(); + TestAliasing(); TestMapping(); + TestDeviceLocalMapped(); TestMappingMultithreaded(); + TestLinearAllocator(); + ManuallyTestLinearAllocator(); + TestLinearAllocatorMultiBlock(); + + BasicTestBuddyAllocator(); + BasicTestAllocatePages(); + + if(g_BufferDeviceAddressEnabled) + TestBufferDeviceAddress(); + + { + FILE* file; + fopen_s(&file, "Algorithms.csv", "w"); + assert(file != NULL); + BenchmarkAlgorithms(file); + fclose(file); + } + TestDefragmentationSimple(); TestDefragmentationFull(); + TestDefragmentationWholePool(); + TestDefragmentationGpu(); + TestDefragmentationIncrementalBasic(); + TestDefragmentationIncrementalComplex(); // # Detailed tests FILE* file; @@ -3144,7 +6459,7 @@ void Test() //PerformCustomPoolTest(file); fclose(file); - + wprintf(L"Done.\n"); } diff --git a/third_party/vkmemalloc/src/Tests.h b/third_party/vkmemalloc/src/Tests.h index 9da4f6f2b0..d259fa94a0 100644 --- a/third_party/vkmemalloc/src/Tests.h +++ b/third_party/vkmemalloc/src/Tests.h @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #ifndef TESTS_H_ #define TESTS_H_ diff --git a/third_party/vkmemalloc/src/VmaReplay/Common.cpp b/third_party/vkmemalloc/src/VmaReplay/Common.cpp index c324245eec..b2bedf1865 100644 --- a/third_party/vkmemalloc/src/VmaReplay/Common.cpp +++ b/third_party/vkmemalloc/src/VmaReplay/Common.cpp @@ -1,5 +1,51 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #include "Common.h" +bool StrRangeToPtrList(const StrRange& s, std::vector& out) +{ + out.clear(); + StrRange currRange = { s.beg, nullptr }; + while(currRange.beg < s.end) + { + currRange.end = currRange.beg; + while(currRange.end < s.end && *currRange.end != ' ') + { + ++currRange.end; + } + + uint64_t ptr = 0; + if(!StrRangeToPtr(currRange, ptr)) + { + return false; + } + out.push_back(ptr); + + currRange.beg = currRange.end + 1; + } + return true; +} + //////////////////////////////////////////////////////////////////////////////// // LineSplit class @@ -12,6 +58,11 @@ bool LineSplit::GetNextLine(StrRange& out) while(currLineEnd < m_NumBytes && m_Data[currLineEnd] != '\n') ++currLineEnd; out.end = m_Data + currLineEnd; + // Ignore trailing '\r' to support Windows end of line. + if(out.end > out.beg && *(out.end - 1) == '\r') + { + --out.end; + } m_NextLineBeg = currLineEnd + 1; // Past '\n' ++m_NextLineIndex; return true; diff --git a/third_party/vkmemalloc/src/VmaReplay/Common.h b/third_party/vkmemalloc/src/VmaReplay/Common.h index 28b5bbc12f..3f966c9ea5 100644 --- a/third_party/vkmemalloc/src/VmaReplay/Common.h +++ b/third_party/vkmemalloc/src/VmaReplay/Common.h @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #pragma once #include "VmaUsage.h" @@ -35,6 +57,11 @@ T ceil_div(T x, T y) { return (x+y-1) / y; } +template +inline T round_div(T x, T y) +{ + return (x+y/(T)2) / y; +} template inline T align_up(T val, T align) @@ -111,6 +138,7 @@ inline bool StrRangeToBool(const StrRange& s, bool& out) return true; } +bool StrRangeToPtrList(const StrRange& s, std::vector& out); class LineSplit { @@ -145,9 +173,16 @@ public: size_t GetCount() const { return m_Count; } StrRange GetRange(size_t index) const { - return StrRange { - m_Line.beg + m_Ranges[index * 2], - m_Line.beg + m_Ranges[index * 2 + 1] }; + if(index < m_Count) + { + return StrRange { + m_Line.beg + m_Ranges[index * 2], + m_Line.beg + m_Ranges[index * 2 + 1] }; + } + else + { + return StrRange{0, 0}; + } } private: diff --git a/third_party/vkmemalloc/src/VmaReplay/Constants.cpp b/third_party/vkmemalloc/src/VmaReplay/Constants.cpp new file mode 100644 index 0000000000..28d9c3b783 --- /dev/null +++ b/third_party/vkmemalloc/src/VmaReplay/Constants.cpp @@ -0,0 +1,795 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#include "Common.h" +#include "Constants.h" + +const int RESULT_EXCEPTION = -1000; +const int RESULT_ERROR_COMMAND_LINE = -1; +const int RESULT_ERROR_SOURCE_FILE = -2; +const int RESULT_ERROR_FORMAT = -3; +const int RESULT_ERROR_VULKAN = -4; + +const char* VMA_FUNCTION_NAMES[] = { + "vmaCreatePool", + "vmaDestroyPool", + "vmaSetAllocationUserData", + "vmaCreateBuffer", + "vmaDestroyBuffer", + "vmaCreateImage", + "vmaDestroyImage", + "vmaFreeMemory", + "vmaFreeMemoryPages", + "vmaCreateLostAllocation", + "vmaAllocateMemory", + "vmaAllocateMemoryPages", + "vmaAllocateMemoryForBuffer", + "vmaAllocateMemoryForImage", + "vmaMapMemory", + "vmaUnmapMemory", + "vmaFlushAllocation", + "vmaInvalidateAllocation", + "vmaTouchAllocation", + "vmaGetAllocationInfo", + "vmaMakePoolAllocationsLost", + "vmaResizeAllocation", + "vmaDefragmentationBegin", + "vmaDefragmentationEnd", + "vmaSetPoolName", +}; +static_assert( + _countof(VMA_FUNCTION_NAMES) == (size_t)VMA_FUNCTION::Count, + "VMA_FUNCTION_NAMES array doesn't match VMA_FUNCTION enum."); + +const char* VMA_POOL_CREATE_FLAG_NAMES[] = { + "VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT", + "VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT", + "VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT", +}; +const uint32_t VMA_POOL_CREATE_FLAG_VALUES[] = { + VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT, + VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT, + VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT, +}; +const size_t VMA_POOL_CREATE_FLAG_COUNT = _countof(VMA_POOL_CREATE_FLAG_NAMES); +static_assert( + _countof(VMA_POOL_CREATE_FLAG_NAMES) == _countof(VMA_POOL_CREATE_FLAG_VALUES), + "VMA_POOL_CREATE_FLAG_NAMES array doesn't match VMA_POOL_CREATE_FLAG_VALUES."); + +const char* VK_BUFFER_CREATE_FLAG_NAMES[] = { + "VK_BUFFER_CREATE_SPARSE_BINDING_BIT", + "VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT", + "VK_BUFFER_CREATE_SPARSE_ALIASED_BIT", + "VK_BUFFER_CREATE_PROTECTED_BIT", +}; +const uint32_t VK_BUFFER_CREATE_FLAG_VALUES[] = { + VK_BUFFER_CREATE_SPARSE_BINDING_BIT, + VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT, + VK_BUFFER_CREATE_SPARSE_ALIASED_BIT, + VK_BUFFER_CREATE_PROTECTED_BIT, +}; +const size_t VK_BUFFER_CREATE_FLAG_COUNT = _countof(VK_BUFFER_CREATE_FLAG_NAMES); +static_assert( + _countof(VK_BUFFER_CREATE_FLAG_NAMES) == _countof(VK_BUFFER_CREATE_FLAG_VALUES), + "VK_BUFFER_CREATE_FLAG_NAMES array doesn't match VK_BUFFER_CREATE_FLAG_VALUES."); + +const char* VK_BUFFER_USAGE_FLAG_NAMES[] = { + "VK_BUFFER_USAGE_TRANSFER_SRC_BIT", + "VK_BUFFER_USAGE_TRANSFER_DST_BIT", + "VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT", + "VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT", + "VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT", + "VK_BUFFER_USAGE_STORAGE_BUFFER_BIT", + "VK_BUFFER_USAGE_INDEX_BUFFER_BIT", + "VK_BUFFER_USAGE_VERTEX_BUFFER_BIT", + "VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT", + "VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT", + //"VK_BUFFER_USAGE_RAYTRACING_BIT_NVX", +}; +const uint32_t VK_BUFFER_USAGE_FLAG_VALUES[] = { + VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + VK_BUFFER_USAGE_TRANSFER_DST_BIT, + VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT, + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT, + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, + VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT, + //VK_BUFFER_USAGE_RAYTRACING_BIT_NVX, +}; +const size_t VK_BUFFER_USAGE_FLAG_COUNT = _countof(VK_BUFFER_USAGE_FLAG_NAMES); +static_assert( + _countof(VK_BUFFER_USAGE_FLAG_NAMES) == _countof(VK_BUFFER_USAGE_FLAG_VALUES), + "VK_BUFFER_USAGE_FLAG_NAMES array doesn't match VK_BUFFER_USAGE_FLAG_VALUES."); + +const char* VK_SHARING_MODE_NAMES[] = { + "VK_SHARING_MODE_EXCLUSIVE", + "VK_SHARING_MODE_CONCURRENT", +}; +const size_t VK_SHARING_MODE_COUNT = _countof(VK_SHARING_MODE_NAMES); + +const char* VK_IMAGE_CREATE_FLAG_NAMES[] = { + "VK_IMAGE_CREATE_SPARSE_BINDING_BIT", + "VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT", + "VK_IMAGE_CREATE_SPARSE_ALIASED_BIT", + "VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT", + "VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT", + "VK_IMAGE_CREATE_ALIAS_BIT", + "VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT", + "VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT", + "VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT", + "VK_IMAGE_CREATE_EXTENDED_USAGE_BIT", + "VK_IMAGE_CREATE_PROTECTED_BIT", + "VK_IMAGE_CREATE_DISJOINT_BIT", + //"VK_IMAGE_CREATE_CORNER_SAMPLED_BIT_NV", + "VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT", +}; +const uint32_t VK_IMAGE_CREATE_FLAG_VALUES[] = { + VK_IMAGE_CREATE_SPARSE_BINDING_BIT, + VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT, + VK_IMAGE_CREATE_SPARSE_ALIASED_BIT, + VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT, + VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT, + VK_IMAGE_CREATE_ALIAS_BIT, + VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT, + VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, + VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT, + VK_IMAGE_CREATE_EXTENDED_USAGE_BIT, + VK_IMAGE_CREATE_PROTECTED_BIT, + VK_IMAGE_CREATE_DISJOINT_BIT, + //VK_IMAGE_CREATE_CORNER_SAMPLED_BIT_NV, + VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT, +}; +const size_t VK_IMAGE_CREATE_FLAG_COUNT = _countof(VK_IMAGE_CREATE_FLAG_NAMES); +static_assert( + _countof(VK_IMAGE_CREATE_FLAG_NAMES) == _countof(VK_IMAGE_CREATE_FLAG_VALUES), + "VK_IMAGE_CREATE_FLAG_NAMES array doesn't match VK_IMAGE_CREATE_FLAG_VALUES."); + +const char* VK_IMAGE_TYPE_NAMES[] = { + "VK_IMAGE_TYPE_1D", + "VK_IMAGE_TYPE_2D", + "VK_IMAGE_TYPE_3D", +}; +const size_t VK_IMAGE_TYPE_COUNT = _countof(VK_IMAGE_TYPE_NAMES); + +const char* VK_FORMAT_NAMES[] = { + "VK_FORMAT_UNDEFINED", + "VK_FORMAT_R4G4_UNORM_PACK8", + "VK_FORMAT_R4G4B4A4_UNORM_PACK16", + "VK_FORMAT_B4G4R4A4_UNORM_PACK16", + "VK_FORMAT_R5G6B5_UNORM_PACK16", + "VK_FORMAT_B5G6R5_UNORM_PACK16", + "VK_FORMAT_R5G5B5A1_UNORM_PACK16", + "VK_FORMAT_B5G5R5A1_UNORM_PACK16", + "VK_FORMAT_A1R5G5B5_UNORM_PACK16", + "VK_FORMAT_R8_UNORM", + "VK_FORMAT_R8_SNORM", + "VK_FORMAT_R8_USCALED", + "VK_FORMAT_R8_SSCALED", + "VK_FORMAT_R8_UINT", + "VK_FORMAT_R8_SINT", + "VK_FORMAT_R8_SRGB", + "VK_FORMAT_R8G8_UNORM", + "VK_FORMAT_R8G8_SNORM", + "VK_FORMAT_R8G8_USCALED", + "VK_FORMAT_R8G8_SSCALED", + "VK_FORMAT_R8G8_UINT", + "VK_FORMAT_R8G8_SINT", + "VK_FORMAT_R8G8_SRGB", + "VK_FORMAT_R8G8B8_UNORM", + "VK_FORMAT_R8G8B8_SNORM", + "VK_FORMAT_R8G8B8_USCALED", + "VK_FORMAT_R8G8B8_SSCALED", + "VK_FORMAT_R8G8B8_UINT", + "VK_FORMAT_R8G8B8_SINT", + "VK_FORMAT_R8G8B8_SRGB", + "VK_FORMAT_B8G8R8_UNORM", + "VK_FORMAT_B8G8R8_SNORM", + "VK_FORMAT_B8G8R8_USCALED", + "VK_FORMAT_B8G8R8_SSCALED", + "VK_FORMAT_B8G8R8_UINT", + "VK_FORMAT_B8G8R8_SINT", + "VK_FORMAT_B8G8R8_SRGB", + "VK_FORMAT_R8G8B8A8_UNORM", + "VK_FORMAT_R8G8B8A8_SNORM", + "VK_FORMAT_R8G8B8A8_USCALED", + "VK_FORMAT_R8G8B8A8_SSCALED", + "VK_FORMAT_R8G8B8A8_UINT", + "VK_FORMAT_R8G8B8A8_SINT", + "VK_FORMAT_R8G8B8A8_SRGB", + "VK_FORMAT_B8G8R8A8_UNORM", + "VK_FORMAT_B8G8R8A8_SNORM", + "VK_FORMAT_B8G8R8A8_USCALED", + "VK_FORMAT_B8G8R8A8_SSCALED", + "VK_FORMAT_B8G8R8A8_UINT", + "VK_FORMAT_B8G8R8A8_SINT", + "VK_FORMAT_B8G8R8A8_SRGB", + "VK_FORMAT_A8B8G8R8_UNORM_PACK32", + "VK_FORMAT_A8B8G8R8_SNORM_PACK32", + "VK_FORMAT_A8B8G8R8_USCALED_PACK32", + "VK_FORMAT_A8B8G8R8_SSCALED_PACK32", + "VK_FORMAT_A8B8G8R8_UINT_PACK32", + "VK_FORMAT_A8B8G8R8_SINT_PACK32", + "VK_FORMAT_A8B8G8R8_SRGB_PACK32", + "VK_FORMAT_A2R10G10B10_UNORM_PACK32", + "VK_FORMAT_A2R10G10B10_SNORM_PACK32", + "VK_FORMAT_A2R10G10B10_USCALED_PACK32", + "VK_FORMAT_A2R10G10B10_SSCALED_PACK32", + "VK_FORMAT_A2R10G10B10_UINT_PACK32", + "VK_FORMAT_A2R10G10B10_SINT_PACK32", + "VK_FORMAT_A2B10G10R10_UNORM_PACK32", + "VK_FORMAT_A2B10G10R10_SNORM_PACK32", + "VK_FORMAT_A2B10G10R10_USCALED_PACK32", + "VK_FORMAT_A2B10G10R10_SSCALED_PACK32", + "VK_FORMAT_A2B10G10R10_UINT_PACK32", + "VK_FORMAT_A2B10G10R10_SINT_PACK32", + "VK_FORMAT_R16_UNORM", + "VK_FORMAT_R16_SNORM", + "VK_FORMAT_R16_USCALED", + "VK_FORMAT_R16_SSCALED", + "VK_FORMAT_R16_UINT", + "VK_FORMAT_R16_SINT", + "VK_FORMAT_R16_SFLOAT", + "VK_FORMAT_R16G16_UNORM", + "VK_FORMAT_R16G16_SNORM", + "VK_FORMAT_R16G16_USCALED", + "VK_FORMAT_R16G16_SSCALED", + "VK_FORMAT_R16G16_UINT", + "VK_FORMAT_R16G16_SINT", + "VK_FORMAT_R16G16_SFLOAT", + "VK_FORMAT_R16G16B16_UNORM", + "VK_FORMAT_R16G16B16_SNORM", + "VK_FORMAT_R16G16B16_USCALED", + "VK_FORMAT_R16G16B16_SSCALED", + "VK_FORMAT_R16G16B16_UINT", + "VK_FORMAT_R16G16B16_SINT", + "VK_FORMAT_R16G16B16_SFLOAT", + "VK_FORMAT_R16G16B16A16_UNORM", + "VK_FORMAT_R16G16B16A16_SNORM", + "VK_FORMAT_R16G16B16A16_USCALED", + "VK_FORMAT_R16G16B16A16_SSCALED", + "VK_FORMAT_R16G16B16A16_UINT", + "VK_FORMAT_R16G16B16A16_SINT", + "VK_FORMAT_R16G16B16A16_SFLOAT", + "VK_FORMAT_R32_UINT", + "VK_FORMAT_R32_SINT", + "VK_FORMAT_R32_SFLOAT", + "VK_FORMAT_R32G32_UINT", + "VK_FORMAT_R32G32_SINT", + "VK_FORMAT_R32G32_SFLOAT", + "VK_FORMAT_R32G32B32_UINT", + "VK_FORMAT_R32G32B32_SINT", + "VK_FORMAT_R32G32B32_SFLOAT", + "VK_FORMAT_R32G32B32A32_UINT", + "VK_FORMAT_R32G32B32A32_SINT", + "VK_FORMAT_R32G32B32A32_SFLOAT", + "VK_FORMAT_R64_UINT", + "VK_FORMAT_R64_SINT", + "VK_FORMAT_R64_SFLOAT", + "VK_FORMAT_R64G64_UINT", + "VK_FORMAT_R64G64_SINT", + "VK_FORMAT_R64G64_SFLOAT", + "VK_FORMAT_R64G64B64_UINT", + "VK_FORMAT_R64G64B64_SINT", + "VK_FORMAT_R64G64B64_SFLOAT", + "VK_FORMAT_R64G64B64A64_UINT", + "VK_FORMAT_R64G64B64A64_SINT", + "VK_FORMAT_R64G64B64A64_SFLOAT", + "VK_FORMAT_B10G11R11_UFLOAT_PACK32", + "VK_FORMAT_E5B9G9R9_UFLOAT_PACK32", + "VK_FORMAT_D16_UNORM", + "VK_FORMAT_X8_D24_UNORM_PACK32", + "VK_FORMAT_D32_SFLOAT", + "VK_FORMAT_S8_UINT", + "VK_FORMAT_D16_UNORM_S8_UINT", + "VK_FORMAT_D24_UNORM_S8_UINT", + "VK_FORMAT_D32_SFLOAT_S8_UINT", + "VK_FORMAT_BC1_RGB_UNORM_BLOCK", + "VK_FORMAT_BC1_RGB_SRGB_BLOCK", + "VK_FORMAT_BC1_RGBA_UNORM_BLOCK", + "VK_FORMAT_BC1_RGBA_SRGB_BLOCK", + "VK_FORMAT_BC2_UNORM_BLOCK", + "VK_FORMAT_BC2_SRGB_BLOCK", + "VK_FORMAT_BC3_UNORM_BLOCK", + "VK_FORMAT_BC3_SRGB_BLOCK", + "VK_FORMAT_BC4_UNORM_BLOCK", + "VK_FORMAT_BC4_SNORM_BLOCK", + "VK_FORMAT_BC5_UNORM_BLOCK", + "VK_FORMAT_BC5_SNORM_BLOCK", + "VK_FORMAT_BC6H_UFLOAT_BLOCK", + "VK_FORMAT_BC6H_SFLOAT_BLOCK", + "VK_FORMAT_BC7_UNORM_BLOCK", + "VK_FORMAT_BC7_SRGB_BLOCK", + "VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK", + "VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK", + "VK_FORMAT_ETC2_R8G8B8A1_UNORM_BLOCK", + "VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK", + "VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK", + "VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK", + "VK_FORMAT_EAC_R11_UNORM_BLOCK", + "VK_FORMAT_EAC_R11_SNORM_BLOCK", + "VK_FORMAT_EAC_R11G11_UNORM_BLOCK", + "VK_FORMAT_EAC_R11G11_SNORM_BLOCK", + "VK_FORMAT_ASTC_4x4_UNORM_BLOCK", + "VK_FORMAT_ASTC_4x4_SRGB_BLOCK", + "VK_FORMAT_ASTC_5x4_UNORM_BLOCK", + "VK_FORMAT_ASTC_5x4_SRGB_BLOCK", + "VK_FORMAT_ASTC_5x5_UNORM_BLOCK", + "VK_FORMAT_ASTC_5x5_SRGB_BLOCK", + "VK_FORMAT_ASTC_6x5_UNORM_BLOCK", + "VK_FORMAT_ASTC_6x5_SRGB_BLOCK", + "VK_FORMAT_ASTC_6x6_UNORM_BLOCK", + "VK_FORMAT_ASTC_6x6_SRGB_BLOCK", + "VK_FORMAT_ASTC_8x5_UNORM_BLOCK", + "VK_FORMAT_ASTC_8x5_SRGB_BLOCK", + "VK_FORMAT_ASTC_8x6_UNORM_BLOCK", + "VK_FORMAT_ASTC_8x6_SRGB_BLOCK", + "VK_FORMAT_ASTC_8x8_UNORM_BLOCK", + "VK_FORMAT_ASTC_8x8_SRGB_BLOCK", + "VK_FORMAT_ASTC_10x5_UNORM_BLOCK", + "VK_FORMAT_ASTC_10x5_SRGB_BLOCK", + "VK_FORMAT_ASTC_10x6_UNORM_BLOCK", + "VK_FORMAT_ASTC_10x6_SRGB_BLOCK", + "VK_FORMAT_ASTC_10x8_UNORM_BLOCK", + "VK_FORMAT_ASTC_10x8_SRGB_BLOCK", + "VK_FORMAT_ASTC_10x10_UNORM_BLOCK", + "VK_FORMAT_ASTC_10x10_SRGB_BLOCK", + "VK_FORMAT_ASTC_12x10_UNORM_BLOCK", + "VK_FORMAT_ASTC_12x10_SRGB_BLOCK", + "VK_FORMAT_ASTC_12x12_UNORM_BLOCK", + "VK_FORMAT_ASTC_12x12_SRGB_BLOCK", + "VK_FORMAT_G8B8G8R8_422_UNORM", + "VK_FORMAT_B8G8R8G8_422_UNORM", + "VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM", + "VK_FORMAT_G8_B8R8_2PLANE_420_UNORM", + "VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM", + "VK_FORMAT_G8_B8R8_2PLANE_422_UNORM", + "VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM", + "VK_FORMAT_R10X6_UNORM_PACK16", + "VK_FORMAT_R10X6G10X6_UNORM_2PACK16", + "VK_FORMAT_R10X6G10X6B10X6A10X6_UNORM_4PACK16", + "VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16", + "VK_FORMAT_B10X6G10X6R10X6G10X6_422_UNORM_4PACK16", + "VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16", + "VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16", + "VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16", + "VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16", + "VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16", + "VK_FORMAT_R12X4_UNORM_PACK16", + "VK_FORMAT_R12X4G12X4_UNORM_2PACK16", + "VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16", + "VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16", + "VK_FORMAT_B12X4G12X4R12X4G12X4_422_UNORM_4PACK16", + "VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16", + "VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16", + "VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16", + "VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16", + "VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16", + "VK_FORMAT_G16B16G16R16_422_UNORM", + "VK_FORMAT_B16G16R16G16_422_UNORM", + "VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM", + "VK_FORMAT_G16_B16R16_2PLANE_420_UNORM", + "VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM", + "VK_FORMAT_G16_B16R16_2PLANE_422_UNORM", + "VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM", + "VK_FORMAT_PVRTC1_2BPP_UNORM_BLOCK_IMG", + "VK_FORMAT_PVRTC1_4BPP_UNORM_BLOCK_IMG", + "VK_FORMAT_PVRTC2_2BPP_UNORM_BLOCK_IMG", + "VK_FORMAT_PVRTC2_4BPP_UNORM_BLOCK_IMG", + "VK_FORMAT_PVRTC1_2BPP_SRGB_BLOCK_IMG", + "VK_FORMAT_PVRTC1_4BPP_SRGB_BLOCK_IMG", + "VK_FORMAT_PVRTC2_2BPP_SRGB_BLOCK_IMG", + "VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG", +}; +const uint32_t VK_FORMAT_VALUES[] = { + VK_FORMAT_UNDEFINED, + VK_FORMAT_R4G4_UNORM_PACK8, + VK_FORMAT_R4G4B4A4_UNORM_PACK16, + VK_FORMAT_B4G4R4A4_UNORM_PACK16, + VK_FORMAT_R5G6B5_UNORM_PACK16, + VK_FORMAT_B5G6R5_UNORM_PACK16, + VK_FORMAT_R5G5B5A1_UNORM_PACK16, + VK_FORMAT_B5G5R5A1_UNORM_PACK16, + VK_FORMAT_A1R5G5B5_UNORM_PACK16, + VK_FORMAT_R8_UNORM, + VK_FORMAT_R8_SNORM, + VK_FORMAT_R8_USCALED, + VK_FORMAT_R8_SSCALED, + VK_FORMAT_R8_UINT, + VK_FORMAT_R8_SINT, + VK_FORMAT_R8_SRGB, + VK_FORMAT_R8G8_UNORM, + VK_FORMAT_R8G8_SNORM, + VK_FORMAT_R8G8_USCALED, + VK_FORMAT_R8G8_SSCALED, + VK_FORMAT_R8G8_UINT, + VK_FORMAT_R8G8_SINT, + VK_FORMAT_R8G8_SRGB, + VK_FORMAT_R8G8B8_UNORM, + VK_FORMAT_R8G8B8_SNORM, + VK_FORMAT_R8G8B8_USCALED, + VK_FORMAT_R8G8B8_SSCALED, + VK_FORMAT_R8G8B8_UINT, + VK_FORMAT_R8G8B8_SINT, + VK_FORMAT_R8G8B8_SRGB, + VK_FORMAT_B8G8R8_UNORM, + VK_FORMAT_B8G8R8_SNORM, + VK_FORMAT_B8G8R8_USCALED, + VK_FORMAT_B8G8R8_SSCALED, + VK_FORMAT_B8G8R8_UINT, + VK_FORMAT_B8G8R8_SINT, + VK_FORMAT_B8G8R8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_SNORM, + VK_FORMAT_R8G8B8A8_USCALED, + VK_FORMAT_R8G8B8A8_SSCALED, + VK_FORMAT_R8G8B8A8_UINT, + VK_FORMAT_R8G8B8A8_SINT, + VK_FORMAT_R8G8B8A8_SRGB, + VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_B8G8R8A8_SNORM, + VK_FORMAT_B8G8R8A8_USCALED, + VK_FORMAT_B8G8R8A8_SSCALED, + VK_FORMAT_B8G8R8A8_UINT, + VK_FORMAT_B8G8R8A8_SINT, + VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_A8B8G8R8_UNORM_PACK32, + VK_FORMAT_A8B8G8R8_SNORM_PACK32, + VK_FORMAT_A8B8G8R8_USCALED_PACK32, + VK_FORMAT_A8B8G8R8_SSCALED_PACK32, + VK_FORMAT_A8B8G8R8_UINT_PACK32, + VK_FORMAT_A8B8G8R8_SINT_PACK32, + VK_FORMAT_A8B8G8R8_SRGB_PACK32, + VK_FORMAT_A2R10G10B10_UNORM_PACK32, + VK_FORMAT_A2R10G10B10_SNORM_PACK32, + VK_FORMAT_A2R10G10B10_USCALED_PACK32, + VK_FORMAT_A2R10G10B10_SSCALED_PACK32, + VK_FORMAT_A2R10G10B10_UINT_PACK32, + VK_FORMAT_A2R10G10B10_SINT_PACK32, + VK_FORMAT_A2B10G10R10_UNORM_PACK32, + VK_FORMAT_A2B10G10R10_SNORM_PACK32, + VK_FORMAT_A2B10G10R10_USCALED_PACK32, + VK_FORMAT_A2B10G10R10_SSCALED_PACK32, + VK_FORMAT_A2B10G10R10_UINT_PACK32, + VK_FORMAT_A2B10G10R10_SINT_PACK32, + VK_FORMAT_R16_UNORM, + VK_FORMAT_R16_SNORM, + VK_FORMAT_R16_USCALED, + VK_FORMAT_R16_SSCALED, + VK_FORMAT_R16_UINT, + VK_FORMAT_R16_SINT, + VK_FORMAT_R16_SFLOAT, + VK_FORMAT_R16G16_UNORM, + VK_FORMAT_R16G16_SNORM, + VK_FORMAT_R16G16_USCALED, + VK_FORMAT_R16G16_SSCALED, + VK_FORMAT_R16G16_UINT, + VK_FORMAT_R16G16_SINT, + VK_FORMAT_R16G16_SFLOAT, + VK_FORMAT_R16G16B16_UNORM, + VK_FORMAT_R16G16B16_SNORM, + VK_FORMAT_R16G16B16_USCALED, + VK_FORMAT_R16G16B16_SSCALED, + VK_FORMAT_R16G16B16_UINT, + VK_FORMAT_R16G16B16_SINT, + VK_FORMAT_R16G16B16_SFLOAT, + VK_FORMAT_R16G16B16A16_UNORM, + VK_FORMAT_R16G16B16A16_SNORM, + VK_FORMAT_R16G16B16A16_USCALED, + VK_FORMAT_R16G16B16A16_SSCALED, + VK_FORMAT_R16G16B16A16_UINT, + VK_FORMAT_R16G16B16A16_SINT, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R32_UINT, + VK_FORMAT_R32_SINT, + VK_FORMAT_R32_SFLOAT, + VK_FORMAT_R32G32_UINT, + VK_FORMAT_R32G32_SINT, + VK_FORMAT_R32G32_SFLOAT, + VK_FORMAT_R32G32B32_UINT, + VK_FORMAT_R32G32B32_SINT, + VK_FORMAT_R32G32B32_SFLOAT, + VK_FORMAT_R32G32B32A32_UINT, + VK_FORMAT_R32G32B32A32_SINT, + VK_FORMAT_R32G32B32A32_SFLOAT, + VK_FORMAT_R64_UINT, + VK_FORMAT_R64_SINT, + VK_FORMAT_R64_SFLOAT, + VK_FORMAT_R64G64_UINT, + VK_FORMAT_R64G64_SINT, + VK_FORMAT_R64G64_SFLOAT, + VK_FORMAT_R64G64B64_UINT, + VK_FORMAT_R64G64B64_SINT, + VK_FORMAT_R64G64B64_SFLOAT, + VK_FORMAT_R64G64B64A64_UINT, + VK_FORMAT_R64G64B64A64_SINT, + VK_FORMAT_R64G64B64A64_SFLOAT, + VK_FORMAT_B10G11R11_UFLOAT_PACK32, + VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, + VK_FORMAT_D16_UNORM, + VK_FORMAT_X8_D24_UNORM_PACK32, + VK_FORMAT_D32_SFLOAT, + VK_FORMAT_S8_UINT, + VK_FORMAT_D16_UNORM_S8_UINT, + VK_FORMAT_D24_UNORM_S8_UINT, + VK_FORMAT_D32_SFLOAT_S8_UINT, + VK_FORMAT_BC1_RGB_UNORM_BLOCK, + VK_FORMAT_BC1_RGB_SRGB_BLOCK, + VK_FORMAT_BC1_RGBA_UNORM_BLOCK, + VK_FORMAT_BC1_RGBA_SRGB_BLOCK, + VK_FORMAT_BC2_UNORM_BLOCK, + VK_FORMAT_BC2_SRGB_BLOCK, + VK_FORMAT_BC3_UNORM_BLOCK, + VK_FORMAT_BC3_SRGB_BLOCK, + VK_FORMAT_BC4_UNORM_BLOCK, + VK_FORMAT_BC4_SNORM_BLOCK, + VK_FORMAT_BC5_UNORM_BLOCK, + VK_FORMAT_BC5_SNORM_BLOCK, + VK_FORMAT_BC6H_UFLOAT_BLOCK, + VK_FORMAT_BC6H_SFLOAT_BLOCK, + VK_FORMAT_BC7_UNORM_BLOCK, + VK_FORMAT_BC7_SRGB_BLOCK, + VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK, + VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK, + VK_FORMAT_ETC2_R8G8B8A1_UNORM_BLOCK, + VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK, + VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK, + VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK, + VK_FORMAT_EAC_R11_UNORM_BLOCK, + VK_FORMAT_EAC_R11_SNORM_BLOCK, + VK_FORMAT_EAC_R11G11_UNORM_BLOCK, + VK_FORMAT_EAC_R11G11_SNORM_BLOCK, + VK_FORMAT_ASTC_4x4_UNORM_BLOCK, + VK_FORMAT_ASTC_4x4_SRGB_BLOCK, + VK_FORMAT_ASTC_5x4_UNORM_BLOCK, + VK_FORMAT_ASTC_5x4_SRGB_BLOCK, + VK_FORMAT_ASTC_5x5_UNORM_BLOCK, + VK_FORMAT_ASTC_5x5_SRGB_BLOCK, + VK_FORMAT_ASTC_6x5_UNORM_BLOCK, + VK_FORMAT_ASTC_6x5_SRGB_BLOCK, + VK_FORMAT_ASTC_6x6_UNORM_BLOCK, + VK_FORMAT_ASTC_6x6_SRGB_BLOCK, + VK_FORMAT_ASTC_8x5_UNORM_BLOCK, + VK_FORMAT_ASTC_8x5_SRGB_BLOCK, + VK_FORMAT_ASTC_8x6_UNORM_BLOCK, + VK_FORMAT_ASTC_8x6_SRGB_BLOCK, + VK_FORMAT_ASTC_8x8_UNORM_BLOCK, + VK_FORMAT_ASTC_8x8_SRGB_BLOCK, + VK_FORMAT_ASTC_10x5_UNORM_BLOCK, + VK_FORMAT_ASTC_10x5_SRGB_BLOCK, + VK_FORMAT_ASTC_10x6_UNORM_BLOCK, + VK_FORMAT_ASTC_10x6_SRGB_BLOCK, + VK_FORMAT_ASTC_10x8_UNORM_BLOCK, + VK_FORMAT_ASTC_10x8_SRGB_BLOCK, + VK_FORMAT_ASTC_10x10_UNORM_BLOCK, + VK_FORMAT_ASTC_10x10_SRGB_BLOCK, + VK_FORMAT_ASTC_12x10_UNORM_BLOCK, + VK_FORMAT_ASTC_12x10_SRGB_BLOCK, + VK_FORMAT_ASTC_12x12_UNORM_BLOCK, + VK_FORMAT_ASTC_12x12_SRGB_BLOCK, + VK_FORMAT_G8B8G8R8_422_UNORM, + VK_FORMAT_B8G8R8G8_422_UNORM, + VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, + VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, + VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, + VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, + VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, + VK_FORMAT_R10X6_UNORM_PACK16, + VK_FORMAT_R10X6G10X6_UNORM_2PACK16, + VK_FORMAT_R10X6G10X6B10X6A10X6_UNORM_4PACK16, + VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, + VK_FORMAT_B10X6G10X6R10X6G10X6_422_UNORM_4PACK16, + VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16, + VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16, + VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, + VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16, + VK_FORMAT_R12X4_UNORM_PACK16, + VK_FORMAT_R12X4G12X4_UNORM_2PACK16, + VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, + VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, + VK_FORMAT_B12X4G12X4R12X4G12X4_422_UNORM_4PACK16, + VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16, + VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, + VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16, + VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, + VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16, + VK_FORMAT_G16B16G16R16_422_UNORM, + VK_FORMAT_B16G16R16G16_422_UNORM, + VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, + VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, + VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, + VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, + VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, + VK_FORMAT_PVRTC1_2BPP_UNORM_BLOCK_IMG, + VK_FORMAT_PVRTC1_4BPP_UNORM_BLOCK_IMG, + VK_FORMAT_PVRTC2_2BPP_UNORM_BLOCK_IMG, + VK_FORMAT_PVRTC2_4BPP_UNORM_BLOCK_IMG, + VK_FORMAT_PVRTC1_2BPP_SRGB_BLOCK_IMG, + VK_FORMAT_PVRTC1_4BPP_SRGB_BLOCK_IMG, + VK_FORMAT_PVRTC2_2BPP_SRGB_BLOCK_IMG, + VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG, +}; +const size_t VK_FORMAT_COUNT = _countof(VK_FORMAT_NAMES); +static_assert( + _countof(VK_FORMAT_NAMES) == _countof(VK_FORMAT_VALUES), + "VK_FORMAT_NAMES array doesn't match VK_FORMAT_VALUES."); + +const char* VK_SAMPLE_COUNT_NAMES[] = { + "VK_SAMPLE_COUNT_1_BIT", + "VK_SAMPLE_COUNT_2_BIT", + "VK_SAMPLE_COUNT_4_BIT", + "VK_SAMPLE_COUNT_8_BIT", + "VK_SAMPLE_COUNT_16_BIT", + "VK_SAMPLE_COUNT_32_BIT", + "VK_SAMPLE_COUNT_64_BIT", +}; +const uint32_t VK_SAMPLE_COUNT_VALUES[] = { + VK_SAMPLE_COUNT_1_BIT, + VK_SAMPLE_COUNT_2_BIT, + VK_SAMPLE_COUNT_4_BIT, + VK_SAMPLE_COUNT_8_BIT, + VK_SAMPLE_COUNT_16_BIT, + VK_SAMPLE_COUNT_32_BIT, + VK_SAMPLE_COUNT_64_BIT, +}; +const size_t VK_SAMPLE_COUNT_COUNT = _countof(VK_SAMPLE_COUNT_NAMES); +static_assert( + _countof(VK_SAMPLE_COUNT_NAMES) == _countof(VK_SAMPLE_COUNT_VALUES), + "VK_SAMPLE_COUNT_NAMES array doesn't match VK_SAMPLE_COUNT_VALUES."); + +const char* VK_IMAGE_TILING_NAMES[] = { + "VK_IMAGE_TILING_OPTIMAL", + "VK_IMAGE_TILING_LINEAR", +}; +const size_t VK_IMAGE_TILING_COUNT = _countof(VK_IMAGE_TILING_NAMES); + +const char* VK_IMAGE_USAGE_FLAG_NAMES[] = { + "VK_IMAGE_USAGE_TRANSFER_SRC_BIT", + "VK_IMAGE_USAGE_TRANSFER_DST_BIT", + "VK_IMAGE_USAGE_SAMPLED_BIT", + "VK_IMAGE_USAGE_STORAGE_BIT", + "VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT", + "VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT", + "VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT", + "VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT", + //"VK_IMAGE_USAGE_SHADING_RATE_IMAGE_BIT_NV", +}; +const uint32_t VK_IMAGE_USAGE_FLAG_VALUES[] = { + VK_IMAGE_USAGE_TRANSFER_SRC_BIT, + VK_IMAGE_USAGE_TRANSFER_DST_BIT, + VK_IMAGE_USAGE_SAMPLED_BIT, + VK_IMAGE_USAGE_STORAGE_BIT, + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, + VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT, + VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT, + VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT, + //VK_IMAGE_USAGE_SHADING_RATE_IMAGE_BIT_NV, +}; +const size_t VK_IMAGE_USAGE_FLAG_COUNT = _countof(VK_IMAGE_USAGE_FLAG_NAMES); +static_assert( + _countof(VK_IMAGE_USAGE_FLAG_NAMES) == _countof(VK_IMAGE_USAGE_FLAG_VALUES), + "VK_IMAGE_USAGE_FLAG_NAMES array doesn't match VK_IMAGE_USAGE_FLAG_VALUES."); + +const char* VK_IMAGE_LAYOUT_NAMES[] = { + "VK_IMAGE_LAYOUT_UNDEFINED", + "VK_IMAGE_LAYOUT_GENERAL", + "VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL", + "VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL", + "VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL", + "VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL", + "VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL", + "VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL", + "VK_IMAGE_LAYOUT_PREINITIALIZED", + "VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL", + "VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL", + "VK_IMAGE_LAYOUT_PRESENT_SRC_KHR", + "VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR", + //"VK_IMAGE_LAYOUT_SHADING_RATE_OPTIMAL_NV", +}; +const uint32_t VK_IMAGE_LAYOUT_VALUES[] = { + VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_IMAGE_LAYOUT_PREINITIALIZED, + VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL, + VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL, + VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR, + //VK_IMAGE_LAYOUT_SHADING_RATE_OPTIMAL_NV, +}; +const size_t VK_IMAGE_LAYOUT_COUNT = _countof(VK_IMAGE_LAYOUT_NAMES); +static_assert( + _countof(VK_IMAGE_LAYOUT_NAMES) == _countof(VK_IMAGE_LAYOUT_VALUES), + "VK_IMAGE_LAYOUT_NAMES array doesn't match VK_IMAGE_LAYOUT_VALUES."); + +const char* VMA_ALLOCATION_CREATE_FLAG_NAMES[] = { + "VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT", + "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT", + "VMA_ALLOCATION_CREATE_MAPPED_BIT", + "VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT", + "VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT", + "VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT", + "VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT", + "VMA_ALLOCATION_CREATE_DONT_BIND_BIT", + "VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT", + "VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT", + "VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT", + "VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT", +}; +const uint32_t VMA_ALLOCATION_CREATE_FLAG_VALUES[] = { + VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, + VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT, + VMA_ALLOCATION_CREATE_MAPPED_BIT, + VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT, + VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT, + VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT, + VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT, + VMA_ALLOCATION_CREATE_DONT_BIND_BIT, + VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT, + VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT, + VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT, +}; +const size_t VMA_ALLOCATION_CREATE_FLAG_COUNT = _countof(VMA_ALLOCATION_CREATE_FLAG_NAMES); +static_assert( + _countof(VMA_ALLOCATION_CREATE_FLAG_NAMES) == _countof(VMA_ALLOCATION_CREATE_FLAG_VALUES), + "VMA_ALLOCATION_CREATE_FLAG_NAMES array doesn't match VMA_ALLOCATION_CREATE_FLAG_VALUES."); + +const char* VMA_MEMORY_USAGE_NAMES[] = { + "VMA_MEMORY_USAGE_UNKNOWN", + "VMA_MEMORY_USAGE_GPU_ONLY", + "VMA_MEMORY_USAGE_CPU_ONLY", + "VMA_MEMORY_USAGE_CPU_TO_GPU", + "VMA_MEMORY_USAGE_GPU_TO_CPU", + "VMA_MEMORY_USAGE_CPU_COPY", + "VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED", +}; +const size_t VMA_MEMORY_USAGE_COUNT = _countof(VMA_MEMORY_USAGE_NAMES); + +const char* VK_MEMORY_PROPERTY_FLAG_NAMES[] = { + "VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT", + "VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT", + "VK_MEMORY_PROPERTY_HOST_COHERENT_BIT", + "VK_MEMORY_PROPERTY_HOST_CACHED_BIT", + "VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT", + "VK_MEMORY_PROPERTY_PROTECTED_BIT", +}; +const uint32_t VK_MEMORY_PROPERTY_FLAG_VALUES[] = { + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + VK_MEMORY_PROPERTY_HOST_CACHED_BIT, + VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT, + VK_MEMORY_PROPERTY_PROTECTED_BIT, +}; +const size_t VK_MEMORY_PROPERTY_FLAG_COUNT = _countof(VK_MEMORY_PROPERTY_FLAG_NAMES); +static_assert( + _countof(VK_MEMORY_PROPERTY_FLAG_NAMES) == _countof(VK_MEMORY_PROPERTY_FLAG_VALUES), + "VK_MEMORY_PROPERTY_FLAG_NAMES array doesn't match VK_MEMORY_PROPERTY_FLAG_VALUES."); diff --git a/third_party/vkmemalloc/src/VmaReplay/Constants.h b/third_party/vkmemalloc/src/VmaReplay/Constants.h new file mode 100644 index 0000000000..5db0858396 --- /dev/null +++ b/third_party/vkmemalloc/src/VmaReplay/Constants.h @@ -0,0 +1,149 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#pragma once + +extern const int RESULT_EXCEPTION; +extern const int RESULT_ERROR_COMMAND_LINE; +extern const int RESULT_ERROR_SOURCE_FILE; +extern const int RESULT_ERROR_FORMAT; +extern const int RESULT_ERROR_VULKAN; + +enum CMD_LINE_OPT +{ + CMD_LINE_OPT_VERBOSITY, + CMD_LINE_OPT_ITERATIONS, + CMD_LINE_OPT_LINES, + CMD_LINE_OPT_PHYSICAL_DEVICE, + CMD_LINE_OPT_USER_DATA, + CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, + CMD_LINE_OPT_VK_EXT_MEMORY_BUDGET, + CMD_LINE_OPT_VK_LAYER_KHRONOS_VALIDATION, + CMD_LINE_OPT_MEM_STATS, + CMD_LINE_OPT_DUMP_STATS_AFTER_LINE, + CMD_LINE_OPT_DEFRAGMENT_AFTER_LINE, + CMD_LINE_OPT_DEFRAGMENTATION_FLAGS, + CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE, +}; + +enum class VERBOSITY +{ + MINIMUM = 0, + DEFAULT, + MAXIMUM, + COUNT, +}; + +enum class VULKAN_EXTENSION_REQUEST +{ + DISABLED, + ENABLED, + DEFAULT +}; + +enum class OBJECT_TYPE { BUFFER, IMAGE }; + +enum class VMA_FUNCTION +{ + CreatePool, + DestroyPool, + SetAllocationUserData, + CreateBuffer, + DestroyBuffer, + CreateImage, + DestroyImage, + FreeMemory, + FreeMemoryPages, + CreateLostAllocation, + AllocateMemory, + AllocateMemoryPages, + AllocateMemoryForBuffer, + AllocateMemoryForImage, + MapMemory, + UnmapMemory, + FlushAllocation, + InvalidateAllocation, + TouchAllocation, + GetAllocationInfo, + MakePoolAllocationsLost, + ResizeAllocation, + DefragmentationBegin, + DefragmentationEnd, + SetPoolName, + Count +}; +extern const char* VMA_FUNCTION_NAMES[]; + +extern const char* VMA_POOL_CREATE_FLAG_NAMES[]; +extern const uint32_t VMA_POOL_CREATE_FLAG_VALUES[]; +extern const size_t VMA_POOL_CREATE_FLAG_COUNT; + +extern const char* VK_BUFFER_CREATE_FLAG_NAMES[]; +extern const uint32_t VK_BUFFER_CREATE_FLAG_VALUES[]; +extern const size_t VK_BUFFER_CREATE_FLAG_COUNT; + +extern const char* VK_BUFFER_USAGE_FLAG_NAMES[]; +extern const uint32_t VK_BUFFER_USAGE_FLAG_VALUES[]; +extern const size_t VK_BUFFER_USAGE_FLAG_COUNT; + +extern const char* VK_SHARING_MODE_NAMES[]; +extern const size_t VK_SHARING_MODE_COUNT; + +extern const char* VK_IMAGE_CREATE_FLAG_NAMES[]; +extern const uint32_t VK_IMAGE_CREATE_FLAG_VALUES[]; +extern const size_t VK_IMAGE_CREATE_FLAG_COUNT; + +extern const char* VK_IMAGE_TYPE_NAMES[]; +extern const size_t VK_IMAGE_TYPE_COUNT; + +extern const char* VK_FORMAT_NAMES[]; +extern const uint32_t VK_FORMAT_VALUES[]; +extern const size_t VK_FORMAT_COUNT; + +extern const char* VK_SAMPLE_COUNT_NAMES[]; +extern const uint32_t VK_SAMPLE_COUNT_VALUES[]; +extern const size_t VK_SAMPLE_COUNT_COUNT; + +extern const char* VK_IMAGE_TILING_NAMES[]; +extern const size_t VK_IMAGE_TILING_COUNT; + +extern const char* VK_IMAGE_USAGE_FLAG_NAMES[]; +extern const uint32_t VK_IMAGE_USAGE_FLAG_VALUES[]; +extern const size_t VK_IMAGE_USAGE_FLAG_COUNT; + +extern const char* VK_IMAGE_TILING_NAMES[]; +extern const size_t VK_IMAGE_TILING_COUNT; + +extern const char* VK_IMAGE_LAYOUT_NAMES[]; +extern const uint32_t VK_IMAGE_LAYOUT_VALUES[]; +extern const size_t VK_IMAGE_LAYOUT_COUNT; + +extern const char* VMA_ALLOCATION_CREATE_FLAG_NAMES[]; +extern const uint32_t VMA_ALLOCATION_CREATE_FLAG_VALUES[]; +extern const size_t VMA_ALLOCATION_CREATE_FLAG_COUNT; + +extern const char* VMA_MEMORY_USAGE_NAMES[]; +extern const size_t VMA_MEMORY_USAGE_COUNT; + +extern const char* VK_MEMORY_PROPERTY_FLAG_NAMES[]; +extern const uint32_t VK_MEMORY_PROPERTY_FLAG_VALUES[]; +extern const size_t VK_MEMORY_PROPERTY_FLAG_COUNT; diff --git a/third_party/vkmemalloc/src/VmaReplay/VmaReplay.cpp b/third_party/vkmemalloc/src/VmaReplay/VmaReplay.cpp index 9635b033e8..774f070259 100644 --- a/third_party/vkmemalloc/src/VmaReplay/VmaReplay.cpp +++ b/third_party/vkmemalloc/src/VmaReplay/VmaReplay.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,92 +22,636 @@ #include "VmaUsage.h" #include "Common.h" +#include "Constants.h" #include +#include +#include -static const int RESULT_EXCEPTION = -1000; -static const int RESULT_ERROR_COMMAND_LINE = -1; -static const int RESULT_ERROR_SOURCE_FILE = -2; -static const int RESULT_ERROR_FORMAT = -3; -static const int RESULT_ERROR_VULKAN = -4; +static VERBOSITY g_Verbosity = VERBOSITY::DEFAULT; -enum CMD_LINE_OPT +static const uint32_t VULKAN_API_VERSION = VK_API_VERSION_1_1; + +namespace DetailedStats { - CMD_LINE_OPT_VERBOSITY, - CMD_LINE_OPT_ITERATIONS, - CMD_LINE_OPT_LINES, - CMD_LINE_OPT_PHYSICAL_DEVICE, - CMD_LINE_OPT_USER_DATA, - CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, - CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, - CMD_LINE_OPT_MEM_STATS, - CMD_LINE_OPT_DUMP_STATS_AFTER_LINE, - CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE, + +struct Flag +{ + uint32_t setCount = 0; + + void PostValue(bool v) + { + if(v) + { + ++setCount; + } + } + + void Print(uint32_t totalCount) const + { + if(setCount) + { + printf(" %u (%.2f%%)\n", setCount, (double)setCount * 100.0 / (double)totalCount); + } + else + { + printf(" 0\n"); + } + } }; -static enum class VERBOSITY +struct Enum { - MINIMUM = 0, - DEFAULT, - MAXIMUM, - COUNT, -} g_Verbosity = VERBOSITY::DEFAULT; + Enum(size_t itemCount, const char* const* itemNames, const uint32_t* itemValues = nullptr) : + m_ItemCount(itemCount), + m_ItemNames(itemNames), + m_ItemValues(itemValues) + { + } -enum class VULKAN_EXTENSION_REQUEST -{ - DISABLED, - ENABLED, - DEFAULT + void PostValue(uint32_t v) + { + if(v < _countof(m_BaseCount)) + { + ++m_BaseCount[v]; + } + else + { + auto it = m_ExtendedCount.find(v); + if(it != m_ExtendedCount.end()) + { + ++it->second; + } + else + { + m_ExtendedCount.insert(std::make_pair(v, 1u)); + } + } + } + + void Print(uint32_t totalCount) const + { + if(totalCount && + (!m_ExtendedCount.empty() || std::count_if(m_BaseCount, m_BaseCount + _countof(m_BaseCount), [](uint32_t v) { return v > 0; }))) + { + printf("\n"); + + for(size_t i = 0; i < _countof(m_BaseCount); ++i) + { + const uint32_t currCount = m_BaseCount[i]; + if(currCount) + { + PrintItem((uint32_t)i, currCount, totalCount); + } + } + + for(const auto& it : m_ExtendedCount) + { + PrintItem(it.first, it.second, totalCount); + } + } + else + { + printf(" 0\n"); + } + } + +private: + const size_t m_ItemCount; + const char* const* const m_ItemNames; + const uint32_t* const m_ItemValues; + + uint32_t m_BaseCount[32] = {}; + std::map m_ExtendedCount; + + void PrintItem(uint32_t value, uint32_t count, uint32_t totalCount) const + { + size_t itemIndex = m_ItemCount; + if(m_ItemValues) + { + for(itemIndex = 0; itemIndex < m_ItemCount; ++itemIndex) + { + if(m_ItemValues[itemIndex] == value) + { + break; + } + } + } + else + { + if(value < m_ItemCount) + { + itemIndex = value; + } + } + + if(itemIndex < m_ItemCount) + { + printf(" %s: ", m_ItemNames[itemIndex]); + } + else + { + printf(" 0x%X: ", value); + } + + printf("%u (%.2f%%)\n", count, (double)count * 100.0 / (double)totalCount); + } }; -enum class OBJECT_TYPE { BUFFER, IMAGE }; - -enum class VMA_FUNCTION +struct FlagSet { - CreatePool, - DestroyPool, - SetAllocationUserData, - CreateBuffer, - DestroyBuffer, - CreateImage, - DestroyImage, - FreeMemory, - CreateLostAllocation, - AllocateMemory, - AllocateMemoryForBuffer, - AllocateMemoryForImage, - MapMemory, - UnmapMemory, - FlushAllocation, - InvalidateAllocation, - TouchAllocation, - GetAllocationInfo, - MakePoolAllocationsLost, - Count + uint32_t count[32] = {}; + + FlagSet(size_t count, const char* const* names, const uint32_t* values = nullptr) : + m_Count(count), + m_Names(names), + m_Values(values) + { + } + + void PostValue(uint32_t v) + { + for(size_t i = 0; i < 32; ++i) + { + if((v & (1u << i)) != 0) + { + ++count[i]; + } + } + } + + void Print(uint32_t totalCount) const + { + if(totalCount && + std::count_if(count, count + _countof(count), [](uint32_t v) { return v > 0; })) + { + printf("\n"); + for(uint32_t bitIndex = 0; bitIndex < 32; ++bitIndex) + { + const uint32_t currCount = count[bitIndex]; + if(currCount) + { + size_t itemIndex = m_Count; + if(m_Values) + { + for(itemIndex = 0; itemIndex < m_Count; ++itemIndex) + { + if(m_Values[itemIndex] == (1u << bitIndex)) + { + break; + } + } + } + else + { + if(bitIndex < m_Count) + { + itemIndex = bitIndex; + } + } + + if(itemIndex < m_Count) + { + printf(" %s: ", m_Names[itemIndex]); + } + else + { + printf(" 0x%X: ", 1u << bitIndex); + } + + printf("%u (%.2f%%)\n", currCount, (double)currCount * 100.0 / (double)totalCount); + } + } + } + else + { + printf(" 0\n"); + } + } + +private: + const size_t m_Count; + const char* const* const m_Names; + const uint32_t* const m_Values; }; -static const char* VMA_FUNCTION_NAMES[] = { - "vmaCreatePool", - "vmaDestroyPool", - "vmaSetAllocationUserData", - "vmaCreateBuffer", - "vmaDestroyBuffer", - "vmaCreateImage", - "vmaDestroyImage", - "vmaFreeMemory", - "vmaCreateLostAllocation", - "vmaAllocateMemory", - "vmaAllocateMemoryForBuffer", - "vmaAllocateMemoryForImage", - "vmaMapMemory", - "vmaUnmapMemory", - "vmaFlushAllocation", - "vmaInvalidateAllocation", - "vmaTouchAllocation", - "vmaGetAllocationInfo", - "vmaMakePoolAllocationsLost", + +// T should be unsigned int +template +struct MinMaxAvg +{ + T min = std::numeric_limits::max(); + T max = 0; + T sum = T(); + + void PostValue(T v) + { + this->min = std::min(this->min, v); + this->max = std::max(this->max, v); + sum += v; + } + + void Print(uint32_t totalCount) const + { + if(totalCount && sum > T()) + { + if(this->min == this->max) + { + printf(" %llu\n", (uint64_t)this->max); + } + else + { + printf("\n Min: %llu\n Max: %llu\n Avg: %llu\n", + (uint64_t)this->min, + (uint64_t)this->max, + round_div(this->sum, totalCount)); + } + } + else + { + printf(" 0\n"); + } + } }; -static_assert( - _countof(VMA_FUNCTION_NAMES) == (size_t)VMA_FUNCTION::Count, - "VMA_FUNCTION_NAMES array doesn't match VMA_FUNCTION enum."); + +template +struct BitMask +{ + uint32_t zeroCount = 0; + uint32_t maxCount = 0; + + void PostValue(T v) + { + if(v) + { + if(v == std::numeric_limits::max()) + { + ++maxCount; + } + } + else + { + ++zeroCount; + } + } + + void Print(uint32_t totalCount) const + { + if(totalCount > 0 && zeroCount < totalCount) + { + const uint32_t otherCount = totalCount - (zeroCount + maxCount); + + printf("\n 0: %u (%.2f%%)\n Max: %u (%.2f%%)\n Other: %u (%.2f%%)\n", + zeroCount, (double)zeroCount * 100.0 / (double)totalCount, + maxCount, (double)maxCount * 100.0 / (double)totalCount, + otherCount, (double)otherCount * 100.0 / (double)totalCount); + } + else + { + printf(" 0\n"); + } + } +}; + +struct CountPerMemType +{ + uint32_t count[VK_MAX_MEMORY_TYPES] = {}; + + void PostValue(uint32_t v) + { + for(uint32_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i) + { + if((v & (1u << i)) != 0) + { + ++count[i]; + } + } + } + + void Print(uint32_t totalCount) const + { + if(totalCount) + { + printf("\n"); + for(uint32_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i) + { + if(count[i]) + { + printf(" %u: %u (%.2f%%)\n", i, count[i], + (double)count[i] * 100.0 / (double)totalCount); + } + } + } + else + { + printf(" 0\n"); + } + } +}; + +struct StructureStats +{ + uint32_t totalCount = 0; +}; + +#define PRINT_FIELD(name) \ + printf(" " #name ":"); \ + (name).Print(totalCount); +#define PRINT_FIELD_NAMED(name, nameStr) \ + printf(" " nameStr ":"); \ + (name).Print(totalCount); + +struct VmaPoolCreateInfoStats : public StructureStats +{ + CountPerMemType memoryTypeIndex; + FlagSet flags; + MinMaxAvg blockSize; + MinMaxAvg minBlockCount; + MinMaxAvg maxBlockCount; + Flag minMaxBlockCountEqual; + MinMaxAvg frameInUseCount; + + VmaPoolCreateInfoStats() : + flags(VMA_POOL_CREATE_FLAG_COUNT, VMA_POOL_CREATE_FLAG_NAMES, VMA_POOL_CREATE_FLAG_VALUES) + { + } + + void PostValue(const VmaPoolCreateInfo& v) + { + ++totalCount; + + memoryTypeIndex.PostValue(v.memoryTypeIndex); + flags.PostValue(v.flags); + blockSize.PostValue(v.blockSize); + minBlockCount.PostValue(v.minBlockCount); + maxBlockCount.PostValue(v.maxBlockCount); + minMaxBlockCountEqual.PostValue(v.minBlockCount == v.maxBlockCount); + frameInUseCount.PostValue(v.frameInUseCount); + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("VmaPoolCreateInfo (%u):\n", totalCount); + + PRINT_FIELD(memoryTypeIndex); + PRINT_FIELD(flags); + PRINT_FIELD(blockSize); + PRINT_FIELD(minBlockCount); + PRINT_FIELD(maxBlockCount); + PRINT_FIELD_NAMED(minMaxBlockCountEqual, "minBlockCount == maxBlockCount"); + PRINT_FIELD(frameInUseCount); + } +}; + +struct VkBufferCreateInfoStats : public StructureStats +{ + FlagSet flags; + MinMaxAvg size; + FlagSet usage; + Enum sharingMode; + + VkBufferCreateInfoStats() : + flags(VK_BUFFER_CREATE_FLAG_COUNT, VK_BUFFER_CREATE_FLAG_NAMES, VK_BUFFER_CREATE_FLAG_VALUES), + usage(VK_BUFFER_USAGE_FLAG_COUNT, VK_BUFFER_USAGE_FLAG_NAMES, VK_BUFFER_USAGE_FLAG_VALUES), + sharingMode(VK_SHARING_MODE_COUNT, VK_SHARING_MODE_NAMES) + { + } + + void PostValue(const VkBufferCreateInfo& v) + { + ++totalCount; + + flags.PostValue(v.flags); + size.PostValue(v.size); + usage.PostValue(v.usage); + sharingMode.PostValue(v.sharingMode); + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("VkBufferCreateInfo (%u):\n", totalCount); + + PRINT_FIELD(flags); + PRINT_FIELD(size); + PRINT_FIELD(usage); + PRINT_FIELD(sharingMode); + } +}; + +struct VkImageCreateInfoStats : public StructureStats +{ + FlagSet flags; + Enum imageType; + Enum format; + MinMaxAvg width, height, depth, mipLevels, arrayLayers; + Flag depthGreaterThanOne, mipLevelsGreaterThanOne, arrayLayersGreaterThanOne; + Enum samples; + Enum tiling; + FlagSet usage; + Enum sharingMode; + Enum initialLayout; + + VkImageCreateInfoStats() : + flags(VK_IMAGE_CREATE_FLAG_COUNT, VK_IMAGE_CREATE_FLAG_NAMES, VK_IMAGE_CREATE_FLAG_VALUES), + imageType(VK_IMAGE_TYPE_COUNT, VK_IMAGE_TYPE_NAMES), + format(VK_FORMAT_COUNT, VK_FORMAT_NAMES, VK_FORMAT_VALUES), + samples(VK_SAMPLE_COUNT_COUNT, VK_SAMPLE_COUNT_NAMES, VK_SAMPLE_COUNT_VALUES), + tiling(VK_IMAGE_TILING_COUNT, VK_IMAGE_TILING_NAMES), + usage(VK_IMAGE_USAGE_FLAG_COUNT, VK_IMAGE_USAGE_FLAG_NAMES, VK_IMAGE_USAGE_FLAG_VALUES), + sharingMode(VK_SHARING_MODE_COUNT, VK_SHARING_MODE_NAMES), + initialLayout(VK_IMAGE_LAYOUT_COUNT, VK_IMAGE_LAYOUT_NAMES, VK_IMAGE_LAYOUT_VALUES) + { + } + + void PostValue(const VkImageCreateInfo& v) + { + ++totalCount; + + flags.PostValue(v.flags); + imageType.PostValue(v.imageType); + format.PostValue(v.format); + width.PostValue(v.extent.width); + height.PostValue(v.extent.height); + depth.PostValue(v.extent.depth); + mipLevels.PostValue(v.mipLevels); + arrayLayers.PostValue(v.arrayLayers); + depthGreaterThanOne.PostValue(v.extent.depth > 1); + mipLevelsGreaterThanOne.PostValue(v.mipLevels > 1); + arrayLayersGreaterThanOne.PostValue(v.arrayLayers > 1); + samples.PostValue(v.samples); + tiling.PostValue(v.tiling); + usage.PostValue(v.usage); + sharingMode.PostValue(v.sharingMode); + initialLayout.PostValue(v.initialLayout); + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("VkImageCreateInfo (%u):\n", totalCount); + + PRINT_FIELD(flags); + PRINT_FIELD(imageType); + PRINT_FIELD(format); + PRINT_FIELD(width); + PRINT_FIELD(height); + PRINT_FIELD(depth); + PRINT_FIELD(mipLevels); + PRINT_FIELD(arrayLayers); + PRINT_FIELD_NAMED(depthGreaterThanOne, "depth > 1"); + PRINT_FIELD_NAMED(mipLevelsGreaterThanOne, "mipLevels > 1"); + PRINT_FIELD_NAMED(arrayLayersGreaterThanOne, "arrayLayers > 1"); + PRINT_FIELD(samples); + PRINT_FIELD(tiling); + PRINT_FIELD(usage); + PRINT_FIELD(sharingMode); + PRINT_FIELD(initialLayout); + } +}; + +struct VmaAllocationCreateInfoStats : public StructureStats +{ + FlagSet flags; + Enum usage; + FlagSet requiredFlags, preferredFlags; + Flag requiredFlagsNotZero, preferredFlagsNotZero; + BitMask memoryTypeBits; + Flag poolNotNull; + Flag userDataNotNull; + + VmaAllocationCreateInfoStats() : + flags(VMA_ALLOCATION_CREATE_FLAG_COUNT, VMA_ALLOCATION_CREATE_FLAG_NAMES, VMA_ALLOCATION_CREATE_FLAG_VALUES), + usage(VMA_MEMORY_USAGE_COUNT, VMA_MEMORY_USAGE_NAMES), + requiredFlags(VK_MEMORY_PROPERTY_FLAG_COUNT, VK_MEMORY_PROPERTY_FLAG_NAMES, VK_MEMORY_PROPERTY_FLAG_VALUES), + preferredFlags(VK_MEMORY_PROPERTY_FLAG_COUNT, VK_MEMORY_PROPERTY_FLAG_NAMES, VK_MEMORY_PROPERTY_FLAG_VALUES) + { + } + + void PostValue(const VmaAllocationCreateInfo& v, size_t count = 1) + { + totalCount += (uint32_t)count; + + for(size_t i = 0; i < count; ++i) + { + flags.PostValue(v.flags); + usage.PostValue(v.usage); + requiredFlags.PostValue(v.requiredFlags); + preferredFlags.PostValue(v.preferredFlags); + requiredFlagsNotZero.PostValue(v.requiredFlags != 0); + preferredFlagsNotZero.PostValue(v.preferredFlags != 0); + memoryTypeBits.PostValue(v.memoryTypeBits); + poolNotNull.PostValue(v.pool != VK_NULL_HANDLE); + userDataNotNull.PostValue(v.pUserData != nullptr); + } + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("VmaAllocationCreateInfo (%u):\n", totalCount); + + PRINT_FIELD(flags); + PRINT_FIELD(usage); + PRINT_FIELD(requiredFlags); + PRINT_FIELD(preferredFlags); + PRINT_FIELD_NAMED(requiredFlagsNotZero, "requiredFlags != 0"); + PRINT_FIELD_NAMED(preferredFlagsNotZero, "preferredFlags != 0"); + PRINT_FIELD(memoryTypeBits); + PRINT_FIELD_NAMED(poolNotNull, "pool != VK_NULL_HANDLE"); + PRINT_FIELD_NAMED(userDataNotNull, "pUserData != nullptr"); + } +}; + +struct VmaAllocateMemoryPagesStats : public StructureStats +{ + MinMaxAvg allocationCount; + + void PostValue(size_t allocationCount) + { + this->allocationCount.PostValue(allocationCount); + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("vmaAllocateMemoryPages (%u):\n", totalCount); + + PRINT_FIELD(allocationCount); + } +}; + +struct VmaDefragmentationInfo2Stats : public StructureStats +{ + BitMask maxCpuBytesToMove; + BitMask maxCpuAllocationsToMove; + BitMask maxGpuBytesToMove; + BitMask maxGpuAllocationsToMove; + Flag commandBufferNotNull; + MinMaxAvg allocationCount; + Flag allocationCountNotZero; + MinMaxAvg poolCount; + Flag poolCountNotZero; + + void PostValue(const VmaDefragmentationInfo2& info) + { + ++totalCount; + + maxCpuBytesToMove.PostValue(info.maxCpuBytesToMove); + maxCpuAllocationsToMove.PostValue(info.maxCpuAllocationsToMove); + maxGpuBytesToMove.PostValue(info.maxGpuBytesToMove); + maxGpuAllocationsToMove.PostValue(info.maxGpuAllocationsToMove); + commandBufferNotNull.PostValue(info.commandBuffer != VK_NULL_HANDLE); + allocationCount.PostValue(info.allocationCount); + allocationCountNotZero.PostValue(info.allocationCount != 0); + poolCount.PostValue(info.poolCount); + poolCountNotZero.PostValue(info.poolCount != 0); + } + + void Print() const + { + if(totalCount == 0) + { + return; + } + + printf("VmaDefragmentationInfo2 (%u):\n", totalCount); + + PRINT_FIELD(maxCpuBytesToMove); + PRINT_FIELD(maxCpuAllocationsToMove); + PRINT_FIELD(maxGpuBytesToMove); + PRINT_FIELD(maxGpuAllocationsToMove); + PRINT_FIELD_NAMED(commandBufferNotNull, "commandBuffer != VK_NULL_HANDLE"); + PRINT_FIELD(allocationCount); + PRINT_FIELD_NAMED(allocationCountNotZero, "allocationCount > 0"); + PRINT_FIELD(poolCount); + PRINT_FIELD_NAMED(poolCountNotZero, "poolCount > 0"); + } +}; + +#undef PRINT_FIELD_NAMED +#undef PRINT_FIELD + +} // namespace DetailedStats // Set this to false to disable deleting leaked VmaAllocation, VmaPool objects // and let VMA report asserts about them. @@ -126,8 +670,9 @@ static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool g_UserDataEnabled = true; static bool g_MemStatsEnabled = false; -VULKAN_EXTENSION_REQUEST g_VK_KHR_dedicated_allocation_request = VULKAN_EXTENSION_REQUEST::DEFAULT; -VULKAN_EXTENSION_REQUEST g_VK_LAYER_LUNARG_standard_validation = VULKAN_EXTENSION_REQUEST::DEFAULT; +VULKAN_EXTENSION_REQUEST g_VK_LAYER_KHRONOS_validation = VULKAN_EXTENSION_REQUEST::DEFAULT; +VULKAN_EXTENSION_REQUEST g_VK_EXT_memory_budget_request = VULKAN_EXTENSION_REQUEST::DEFAULT; +VULKAN_EXTENSION_REQUEST g_VK_AMD_device_coherent_memory_request = VULKAN_EXTENSION_REQUEST::DEFAULT; struct StatsAfterLineEntry { @@ -138,12 +683,15 @@ struct StatsAfterLineEntry bool operator==(const StatsAfterLineEntry& rhs) const { return line == rhs.line; } }; static std::vector g_DumpStatsAfterLine; +static std::vector g_DefragmentAfterLine; +static uint32_t g_DefragmentationFlags = 0; static size_t g_DumpStatsAfterLineNextIndex = 0; +static size_t g_DefragmentAfterLineNextIndex = 0; static bool ValidateFileVersion() { if(GetVersionMajor(g_FileVersion) == 1 && - GetVersionMinor(g_FileVersion) <= 3) + GetVersionMinor(g_FileVersion) <= 8) { return true; } @@ -179,22 +727,30 @@ public: static uint32_t ImageUsageToClass(uint32_t usage); Statistics(); + ~Statistics(); void Init(uint32_t memHeapCount, uint32_t memTypeCount); + void PrintDeviceMemStats() const; void PrintMemStats() const; + void PrintDetailedStats() const; const size_t* GetFunctionCallCount() const { return m_FunctionCallCount; } size_t GetImageCreationCount(uint32_t imgClass) const { return m_ImageCreationCount[imgClass]; } size_t GetLinearImageCreationCount() const { return m_LinearImageCreationCount; } size_t GetBufferCreationCount(uint32_t bufClass) const { return m_BufferCreationCount[bufClass]; } - size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } - size_t GetPoolCreationCount() const { return m_PoolCreationCount; } + size_t GetAllocationCreationCount() const { return (size_t)m_VmaAllocationCreateInfo.totalCount + m_CreateLostAllocationCount; } + size_t GetPoolCreationCount() const { return m_VmaPoolCreateInfo.totalCount; } + size_t GetBufferCreationCount() const { return (size_t)m_VkBufferCreateInfo.totalCount; } void RegisterFunctionCall(VMA_FUNCTION func); - void RegisterCreateImage(uint32_t usage, uint32_t tiling); - void RegisterCreateBuffer(uint32_t usage); - void RegisterCreatePool(); - void RegisterCreateAllocation(); + void RegisterCreateImage(const VkImageCreateInfo& info); + void RegisterCreateBuffer(const VkBufferCreateInfo& info); + void RegisterCreatePool(const VmaPoolCreateInfo& info); + void RegisterCreateAllocation(const VmaAllocationCreateInfo& info, size_t allocCount = 1); + void RegisterCreateLostAllocation() { ++m_CreateLostAllocationCount; } + void RegisterAllocateMemoryPages(size_t allocCount) { m_VmaAllocateMemoryPages.PostValue(allocCount); } + void RegisterDefragmentation(const VmaDefragmentationInfo2& info); + void RegisterDeviceMemoryAllocation(uint32_t memoryType, VkDeviceSize size); void UpdateMemStats(const VmaStats& currStats); private: @@ -205,8 +761,17 @@ private: size_t m_ImageCreationCount[4] = { }; size_t m_LinearImageCreationCount = 0; size_t m_BufferCreationCount[4] = { }; - size_t m_AllocationCreationCount = 0; // Also includes buffers and images, and lost allocations. - size_t m_PoolCreationCount = 0; + + struct DeviceMemStatInfo + { + size_t allocationCount; + VkDeviceSize allocationTotalSize; + }; + struct DeviceMemStats + { + DeviceMemStatInfo memoryType[VK_MAX_MEMORY_TYPES]; + DeviceMemStatInfo total; + } m_DeviceMemStats; // Structure similar to VmaStatInfo, but not the same. struct MemStatInfo @@ -225,10 +790,42 @@ private: MemStatInfo total; } m_PeakMemStats; + DetailedStats::VmaPoolCreateInfoStats m_VmaPoolCreateInfo; + DetailedStats::VkBufferCreateInfoStats m_VkBufferCreateInfo; + DetailedStats::VkImageCreateInfoStats m_VkImageCreateInfo; + DetailedStats::VmaAllocationCreateInfoStats m_VmaAllocationCreateInfo; + size_t m_CreateLostAllocationCount = 0; + DetailedStats::VmaAllocateMemoryPagesStats m_VmaAllocateMemoryPages; + DetailedStats::VmaDefragmentationInfo2Stats m_VmaDefragmentationInfo2; + void UpdateMemStatInfo(MemStatInfo& inoutPeakInfo, const VmaStatInfo& currInfo); static void PrintMemStatInfo(const MemStatInfo& info); }; +// Hack for global AllocateDeviceMemoryCallback. +static Statistics* g_Statistics; + +static void VKAPI_CALL AllocateDeviceMemoryCallback( + VmaAllocator allocator, + uint32_t memoryType, + VkDeviceMemory memory, + VkDeviceSize size, + void* pUserData) +{ + g_Statistics->RegisterDeviceMemoryAllocation(memoryType, size); +} + +/// Callback function called before vkFreeMemory. +static void VKAPI_CALL FreeDeviceMemoryCallback( + VmaAllocator allocator, + uint32_t memoryType, + VkDeviceMemory memory, + VkDeviceSize size, + void* pUserData) +{ + // Nothing. +} + uint32_t Statistics::BufferUsageToClass(uint32_t usage) { // Buffer is used as source of data for fixed-function stage of graphics pipeline. @@ -294,7 +891,17 @@ uint32_t Statistics::ImageUsageToClass(uint32_t usage) Statistics::Statistics() { + ZeroMemory(&m_DeviceMemStats, sizeof(m_DeviceMemStats)); ZeroMemory(&m_PeakMemStats, sizeof(m_PeakMemStats)); + + assert(g_Statistics == nullptr); + g_Statistics = this; +} + +Statistics::~Statistics() +{ + assert(g_Statistics == this); + g_Statistics = nullptr; } void Statistics::Init(uint32_t memHeapCount, uint32_t memTypeCount) @@ -303,6 +910,18 @@ void Statistics::Init(uint32_t memHeapCount, uint32_t memTypeCount) m_MemTypeCount = memTypeCount; } +void Statistics::PrintDeviceMemStats() const +{ + printf("Successful device memory allocations:\n"); + printf(" Total: count = %zu, total size = %llu\n", + m_DeviceMemStats.total.allocationCount, m_DeviceMemStats.total.allocationTotalSize); + for(uint32_t i = 0; i < m_MemTypeCount; ++i) + { + printf(" Memory type %u: count = %zu, total size = %llu\n", + i, m_DeviceMemStats.memoryType[i].allocationCount, m_DeviceMemStats.memoryType[i].allocationTotalSize); + } +} + void Statistics::PrintMemStats() const { printf("Memory statistics:\n"); @@ -331,40 +950,55 @@ void Statistics::PrintMemStats() const } } +void Statistics::PrintDetailedStats() const +{ + m_VmaPoolCreateInfo.Print(); + m_VmaAllocationCreateInfo.Print(); + m_VmaAllocateMemoryPages.Print(); + m_VkBufferCreateInfo.Print(); + m_VkImageCreateInfo.Print(); + m_VmaDefragmentationInfo2.Print(); +} + void Statistics::RegisterFunctionCall(VMA_FUNCTION func) { ++m_FunctionCallCount[(size_t)func]; } -void Statistics::RegisterCreateImage(uint32_t usage, uint32_t tiling) +void Statistics::RegisterCreateImage(const VkImageCreateInfo& info) { - if(tiling == VK_IMAGE_TILING_LINEAR) + if(info.tiling == VK_IMAGE_TILING_LINEAR) ++m_LinearImageCreationCount; else { - const uint32_t imgClass = ImageUsageToClass(usage); + const uint32_t imgClass = ImageUsageToClass(info.usage); ++m_ImageCreationCount[imgClass]; } - ++m_AllocationCreationCount; + m_VkImageCreateInfo.PostValue(info); } -void Statistics::RegisterCreateBuffer(uint32_t usage) +void Statistics::RegisterCreateBuffer(const VkBufferCreateInfo& info) { - const uint32_t bufClass = BufferUsageToClass(usage); + const uint32_t bufClass = BufferUsageToClass(info.usage); ++m_BufferCreationCount[bufClass]; - ++m_AllocationCreationCount; + m_VkBufferCreateInfo.PostValue(info); } -void Statistics::RegisterCreatePool() +void Statistics::RegisterCreatePool(const VmaPoolCreateInfo& info) { - ++m_PoolCreationCount; + m_VmaPoolCreateInfo.PostValue(info); } -void Statistics::RegisterCreateAllocation() +void Statistics::RegisterCreateAllocation(const VmaAllocationCreateInfo& info, size_t allocCount) { - ++m_AllocationCreationCount; + m_VmaAllocationCreateInfo.PostValue(info, allocCount); +} + +void Statistics::RegisterDefragmentation(const VmaDefragmentationInfo2& info) +{ + m_VmaDefragmentationInfo2.PostValue(info); } void Statistics::UpdateMemStats(const VmaStats& currStats) @@ -382,6 +1016,15 @@ void Statistics::UpdateMemStats(const VmaStats& currStats) } } +void Statistics::RegisterDeviceMemoryAllocation(uint32_t memoryType, VkDeviceSize size) +{ + ++m_DeviceMemStats.total.allocationCount; + m_DeviceMemStats.total.allocationTotalSize += size; + + ++m_DeviceMemStats.memoryType[memoryType].allocationCount; + m_DeviceMemStats.memoryType[memoryType].allocationTotalSize += size; +} + void Statistics::UpdateMemStatInfo(MemStatInfo& inoutPeakInfo, const VmaStatInfo& currInfo) { #define SET_PEAK(inoutDst, src) \ @@ -425,11 +1068,13 @@ public: void Compare( const VkPhysicalDeviceProperties& currDevProps, const VkPhysicalDeviceMemoryProperties& currMemProps, - bool currDedicatedAllocationExtensionEnabled); + uint32_t vulkanApiVersion, + bool currMemoryBudgetEnabled); private: enum class OPTION { + VulkanApiVersion, PhysicalDevice_apiVersion, PhysicalDevice_driverVersion, PhysicalDevice_vendorID, @@ -440,6 +1085,9 @@ private: PhysicalDeviceLimits_bufferImageGranularity, PhysicalDeviceLimits_nonCoherentAtomSize, Extension_VK_KHR_dedicated_allocation, + Extension_VK_KHR_bind_memory2, + Extension_VK_EXT_memory_budget, + Extension_VK_AMD_device_coherent_memory, Macro_VMA_DEBUG_ALWAYS_DEDICATED_MEMORY, Macro_VMA_DEBUG_ALIGNMENT, Macro_VMA_DEBUG_MARGIN, @@ -517,7 +1165,11 @@ bool ConfigurationParser::Parse(LineSplit& lineSplit) } const StrRange optionName = csvSplit.GetRange(0); - if(StrRangeEq(optionName, "PhysicalDevice")) + if(StrRangeEq(optionName, "VulkanApiVersion")) + { + SetOption(currLineNumber, OPTION::VulkanApiVersion, StrRange{csvSplit.GetRange(1).beg, csvSplit.GetRange(2).end}); + } + else if(StrRangeEq(optionName, "PhysicalDevice")) { if(csvSplit.GetCount() >= 3) { @@ -563,7 +1215,15 @@ bool ConfigurationParser::Parse(LineSplit& lineSplit) { const StrRange subOptionName = csvSplit.GetRange(1); if(StrRangeEq(subOptionName, "VK_KHR_dedicated_allocation")) - SetOption(currLineNumber, OPTION::Extension_VK_KHR_dedicated_allocation, csvSplit.GetRange(2)); + { + // Ignore because this extension is promoted to Vulkan 1.1. + } + else if(StrRangeEq(subOptionName, "VK_KHR_bind_memory2")) + SetOption(currLineNumber, OPTION::Extension_VK_KHR_bind_memory2, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VK_EXT_memory_budget")) + SetOption(currLineNumber, OPTION::Extension_VK_EXT_memory_budget, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VK_AMD_device_coherent_memory")) + SetOption(currLineNumber, OPTION::Extension_VK_AMD_device_coherent_memory, csvSplit.GetRange(2)); else printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); } @@ -659,8 +1319,14 @@ bool ConfigurationParser::Parse(LineSplit& lineSplit) void ConfigurationParser::Compare( const VkPhysicalDeviceProperties& currDevProps, const VkPhysicalDeviceMemoryProperties& currMemProps, - bool currDedicatedAllocationExtensionEnabled) + uint32_t vulkanApiVersion, + bool currMemoryBudgetEnabled) { + char vulkanApiVersionStr[32]; + sprintf_s(vulkanApiVersionStr, "%u,%u", VK_VERSION_MAJOR(vulkanApiVersion), VK_VERSION_MINOR(vulkanApiVersion)); + CompareOption(VERBOSITY::DEFAULT, "VulkanApiVersion", + OPTION::VulkanApiVersion, vulkanApiVersionStr); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice apiVersion", OPTION::PhysicalDevice_apiVersion, currDevProps.apiVersion); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice driverVersion", @@ -680,8 +1346,6 @@ void ConfigurationParser::Compare( OPTION::PhysicalDeviceLimits_bufferImageGranularity, currDevProps.limits.bufferImageGranularity); CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits nonCoherentAtomSize", OPTION::PhysicalDeviceLimits_nonCoherentAtomSize, currDevProps.limits.nonCoherentAtomSize); - CompareOption(VERBOSITY::DEFAULT, "Extension VK_KHR_dedicated_allocation", - OPTION::Extension_VK_KHR_dedicated_allocation, currDedicatedAllocationExtensionEnabled); CompareMemProps(currMemProps); } @@ -830,53 +1494,16 @@ void ConfigurationParser::CompareMemProps( //////////////////////////////////////////////////////////////////////////////// // class Player -static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; +static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_KHRONOS_validation"; -static bool g_MemoryAliasingWarningEnabled = true; - -static VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( - VkDebugReportFlagsEXT flags, - VkDebugReportObjectTypeEXT objectType, - uint64_t object, - size_t location, - int32_t messageCode, - const char* pLayerPrefix, - const char* pMessage, - void* pUserData) +static VkBool32 VKAPI_PTR MyDebugReportCallback( + VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, + VkDebugUtilsMessageTypeFlagsEXT messageTypes, + const VkDebugUtilsMessengerCallbackDataEXT* pCallbackData, + void* pUserData) { - // "Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug." - if(!g_MemoryAliasingWarningEnabled && flags == VK_DEBUG_REPORT_WARNING_BIT_EXT && - (strstr(pMessage, " is aliased with non-linear ") || strstr(pMessage, " is aliased with linear "))) - { - return VK_FALSE; - } - - // Ignoring because when VK_KHR_dedicated_allocation extension is enabled, - // vkGetBufferMemoryRequirements2KHR function is used instead, while Validation - // Layer seems to be unaware of it. - if (strstr(pMessage, "but vkGetBufferMemoryRequirements() has not been called on that buffer") != nullptr) - { - return VK_FALSE; - } - if (strstr(pMessage, "but vkGetImageMemoryRequirements() has not been called on that image") != nullptr) - { - return VK_FALSE; - } - - /* - "Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used." - Ignoring because we map entire VkDeviceMemory blocks, where different types of - images and buffers may end up together, especially on GPUs with unified memory - like Intel. - */ - if(strstr(pMessage, "Mapping an image with layout") != nullptr && - strstr(pMessage, "can result in undefined behavior if this memory is used by the device") != nullptr) - { - return VK_FALSE; - } - - printf("%s \xBA %s\n", pLayerPrefix, pMessage); - + assert(pCallbackData && pCallbackData->pMessageIdName && pCallbackData->pMessage); + printf("%s \xBA %s\n", pCallbackData->pMessageIdName, pCallbackData->pMessage); return VK_FALSE; } @@ -921,6 +1548,7 @@ public: void ApplyConfig(ConfigurationParser& configParser); void ExecuteLine(size_t lineNumber, const StrRange& line); void DumpStats(const char* fileNameFormat, size_t lineNumber, bool detailed); + void Defragment(); void PrintStats(); @@ -932,17 +1560,21 @@ private: VkInstance m_VulkanInstance = VK_NULL_HANDLE; VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE; - uint32_t m_GraphicsQueueFamilyIndex = UINT_MAX; + uint32_t m_GraphicsQueueFamilyIndex = UINT32_MAX; + uint32_t m_TransferQueueFamilyIndex = UINT32_MAX; VkDevice m_Device = VK_NULL_HANDLE; + VkQueue m_GraphicsQueue = VK_NULL_HANDLE; + VkQueue m_TransferQueue = VK_NULL_HANDLE; VmaAllocator m_Allocator = VK_NULL_HANDLE; - bool m_DedicatedAllocationEnabled = false; + VkCommandPool m_CommandPool = VK_NULL_HANDLE; + VkCommandBuffer m_CommandBuffer = VK_NULL_HANDLE; + bool m_MemoryBudgetEnabled = false; const VkPhysicalDeviceProperties* m_DevProps = nullptr; const VkPhysicalDeviceMemoryProperties* m_MemProps = nullptr; - PFN_vkCreateDebugReportCallbackEXT m_pvkCreateDebugReportCallbackEXT; - PFN_vkDebugReportMessageEXT m_pvkDebugReportMessageEXT; - PFN_vkDestroyDebugReportCallbackEXT m_pvkDestroyDebugReportCallbackEXT; - VkDebugReportCallbackEXT m_hCallback; + PFN_vkCreateDebugUtilsMessengerEXT m_vkCreateDebugUtilsMessengerEXT = nullptr; + PFN_vkDestroyDebugUtilsMessengerEXT m_vkDestroyDebugUtilsMessengerEXT = nullptr; + VkDebugUtilsMessengerEXT m_DebugUtilsMessenger = VK_NULL_HANDLE; uint32_t m_VmaFrameIndex = 0; @@ -953,13 +1585,14 @@ private: }; struct Allocation { - uint32_t allocationFlags; - VmaAllocation allocation; - VkBuffer buffer; - VkImage image; + uint32_t allocationFlags = 0; + VmaAllocation allocation = VK_NULL_HANDLE; + VkBuffer buffer = VK_NULL_HANDLE; + VkImage image = VK_NULL_HANDLE; }; std::unordered_map m_Pools; std::unordered_map m_Allocations; + std::unordered_map m_DefragmentationContexts; struct Thread { @@ -988,6 +1621,7 @@ private: int InitVulkan(); void FinalizeVulkan(); void RegisterDebugCallbacks(); + void UnregisterDebugCallbacks(); // If parmeter count doesn't match, issues warning and returns false. bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); @@ -1001,12 +1635,14 @@ private: void ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); - void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyBuffer); DestroyAllocation(lineNumber, csvSplit); } + void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyBuffer); DestroyAllocation(lineNumber, csvSplit, "vmaDestroyBuffer"); } void ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit); - void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyImage); DestroyAllocation(lineNumber, csvSplit); } - void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::FreeMemory); DestroyAllocation(lineNumber, csvSplit); } + void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyImage); DestroyAllocation(lineNumber, csvSplit, "vmaDestroyImage"); } + void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::FreeMemory); DestroyAllocation(lineNumber, csvSplit, "vmaFreeMemory"); } + void ExecuteFreeMemoryPages(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteAllocateMemoryPages(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType); void ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit); @@ -1015,8 +1651,15 @@ private: void ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteMakePoolAllocationsLost(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteResizeAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteDefragmentationBegin(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteDefragmentationEnd(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteSetPoolName(size_t lineNumber, const CsvSplit& csvSplit); - void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit, const char* functionName); + + void PrintStats(const VmaStats& stats, const char* suffix); + void PrintStatInfo(const VmaStatInfo& info); }; Player::Player() @@ -1046,7 +1689,9 @@ Player::~Player() void Player::ApplyConfig(ConfigurationParser& configParser) { - configParser.Compare(*m_DevProps, *m_MemProps, m_DedicatedAllocationEnabled); + configParser.Compare(*m_DevProps, *m_MemProps, + VULKAN_API_VERSION, + m_MemoryBudgetEnabled); } void Player::ExecuteLine(size_t lineNumber, const StrRange& line) @@ -1118,44 +1763,56 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) // Nothing. } } - else if(StrRangeEq(functionName, "vmaCreatePool")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::CreatePool])) ExecuteCreatePool(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaDestroyPool")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::DestroyPool])) ExecuteDestroyPool(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaSetAllocationUserData")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::SetAllocationUserData])) ExecuteSetAllocationUserData(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaCreateBuffer")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::CreateBuffer])) ExecuteCreateBuffer(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaDestroyBuffer")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::DestroyBuffer])) ExecuteDestroyBuffer(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaCreateImage")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::CreateImage])) ExecuteCreateImage(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaDestroyImage")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::DestroyImage])) ExecuteDestroyImage(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaFreeMemory")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::FreeMemory])) ExecuteFreeMemory(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaCreateLostAllocation")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::FreeMemoryPages])) + ExecuteFreeMemoryPages(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::CreateLostAllocation])) ExecuteCreateLostAllocation(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaAllocateMemory")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::AllocateMemory])) ExecuteAllocateMemory(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaAllocateMemoryForBuffer")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::AllocateMemoryPages])) + ExecuteAllocateMemoryPages(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::AllocateMemoryForBuffer])) ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::BUFFER); - else if(StrRangeEq(functionName, "vmaAllocateMemoryForImage")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::AllocateMemoryForImage])) ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::IMAGE); - else if(StrRangeEq(functionName, "vmaMapMemory")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::MapMemory])) ExecuteMapMemory(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaUnmapMemory")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::UnmapMemory])) ExecuteUnmapMemory(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaFlushAllocation")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::FlushAllocation])) ExecuteFlushAllocation(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaInvalidateAllocation")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::InvalidateAllocation])) ExecuteInvalidateAllocation(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaTouchAllocation")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::TouchAllocation])) ExecuteTouchAllocation(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaGetAllocationInfo")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::GetAllocationInfo])) ExecuteGetAllocationInfo(lineNumber, csvSplit); - else if(StrRangeEq(functionName, "vmaMakePoolAllocationsLost")) + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::MakePoolAllocationsLost])) ExecuteMakePoolAllocationsLost(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::ResizeAllocation])) + ExecuteResizeAllocation(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::DefragmentationBegin])) + ExecuteDefragmentationBegin(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::DefragmentationEnd])) + ExecuteDefragmentationEnd(lineNumber, csvSplit); + else if(StrRangeEq(functionName, VMA_FUNCTION_NAMES[(uint32_t)VMA_FUNCTION::SetPoolName])) + ExecuteSetPoolName(lineNumber, csvSplit); else { if(IssueWarning()) @@ -1324,7 +1981,7 @@ int Player::InitVulkan() IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME); bool validationLayersEnabled = false; - switch(g_VK_LAYER_LUNARG_standard_validation) + switch(g_VK_LAYER_KHRONOS_validation) { case VULKAN_EXTENSION_REQUEST::DISABLED: break; @@ -1341,28 +1998,56 @@ int Player::InitVulkan() default: assert(0); } - std::vector instanceExtensions; - //instanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); - //instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); + uint32_t availableInstanceExtensionCount = 0; + res = vkEnumerateInstanceExtensionProperties(nullptr, &availableInstanceExtensionCount, nullptr); + assert(res == VK_SUCCESS); + std::vector availableInstanceExtensions(availableInstanceExtensionCount); + if(availableInstanceExtensionCount > 0) + { + res = vkEnumerateInstanceExtensionProperties(nullptr, &availableInstanceExtensionCount, availableInstanceExtensions.data()); + assert(res == VK_SUCCESS); + } + + std::vector enabledInstanceExtensions; + //enabledInstanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); + //enabledInstanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); std::vector instanceLayers; if(validationLayersEnabled) { instanceLayers.push_back(VALIDATION_LAYER_NAME); - instanceExtensions.push_back("VK_EXT_debug_report"); + } + + bool VK_KHR_get_physical_device_properties2_enabled = false; + bool VK_EXT_debug_utils_enabled = false; + for(const auto& extensionProperties : availableInstanceExtensions) + { + if(strcmp(extensionProperties.extensionName, VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME) == 0) + { + enabledInstanceExtensions.push_back(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME); + VK_KHR_get_physical_device_properties2_enabled = true; + } + else if(strcmp(extensionProperties.extensionName, VK_EXT_DEBUG_UTILS_EXTENSION_NAME) == 0) + { + if(validationLayersEnabled) + { + enabledInstanceExtensions.push_back("VK_EXT_debug_utils"); + VK_EXT_debug_utils_enabled = true; + } + } } VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; appInfo.pApplicationName = "VmaReplay"; - appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0); + appInfo.applicationVersion = VK_MAKE_VERSION(2, 3, 0); appInfo.pEngineName = "Vulkan Memory Allocator"; - appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0); - appInfo.apiVersion = VK_API_VERSION_1_0; + appInfo.engineVersion = VK_MAKE_VERSION(2, 3, 0); + appInfo.apiVersion = VULKAN_API_VERSION; VkInstanceCreateInfo instInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; instInfo.pApplicationInfo = &appInfo; - instInfo.enabledExtensionCount = (uint32_t)instanceExtensions.size(); - instInfo.ppEnabledExtensionNames = instanceExtensions.data(); + instInfo.enabledExtensionCount = (uint32_t)enabledInstanceExtensions.size(); + instInfo.ppEnabledExtensionNames = enabledInstanceExtensions.data(); instInfo.enabledLayerCount = (uint32_t)instanceLayers.size(); instInfo.ppEnabledLayerNames = instanceLayers.data(); @@ -1373,7 +2058,7 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } - if(validationLayersEnabled) + if(VK_EXT_debug_utils_enabled) { RegisterDebugCallbacks(); } @@ -1413,11 +2098,18 @@ int Player::InitVulkan() vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, queueFamilies.data()); for(uint32_t i = 0; i < queueFamilyCount; ++i) { - if(queueFamilies[i].queueCount > 0 && - (queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) + if(queueFamilies[i].queueCount > 0) { - m_GraphicsQueueFamilyIndex = i; - break; + if(m_GraphicsQueueFamilyIndex == UINT32_MAX && + (queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) + { + m_GraphicsQueueFamilyIndex = i; + } + if(m_TransferQueueFamilyIndex == UINT32_MAX && + (queueFamilies[i].queueFlags & VK_QUEUE_TRANSFER_BIT) != 0) + { + m_TransferQueueFamilyIndex = i; + } } } } @@ -1426,6 +2118,11 @@ int Player::InitVulkan() printf("ERROR: Couldn't find graphics queue.\n"); return RESULT_ERROR_VULKAN; } + if(m_TransferQueueFamilyIndex == UINT_MAX) + { + printf("ERROR: Couldn't find transfer queue.\n"); + return RESULT_ERROR_VULKAN; + } VkPhysicalDeviceFeatures supportedFeatures; vkGetPhysicalDeviceFeatures(m_PhysicalDevice, &supportedFeatures); @@ -1434,21 +2131,30 @@ int Player::InitVulkan() const float queuePriority = 1.f; - VkDeviceQueueCreateInfo deviceQueueCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; - deviceQueueCreateInfo.queueFamilyIndex = m_GraphicsQueueFamilyIndex; - deviceQueueCreateInfo.queueCount = 1; - deviceQueueCreateInfo.pQueuePriorities = &queuePriority; + VkDeviceQueueCreateInfo deviceQueueCreateInfo[2] = {}; + deviceQueueCreateInfo[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + deviceQueueCreateInfo[0].queueFamilyIndex = m_GraphicsQueueFamilyIndex; + deviceQueueCreateInfo[0].queueCount = 1; + deviceQueueCreateInfo[0].pQueuePriorities = &queuePriority; + + if(m_TransferQueueFamilyIndex != m_GraphicsQueueFamilyIndex) + { + deviceQueueCreateInfo[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + deviceQueueCreateInfo[1].queueFamilyIndex = m_TransferQueueFamilyIndex; + deviceQueueCreateInfo[1].queueCount = 1; + deviceQueueCreateInfo[1].pQueuePriorities = &queuePriority; + } // Enable something what may interact with memory/buffer/image support. VkPhysicalDeviceFeatures enabledFeatures; InitVulkanFeatures(enabledFeatures, supportedFeatures); bool VK_KHR_get_memory_requirements2_available = false; - bool VK_KHR_dedicated_allocation_available = false; // Determine list of device extensions to enable. std::vector enabledDeviceExtensions; //enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); + bool memoryBudgetAvailable = false; { uint32_t propertyCount = 0; res = vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, nullptr); @@ -1466,45 +2172,49 @@ int Player::InitVulkan() { VK_KHR_get_memory_requirements2_available = true; } - else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) + else if(strcmp(properties[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME) == 0) { - VK_KHR_dedicated_allocation_available = true; + if(VK_KHR_get_physical_device_properties2_enabled) + { + memoryBudgetAvailable = true; + } } } } } - const bool dedicatedAllocationAvailable = - VK_KHR_get_memory_requirements2_available && VK_KHR_dedicated_allocation_available; - - switch(g_VK_KHR_dedicated_allocation_request) + switch(g_VK_EXT_memory_budget_request) { case VULKAN_EXTENSION_REQUEST::DISABLED: break; case VULKAN_EXTENSION_REQUEST::DEFAULT: - m_DedicatedAllocationEnabled = dedicatedAllocationAvailable; + m_MemoryBudgetEnabled = memoryBudgetAvailable; break; case VULKAN_EXTENSION_REQUEST::ENABLED: - m_DedicatedAllocationEnabled = dedicatedAllocationAvailable; - if(!dedicatedAllocationAvailable) + m_MemoryBudgetEnabled = memoryBudgetAvailable; + if(!memoryBudgetAvailable) { - printf("WARNING: VK_KHR_dedicated_allocation extension cannot be enabled.\n"); + printf("WARNING: VK_EXT_memory_budget extension cannot be enabled.\n"); } break; default: assert(0); } - if(m_DedicatedAllocationEnabled) + if(g_VK_AMD_device_coherent_memory_request == VULKAN_EXTENSION_REQUEST::ENABLED) { - enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); - enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); + printf("WARNING: AMD_device_coherent_memory requested but not currently supported by the player.\n"); + } + + if(m_MemoryBudgetEnabled) + { + enabledDeviceExtensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); } VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; - deviceCreateInfo.queueCreateInfoCount = 1; - deviceCreateInfo.pQueueCreateInfos = &deviceQueueCreateInfo; + deviceCreateInfo.queueCreateInfoCount = m_TransferQueueFamilyIndex != m_GraphicsQueueFamilyIndex ? 2 : 1; + deviceCreateInfo.pQueueCreateInfos = deviceQueueCreateInfo; deviceCreateInfo.pEnabledFeatures = &enabledFeatures; res = vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device); @@ -1514,16 +2224,27 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } + // Fetch queues + vkGetDeviceQueue(m_Device, m_GraphicsQueueFamilyIndex, 0, &m_GraphicsQueue); + vkGetDeviceQueue(m_Device, m_TransferQueueFamilyIndex, 0, &m_TransferQueue); + // Create memory allocator + VmaDeviceMemoryCallbacks deviceMemoryCallbacks = {}; + deviceMemoryCallbacks.pfnAllocate = AllocateDeviceMemoryCallback; + deviceMemoryCallbacks.pfnFree = FreeDeviceMemoryCallback; + VmaAllocatorCreateInfo allocatorInfo = {}; + allocatorInfo.instance = m_VulkanInstance; allocatorInfo.physicalDevice = m_PhysicalDevice; allocatorInfo.device = m_Device; allocatorInfo.flags = VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT; + allocatorInfo.pDeviceMemoryCallbacks = &deviceMemoryCallbacks; + allocatorInfo.vulkanApiVersion = VULKAN_API_VERSION; - if(m_DedicatedAllocationEnabled) + if(m_MemoryBudgetEnabled) { - allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; + allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; } res = vmaCreateAllocator(&allocatorInfo, &m_Allocator); @@ -1536,11 +2257,52 @@ int Player::InitVulkan() vmaGetPhysicalDeviceProperties(m_Allocator, &m_DevProps); vmaGetMemoryProperties(m_Allocator, &m_MemProps); + // Create command pool + + VkCommandPoolCreateInfo cmdPoolCreateInfo = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; + cmdPoolCreateInfo.queueFamilyIndex = m_TransferQueueFamilyIndex; + cmdPoolCreateInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + + res = vkCreateCommandPool(m_Device, &cmdPoolCreateInfo, nullptr, &m_CommandPool); + if(res != VK_SUCCESS) + { + printf("ERROR: vkCreateCommandPool failed (%d)\n", res); + return RESULT_ERROR_VULKAN; + } + + // Create command buffer + + VkCommandBufferAllocateInfo cmdBufAllocInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + cmdBufAllocInfo.commandBufferCount = 1; + cmdBufAllocInfo.commandPool = m_CommandPool; + cmdBufAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + res = vkAllocateCommandBuffers(m_Device, &cmdBufAllocInfo, &m_CommandBuffer); + if(res != VK_SUCCESS) + { + printf("ERROR: vkAllocateCommandBuffers failed (%d)\n", res); + return RESULT_ERROR_VULKAN; + } + return 0; } void Player::FinalizeVulkan() { + if(!m_DefragmentationContexts.empty()) + { + printf("WARNING: Defragmentation contexts not destroyed: %zu.\n", m_DefragmentationContexts.size()); + + if(CLEANUP_LEAKED_OBJECTS) + { + for(const auto& it : m_DefragmentationContexts) + { + vmaDefragmentationEnd(m_Allocator, it.second); + } + } + + m_DefragmentationContexts.clear(); + } + if(!m_Allocations.empty()) { printf("WARNING: Allocations not destroyed: %zu.\n", m_Allocations.size()); @@ -1573,6 +2335,18 @@ void Player::FinalizeVulkan() vkDeviceWaitIdle(m_Device); + if(m_CommandBuffer != VK_NULL_HANDLE) + { + vkFreeCommandBuffers(m_Device, m_CommandPool, 1, &m_CommandBuffer); + m_CommandBuffer = VK_NULL_HANDLE; + } + + if(m_CommandPool != VK_NULL_HANDLE) + { + vkDestroyCommandPool(m_Device, m_CommandPool, nullptr); + m_CommandPool = VK_NULL_HANDLE; + } + if(m_Allocator != VK_NULL_HANDLE) { vmaDestroyAllocator(m_Allocator); @@ -1585,11 +2359,7 @@ void Player::FinalizeVulkan() m_Device = nullptr; } - if(m_pvkDestroyDebugReportCallbackEXT && m_hCallback != VK_NULL_HANDLE) - { - m_pvkDestroyDebugReportCallbackEXT(m_VulkanInstance, m_hCallback, nullptr); - m_hCallback = VK_NULL_HANDLE; - } + UnregisterDebugCallbacks(); if(m_VulkanInstance != VK_NULL_HANDLE) { @@ -1600,29 +2370,169 @@ void Player::FinalizeVulkan() void Player::RegisterDebugCallbacks() { - m_pvkCreateDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(m_VulkanInstance, "vkCreateDebugReportCallbackEXT")); - m_pvkDebugReportMessageEXT = - reinterpret_cast - (vkGetInstanceProcAddr(m_VulkanInstance, "vkDebugReportMessageEXT")); - m_pvkDestroyDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(m_VulkanInstance, "vkDestroyDebugReportCallbackEXT")); - assert(m_pvkCreateDebugReportCallbackEXT); - assert(m_pvkDebugReportMessageEXT); - assert(m_pvkDestroyDebugReportCallbackEXT); + static const VkDebugUtilsMessageSeverityFlagsEXT DEBUG_UTILS_MESSENGER_MESSAGE_SEVERITY = + //VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT | + //VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + static const VkDebugUtilsMessageTypeFlagsEXT DEBUG_UTILS_MESSENGER_MESSAGE_TYPE = + VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT; - VkDebugReportCallbackCreateInfoEXT callbackCreateInfo = { VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT }; - callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | - VK_DEBUG_REPORT_ERROR_BIT_EXT | - VK_DEBUG_REPORT_WARNING_BIT_EXT | - VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| - VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; - callbackCreateInfo.pfnCallback = &MyDebugReportCallback; + m_vkCreateDebugUtilsMessengerEXT = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr( + m_VulkanInstance, "vkCreateDebugUtilsMessengerEXT"); + m_vkDestroyDebugUtilsMessengerEXT = (PFN_vkDestroyDebugUtilsMessengerEXT)vkGetInstanceProcAddr( + m_VulkanInstance, "vkDestroyDebugUtilsMessengerEXT"); + assert(m_vkCreateDebugUtilsMessengerEXT); + assert(m_vkDestroyDebugUtilsMessengerEXT); - VkResult res = m_pvkCreateDebugReportCallbackEXT(m_VulkanInstance, &callbackCreateInfo, nullptr, &m_hCallback); - assert(res == VK_SUCCESS); + VkDebugUtilsMessengerCreateInfoEXT messengerCreateInfo = { VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT }; + messengerCreateInfo.messageSeverity = DEBUG_UTILS_MESSENGER_MESSAGE_SEVERITY; + messengerCreateInfo.messageType = DEBUG_UTILS_MESSENGER_MESSAGE_TYPE; + messengerCreateInfo.pfnUserCallback = MyDebugReportCallback; + VkResult res = m_vkCreateDebugUtilsMessengerEXT(m_VulkanInstance, &messengerCreateInfo, nullptr, &m_DebugUtilsMessenger); + if(res != VK_SUCCESS) + { + printf("ERROR: vkCreateDebugUtilsMessengerEXT failed (%d)\n", res); + m_DebugUtilsMessenger = VK_NULL_HANDLE; + } +} + +void Player::UnregisterDebugCallbacks() +{ + if(m_DebugUtilsMessenger) + { + m_vkDestroyDebugUtilsMessengerEXT(m_VulkanInstance, m_DebugUtilsMessenger, nullptr); + } +} + +void Player::Defragment() +{ + VmaStats stats; + vmaCalculateStats(m_Allocator, &stats); + PrintStats(stats, "before defragmentation"); + + const size_t allocCount = m_Allocations.size(); + std::vector allocations(allocCount); + size_t notNullAllocCount = 0; + for(const auto& it : m_Allocations) + { + if(it.second.allocation != VK_NULL_HANDLE) + { + allocations[notNullAllocCount] = it.second.allocation; + ++notNullAllocCount; + } + } + if(notNullAllocCount == 0) + { + printf(" Nothing to defragment.\n"); + return; + } + + allocations.resize(notNullAllocCount); + std::vector allocationsChanged(notNullAllocCount); + + VmaDefragmentationStats defragStats = {}; + + VkCommandBufferBeginInfo cmdBufBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + VkResult res = vkBeginCommandBuffer(m_CommandBuffer, &cmdBufBeginInfo); + if(res != VK_SUCCESS) + { + printf("ERROR: vkBeginCommandBuffer failed (%d)\n", res); + return; + } + + const time_point timeBeg = std::chrono::high_resolution_clock::now(); + + VmaDefragmentationInfo2 defragInfo = {}; + defragInfo.allocationCount = (uint32_t)notNullAllocCount; + defragInfo.pAllocations = allocations.data(); + defragInfo.pAllocationsChanged = allocationsChanged.data(); + defragInfo.maxCpuAllocationsToMove = UINT32_MAX; + defragInfo.maxCpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.maxGpuAllocationsToMove = UINT32_MAX; + defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; + defragInfo.flags = g_DefragmentationFlags; + defragInfo.commandBuffer = m_CommandBuffer; + + VmaDefragmentationContext defragCtx = VK_NULL_HANDLE; + res = vmaDefragmentationBegin(m_Allocator, &defragInfo, &defragStats, &defragCtx); + + const time_point timeAfterDefragBegin = std::chrono::high_resolution_clock::now(); + + vkEndCommandBuffer(m_CommandBuffer); + + if(res >= VK_SUCCESS) + { + VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &m_CommandBuffer; + vkQueueSubmit(m_TransferQueue, 1, &submitInfo, VK_NULL_HANDLE); + vkQueueWaitIdle(m_TransferQueue); + + const time_point timeAfterGpu = std::chrono::high_resolution_clock::now(); + + vmaDefragmentationEnd(m_Allocator, defragCtx); + + const time_point timeAfterDefragEnd = std::chrono::high_resolution_clock::now(); + + const duration defragDurationBegin = timeAfterDefragBegin - timeBeg; + const duration defragDurationGpu = timeAfterGpu - timeAfterDefragBegin; + const duration defragDurationEnd = timeAfterDefragEnd - timeAfterGpu; + + // If anything changed. + if(defragStats.allocationsMoved > 0) + { + // Go over allocation that changed and destroy their buffers and images. + size_t i = 0; + for(auto& it : m_Allocations) + { + if(allocationsChanged[i] != VK_FALSE) + { + if(it.second.buffer != VK_NULL_HANDLE) + { + vkDestroyBuffer(m_Device, it.second.buffer, nullptr); + it.second.buffer = VK_NULL_HANDLE; + } + if(it.second.image != VK_NULL_HANDLE) + { + vkDestroyImage(m_Device, it.second.image, nullptr); + it.second.image = VK_NULL_HANDLE; + } + } + ++i; + } + } + + // Print statistics + std::string defragDurationBeginStr; + std::string defragDurationGpuStr; + std::string defragDurationEndStr; + SecondsToFriendlyStr(ToFloatSeconds(defragDurationBegin), defragDurationBeginStr); + SecondsToFriendlyStr(ToFloatSeconds(defragDurationGpu), defragDurationGpuStr); + SecondsToFriendlyStr(ToFloatSeconds(defragDurationEnd), defragDurationEndStr); + + printf(" Defragmentation took:\n"); + printf(" vmaDefragmentationBegin: %s\n", defragDurationBeginStr.c_str()); + printf(" GPU: %s\n", defragDurationGpuStr.c_str()); + printf(" vmaDefragmentationEnd: %s\n", defragDurationEndStr.c_str()); + printf(" VmaDefragmentationStats:\n"); + printf(" bytesMoved: %llu\n", defragStats.bytesMoved); + printf(" bytesFreed: %llu\n", defragStats.bytesFreed); + printf(" allocationsMoved: %u\n", defragStats.allocationsMoved); + printf(" deviceMemoryBlocksFreed: %u\n", defragStats.deviceMemoryBlocksFreed); + + vmaCalculateStats(m_Allocator, &stats); + PrintStats(stats, "after defragmentation"); + } + else + { + printf("vmaDefragmentationBegin failed (%d).\n", res); + } + + vkResetCommandPool(m_Device, m_CommandPool, 0); } void Player::PrintStats() @@ -1632,6 +2542,8 @@ void Player::PrintStats() return; } + m_Stats.PrintDeviceMemStats(); + printf("Statistics:\n"); if(m_Stats.GetAllocationCreationCount() > 0) { @@ -1639,14 +2551,9 @@ void Player::PrintStats() } // Buffers - const size_t bufferCreationCount = - m_Stats.GetBufferCreationCount(0) + - m_Stats.GetBufferCreationCount(1) + - m_Stats.GetBufferCreationCount(2) + - m_Stats.GetBufferCreationCount(3); - if(bufferCreationCount > 0) + if(m_Stats.GetBufferCreationCount()) { - printf(" Total buffers created: %zu\n", bufferCreationCount); + printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); if(g_Verbosity == VERBOSITY::MAXIMUM) { printf(" Class 0 (indirect/vertex/index): %zu\n", m_Stats.GetBufferCreationCount(0)); @@ -1726,6 +2633,12 @@ void Player::PrintStats() } } + // Detailed stats + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + m_Stats.PrintDetailedStats(); + } + if(g_MemStatsEnabled) { m_Stats.PrintMemStats(); @@ -1817,7 +2730,7 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), poolCreateInfo.frameInUseCount) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), origPtr)) { - m_Stats.RegisterCreatePool(); + m_Stats.RegisterCreatePool(poolCreateInfo); Pool poolDesc = {}; VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); @@ -2006,7 +2919,11 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) allocCreateInfo.pUserData); } - m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); + m_Stats.RegisterCreateBuffer(bufCreateInfo); + m_Stats.RegisterCreateAllocation(allocCreateInfo); + + // Forcing VK_SHARING_MODE_EXCLUSIVE because we use only one queue anyway. + bufCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Allocation allocDesc = { }; allocDesc.allocationFlags = allocCreateInfo.flags; @@ -2024,7 +2941,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) +void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit, const char* functionName) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { @@ -2054,12 +2971,48 @@ void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) { if(IssueWarning()) { - printf("Line %zu: Invalid parameters for vmaDestroyBuffer.\n", lineNumber); + printf("Line %zu: Invalid parameters for %s.\n", lineNumber, functionName); } } } } +void Player::PrintStats(const VmaStats& stats, const char* suffix) +{ + printf(" VmaStats %s:\n", suffix); + printf(" total:\n"); + PrintStatInfo(stats.total); + + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + for(uint32_t i = 0; i < m_MemProps->memoryHeapCount; ++i) + { + printf(" memoryHeap[%u]:\n", i); + PrintStatInfo(stats.memoryHeap[i]); + } + for(uint32_t i = 0; i < m_MemProps->memoryTypeCount; ++i) + { + printf(" memoryType[%u]:\n", i); + PrintStatInfo(stats.memoryType[i]); + } + } +} + +void Player::PrintStatInfo(const VmaStatInfo& info) +{ + printf(" blockCount: %u\n", info.blockCount); + printf(" allocationCount: %u\n", info.allocationCount); + printf(" unusedRangeCount: %u\n", info.unusedRangeCount); + printf(" usedBytes: %llu\n", info.usedBytes); + printf(" unusedBytes: %llu\n", info.unusedBytes); + printf(" allocationSizeMin: %llu\n", info.allocationSizeMin); + printf(" allocationSizeAvg: %llu\n", info.allocationSizeAvg); + printf(" allocationSizeMax: %llu\n", info.allocationSizeMax); + printf(" unusedRangeSizeMin: %llu\n", info.unusedRangeSizeMin); + printf(" unusedRangeSizeAvg: %llu\n", info.unusedRangeSizeAvg); + printf(" unusedRangeSizeMax: %llu\n", info.unusedRangeSizeMax); +} + void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreateImage); @@ -2104,7 +3057,11 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) allocCreateInfo.pUserData); } - m_Stats.RegisterCreateImage(imageCreateInfo.usage, imageCreateInfo.tiling); + m_Stats.RegisterCreateImage(imageCreateInfo); + m_Stats.RegisterCreateAllocation(allocCreateInfo); + + // Forcing VK_SHARING_MODE_EXCLUSIVE because we use only one queue anyway. + imageCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Allocation allocDesc = {}; allocDesc.allocationFlags = allocCreateInfo.flags; @@ -2122,6 +3079,53 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) } } +void Player::ExecuteFreeMemoryPages(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::FreeMemoryPages); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + std::vector origAllocPtrs; + if(StrRangeToPtrList(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtrs)) + { + const size_t allocCount = origAllocPtrs.size(); + size_t notNullCount = 0; + for(size_t i = 0; i < allocCount; ++i) + { + const uint64_t origAllocPtr = origAllocPtrs[i]; + if(origAllocPtr != 0) + { + const auto it = m_Allocations.find(origAllocPtr); + if(it != m_Allocations.end()) + { + Destroy(it->second); + m_Allocations.erase(it); + ++notNullCount; + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origAllocPtr); + } + } + } + } + if(notNullCount) + { + UpdateMemStats(); + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaFreeMemoryPages.\n", lineNumber); + } + } + } +} + void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreateLostAllocation); @@ -2135,7 +3139,7 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS Allocation allocDesc = {}; vmaCreateLostAllocation(m_Allocator, &allocDesc.allocation); UpdateMemStats(); - m_Stats.RegisterCreateAllocation(); + m_Stats.RegisterCreateLostAllocation(); AddAllocation(lineNumber, origPtr, VK_SUCCESS, "vmaCreateLostAllocation", std::move(allocDesc)); } @@ -2184,7 +3188,7 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) } UpdateMemStats(); - m_Stats.RegisterCreateAllocation(); + m_Stats.RegisterCreateAllocation(allocCreateInfo); Allocation allocDesc = {}; allocDesc.allocationFlags = allocCreateInfo.flags; @@ -2201,6 +3205,69 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) } } +void Player::ExecuteAllocateMemoryPages(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::AllocateMemoryPages); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 11, true)) + { + VkMemoryRequirements memReq = {}; + VmaAllocationCreateInfo allocCreateInfo = {}; + uint64_t origPool = 0; + std::vector origPtrs; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), memReq.size) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), memReq.alignment) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), memReq.memoryTypeBits) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), allocCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), (uint32_t&)allocCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), allocCreateInfo.requiredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), allocCreateInfo.preferredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), allocCreateInfo.memoryTypeBits) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), origPool) && + StrRangeToPtrList(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), origPtrs)) + { + const size_t allocCount = origPtrs.size(); + if(allocCount > 0) + { + FindPool(lineNumber, origPool, allocCreateInfo.pool); + + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 10) + { + PrepareUserData( + lineNumber, + allocCreateInfo.flags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 10), + csvSplit.GetLine(), + allocCreateInfo.pUserData); + } + + UpdateMemStats(); + m_Stats.RegisterCreateAllocation(allocCreateInfo, allocCount); + m_Stats.RegisterAllocateMemoryPages(allocCount); + + std::vector allocations(allocCount); + + VkResult res = vmaAllocateMemoryPages(m_Allocator, &memReq, &allocCreateInfo, allocCount, allocations.data(), nullptr); + for(size_t i = 0; i < allocCount; ++i) + { + Allocation allocDesc = {}; + allocDesc.allocationFlags = allocCreateInfo.flags; + allocDesc.allocation = allocations[i]; + AddAllocation(lineNumber, origPtrs[i], res, "vmaAllocateMemoryPages", std::move(allocDesc)); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaAllocateMemoryPages.\n", lineNumber); + } + } + } +} + void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType) { switch(objType) @@ -2248,6 +3315,9 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS allocCreateInfo.pUserData); } + UpdateMemStats(); + m_Stats.RegisterCreateAllocation(allocCreateInfo); + if(requiresDedicatedAllocation || prefersDedicatedAllocation) { allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; @@ -2262,9 +3332,6 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS m_AllocateForBufferImageWarningIssued = true; } - UpdateMemStats(); - m_Stats.RegisterCreateAllocation(); - Allocation allocDesc = {}; allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); @@ -2599,6 +3666,281 @@ void Player::ExecuteMakePoolAllocationsLost(size_t lineNumber, const CsvSplit& c } } +void Player::ExecuteResizeAllocation(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::ResizeAllocation); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, false)) + { + uint64_t origPtr = 0; + uint64_t newSize = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), newSize)) + { + if(origPtr != 0) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + vmaResizeAllocation(m_Allocator, it->second.allocation, newSize); + UpdateMemStats(); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaResizeAllocation.\n", lineNumber); + } + } + } +} + +void Player::ExecuteDefragmentationBegin(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::DefragmentationBegin); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 9, false)) + { + VmaDefragmentationInfo2 defragInfo = {}; + std::vector allocationOrigPtrs; + std::vector poolOrigPtrs; + uint64_t cmdBufOrigPtr = 0; + uint64_t defragCtxOrigPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), defragInfo.flags) && + StrRangeToPtrList(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), allocationOrigPtrs) && + StrRangeToPtrList(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), poolOrigPtrs) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), defragInfo.maxCpuBytesToMove) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), defragInfo.maxCpuAllocationsToMove) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), defragInfo.maxGpuBytesToMove) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), defragInfo.maxGpuAllocationsToMove) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), cmdBufOrigPtr) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), defragCtxOrigPtr)) + { + const size_t allocationOrigPtrCount = allocationOrigPtrs.size(); + std::vector allocations; + allocations.reserve(allocationOrigPtrCount); + for(size_t i = 0; i < allocationOrigPtrCount; ++i) + { + const auto it = m_Allocations.find(allocationOrigPtrs[i]); + if(it != m_Allocations.end() && it->second.allocation) + { + allocations.push_back(it->second.allocation); + } + } + if(!allocations.empty()) + { + defragInfo.allocationCount = (uint32_t)allocations.size(); + defragInfo.pAllocations = allocations.data(); + } + + const size_t poolOrigPtrCount = poolOrigPtrs.size(); + std::vector pools; + pools.reserve(poolOrigPtrCount); + for(size_t i = 0; i < poolOrigPtrCount; ++i) + { + const auto it = m_Pools.find(poolOrigPtrs[i]); + if(it != m_Pools.end() && it->second.pool) + { + pools.push_back(it->second.pool); + } + } + if(!pools.empty()) + { + defragInfo.poolCount = (uint32_t)pools.size(); + defragInfo.pPools = pools.data(); + } + + if(allocations.size() != allocationOrigPtrCount || + pools.size() != poolOrigPtrCount) + { + if(IssueWarning()) + { + printf("Line %zu: Passing %zu allocations and %zu pools to vmaDefragmentationBegin, while originally %zu allocations and %zu pools were passed.\n", + lineNumber, + allocations.size(), pools.size(), + allocationOrigPtrCount, poolOrigPtrCount); + } + } + + if(cmdBufOrigPtr) + { + VkCommandBufferBeginInfo cmdBufBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + VkResult res = vkBeginCommandBuffer(m_CommandBuffer, &cmdBufBeginInfo); + if(res == VK_SUCCESS) + { + defragInfo.commandBuffer = m_CommandBuffer; + } + else + { + printf("Line %zu: vkBeginCommandBuffer failed (%d)\n", lineNumber, res); + } + } + + m_Stats.RegisterDefragmentation(defragInfo); + + VmaDefragmentationContext defragCtx = nullptr; + VkResult res = vmaDefragmentationBegin(m_Allocator, &defragInfo, nullptr, &defragCtx); + + if(defragInfo.commandBuffer) + { + vkEndCommandBuffer(m_CommandBuffer); + + VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &m_CommandBuffer; + vkQueueSubmit(m_TransferQueue, 1, &submitInfo, VK_NULL_HANDLE); + vkQueueWaitIdle(m_TransferQueue); + } + + if(res >= VK_SUCCESS) + { + if(defragCtx) + { + if(defragCtxOrigPtr) + { + // We have defragmentation context, originally had defragmentation context: Store it. + m_DefragmentationContexts[defragCtxOrigPtr] = defragCtx; + } + else + { + // We have defragmentation context, originally it was null: End immediately. + vmaDefragmentationEnd(m_Allocator, defragCtx); + } + } + else + { + if(defragCtxOrigPtr) + { + // We have no defragmentation context, originally there was one: Store null. + m_DefragmentationContexts[defragCtxOrigPtr] = nullptr; + } + else + { + // We have no defragmentation context, originally there wasn't as well - nothing to do. + } + } + } + else + { + if(defragCtxOrigPtr) + { + // Currently failed, originally succeeded. + if(IssueWarning()) + { + printf("Line %zu: vmaDefragmentationBegin failed (%d), while originally succeeded.\n", lineNumber, res); + } + } + else + { + // Currently failed, originally don't know. + if(IssueWarning()) + { + printf("Line %zu: vmaDefragmentationBegin failed (%d).\n", lineNumber, res); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaDefragmentationBegin.\n", lineNumber); + } + } + } +} + +void Player::ExecuteDefragmentationEnd(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::DefragmentationEnd); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + if(origPtr != 0) + { + const auto it = m_DefragmentationContexts.find(origPtr); + if(it != m_DefragmentationContexts.end()) + { + vmaDefragmentationEnd(m_Allocator, it->second); + m_DefragmentationContexts.erase(it); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Defragmentation context %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaDefragmentationEnd.\n", lineNumber); + } + } + } +} + +void Player::ExecuteSetPoolName(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::SetPoolName); + + if(!g_UserDataEnabled) + { + return; + } + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) + { + uint64_t origPtr = 0; + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + if(origPtr != 0) + { + const auto it = m_Pools.find(origPtr); + if(it != m_Pools.end()) + { + std::string poolName; + csvSplit.GetRange(FIRST_PARAM_INDEX + 1).to_str(poolName); + vmaSetPoolName(m_Allocator, it->second.pool, !poolName.empty() ? poolName.c_str() : nullptr); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaSetPoolName.\n", lineNumber); + } + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Main functions @@ -2620,14 +3962,17 @@ static void PrintCommandLineSyntax() " File is written to current directory with name: VmaReplay_Line####.json.\n" " This parameter can be repeated.\n" " --DumpDetailedStatsAfterLine - Like command above, but includes detailed map.\n" + " --DefragmentAfterLine - Defragment memory after specified source file line and print statistics.\n" + " It also prints detailed statistics to files VmaReplay_Line####_Defragment*.json\n" + " --DefragmentationFlags - Flags to be applied when using DefragmentAfterLine.\n" " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" " --UserData - 0 to disable or 1 to enable setting pUserData during playback.\n" " Default is 1. Affects both creation of buffers and images, as well as calls to vmaSetAllocationUserData.\n" - " --VK_LAYER_LUNARG_standard_validation - 0 to disable or 1 to enable validation layers.\n" + " --VK_LAYER_KHRONOS_validation - 0 to disable or 1 to enable validation layers.\n" " By default the layers are silently enabled if available.\n" - " --VK_KHR_dedicated_allocation - 0 to disable or 1 to enable this extension.\n" + " --VK_EXT_memory_budget - 0 to disable or 1 to enable this extension.\n" " By default the extension is silently enabled if available.\n" ); } @@ -2638,6 +3983,7 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, const bool useLineRanges = !g_LineRanges.IsEmpty(); const bool useDumpStatsAfterLine = !g_DumpStatsAfterLine.empty(); + const bool useDefragmentAfterLine = !g_DefragmentAfterLine.empty(); LineSplit lineSplit(data, numBytes); StrRange line; @@ -2733,6 +4079,25 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, ++g_DumpStatsAfterLineNextIndex; } + + while(useDefragmentAfterLine && + g_DefragmentAfterLineNextIndex < g_DefragmentAfterLine.size() && + currLineNumber >= g_DefragmentAfterLine[g_DefragmentAfterLineNextIndex]) + { + const size_t requestedLine = g_DefragmentAfterLine[g_DefragmentAfterLineNextIndex]; + if(g_Verbosity >= VERBOSITY::DEFAULT) + { + printf("Defragmenting after line %zu actual line %zu...\n", + requestedLine, + currLineNumber); + } + + player.DumpStats("VmaReplay_Line%04zu_Defragment_1Before.json", requestedLine, true); + player.Defragment(); + player.DumpStats("VmaReplay_Line%04zu_Defragment_2After.json", requestedLine, true); + + ++g_DefragmentAfterLineNextIndex; + } } const duration playDuration = std::chrono::high_resolution_clock::now() - timeBeg; @@ -2827,10 +4192,12 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_PHYSICAL_DEVICE, "PhysicalDevice", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_USER_DATA, "UserData", true); - cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, "VK_KHR_dedicated_allocation", true); - cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, VALIDATION_LAYER_NAME, true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_EXT_MEMORY_BUDGET, "VK_EXT_memory_budget", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_LAYER_KHRONOS_VALIDATION, VALIDATION_LAYER_NAME, true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_MEM_STATS, "MemStats", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_DUMP_STATS_AFTER_LINE, "DumpStatsAfterLine", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_DEFRAGMENT_AFTER_LINE, "DefragmentAfterLine", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_DEFRAGMENTATION_FLAGS, "DefragmentationFlags", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE, "DumpDetailedStatsAfterLine", true); CmdLineParser::RESULT res; @@ -2884,12 +4251,12 @@ static int main2(int argc, char** argv) return RESULT_ERROR_COMMAND_LINE; } break; - case CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION: + case CMD_LINE_OPT_VK_EXT_MEMORY_BUDGET: { bool newValue; if(StrRangeToBool(StrRange(cmdLineParser.GetParameter()), newValue)) { - g_VK_KHR_dedicated_allocation_request = newValue ? + g_VK_EXT_memory_budget_request = newValue ? VULKAN_EXTENSION_REQUEST::ENABLED : VULKAN_EXTENSION_REQUEST::DISABLED; } @@ -2900,12 +4267,12 @@ static int main2(int argc, char** argv) } } break; - case CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION: + case CMD_LINE_OPT_VK_LAYER_KHRONOS_VALIDATION: { bool newValue; if(StrRangeToBool(StrRange(cmdLineParser.GetParameter()), newValue)) { - g_VK_LAYER_LUNARG_standard_validation = newValue ? + g_VK_LAYER_KHRONOS_validation = newValue ? VULKAN_EXTENSION_REQUEST::ENABLED : VULKAN_EXTENSION_REQUEST::DISABLED; } @@ -2940,6 +4307,29 @@ static int main2(int argc, char** argv) } } break; + case CMD_LINE_OPT_DEFRAGMENT_AFTER_LINE: + { + size_t line; + if(StrRangeToUint(StrRange(cmdLineParser.GetParameter()), line)) + { + g_DefragmentAfterLine.push_back(line); + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; + case CMD_LINE_OPT_DEFRAGMENTATION_FLAGS: + { + if(!StrRangeToUint(StrRange(cmdLineParser.GetParameter()), g_DefragmentationFlags)) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; default: assert(0); } @@ -2978,6 +4368,12 @@ static int main2(int argc, char** argv) std::unique(g_DumpStatsAfterLine.begin(), g_DumpStatsAfterLine.end()), g_DumpStatsAfterLine.end()); + // Sort g_DefragmentAfterLine and make unique. + std::sort(g_DefragmentAfterLine.begin(), g_DefragmentAfterLine.end()); + g_DefragmentAfterLine.erase( + std::unique(g_DefragmentAfterLine.begin(), g_DefragmentAfterLine.end()), + g_DefragmentAfterLine.end()); + return ProcessFile(); } diff --git a/third_party/vkmemalloc/src/VmaReplay/VmaUsage.cpp b/third_party/vkmemalloc/src/VmaReplay/VmaUsage.cpp index d2d035b211..20555a4d69 100644 --- a/third_party/vkmemalloc/src/VmaReplay/VmaUsage.cpp +++ b/third_party/vkmemalloc/src/VmaReplay/VmaUsage.cpp @@ -1,2 +1,24 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #define VMA_IMPLEMENTATION #include "VmaUsage.h" diff --git a/third_party/vkmemalloc/src/VmaReplay/VmaUsage.h b/third_party/vkmemalloc/src/VmaReplay/VmaUsage.h index 182132b06c..5c1b4815db 100644 --- a/third_party/vkmemalloc/src/VmaReplay/VmaUsage.h +++ b/third_party/vkmemalloc/src/VmaReplay/VmaUsage.h @@ -1,10 +1,32 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #pragma once #define NOMINMAX #define WIN32_LEAN_AND_MEAN #include -#define VK_USE_PLATFORM_WIN32_KHR +#define VK_USE_PLATFORM_WIN32_KHR #include //#define VMA_USE_STL_CONTAINERS 1 @@ -21,6 +43,7 @@ #pragma warning(disable: 4127) // conditional expression is constant #pragma warning(disable: 4100) // unreferenced formal parameter #pragma warning(disable: 4189) // local variable is initialized but not referenced +#pragma warning(disable: 4324) // structure was padded due to alignment specifier #include "../vk_mem_alloc.h" diff --git a/third_party/vkmemalloc/src/VmaUsage.cpp b/third_party/vkmemalloc/src/VmaUsage.cpp index 2ebf69c8f4..5dc0ded2ee 100644 --- a/third_party/vkmemalloc/src/VmaUsage.cpp +++ b/third_party/vkmemalloc/src/VmaUsage.cpp @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + /* In exactly one CPP file define macro VMA_IMPLEMENTATION and then include vk_mem_alloc.h to include definitions of its internal implementation diff --git a/third_party/vkmemalloc/src/VmaUsage.h b/third_party/vkmemalloc/src/VmaUsage.h index e788a30b98..59df6c3f06 100644 --- a/third_party/vkmemalloc/src/VmaUsage.h +++ b/third_party/vkmemalloc/src/VmaUsage.h @@ -1,3 +1,25 @@ +// +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + #ifndef VMA_USAGE_H_ #define VMA_USAGE_H_ @@ -8,39 +30,73 @@ #include #define VK_USE_PLATFORM_WIN32_KHR +#else // #ifdef _WIN32 + #include +#endif // #ifdef _WIN32 + +#ifdef _MSVC_LANG + +// Uncomment to test including `vulkan.h` on your own before including VMA. +//#include + /* In every place where you want to use Vulkan Memory Allocator, define appropriate macros if you want to configure the library and then include its header to include all public interface declarations. Example: */ -//#define VMA_USE_STL_CONTAINERS 1 - //#define VMA_HEAVY_ASSERT(expr) assert(expr) - +//#define VMA_USE_STL_CONTAINERS 1 //#define VMA_DEDICATED_ALLOCATION 0 - //#define VMA_DEBUG_MARGIN 16 //#define VMA_DEBUG_DETECT_CORRUPTION 1 //#define VMA_DEBUG_INITIALIZE_ALLOCATIONS 1 -//#define VMA_RECORDING_ENABLED 0 +//#define VMA_RECORDING_ENABLED 1 +//#define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY 256 +//#define VMA_USE_STL_SHARED_MUTEX 0 +//#define VMA_DEBUG_GLOBAL_MUTEX 1 +//#define VMA_MEMORY_BUDGET 0 +#define VMA_STATIC_VULKAN_FUNCTIONS 0 +#define VMA_DYNAMIC_VULKAN_FUNCTIONS 1 + +//#define VMA_VULKAN_VERSION 1002000 // Vulkan 1.2 +#define VMA_VULKAN_VERSION 1001000 // Vulkan 1.1 +//#define VMA_VULKAN_VERSION 1000000 // Vulkan 1.0 + +/* +#define VMA_DEBUG_LOG(format, ...) do { \ + printf(format, __VA_ARGS__); \ + printf("\n"); \ + } while(false) +*/ #pragma warning(push, 4) #pragma warning(disable: 4127) // conditional expression is constant #pragma warning(disable: 4100) // unreferenced formal parameter #pragma warning(disable: 4189) // local variable is initialized but not referenced +#pragma warning(disable: 4324) // structure was padded due to alignment specifier + +#endif // #ifdef _MSVC_LANG + +#ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wtautological-compare" // comparison of unsigned expression < 0 is always false + #pragma clang diagnostic ignored "-Wunused-private-field" + #pragma clang diagnostic ignored "-Wunused-parameter" + #pragma clang diagnostic ignored "-Wmissing-field-initializers" + #pragma clang diagnostic ignored "-Wnullability-completeness" +#endif #include "vk_mem_alloc.h" -#pragma warning(pop) +#ifdef __clang__ + #pragma clang diagnostic pop +#endif -#else // #ifdef _WIN32 - -#include -#include "vk_mem_alloc.h" - -#endif // #ifdef _WIN32 +#ifdef _MSVC_LANG + #pragma warning(pop) +#endif #endif diff --git a/third_party/vkmemalloc/src/VulkanSample.cpp b/third_party/vkmemalloc/src/VulkanSample.cpp index d33861e68f..1c31f0795d 100644 --- a/third_party/vkmemalloc/src/VulkanSample.cpp +++ b/third_party/vkmemalloc/src/VulkanSample.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,35 +22,48 @@ #ifdef _WIN32 +#include "SparseBindingTest.h" #include "Tests.h" #include "VmaUsage.h" #include "Common.h" +#include static const char* const SHADER_PATH1 = "./"; static const char* const SHADER_PATH2 = "../bin/"; static const wchar_t* const WINDOW_CLASS_NAME = L"VULKAN_MEMORY_ALLOCATOR_SAMPLE"; -static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; -static const char* const APP_TITLE_A = "Vulkan Memory Allocator Sample 2.0"; -static const wchar_t* const APP_TITLE_W = L"Vulkan Memory Allocator Sample 2.0"; +static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_KHRONOS_validation"; +static const char* const APP_TITLE_A = "Vulkan Memory Allocator Sample 2.4.0"; +static const wchar_t* const APP_TITLE_W = L"Vulkan Memory Allocator Sample 2.4.0"; static const bool VSYNC = true; static const uint32_t COMMAND_BUFFER_COUNT = 2; static void* const CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA = (void*)(intptr_t)43564544; -static const bool USE_CUSTOM_CPU_ALLOCATION_CALLBACKS = false; +static const bool USE_CUSTOM_CPU_ALLOCATION_CALLBACKS = true; VkPhysicalDevice g_hPhysicalDevice; VkDevice g_hDevice; VmaAllocator g_hAllocator; -bool g_MemoryAliasingWarningEnabled = true; +VkInstance g_hVulkanInstance; -static bool g_EnableValidationLayer = true; -static bool VK_KHR_get_memory_requirements2_enabled = false; -static bool VK_KHR_dedicated_allocation_enabled = false; +bool g_EnableValidationLayer = true; +bool VK_KHR_get_memory_requirements2_enabled = false; +bool VK_KHR_get_physical_device_properties2_enabled = false; +bool VK_KHR_dedicated_allocation_enabled = false; +bool VK_KHR_bind_memory2_enabled = false; +bool VK_EXT_memory_budget_enabled = false; +bool VK_AMD_device_coherent_memory_enabled = false; +bool VK_EXT_buffer_device_address_enabled = false; +bool VK_KHR_buffer_device_address_enabled = false; +bool VK_EXT_debug_utils_enabled = false; +bool g_SparseBindingEnabled = false; +bool g_BufferDeviceAddressEnabled = false; + +// # Pointers to functions from extensions +PFN_vkGetBufferDeviceAddressEXT g_vkGetBufferDeviceAddressEXT; static HINSTANCE g_hAppInstance; static HWND g_hWnd; static LONG g_SizeX = 1280, g_SizeY = 720; -static VkInstance g_hVulkanInstance; static VkSurfaceKHR g_hSurface; static VkQueue g_hPresentQueue; static VkSurfaceFormatKHR g_SurfaceFormat; @@ -62,11 +75,13 @@ static std::vector g_Framebuffers; static VkCommandPool g_hCommandPool; static VkCommandBuffer g_MainCommandBuffers[COMMAND_BUFFER_COUNT]; static VkFence g_MainCommandBufferExecutedFances[COMMAND_BUFFER_COUNT]; +VkFence g_ImmediateFence; static uint32_t g_NextCommandBufferIndex; static VkSemaphore g_hImageAvailableSemaphore; static VkSemaphore g_hRenderFinishedSemaphore; static uint32_t g_GraphicsQueueFamilyIndex = UINT_MAX; static uint32_t g_PresentQueueFamilyIndex = UINT_MAX; +static uint32_t g_SparseBindingQueueFamilyIndex = UINT_MAX; static VkDescriptorSetLayout g_hDescriptorSetLayout; static VkDescriptorPool g_hDescriptorPool; static VkDescriptorSet g_hDescriptorSet; // Automatically destroyed with m_DescriptorPool. @@ -80,13 +95,22 @@ static VkSurfaceCapabilitiesKHR g_SurfaceCapabilities; static std::vector g_SurfaceFormats; static std::vector g_PresentModes; -static PFN_vkCreateDebugReportCallbackEXT g_pvkCreateDebugReportCallbackEXT; -static PFN_vkDebugReportMessageEXT g_pvkDebugReportMessageEXT; -static PFN_vkDestroyDebugReportCallbackEXT g_pvkDestroyDebugReportCallbackEXT; -static VkDebugReportCallbackEXT g_hCallback; +static const VkDebugUtilsMessageSeverityFlagsEXT DEBUG_UTILS_MESSENGER_MESSAGE_SEVERITY = + //VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT | + //VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; +static const VkDebugUtilsMessageTypeFlagsEXT DEBUG_UTILS_MESSENGER_MESSAGE_TYPE = + VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT; +static PFN_vkCreateDebugUtilsMessengerEXT vkCreateDebugUtilsMessengerEXT_Func; +static PFN_vkDestroyDebugUtilsMessengerEXT vkDestroyDebugUtilsMessengerEXT_Func; +static VkDebugUtilsMessengerEXT g_DebugUtilsMessenger; static VkQueue g_hGraphicsQueue; -static VkCommandBuffer g_hTemporaryCommandBuffer; +VkQueue g_hSparseBindingQueue; +VkCommandBuffer g_hTemporaryCommandBuffer; static VkPipelineLayout g_hPipelineLayout; static VkRenderPass g_hRenderPass; @@ -103,12 +127,19 @@ static VkImage g_hTextureImage; static VmaAllocation g_hTextureImageAlloc; static VkImageView g_hTextureImageView; +static std::atomic_uint32_t g_CpuAllocCount; + static void* CustomCpuAllocation( void* pUserData, size_t size, size_t alignment, VkSystemAllocationScope allocationScope) { assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - return _aligned_malloc(size, alignment); + void* const result = _aligned_malloc(size, alignment); + if(result) + { + ++g_CpuAllocCount; + } + return result; } static void* CustomCpuReallocation( @@ -116,23 +147,46 @@ static void* CustomCpuReallocation( VkSystemAllocationScope allocationScope) { assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - return _aligned_realloc(pOriginal, size, alignment); + void* const result = _aligned_realloc(pOriginal, size, alignment); + if(pOriginal && !result) + { + --g_CpuAllocCount; + } + else if(!pOriginal && result) + { + ++g_CpuAllocCount; + } + return result; } static void CustomCpuFree(void* pUserData, void* pMemory) { assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - _aligned_free(pMemory); + if(pMemory) + { + const uint32_t oldAllocCount = g_CpuAllocCount.fetch_sub(1); + TEST(oldAllocCount > 0); + _aligned_free(pMemory); + } } -static void BeginSingleTimeCommands() +static const VkAllocationCallbacks g_CpuAllocationCallbacks = { + CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA, // pUserData + &CustomCpuAllocation, // pfnAllocation + &CustomCpuReallocation, // pfnReallocation + &CustomCpuFree // pfnFree +}; + +const VkAllocationCallbacks* g_Allocs; + +void BeginSingleTimeCommands() { VkCommandBufferBeginInfo cmdBufBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; ERR_GUARD_VULKAN( vkBeginCommandBuffer(g_hTemporaryCommandBuffer, &cmdBufBeginInfo) ); } -static void EndSingleTimeCommands() +void EndSingleTimeCommands() { ERR_GUARD_VULKAN( vkEndCommandBuffer(g_hTemporaryCommandBuffer) ); @@ -144,7 +198,7 @@ static void EndSingleTimeCommands() ERR_GUARD_VULKAN( vkQueueWaitIdle(g_hGraphicsQueue) ); } -static void LoadShader(std::vector& out, const char* fileName) +void LoadShader(std::vector& out, const char* fileName) { std::ifstream file(std::string(SHADER_PATH1) + fileName, std::ios::ate | std::ios::binary); if(file.is_open() == false) @@ -162,67 +216,37 @@ static void LoadShader(std::vector& out, const char* fileName) out.clear(); } -VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( - VkDebugReportFlagsEXT flags, - VkDebugReportObjectTypeEXT objectType, - uint64_t object, - size_t location, - int32_t messageCode, - const char* pLayerPrefix, - const char* pMessage, - void* pUserData) +static VkBool32 VKAPI_PTR MyDebugReportCallback( + VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, + VkDebugUtilsMessageTypeFlagsEXT messageTypes, + const VkDebugUtilsMessengerCallbackDataEXT* pCallbackData, + void* pUserData) { - // "Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug." - if(!g_MemoryAliasingWarningEnabled && flags == VK_DEBUG_REPORT_WARNING_BIT_EXT && - (strstr(pMessage, " is aliased with non-linear ") || strstr(pMessage, " is aliased with linear "))) - { - return VK_FALSE; - } + assert(pCallbackData && pCallbackData->pMessageIdName && pCallbackData->pMessage); - // Ignoring because when VK_KHR_dedicated_allocation extension is enabled, - // vkGetBufferMemoryRequirements2KHR function is used instead, while Validation - // Layer seems to be unaware of it. - if (strstr(pMessage, "but vkGetBufferMemoryRequirements() has not been called on that buffer") != nullptr) + switch(messageSeverity) { - return VK_FALSE; - } - if (strstr(pMessage, "but vkGetImageMemoryRequirements() has not been called on that image") != nullptr) - { - return VK_FALSE; - } - - /* - "Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used." - Ignoring because we map entire VkDeviceMemory blocks, where different types of - images and buffers may end up together, especially on GPUs with unified memory - like Intel. - */ - if(strstr(pMessage, "Mapping an image with layout") != nullptr && - strstr(pMessage, "can result in undefined behavior if this memory is used by the device") != nullptr) - { - return VK_FALSE; - } - - switch(flags) - { - case VK_DEBUG_REPORT_WARNING_BIT_EXT: + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: SetConsoleColor(CONSOLE_COLOR::WARNING); break; - case VK_DEBUG_REPORT_ERROR_BIT_EXT: + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: SetConsoleColor(CONSOLE_COLOR::ERROR_); break; - default: + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: + SetConsoleColor(CONSOLE_COLOR::NORMAL); + break; + default: // VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT SetConsoleColor(CONSOLE_COLOR::INFO); } - printf("%s \xBA %s\n", pLayerPrefix, pMessage); + printf("%s \xBA %s\n", pCallbackData->pMessageIdName, pCallbackData->pMessage); SetConsoleColor(CONSOLE_COLOR::NORMAL); - if(flags == VK_DEBUG_REPORT_WARNING_BIT_EXT || - flags == VK_DEBUG_REPORT_ERROR_BIT_EXT) + if(messageSeverity == VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT || + messageSeverity == VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) { - OutputDebugStringA(pMessage); + OutputDebugStringA(pCallbackData->pMessage); OutputDebugStringA("\n"); } @@ -407,44 +431,25 @@ static void CreateMesh() static void CreateTexture(uint32_t sizeX, uint32_t sizeY) { - // Create Image + // Create staging buffer. const VkDeviceSize imageSize = sizeX * sizeY * 4; - VkImageCreateInfo stagingImageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; - stagingImageInfo.imageType = VK_IMAGE_TYPE_2D; - stagingImageInfo.extent.width = sizeX; - stagingImageInfo.extent.height = sizeY; - stagingImageInfo.extent.depth = 1; - stagingImageInfo.mipLevels = 1; - stagingImageInfo.arrayLayers = 1; - stagingImageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; - stagingImageInfo.tiling = VK_IMAGE_TILING_LINEAR; - stagingImageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; - stagingImageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT; - stagingImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - stagingImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; - stagingImageInfo.flags = 0; + VkBufferCreateInfo stagingBufInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + stagingBufInfo.size = imageSize; + stagingBufInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VmaAllocationCreateInfo stagingBufAllocCreateInfo = {}; + stagingBufAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + stagingBufAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; - VmaAllocationCreateInfo stagingImageAllocCreateInfo = {}; - stagingImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; - stagingImageAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; - - VkImage stagingImage = VK_NULL_HANDLE; - VmaAllocation stagingImageAlloc = VK_NULL_HANDLE; - VmaAllocationInfo stagingImageAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &stagingImageInfo, &stagingImageAllocCreateInfo, &stagingImage, &stagingImageAlloc, &stagingImageAllocInfo) ); + VkBuffer stagingBuf = VK_NULL_HANDLE; + VmaAllocation stagingBufAlloc = VK_NULL_HANDLE; + VmaAllocationInfo stagingBufAllocInfo = {}; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &stagingBufInfo, &stagingBufAllocCreateInfo, &stagingBuf, &stagingBufAlloc, &stagingBufAllocInfo) ); - VkImageSubresource imageSubresource = {}; - imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageSubresource.mipLevel = 0; - imageSubresource.arrayLayer = 0; - - VkSubresourceLayout imageLayout = {}; - vkGetImageSubresourceLayout(g_hDevice, stagingImage, &imageSubresource, &imageLayout); - - char* const pMipLevelData = (char*)stagingImageAllocInfo.pMappedData + imageLayout.offset; - uint8_t* pRowData = (uint8_t*)pMipLevelData; + char* const pImageData = (char*)stagingBufAllocInfo.pMappedData; + uint8_t* pRowData = (uint8_t*)pImageData; for(uint32_t y = 0; y < sizeY; ++y) { uint32_t* pPixelData = (uint32_t*)pRowData; @@ -457,7 +462,7 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) ((y & 0x18) == 0x10 ? 0x00FF0000 : 0x00000000); ++pPixelData; } - pRowData += imageLayout.rowPitch; + pRowData += sizeX * 4; } // No need to flush stagingImage memory because CPU_ONLY memory is always HOST_COHERENT. @@ -489,28 +494,13 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) BeginSingleTimeCommands(); VkImageMemoryBarrier imgMemBarrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; - imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; - imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; imgMemBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; imgMemBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - imgMemBarrier.image = stagingImage; imgMemBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imgMemBarrier.subresourceRange.baseMipLevel = 0; imgMemBarrier.subresourceRange.levelCount = 1; imgMemBarrier.subresourceRange.baseArrayLayer = 0; imgMemBarrier.subresourceRange.layerCount = 1; - imgMemBarrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - - vkCmdPipelineBarrier( - g_hTemporaryCommandBuffer, - VK_PIPELINE_STAGE_HOST_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, - 0, nullptr, - 0, nullptr, - 1, &imgMemBarrier); - imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; imgMemBarrier.image = g_hTextureImage; @@ -526,29 +516,14 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) 0, nullptr, 1, &imgMemBarrier); - VkImageCopy imageCopy = {}; - imageCopy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageCopy.srcSubresource.baseArrayLayer = 0; - imageCopy.srcSubresource.mipLevel = 0; - imageCopy.srcSubresource.layerCount = 1; - imageCopy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageCopy.dstSubresource.baseArrayLayer = 0; - imageCopy.dstSubresource.mipLevel = 0; - imageCopy.dstSubresource.layerCount = 1; - imageCopy.srcOffset.x = 0; - imageCopy.srcOffset.y = 0; - imageCopy.srcOffset.z = 0; - imageCopy.dstOffset.x = 0; - imageCopy.dstOffset.y = 0; - imageCopy.dstOffset.z = 0; - imageCopy.extent.width = sizeX; - imageCopy.extent.height = sizeY; - imageCopy.extent.depth = 1; - vkCmdCopyImage( - g_hTemporaryCommandBuffer, - stagingImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - g_hTextureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - 1, &imageCopy); + VkBufferImageCopy region = {}; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.layerCount = 1; + region.imageExtent.width = sizeX; + region.imageExtent.height = sizeY; + region.imageExtent.depth = 1; + + vkCmdCopyBufferToImage(g_hTemporaryCommandBuffer, stagingBuf, g_hTextureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; @@ -567,7 +542,7 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) EndSingleTimeCommands(); - vmaDestroyImage(g_hAllocator, stagingImage, stagingImageAlloc); + vmaDestroyBuffer(g_hAllocator, stagingBuf, stagingBufAlloc); // Create ImageView @@ -580,7 +555,7 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) textureImageViewInfo.subresourceRange.levelCount = 1; textureImageViewInfo.subresourceRange.baseArrayLayer = 0; textureImageViewInfo.subresourceRange.layerCount = 1; - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &textureImageViewInfo, nullptr, &g_hTextureImageView) ); + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &textureImageViewInfo, g_Allocs, &g_hTextureImageView) ); } struct UniformBufferObject @@ -590,31 +565,26 @@ struct UniformBufferObject static void RegisterDebugCallbacks() { - g_pvkCreateDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkCreateDebugReportCallbackEXT")); - g_pvkDebugReportMessageEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDebugReportMessageEXT")); - g_pvkDestroyDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDestroyDebugReportCallbackEXT")); - assert(g_pvkCreateDebugReportCallbackEXT); - assert(g_pvkDebugReportMessageEXT); - assert(g_pvkDestroyDebugReportCallbackEXT); + vkCreateDebugUtilsMessengerEXT_Func = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr( + g_hVulkanInstance, "vkCreateDebugUtilsMessengerEXT"); + vkDestroyDebugUtilsMessengerEXT_Func = (PFN_vkDestroyDebugUtilsMessengerEXT)vkGetInstanceProcAddr( + g_hVulkanInstance, "vkDestroyDebugUtilsMessengerEXT"); + assert(vkCreateDebugUtilsMessengerEXT_Func); + assert(vkDestroyDebugUtilsMessengerEXT_Func); - VkDebugReportCallbackCreateInfoEXT callbackCreateInfo; - callbackCreateInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT; - callbackCreateInfo.pNext = nullptr; - callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | - VK_DEBUG_REPORT_ERROR_BIT_EXT | - VK_DEBUG_REPORT_WARNING_BIT_EXT | - VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| - VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; - callbackCreateInfo.pfnCallback = &MyDebugReportCallback; - callbackCreateInfo.pUserData = nullptr; + VkDebugUtilsMessengerCreateInfoEXT messengerCreateInfo = { VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT }; + messengerCreateInfo.messageSeverity = DEBUG_UTILS_MESSENGER_MESSAGE_SEVERITY; + messengerCreateInfo.messageType = DEBUG_UTILS_MESSENGER_MESSAGE_TYPE; + messengerCreateInfo.pfnUserCallback = MyDebugReportCallback; + ERR_GUARD_VULKAN( vkCreateDebugUtilsMessengerEXT_Func(g_hVulkanInstance, &messengerCreateInfo, g_Allocs, &g_DebugUtilsMessenger) ); +} - ERR_GUARD_VULKAN( g_pvkCreateDebugReportCallbackEXT(g_hVulkanInstance, &callbackCreateInfo, nullptr, &g_hCallback) ); +static void UnregisterDebugCallbacks() +{ + if(g_DebugUtilsMessenger) + { + vkDestroyDebugUtilsMessengerEXT_Func(g_hVulkanInstance, g_DebugUtilsMessenger, g_Allocs); + } } static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, const char* pLayerName) @@ -721,9 +691,9 @@ static void CreateSwapchain() } VkSwapchainKHR hNewSwapchain = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateSwapchainKHR(g_hDevice, &swapChainInfo, nullptr, &hNewSwapchain) ); + ERR_GUARD_VULKAN( vkCreateSwapchainKHR(g_hDevice, &swapChainInfo, g_Allocs, &hNewSwapchain) ); if(g_hSwapchain != VK_NULL_HANDLE) - vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); + vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, g_Allocs); g_hSwapchain = hNewSwapchain; // Retrieve swapchain images. @@ -736,7 +706,7 @@ static void CreateSwapchain() // Create swapchain image views. for(size_t i = g_SwapchainImageViews.size(); i--; ) - vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); + vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], g_Allocs); g_SwapchainImageViews.clear(); VkImageViewCreateInfo swapchainImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; @@ -755,7 +725,7 @@ static void CreateSwapchain() swapchainImageViewInfo.subresourceRange.levelCount = 1; swapchainImageViewInfo.subresourceRange.baseArrayLayer = 0; swapchainImageViewInfo.subresourceRange.layerCount = 1; - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &swapchainImageViewInfo, nullptr, &g_SwapchainImageViews[i]) ); + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &swapchainImageViewInfo, g_Allocs, &g_SwapchainImageViews[i]) ); } // Create depth buffer @@ -793,13 +763,13 @@ static void CreateSwapchain() depthImageViewInfo.subresourceRange.baseArrayLayer = 0; depthImageViewInfo.subresourceRange.layerCount = 1; - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &depthImageViewInfo, nullptr, &g_hDepthImageView) ); + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &depthImageViewInfo, g_Allocs, &g_hDepthImageView) ); // Create pipeline layout { if(g_hPipelineLayout != VK_NULL_HANDLE) { - vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); + vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, g_Allocs); g_hPipelineLayout = VK_NULL_HANDLE; } @@ -815,14 +785,14 @@ static void CreateSwapchain() pipelineLayoutInfo.pSetLayouts = descriptorSetLayouts; pipelineLayoutInfo.pushConstantRangeCount = 1; pipelineLayoutInfo.pPushConstantRanges = pushConstantRanges; - ERR_GUARD_VULKAN( vkCreatePipelineLayout(g_hDevice, &pipelineLayoutInfo, nullptr, &g_hPipelineLayout) ); + ERR_GUARD_VULKAN( vkCreatePipelineLayout(g_hDevice, &pipelineLayoutInfo, g_Allocs, &g_hPipelineLayout) ); } // Create render pass { if(g_hRenderPass != VK_NULL_HANDLE) { - vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); + vkDestroyRenderPass(g_hDevice, g_hRenderPass, g_Allocs); g_hRenderPass = VK_NULL_HANDLE; } @@ -867,7 +837,7 @@ static void CreateSwapchain() renderPassInfo.subpassCount = 1; renderPassInfo.pSubpasses = &subpassDesc; renderPassInfo.dependencyCount = 0; - ERR_GUARD_VULKAN( vkCreateRenderPass(g_hDevice, &renderPassInfo, nullptr, &g_hRenderPass) ); + ERR_GUARD_VULKAN( vkCreateRenderPass(g_hDevice, &renderPassInfo, g_Allocs, &g_hRenderPass) ); } // Create pipeline @@ -878,14 +848,14 @@ static void CreateSwapchain() shaderModuleInfo.codeSize = vertShaderCode.size(); shaderModuleInfo.pCode = (const uint32_t*)vertShaderCode.data(); VkShaderModule hVertShaderModule = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &hVertShaderModule) ); + ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, g_Allocs, &hVertShaderModule) ); std::vector hFragShaderCode; LoadShader(hFragShaderCode, "Shader.frag.spv"); shaderModuleInfo.codeSize = hFragShaderCode.size(); shaderModuleInfo.pCode = (const uint32_t*)hFragShaderCode.data(); VkShaderModule fragShaderModule = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &fragShaderModule) ); + ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, g_Allocs, &fragShaderModule) ); VkPipelineShaderStageCreateInfo vertPipelineShaderStageInfo = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO }; vertPipelineShaderStageInfo.stage = VK_SHADER_STAGE_VERTEX_BIT; @@ -1021,17 +991,18 @@ static void CreateSwapchain() g_hDevice, VK_NULL_HANDLE, 1, - &pipelineInfo, nullptr, + &pipelineInfo, + g_Allocs, &g_hPipeline) ); - vkDestroyShaderModule(g_hDevice, fragShaderModule, nullptr); - vkDestroyShaderModule(g_hDevice, hVertShaderModule, nullptr); + vkDestroyShaderModule(g_hDevice, fragShaderModule, g_Allocs); + vkDestroyShaderModule(g_hDevice, hVertShaderModule, g_Allocs); } // Create frambuffers for(size_t i = g_Framebuffers.size(); i--; ) - vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); + vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], g_Allocs); g_Framebuffers.clear(); g_Framebuffers.resize(g_SwapchainImageViews.size()); @@ -1046,47 +1017,47 @@ static void CreateSwapchain() framebufferInfo.width = g_Extent.width; framebufferInfo.height = g_Extent.height; framebufferInfo.layers = 1; - ERR_GUARD_VULKAN( vkCreateFramebuffer(g_hDevice, &framebufferInfo, nullptr, &g_Framebuffers[i]) ); + ERR_GUARD_VULKAN( vkCreateFramebuffer(g_hDevice, &framebufferInfo, g_Allocs, &g_Framebuffers[i]) ); } // Create semaphores if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) { - vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); + vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, g_Allocs); g_hImageAvailableSemaphore = VK_NULL_HANDLE; } if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) { - vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); + vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, g_Allocs); g_hRenderFinishedSemaphore = VK_NULL_HANDLE; } VkSemaphoreCreateInfo semaphoreInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO }; - ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hImageAvailableSemaphore) ); - ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hRenderFinishedSemaphore) ); + ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, g_Allocs, &g_hImageAvailableSemaphore) ); + ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, g_Allocs, &g_hRenderFinishedSemaphore) ); } static void DestroySwapchain(bool destroyActualSwapchain) { if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) { - vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); + vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, g_Allocs); g_hImageAvailableSemaphore = VK_NULL_HANDLE; } if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) { - vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); + vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, g_Allocs); g_hRenderFinishedSemaphore = VK_NULL_HANDLE; } for(size_t i = g_Framebuffers.size(); i--; ) - vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); + vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], g_Allocs); g_Framebuffers.clear(); if(g_hDepthImageView != VK_NULL_HANDLE) { - vkDestroyImageView(g_hDevice, g_hDepthImageView, nullptr); + vkDestroyImageView(g_hDevice, g_hDepthImageView, g_Allocs); g_hDepthImageView = VK_NULL_HANDLE; } if(g_hDepthImage != VK_NULL_HANDLE) @@ -1097,35 +1068,136 @@ static void DestroySwapchain(bool destroyActualSwapchain) if(g_hPipeline != VK_NULL_HANDLE) { - vkDestroyPipeline(g_hDevice, g_hPipeline, nullptr); + vkDestroyPipeline(g_hDevice, g_hPipeline, g_Allocs); g_hPipeline = VK_NULL_HANDLE; } if(g_hRenderPass != VK_NULL_HANDLE) { - vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); + vkDestroyRenderPass(g_hDevice, g_hRenderPass, g_Allocs); g_hRenderPass = VK_NULL_HANDLE; } if(g_hPipelineLayout != VK_NULL_HANDLE) { - vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); + vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, g_Allocs); g_hPipelineLayout = VK_NULL_HANDLE; } for(size_t i = g_SwapchainImageViews.size(); i--; ) - vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); + vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], g_Allocs); g_SwapchainImageViews.clear(); if(destroyActualSwapchain && (g_hSwapchain != VK_NULL_HANDLE)) { - vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); + vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, g_Allocs); g_hSwapchain = VK_NULL_HANDLE; } } +static constexpr uint32_t GetVulkanApiVersion() +{ +#if VMA_VULKAN_VERSION == 1002000 + return VK_API_VERSION_1_2; +#elif VMA_VULKAN_VERSION == 1001000 + return VK_API_VERSION_1_1; +#elif VMA_VULKAN_VERSION == 1000000 + return VK_API_VERSION_1_0; +#else + #error Invalid VMA_VULKAN_VERSION. + return UINT32_MAX; +#endif +} + +static void PrintEnabledFeatures() +{ + wprintf(L"Validation layer: %d\n", g_EnableValidationLayer ? 1 : 0); + wprintf(L"Sparse binding: %d\n", g_SparseBindingEnabled ? 1 : 0); + wprintf(L"Buffer device address: %d\n", g_BufferDeviceAddressEnabled ? 1 : 0); + if(GetVulkanApiVersion() == VK_API_VERSION_1_0) + { + wprintf(L"VK_KHR_get_memory_requirements2: %d\n", VK_KHR_get_memory_requirements2_enabled ? 1 : 0); + wprintf(L"VK_KHR_get_physical_device_properties2: %d\n", VK_KHR_get_physical_device_properties2_enabled ? 1 : 0); + wprintf(L"VK_KHR_dedicated_allocation: %d\n", VK_KHR_dedicated_allocation_enabled ? 1 : 0); + wprintf(L"VK_KHR_bind_memory2: %d\n", VK_KHR_bind_memory2_enabled ? 1 : 0); + } + wprintf(L"VK_EXT_memory_budget: %d\n", VK_EXT_memory_budget_enabled ? 1 : 0); + wprintf(L"VK_AMD_device_coherent_memory: %d\n", VK_AMD_device_coherent_memory_enabled ? 1 : 0); + wprintf(L"VK_KHR_buffer_device_address: %d\n", VK_KHR_buffer_device_address_enabled ? 1 : 0); + wprintf(L"VK_EXT_buffer_device_address: %d\n", VK_EXT_buffer_device_address_enabled ? 1 : 0); +} + +void SetAllocatorCreateInfo(VmaAllocatorCreateInfo& outInfo) +{ + outInfo = {}; + + outInfo.physicalDevice = g_hPhysicalDevice; + outInfo.device = g_hDevice; + outInfo.instance = g_hVulkanInstance; + outInfo.vulkanApiVersion = GetVulkanApiVersion(); + + if(VK_KHR_dedicated_allocation_enabled) + { + outInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; + } + if(VK_KHR_bind_memory2_enabled) + { + outInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT; + } +#if !defined(VMA_MEMORY_BUDGET) || VMA_MEMORY_BUDGET == 1 + if(VK_EXT_memory_budget_enabled && ( + GetVulkanApiVersion() >= VK_API_VERSION_1_1 || VK_KHR_get_physical_device_properties2_enabled)) + { + outInfo.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; + } +#endif + if(VK_AMD_device_coherent_memory_enabled) + { + outInfo.flags |= VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT; + } + if(g_BufferDeviceAddressEnabled) + { + outInfo.flags |= VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; + } + + if(USE_CUSTOM_CPU_ALLOCATION_CALLBACKS) + { + outInfo.pAllocationCallbacks = &g_CpuAllocationCallbacks; + } + + // Uncomment to enable recording to CSV file. + /* + static VmaRecordSettings recordSettings = {}; + recordSettings.pFilePath = "VulkanSample.csv"; + outInfo.pRecordSettings = &recordSettings; + */ + + // Uncomment to enable HeapSizeLimit. + /* + static std::array heapSizeLimit; + std::fill(heapSizeLimit.begin(), heapSizeLimit.end(), VK_WHOLE_SIZE); + heapSizeLimit[0] = 512ull * 1024 * 1024; + outInfo.pHeapSizeLimit = heapSizeLimit.data(); + */ +} + +static void PrintPhysicalDeviceProperties(const VkPhysicalDeviceProperties& properties) +{ + wprintf(L"Physical device:\n"); + wprintf(L" Driver version: 0x%X\n", properties.driverVersion); + wprintf(L" Vendor ID: 0x%X\n", properties.vendorID); + wprintf(L" Device ID: 0x%X\n", properties.deviceID); + wprintf(L" Device type: %u\n", properties.deviceType); + wprintf(L" Device name: %hs\n", properties.deviceName); +} + static void InitializeApplication() { + if(USE_CUSTOM_CPU_ALLOCATION_CALLBACKS) + { + g_Allocs = &g_CpuAllocationCallbacks; + } + uint32_t instanceLayerPropCount = 0; ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr) ); std::vector instanceLayerProps(instanceLayerPropCount); @@ -1134,24 +1206,48 @@ static void InitializeApplication() ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()) ); } - if(g_EnableValidationLayer == true) + if(g_EnableValidationLayer) { if(IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME) == false) { - printf("Layer \"%s\" not supported.", VALIDATION_LAYER_NAME); + wprintf(L"Layer \"%hs\" not supported.", VALIDATION_LAYER_NAME); g_EnableValidationLayer = false; } } - std::vector instanceExtensions; - instanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); - instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); + uint32_t availableInstanceExtensionCount = 0; + ERR_GUARD_VULKAN( vkEnumerateInstanceExtensionProperties(nullptr, &availableInstanceExtensionCount, nullptr) ); + std::vector availableInstanceExtensions(availableInstanceExtensionCount); + if(availableInstanceExtensionCount > 0) + { + ERR_GUARD_VULKAN( vkEnumerateInstanceExtensionProperties(nullptr, &availableInstanceExtensionCount, availableInstanceExtensions.data()) ); + } + + std::vector enabledInstanceExtensions; + enabledInstanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); + enabledInstanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); std::vector instanceLayers; - if(g_EnableValidationLayer == true) + if(g_EnableValidationLayer) { instanceLayers.push_back(VALIDATION_LAYER_NAME); - instanceExtensions.push_back("VK_EXT_debug_report"); + } + + for(const auto& extensionProperties : availableInstanceExtensions) + { + if(strcmp(extensionProperties.extensionName, VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() == VK_API_VERSION_1_0) + { + enabledInstanceExtensions.push_back(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME); + VK_KHR_get_physical_device_properties2_enabled = true; + } + } + else if(strcmp(extensionProperties.extensionName, VK_EXT_DEBUG_UTILS_EXTENSION_NAME) == 0) + { + enabledInstanceExtensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + VK_EXT_debug_utils_enabled = true; + } } VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; @@ -1159,27 +1255,38 @@ static void InitializeApplication() appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0); appInfo.pEngineName = "Adam Sawicki Engine"; appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0); - appInfo.apiVersion = VK_API_VERSION_1_0; + appInfo.apiVersion = GetVulkanApiVersion(); VkInstanceCreateInfo instInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; instInfo.pApplicationInfo = &appInfo; - instInfo.enabledExtensionCount = static_cast(instanceExtensions.size()); - instInfo.ppEnabledExtensionNames = instanceExtensions.data(); + instInfo.enabledExtensionCount = static_cast(enabledInstanceExtensions.size()); + instInfo.ppEnabledExtensionNames = enabledInstanceExtensions.data(); instInfo.enabledLayerCount = static_cast(instanceLayers.size()); instInfo.ppEnabledLayerNames = instanceLayers.data(); - ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &g_hVulkanInstance) ); + wprintf(L"Vulkan API version: "); + switch(appInfo.apiVersion) + { + case VK_API_VERSION_1_0: wprintf(L"1.0\n"); break; + case VK_API_VERSION_1_1: wprintf(L"1.1\n"); break; + case VK_API_VERSION_1_2: wprintf(L"1.2\n"); break; + default: assert(0); + } + + ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, g_Allocs, &g_hVulkanInstance) ); + + if(VK_EXT_debug_utils_enabled) + { + RegisterDebugCallbacks(); + } // Create VkSurfaceKHR. VkWin32SurfaceCreateInfoKHR surfaceInfo = { VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR }; surfaceInfo.hinstance = g_hAppInstance; surfaceInfo.hwnd = g_hWnd; - VkResult result = vkCreateWin32SurfaceKHR(g_hVulkanInstance, &surfaceInfo, NULL, &g_hSurface); + VkResult result = vkCreateWin32SurfaceKHR(g_hVulkanInstance, &surfaceInfo, g_Allocs, &g_hSurface); assert(result == VK_SUCCESS); - if(g_EnableValidationLayer == true) - RegisterDebugCallbacks(); - // Find physical device uint32_t deviceCount = 0; @@ -1191,13 +1298,96 @@ static void InitializeApplication() g_hPhysicalDevice = physicalDevices[0]; + // Query for device extensions + + uint32_t physicalDeviceExtensionPropertyCount = 0; + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &physicalDeviceExtensionPropertyCount, nullptr) ); + std::vector physicalDeviceExtensionProperties{physicalDeviceExtensionPropertyCount}; + if(physicalDeviceExtensionPropertyCount) + { + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties( + g_hPhysicalDevice, + nullptr, + &physicalDeviceExtensionPropertyCount, + physicalDeviceExtensionProperties.data()) ); + } + + for(uint32_t i = 0; i < physicalDeviceExtensionPropertyCount; ++i) + { + if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() == VK_API_VERSION_1_0) + { + VK_KHR_get_memory_requirements2_enabled = true; + } + } + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() == VK_API_VERSION_1_0) + { + VK_KHR_dedicated_allocation_enabled = true; + } + } + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_KHR_BIND_MEMORY_2_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() == VK_API_VERSION_1_0) + { + VK_KHR_bind_memory2_enabled = true; + } + } + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME) == 0) + VK_EXT_memory_budget_enabled = true; + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_AMD_DEVICE_COHERENT_MEMORY_EXTENSION_NAME) == 0) + VK_AMD_device_coherent_memory_enabled = true; + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() < VK_API_VERSION_1_2) + { + VK_KHR_buffer_device_address_enabled = true; + } + } + else if(strcmp(physicalDeviceExtensionProperties[i].extensionName, VK_EXT_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME) == 0) + { + if(GetVulkanApiVersion() < VK_API_VERSION_1_2) + { + VK_EXT_buffer_device_address_enabled = true; + } + } + } + + if(VK_EXT_buffer_device_address_enabled && VK_KHR_buffer_device_address_enabled) + VK_EXT_buffer_device_address_enabled = false; + // Query for features VkPhysicalDeviceProperties physicalDeviceProperties = {}; vkGetPhysicalDeviceProperties(g_hPhysicalDevice, &physicalDeviceProperties); - //VkPhysicalDeviceFeatures physicalDeviceFreatures = {}; - //vkGetPhysicalDeviceFeatures(g_PhysicalDevice, &physicalDeviceFreatures); + PrintPhysicalDeviceProperties(physicalDeviceProperties); + + VkPhysicalDeviceFeatures2 physicalDeviceFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 }; + + VkPhysicalDeviceCoherentMemoryFeaturesAMD physicalDeviceCoherentMemoryFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COHERENT_MEMORY_FEATURES_AMD }; + if(VK_AMD_device_coherent_memory_enabled) + { + PnextChainPushFront(&physicalDeviceFeatures, &physicalDeviceCoherentMemoryFeatures); + } + + VkPhysicalDeviceBufferDeviceAddressFeaturesEXT physicalDeviceBufferDeviceAddressFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_EXT }; + if(VK_KHR_buffer_device_address_enabled || VK_EXT_buffer_device_address_enabled || GetVulkanApiVersion() >= VK_API_VERSION_1_2) + { + PnextChainPushFront(&physicalDeviceFeatures, &physicalDeviceBufferDeviceAddressFeatures); + } + + vkGetPhysicalDeviceFeatures2(g_hPhysicalDevice, &physicalDeviceFeatures); + + g_SparseBindingEnabled = physicalDeviceFeatures.features.sparseBinding != 0; + + // The extension is supported as fake with no real support for this feature? Don't use it. + if(VK_AMD_device_coherent_memory_enabled && !physicalDeviceCoherentMemoryFeatures.deviceCoherentMemory) + VK_AMD_device_coherent_memory_enabled = false; + if(VK_KHR_buffer_device_address_enabled || VK_EXT_buffer_device_address_enabled || GetVulkanApiVersion() >= VK_API_VERSION_1_2) + g_BufferDeviceAddressEnabled = physicalDeviceBufferDeviceAddressFeatures.bufferDeviceAddress != VK_FALSE; // Find queue family index @@ -1208,13 +1398,16 @@ static void InitializeApplication() vkGetPhysicalDeviceQueueFamilyProperties(g_hPhysicalDevice, &queueFamilyCount, queueFamilies.data()); for(uint32_t i = 0; (i < queueFamilyCount) && - (g_GraphicsQueueFamilyIndex == UINT_MAX || g_PresentQueueFamilyIndex == UINT_MAX); + (g_GraphicsQueueFamilyIndex == UINT_MAX || + g_PresentQueueFamilyIndex == UINT_MAX || + (g_SparseBindingEnabled && g_SparseBindingQueueFamilyIndex == UINT_MAX)); ++i) { if(queueFamilies[i].queueCount > 0) { + const uint32_t flagsForGraphicsQueue = VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT; if((g_GraphicsQueueFamilyIndex != 0) && - ((queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0)) + ((queueFamilies[i].queueFlags & flagsForGraphicsQueue) == flagsForGraphicsQueue)) { g_GraphicsQueueFamilyIndex = i; } @@ -1225,102 +1418,168 @@ static void InitializeApplication() { g_PresentQueueFamilyIndex = i; } + + if(g_SparseBindingEnabled && + g_SparseBindingQueueFamilyIndex == UINT32_MAX && + (queueFamilies[i].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0) + { + g_SparseBindingQueueFamilyIndex = i; + } } } assert(g_GraphicsQueueFamilyIndex != UINT_MAX); + g_SparseBindingEnabled = g_SparseBindingEnabled && g_SparseBindingQueueFamilyIndex != UINT32_MAX; + // Create logical device const float queuePriority = 1.f; - VkDeviceQueueCreateInfo deviceQueueCreateInfo[2] = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; - deviceQueueCreateInfo[0].queueFamilyIndex = g_GraphicsQueueFamilyIndex; - deviceQueueCreateInfo[0].queueCount = 1; - deviceQueueCreateInfo[0].pQueuePriorities = &queuePriority; - deviceQueueCreateInfo[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; - deviceQueueCreateInfo[1].queueFamilyIndex = g_PresentQueueFamilyIndex; - deviceQueueCreateInfo[1].queueCount = 1; - deviceQueueCreateInfo[1].pQueuePriorities = &queuePriority; + VkDeviceQueueCreateInfo queueCreateInfo[3] = {}; + uint32_t queueCount = 1; + queueCreateInfo[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queueCreateInfo[0].queueFamilyIndex = g_GraphicsQueueFamilyIndex; + queueCreateInfo[0].queueCount = 1; + queueCreateInfo[0].pQueuePriorities = &queuePriority; + + if(g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex) + { - VkPhysicalDeviceFeatures deviceFeatures = {}; - deviceFeatures.fillModeNonSolid = VK_TRUE; - deviceFeatures.samplerAnisotropy = VK_TRUE; + queueCreateInfo[queueCount].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queueCreateInfo[queueCount].queueFamilyIndex = g_PresentQueueFamilyIndex; + queueCreateInfo[queueCount].queueCount = 1; + queueCreateInfo[queueCount].pQueuePriorities = &queuePriority; + ++queueCount; + } + + if(g_SparseBindingEnabled && + g_SparseBindingQueueFamilyIndex != g_GraphicsQueueFamilyIndex && + g_SparseBindingQueueFamilyIndex != g_PresentQueueFamilyIndex) + { + + queueCreateInfo[queueCount].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queueCreateInfo[queueCount].queueFamilyIndex = g_SparseBindingQueueFamilyIndex; + queueCreateInfo[queueCount].queueCount = 1; + queueCreateInfo[queueCount].pQueuePriorities = &queuePriority; + ++queueCount; + } - // Determine list of device extensions to enable. std::vector enabledDeviceExtensions; enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); + if(VK_KHR_get_memory_requirements2_enabled) + enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); + if(VK_KHR_dedicated_allocation_enabled) + enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); + if(VK_KHR_bind_memory2_enabled) + enabledDeviceExtensions.push_back(VK_KHR_BIND_MEMORY_2_EXTENSION_NAME); + if(VK_EXT_memory_budget_enabled) + enabledDeviceExtensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); + if(VK_AMD_device_coherent_memory_enabled) + enabledDeviceExtensions.push_back(VK_AMD_DEVICE_COHERENT_MEMORY_EXTENSION_NAME); + if(VK_KHR_buffer_device_address_enabled) + enabledDeviceExtensions.push_back(VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); + if(VK_EXT_buffer_device_address_enabled) + enabledDeviceExtensions.push_back(VK_EXT_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); + + VkPhysicalDeviceFeatures2 deviceFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 }; + deviceFeatures.features.samplerAnisotropy = VK_TRUE; + deviceFeatures.features.sparseBinding = g_SparseBindingEnabled ? VK_TRUE : VK_FALSE; + + if(VK_AMD_device_coherent_memory_enabled) { - uint32_t propertyCount = 0; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, nullptr) ); - - if(propertyCount) - { - std::vector properties{propertyCount}; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, properties.data()) ); - - for(uint32_t i = 0; i < propertyCount; ++i) - { - if(strcmp(properties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) - { - enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); - VK_KHR_get_memory_requirements2_enabled = true; - } - else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) - { - enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); - VK_KHR_dedicated_allocation_enabled = true; - } - } - } + physicalDeviceCoherentMemoryFeatures.deviceCoherentMemory = VK_TRUE; + PnextChainPushBack(&deviceFeatures, &physicalDeviceCoherentMemoryFeatures); + } + if(g_BufferDeviceAddressEnabled) + { + physicalDeviceBufferDeviceAddressFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_EXT }; + physicalDeviceBufferDeviceAddressFeatures.bufferDeviceAddress = VK_TRUE; + PnextChainPushBack(&deviceFeatures, &physicalDeviceBufferDeviceAddressFeatures); } VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; + deviceCreateInfo.pNext = &deviceFeatures; deviceCreateInfo.enabledLayerCount = 0; deviceCreateInfo.ppEnabledLayerNames = nullptr; deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; - deviceCreateInfo.queueCreateInfoCount = g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex ? 2 : 1; - deviceCreateInfo.pQueueCreateInfos = deviceQueueCreateInfo; - deviceCreateInfo.pEnabledFeatures = &deviceFeatures; + deviceCreateInfo.queueCreateInfoCount = queueCount; + deviceCreateInfo.pQueueCreateInfos = queueCreateInfo; - ERR_GUARD_VULKAN( vkCreateDevice(g_hPhysicalDevice, &deviceCreateInfo, nullptr, &g_hDevice) ); + ERR_GUARD_VULKAN( vkCreateDevice(g_hPhysicalDevice, &deviceCreateInfo, g_Allocs, &g_hDevice) ); + + // Fetch pointers to extension functions + if(g_BufferDeviceAddressEnabled) + { + if(GetVulkanApiVersion() >= VK_API_VERSION_1_2) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetDeviceProcAddr(g_hDevice, "vkGetBufferDeviceAddress"); + //assert(g_vkGetBufferDeviceAddressEXT != nullptr); + /* + For some reason this doesn't work, the pointer is NULL :( None of the below methods help. + + Validation layers also report following error: + [ VUID-VkMemoryAllocateInfo-flags-03331 ] Object: VK_NULL_HANDLE (Type = 0) | If VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR is set, bufferDeviceAddress must be enabled. The Vulkan spec states: If VkMemoryAllocateFlagsInfo::flags includes VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, the bufferDeviceAddress feature must be enabled (https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/vkspec.html#VUID-VkMemoryAllocateInfo-flags-03331) + Despite I'm posting VkPhysicalDeviceBufferDeviceAddressFeaturesEXT::bufferDeviceAddress = VK_TRUE in VkDeviceCreateInfo::pNext chain. + + if(g_vkGetBufferDeviceAddressEXT == nullptr) + { + g_vkGetBufferDeviceAddressEXT = &vkGetBufferDeviceAddress; // Doesn't run, cannot find entry point... + } + + if(g_vkGetBufferDeviceAddressEXT == nullptr) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetInstanceProcAddr(g_hVulkanInstance, "vkGetBufferDeviceAddress"); + } + if(g_vkGetBufferDeviceAddressEXT == nullptr) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetDeviceProcAddr(g_hDevice, "vkGetBufferDeviceAddressKHR"); + } + if(g_vkGetBufferDeviceAddressEXT == nullptr) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetDeviceProcAddr(g_hDevice, "vkGetBufferDeviceAddressEXT"); + } + */ + } + else if(VK_KHR_buffer_device_address_enabled) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetDeviceProcAddr(g_hDevice, "vkGetBufferDeviceAddressKHR"); + assert(g_vkGetBufferDeviceAddressEXT != nullptr); + } + else if(VK_EXT_buffer_device_address_enabled) + { + g_vkGetBufferDeviceAddressEXT = (PFN_vkGetBufferDeviceAddressEXT)vkGetDeviceProcAddr(g_hDevice, "vkGetBufferDeviceAddressEXT"); + assert(g_vkGetBufferDeviceAddressEXT != nullptr); + } + } // Create memory allocator VmaAllocatorCreateInfo allocatorInfo = {}; - allocatorInfo.physicalDevice = g_hPhysicalDevice; - allocatorInfo.device = g_hDevice; - - if(VK_KHR_dedicated_allocation_enabled) - { - allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; - } - - VkAllocationCallbacks cpuAllocationCallbacks = {}; - if(USE_CUSTOM_CPU_ALLOCATION_CALLBACKS) - { - cpuAllocationCallbacks.pUserData = CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA; - cpuAllocationCallbacks.pfnAllocation = &CustomCpuAllocation; - cpuAllocationCallbacks.pfnReallocation = &CustomCpuReallocation; - cpuAllocationCallbacks.pfnFree = &CustomCpuFree; - allocatorInfo.pAllocationCallbacks = &cpuAllocationCallbacks; - } - + SetAllocatorCreateInfo(allocatorInfo); ERR_GUARD_VULKAN( vmaCreateAllocator(&allocatorInfo, &g_hAllocator) ); - // Retrieve queue (doesn't need to be destroyed) + PrintEnabledFeatures(); + + // Retrieve queues (don't need to be destroyed). vkGetDeviceQueue(g_hDevice, g_GraphicsQueueFamilyIndex, 0, &g_hGraphicsQueue); vkGetDeviceQueue(g_hDevice, g_PresentQueueFamilyIndex, 0, &g_hPresentQueue); assert(g_hGraphicsQueue); assert(g_hPresentQueue); + if(g_SparseBindingEnabled) + { + vkGetDeviceQueue(g_hDevice, g_SparseBindingQueueFamilyIndex, 0, &g_hSparseBindingQueue); + assert(g_hSparseBindingQueue); + } + // Create command pool VkCommandPoolCreateInfo commandPoolInfo = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; commandPoolInfo.queueFamilyIndex = g_GraphicsQueueFamilyIndex; commandPoolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - ERR_GUARD_VULKAN( vkCreateCommandPool(g_hDevice, &commandPoolInfo, nullptr, &g_hCommandPool) ); + ERR_GUARD_VULKAN( vkCreateCommandPool(g_hDevice, &commandPoolInfo, g_Allocs, &g_hCommandPool) ); VkCommandBufferAllocateInfo commandBufferInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; commandBufferInfo.commandPool = g_hCommandPool; @@ -1332,9 +1591,11 @@ static void InitializeApplication() fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT; for(size_t i = 0; i < COMMAND_BUFFER_COUNT; ++i) { - ERR_GUARD_VULKAN( vkCreateFence(g_hDevice, &fenceInfo, nullptr, &g_MainCommandBufferExecutedFances[i]) ); + ERR_GUARD_VULKAN( vkCreateFence(g_hDevice, &fenceInfo, g_Allocs, &g_MainCommandBufferExecutedFances[i]) ); } + ERR_GUARD_VULKAN( vkCreateFence(g_hDevice, &fenceInfo, g_Allocs, &g_ImmediateFence) ); + commandBufferInfo.commandBufferCount = 1; ERR_GUARD_VULKAN( vkAllocateCommandBuffers(g_hDevice, &commandBufferInfo, &g_hTemporaryCommandBuffer) ); @@ -1356,7 +1617,7 @@ static void InitializeApplication() samplerInfo.mipLodBias = 0.f; samplerInfo.minLod = 0.f; samplerInfo.maxLod = FLT_MAX; - ERR_GUARD_VULKAN( vkCreateSampler(g_hDevice, &samplerInfo, nullptr, &g_hSampler) ); + ERR_GUARD_VULKAN( vkCreateSampler(g_hDevice, &samplerInfo, g_Allocs, &g_hSampler) ); CreateTexture(128, 128); CreateMesh(); @@ -1370,7 +1631,7 @@ static void InitializeApplication() VkDescriptorSetLayoutCreateInfo descriptorSetLayoutInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO }; descriptorSetLayoutInfo.bindingCount = 1; descriptorSetLayoutInfo.pBindings = &samplerLayoutBinding; - ERR_GUARD_VULKAN( vkCreateDescriptorSetLayout(g_hDevice, &descriptorSetLayoutInfo, nullptr, &g_hDescriptorSetLayout) ); + ERR_GUARD_VULKAN( vkCreateDescriptorSetLayout(g_hDevice, &descriptorSetLayoutInfo, g_Allocs, &g_hDescriptorSetLayout) ); // Create descriptor pool @@ -1385,7 +1646,7 @@ static void InitializeApplication() descriptorPoolInfo.poolSizeCount = (uint32_t)_countof(descriptorPoolSizes); descriptorPoolInfo.pPoolSizes = descriptorPoolSizes; descriptorPoolInfo.maxSets = 1; - ERR_GUARD_VULKAN( vkCreateDescriptorPool(g_hDevice, &descriptorPoolInfo, nullptr, &g_hDescriptorPool) ); + ERR_GUARD_VULKAN( vkCreateDescriptorPool(g_hDevice, &descriptorPoolInfo, g_Allocs, &g_hDescriptorPool) ); // Create descriptor set layout @@ -1422,19 +1683,19 @@ static void FinalizeApplication() if(g_hDescriptorPool != VK_NULL_HANDLE) { - vkDestroyDescriptorPool(g_hDevice, g_hDescriptorPool, nullptr); + vkDestroyDescriptorPool(g_hDevice, g_hDescriptorPool, g_Allocs); g_hDescriptorPool = VK_NULL_HANDLE; } if(g_hDescriptorSetLayout != VK_NULL_HANDLE) { - vkDestroyDescriptorSetLayout(g_hDevice, g_hDescriptorSetLayout, nullptr); + vkDestroyDescriptorSetLayout(g_hDevice, g_hDescriptorSetLayout, g_Allocs); g_hDescriptorSetLayout = VK_NULL_HANDLE; } if(g_hTextureImageView != VK_NULL_HANDLE) { - vkDestroyImageView(g_hDevice, g_hTextureImageView, nullptr); + vkDestroyImageView(g_hDevice, g_hTextureImageView, g_Allocs); g_hTextureImageView = VK_NULL_HANDLE; } if(g_hTextureImage != VK_NULL_HANDLE) @@ -1456,15 +1717,21 @@ static void FinalizeApplication() if(g_hSampler != VK_NULL_HANDLE) { - vkDestroySampler(g_hDevice, g_hSampler, nullptr); + vkDestroySampler(g_hDevice, g_hSampler, g_Allocs); g_hSampler = VK_NULL_HANDLE; } + if(g_ImmediateFence) + { + vkDestroyFence(g_hDevice, g_ImmediateFence, g_Allocs); + g_ImmediateFence = VK_NULL_HANDLE; + } + for(size_t i = COMMAND_BUFFER_COUNT; i--; ) { if(g_MainCommandBufferExecutedFances[i] != VK_NULL_HANDLE) { - vkDestroyFence(g_hDevice, g_MainCommandBufferExecutedFances[i], nullptr); + vkDestroyFence(g_hDevice, g_MainCommandBufferExecutedFances[i], g_Allocs); g_MainCommandBufferExecutedFances[i] = VK_NULL_HANDLE; } } @@ -1481,7 +1748,7 @@ static void FinalizeApplication() if(g_hCommandPool != VK_NULL_HANDLE) { - vkDestroyCommandPool(g_hDevice, g_hCommandPool, nullptr); + vkDestroyCommandPool(g_hDevice, g_hCommandPool, g_Allocs); g_hCommandPool = VK_NULL_HANDLE; } @@ -1493,25 +1760,21 @@ static void FinalizeApplication() if(g_hDevice != VK_NULL_HANDLE) { - vkDestroyDevice(g_hDevice, nullptr); + vkDestroyDevice(g_hDevice, g_Allocs); g_hDevice = nullptr; } - if(g_pvkDestroyDebugReportCallbackEXT && g_hCallback != VK_NULL_HANDLE) - { - g_pvkDestroyDebugReportCallbackEXT(g_hVulkanInstance, g_hCallback, nullptr); - g_hCallback = VK_NULL_HANDLE; - } - if(g_hSurface != VK_NULL_HANDLE) { - vkDestroySurfaceKHR(g_hVulkanInstance, g_hSurface, NULL); + vkDestroySurfaceKHR(g_hVulkanInstance, g_hSurface, g_Allocs); g_hSurface = VK_NULL_HANDLE; } + UnregisterDebugCallbacks(); + if(g_hVulkanInstance != VK_NULL_HANDLE) { - vkDestroyInstance(g_hVulkanInstance, NULL); + vkDestroyInstance(g_hVulkanInstance, g_Allocs); g_hVulkanInstance = VK_NULL_HANDLE; } } @@ -1714,7 +1977,31 @@ static LRESULT WINAPI WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) PostMessage(hWnd, WM_CLOSE, 0, 0); break; case 'T': - Test(); + try + { + Test(); + } + catch(const std::exception& ex) + { + printf("ERROR: %s\n", ex.what()); + } + break; + case 'S': + try + { + if(g_SparseBindingEnabled) + { + TestSparseBinding(); + } + else + { + printf("Sparse binding not supported.\n"); + } + } + catch(const std::exception& ex) + { + printf("ERROR: %s\n", ex.what()); + } break; } return 0; @@ -1767,6 +2054,8 @@ int main() DrawFrame(); } + TEST(g_CpuAllocCount.load() == 0); + return 0; } diff --git a/third_party/vkmemalloc/src/vk_mem_alloc.h b/third_party/vkmemalloc/src/vk_mem_alloc.h index 5b8fcc0c25..2d1134daaa 100644 --- a/third_party/vkmemalloc/src/vk_mem_alloc.h +++ b/third_party/vkmemalloc/src/vk_mem_alloc.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,15 +23,11 @@ #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H #define AMD_VULKAN_MEMORY_ALLOCATOR_H -#ifdef __cplusplus -extern "C" { -#endif - /** \mainpage Vulkan Memory Allocator -Version 2.1.0-alpha.4 (2018-08-22) +Version 3.0.0-development (2020-06-24) -Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. \n +Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT Documentation of all members: vk_mem_alloc.h @@ -48,14 +44,29 @@ Documentation of all members: vk_mem_alloc.h - [Required and preferred flags](@ref choosing_memory_type_required_preferred_flags) - [Explicit memory types](@ref choosing_memory_type_explicit_memory_types) - [Custom memory pools](@ref choosing_memory_type_custom_memory_pools) + - [Dedicated allocations](@ref choosing_memory_type_dedicated_allocations) - \subpage memory_mapping - [Mapping functions](@ref memory_mapping_mapping_functions) - [Persistently mapped memory](@ref memory_mapping_persistently_mapped_memory) - - [Cache control](@ref memory_mapping_cache_control) + - [Cache flush and invalidate](@ref memory_mapping_cache_control) - [Finding out if memory is mappable](@ref memory_mapping_finding_if_memory_mappable) + - \subpage staying_within_budget + - [Querying for budget](@ref staying_within_budget_querying_for_budget) + - [Controlling memory usage](@ref staying_within_budget_controlling_memory_usage) + - \subpage resource_aliasing - \subpage custom_memory_pools - [Choosing memory type index](@ref custom_memory_pools_MemTypeIndex) + - [Linear allocation algorithm](@ref linear_algorithm) + - [Free-at-once](@ref linear_algorithm_free_at_once) + - [Stack](@ref linear_algorithm_stack) + - [Double stack](@ref linear_algorithm_double_stack) + - [Ring buffer](@ref linear_algorithm_ring_buffer) + - [Buddy allocation algorithm](@ref buddy_algorithm) - \subpage defragmentation + - [Defragmenting CPU memory](@ref defragmentation_cpu) + - [Defragmenting GPU memory](@ref defragmentation_gpu) + - [Additional notes](@ref defragmentation_additional_notes) + - [Writing custom allocation algorithm](@ref defragmentation_custom_algorithm) - \subpage lost_allocations - \subpage statistics - [Numeric statistics](@ref statistics_numeric_statistics) @@ -69,6 +80,7 @@ Documentation of all members: vk_mem_alloc.h - [Corruption detection](@ref debugging_memory_usage_corruption_detection) - \subpage record_and_replay - \subpage usage_patterns + - [Common mistakes](@ref usage_patterns_common_mistakes) - [Simple patterns](@ref usage_patterns_simple) - [Advanced patterns](@ref usage_patterns_advanced) - \subpage configuration @@ -77,8 +89,11 @@ Documentation of all members: vk_mem_alloc.h - [Device memory allocation callbacks](@ref allocation_callbacks) - [Device heap memory limit](@ref heap_memory_limit) - \subpage vk_khr_dedicated_allocation + - \subpage enabling_buffer_device_address + - \subpage vk_amd_device_coherent_memory - \subpage general_considerations - [Thread safety](@ref general_considerations_thread_safety) + - [Validation layer warnings](@ref general_considerations_validation_layer_warnings) - [Allocation algorithm](@ref general_considerations_allocation_algorithm) - [Features not supported](@ref general_considerations_features_not_supported) @@ -94,7 +109,7 @@ Documentation of all members: vk_mem_alloc.h \section quick_start_project_setup Project setup -Vulkan Memory Allocator comes in form of a single header file. +Vulkan Memory Allocator comes in form of a "stb-style" single header file. You don't need to build it as a separate library project. You can add this file directly to your project and submit it to code repository next to your other source files. @@ -117,11 +132,22 @@ To do it properly: It may be a good idea to create dedicated CPP file just for this purpose. +Note on language: This library is written in C++, but has C-compatible interface. +Thus you can include and use vk_mem_alloc.h in C or C++ code, but full +implementation with `VMA_IMPLEMENTATION` macro must be compiled as C++, NOT as C. + +Please note that this library includes header ``, which in turn +includes `` on Windows. If you need some specific macros defined +before including these headers (like `WIN32_LEAN_AND_MEAN` or +`WINVER` for Windows, `VK_USE_PLATFORM_WIN32_KHR` for Vulkan), you must define +them before every `#include` of this library. + + \section quick_start_initialization Initialization At program startup: --# Initialize Vulkan to have `VkPhysicalDevice` and `VkDevice` object. +-# Initialize Vulkan to have `VkPhysicalDevice`, `VkDevice` and `VkInstance` object. -# Fill VmaAllocatorCreateInfo structure and create #VmaAllocator object by calling vmaCreateAllocator(). @@ -129,6 +155,7 @@ At program startup: VmaAllocatorCreateInfo allocatorInfo = {}; allocatorInfo.physicalDevice = physicalDevice; allocatorInfo.device = device; +allocatorInfo.instance = instance; VmaAllocator allocator; vmaCreateAllocator(&allocatorInfo, &allocator); @@ -173,17 +200,21 @@ appropriate members of VmaAllocationCreateInfo structure, as described below. You can also combine multiple methods. -# If you just want to find memory type index that meets your requirements, you - can use function vmaFindMemoryTypeIndex(). + can use function: vmaFindMemoryTypeIndex(), vmaFindMemoryTypeIndexForBufferInfo(), + vmaFindMemoryTypeIndexForImageInfo(). -# If you want to allocate a region of device memory without association with any specific image or buffer, you can use function vmaAllocateMemory(). Usage of this function is not recommended and usually not needed. + vmaAllocateMemoryPages() function is also provided for creating multiple allocations at once, + which may be useful for sparse binding. -# If you already have a buffer or an image created, you want to allocate memory for it and then you will bind it yourself, you can use function vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(). - For binding you should use functions: vmaBindBufferMemory(), vmaBindImageMemory(). + For binding you should use functions: vmaBindBufferMemory(), vmaBindImageMemory() + or their extended versions: vmaBindBufferMemory2(), vmaBindImageMemory2(). -# If you want to create a buffer or an image, allocate memory for it and bind them together, all in one call, you can use function vmaCreateBuffer(), - vmaCreateImage(). This is the recommended way to use this library. + vmaCreateImage(). This is the easiest and recommended way to use this library. When using 3. or 4., the library internally queries Vulkan for memory types supported for that buffer or image (function `vkGetBufferMemoryRequirements()`) @@ -271,6 +302,7 @@ VmaAllocation allocation; vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr); \endcode + \section choosing_memory_type_custom_memory_pools Custom memory pools If you allocate from custom memory pool, all the ways of specifying memory @@ -279,6 +311,23 @@ of VmaAllocationCreateInfo structure are ignored. Memory type is selected explicitly when creating the pool and then used to make all the allocations from that pool. For further details, see \ref custom_memory_pools. +\section choosing_memory_type_dedicated_allocations Dedicated allocations + +Memory for allocations is reserved out of larger block of `VkDeviceMemory` +allocated from Vulkan internally. That's the main feature of this whole library. +You can still request a separate memory block to be created for an allocation, +just like you would do in a trivial solution without using any allocator. +In that case, a buffer or image is always bound to that memory at offset 0. +This is called a "dedicated allocation". +You can explicitly request it by using flag #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. +The library can also internally decide to use dedicated allocation in some cases, e.g.: + +- When the size of the allocation is large. +- When [VK_KHR_dedicated_allocation](@ref vk_khr_dedicated_allocation) extension is enabled + and it reports that dedicated allocation is required or recommended for the resource. +- When allocation of next big memory block fails due to not enough device memory, + but allocation with the exact requested size succeeds. + \page memory_mapping Memory mapping @@ -373,19 +422,23 @@ There are some exceptions though, when you should consider mapping memory only f for the time of any call to `vkQueueSubmit()` or `vkQueuePresentKHR()`, this block is migrated by WDDM to system RAM, which degrades performance. It doesn't matter if that particular memory block is actually used by the command buffer - being submitted. + being submitted. +- On Mac/MoltenVK there is a known bug - [Issue #175](https://github.com/KhronosGroup/MoltenVK/issues/175) + which requires unmapping before GPU can see updated texture. - Keeping many large memory blocks mapped may impact performance or stability of some debugging tools. -\section memory_mapping_cache_control Cache control +\section memory_mapping_cache_control Cache flush and invalidate Memory in Vulkan doesn't need to be unmapped before using it on GPU, but unless a memory types has `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag set, -you need to manually invalidate cache before reading of mapped pointer -and flush cache after writing to mapped pointer. +you need to manually **invalidate** cache before reading of mapped pointer +and **flush** cache after writing to mapped pointer. +Map/unmap operations don't do that automatically. Vulkan provides following functions for this purpose `vkFlushMappedMemoryRanges()`, `vkInvalidateMappedMemoryRanges()`, but this library provides more convenient functions that refer to given allocation object: vmaFlushAllocation(), -vmaInvalidateAllocation(). +vmaInvalidateAllocation(), +or multiple objects at once: vmaFlushAllocations(), vmaInvalidateAllocations(). Regions of memory specified for flush/invalidate must be aligned to `VkPhysicalDeviceLimits::nonCoherentAtomSize`. This is automatically ensured by the library. @@ -395,7 +448,7 @@ within blocks are aligned to this value, so their offsets are always multiply of Please note that memory allocated with #VMA_MEMORY_USAGE_CPU_ONLY is guaranteed to be `HOST_COHERENT`. -Also, Windows drivers from all 3 PC GPU vendors (AMD, Intel, NVIDIA) +Also, Windows drivers from all 3 **PC** GPU vendors (AMD, Intel, NVIDIA) currently provide `HOST_COHERENT` flag on all memory types that are `HOST_VISIBLE`, so on this platform you may not need to bother. @@ -427,7 +480,7 @@ vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allo VkMemoryPropertyFlags memFlags; vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags); -if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) +if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { // Allocation ended up in mappable memory. You can map it and access it directly. void* mappedData; @@ -462,7 +515,7 @@ VmaAllocation alloc; VmaAllocationInfo allocInfo; vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); -if(allocInfo.pUserData != nullptr) +if(allocInfo.pMappedData != nullptr) { // Allocation ended up in mappable memory. // It's persistently mapped. You can access it directly. @@ -476,6 +529,186 @@ else \endcode +\page staying_within_budget Staying within budget + +When developing a graphics-intensive game or program, it is important to avoid allocating +more GPU memory than it's physically available. When the memory is over-committed, +various bad things can happen, depending on the specific GPU, graphics driver, and +operating system: + +- It may just work without any problems. +- The application may slow down because some memory blocks are moved to system RAM + and the GPU has to access them through PCI Express bus. +- A new allocation may take very long time to complete, even few seconds, and possibly + freeze entire system. +- The new allocation may fail with `VK_ERROR_OUT_OF_DEVICE_MEMORY`. +- It may even result in GPU crash (TDR), observed as `VK_ERROR_DEVICE_LOST` + returned somewhere later. + +\section staying_within_budget_querying_for_budget Querying for budget + +To query for current memory usage and available budget, use function vmaGetBudget(). +Returned structure #VmaBudget contains quantities expressed in bytes, per Vulkan memory heap. + +Please note that this function returns different information and works faster than +vmaCalculateStats(). vmaGetBudget() can be called every frame or even before every +allocation, while vmaCalculateStats() is intended to be used rarely, +only to obtain statistical information, e.g. for debugging purposes. + +It is recommended to use VK_EXT_memory_budget device extension to obtain information +about the budget from Vulkan device. VMA is able to use this extension automatically. +When not enabled, the allocator behaves same way, but then it estimates current usage +and available budget based on its internal information and Vulkan memory heap sizes, +which may be less precise. In order to use this extension: + +1. Make sure extensions VK_EXT_memory_budget and VK_KHR_get_physical_device_properties2 + required by it are available and enable them. Please note that the first is a device + extension and the second is instance extension! +2. Use flag #VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT when creating #VmaAllocator object. +3. Make sure to call vmaSetCurrentFrameIndex() every frame. Budget is queried from + Vulkan inside of it to avoid overhead of querying it with every allocation. + +\section staying_within_budget_controlling_memory_usage Controlling memory usage + +There are many ways in which you can try to stay within the budget. + +First, when making new allocation requires allocating a new memory block, the library +tries not to exceed the budget automatically. If a block with default recommended size +(e.g. 256 MB) would go over budget, a smaller block is allocated, possibly even +dedicated memory for just this resource. + +If the size of the requested resource plus current memory usage is more than the +budget, by default the library still tries to create it, leaving it to the Vulkan +implementation whether the allocation succeeds or fails. You can change this behavior +by using #VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT flag. With it, the allocation is +not made if it would exceed the budget or if the budget is already exceeded. +Some other allocations become lost instead to make room for it, if the mechanism of +[lost allocations](@ref lost_allocations) is used. +If that is not possible, the allocation fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY`. +Example usage pattern may be to pass the #VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT flag +when creating resources that are not essential for the application (e.g. the texture +of a specific object) and not to pass it when creating critically important resources +(e.g. render targets). + +Finally, you can also use #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT flag to make sure +a new allocation is created only when it fits inside one of the existing memory blocks. +If it would require to allocate a new block, if fails instead with `VK_ERROR_OUT_OF_DEVICE_MEMORY`. +This also ensures that the function call is very fast because it never goes to Vulkan +to obtain a new block. + +Please note that creating \ref custom_memory_pools with VmaPoolCreateInfo::minBlockCount +set to more than 0 will try to allocate memory blocks without checking whether they +fit within budget. + + +\page resource_aliasing Resource aliasing (overlap) + +New explicit graphics APIs (Vulkan and Direct3D 12), thanks to manual memory +management, give an opportunity to alias (overlap) multiple resources in the +same region of memory - a feature not available in the old APIs (Direct3D 11, OpenGL). +It can be useful to save video memory, but it must be used with caution. + +For example, if you know the flow of your whole render frame in advance, you +are going to use some intermediate textures or buffers only during a small range of render passes, +and you know these ranges don't overlap in time, you can bind these resources to +the same place in memory, even if they have completely different parameters (width, height, format etc.). + +![Resource aliasing (overlap)](../gfx/Aliasing.png) + +Such scenario is possible using VMA, but you need to create your images manually. +Then you need to calculate parameters of an allocation to be made using formula: + +- allocation size = max(size of each image) +- allocation alignment = max(alignment of each image) +- allocation memoryTypeBits = bitwise AND(memoryTypeBits of each image) + +Following example shows two different images bound to the same place in memory, +allocated to fit largest of them. + +\code +// A 512x512 texture to be sampled. +VkImageCreateInfo img1CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; +img1CreateInfo.imageType = VK_IMAGE_TYPE_2D; +img1CreateInfo.extent.width = 512; +img1CreateInfo.extent.height = 512; +img1CreateInfo.extent.depth = 1; +img1CreateInfo.mipLevels = 10; +img1CreateInfo.arrayLayers = 1; +img1CreateInfo.format = VK_FORMAT_R8G8B8A8_SRGB; +img1CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; +img1CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; +img1CreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; +img1CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + +// A full screen texture to be used as color attachment. +VkImageCreateInfo img2CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; +img2CreateInfo.imageType = VK_IMAGE_TYPE_2D; +img2CreateInfo.extent.width = 1920; +img2CreateInfo.extent.height = 1080; +img2CreateInfo.extent.depth = 1; +img2CreateInfo.mipLevels = 1; +img2CreateInfo.arrayLayers = 1; +img2CreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; +img2CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; +img2CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; +img2CreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; +img2CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + +VkImage img1; +res = vkCreateImage(device, &img1CreateInfo, nullptr, &img1); +VkImage img2; +res = vkCreateImage(device, &img2CreateInfo, nullptr, &img2); + +VkMemoryRequirements img1MemReq; +vkGetImageMemoryRequirements(device, img1, &img1MemReq); +VkMemoryRequirements img2MemReq; +vkGetImageMemoryRequirements(device, img2, &img2MemReq); + +VkMemoryRequirements finalMemReq = {}; +finalMemReq.size = std::max(img1MemReq.size, img2MemReq.size); +finalMemReq.alignment = std::max(img1MemReq.alignment, img2MemReq.alignment); +finalMemReq.memoryTypeBits = img1MemReq.memoryTypeBits & img2MemReq.memoryTypeBits; +// Validate if(finalMemReq.memoryTypeBits != 0) + +VmaAllocationCreateInfo allocCreateInfo = {}; +allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + +VmaAllocation alloc; +res = vmaAllocateMemory(allocator, &finalMemReq, &allocCreateInfo, &alloc, nullptr); + +res = vmaBindImageMemory(allocator, alloc, img1); +res = vmaBindImageMemory(allocator, alloc, img2); + +// You can use img1, img2 here, but not at the same time! + +vmaFreeMemory(allocator, alloc); +vkDestroyImage(allocator, img2, nullptr); +vkDestroyImage(allocator, img1, nullptr); +\endcode + +Remember that using resouces that alias in memory requires proper synchronization. +You need to issue a memory barrier to make sure commands that use `img1` and `img2` +don't overlap on GPU timeline. +You also need to treat a resource after aliasing as uninitialized - containing garbage data. +For example, if you use `img1` and then want to use `img2`, you need to issue +an image memory barrier for `img2` with `oldLayout` = `VK_IMAGE_LAYOUT_UNDEFINED`. + +Additional considerations: + +- Vulkan also allows to interpret contents of memory between aliasing resources consistently in some cases. +See chapter 11.8. "Memory Aliasing" of Vulkan specification or `VK_IMAGE_CREATE_ALIAS_BIT` flag. +- You can create more complex layout where different images and buffers are bound +at different offsets inside one large allocation. For example, one can imagine +a big texture used in some render passes, aliasing with a set of many small buffers +used between in some further passes. To bind a resource at non-zero offset of an allocation, +use vmaBindBufferMemory2() / vmaBindImageMemory2(). +- Before allocating memory for the resources you want to alias, check `memoryTypeBits` +returned in memory requirements of each resource to make sure the bits overlap. +Some GPUs may expose multiple memory types suitable e.g. only for buffers or +images with `COLOR_ATTACHMENT` usage, so the sets of memory types supported by your +resources may be disjoint. Aliasing them is not possible in that case. + + \page custom_memory_pools Custom memory pools A memory pool contains a number of `VkDeviceMemory` blocks. @@ -496,7 +729,7 @@ To use custom memory pools: -# Fill VmaPoolCreateInfo structure. -# Call vmaCreatePool() to obtain #VmaPool handle. -# When making an allocation, set VmaAllocationCreateInfo::pool to this handle. - You don't need to specify any other parameters of this structure, like usage. + You don't need to specify any other parameters of this structure, like `usage`. Example: @@ -564,26 +797,347 @@ When creating buffers/images allocated in that pool, provide following parameter - VmaAllocationCreateInfo: You don't need to pass same parameters. Fill only `pool` member. Other members are ignored anyway. +\section linear_algorithm Linear allocation algorithm + +Each Vulkan memory block managed by this library has accompanying metadata that +keeps track of used and unused regions. By default, the metadata structure and +algorithm tries to find best place for new allocations among free regions to +optimize memory usage. This way you can allocate and free objects in any order. + +![Default allocation algorithm](../gfx/Linear_allocator_1_algo_default.png) + +Sometimes there is a need to use simpler, linear allocation algorithm. You can +create custom pool that uses such algorithm by adding flag +#VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT to VmaPoolCreateInfo::flags while creating +#VmaPool object. Then an alternative metadata management is used. It always +creates new allocations after last one and doesn't reuse free regions after +allocations freed in the middle. It results in better allocation performance and +less memory consumed by metadata. + +![Linear allocation algorithm](../gfx/Linear_allocator_2_algo_linear.png) + +With this one flag, you can create a custom pool that can be used in many ways: +free-at-once, stack, double stack, and ring buffer. See below for details. + +\subsection linear_algorithm_free_at_once Free-at-once + +In a pool that uses linear algorithm, you still need to free all the allocations +individually, e.g. by using vmaFreeMemory() or vmaDestroyBuffer(). You can free +them in any order. New allocations are always made after last one - free space +in the middle is not reused. However, when you release all the allocation and +the pool becomes empty, allocation starts from the beginning again. This way you +can use linear algorithm to speed up creation of allocations that you are going +to release all at once. + +![Free-at-once](../gfx/Linear_allocator_3_free_at_once.png) + +This mode is also available for pools created with VmaPoolCreateInfo::maxBlockCount +value that allows multiple memory blocks. + +\subsection linear_algorithm_stack Stack + +When you free an allocation that was created last, its space can be reused. +Thanks to this, if you always release allocations in the order opposite to their +creation (LIFO - Last In First Out), you can achieve behavior of a stack. + +![Stack](../gfx/Linear_allocator_4_stack.png) + +This mode is also available for pools created with VmaPoolCreateInfo::maxBlockCount +value that allows multiple memory blocks. + +\subsection linear_algorithm_double_stack Double stack + +The space reserved by a custom pool with linear algorithm may be used by two +stacks: + +- First, default one, growing up from offset 0. +- Second, "upper" one, growing down from the end towards lower offsets. + +To make allocation from upper stack, add flag #VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT +to VmaAllocationCreateInfo::flags. + +![Double stack](../gfx/Linear_allocator_7_double_stack.png) + +Double stack is available only in pools with one memory block - +VmaPoolCreateInfo::maxBlockCount must be 1. Otherwise behavior is undefined. + +When the two stacks' ends meet so there is not enough space between them for a +new allocation, such allocation fails with usual +`VK_ERROR_OUT_OF_DEVICE_MEMORY` error. + +\subsection linear_algorithm_ring_buffer Ring buffer + +When you free some allocations from the beginning and there is not enough free space +for a new one at the end of a pool, allocator's "cursor" wraps around to the +beginning and starts allocation there. Thanks to this, if you always release +allocations in the same order as you created them (FIFO - First In First Out), +you can achieve behavior of a ring buffer / queue. + +![Ring buffer](../gfx/Linear_allocator_5_ring_buffer.png) + +Pools with linear algorithm support [lost allocations](@ref lost_allocations) when used as ring buffer. +If there is not enough free space for a new allocation, but existing allocations +from the front of the queue can become lost, they become lost and the allocation +succeeds. + +![Ring buffer with lost allocations](../gfx/Linear_allocator_6_ring_buffer_lost.png) + +Ring buffer is available only in pools with one memory block - +VmaPoolCreateInfo::maxBlockCount must be 1. Otherwise behavior is undefined. + +\section buddy_algorithm Buddy allocation algorithm + +There is another allocation algorithm that can be used with custom pools, called +"buddy". Its internal data structure is based on a tree of blocks, each having +size that is a power of two and a half of its parent's size. When you want to +allocate memory of certain size, a free node in the tree is located. If it's too +large, it is recursively split into two halves (called "buddies"). However, if +requested allocation size is not a power of two, the size of a tree node is +aligned up to the nearest power of two and the remaining space is wasted. When +two buddy nodes become free, they are merged back into one larger node. + +![Buddy allocator](../gfx/Buddy_allocator.png) + +The advantage of buddy allocation algorithm over default algorithm is faster +allocation and deallocation, as well as smaller external fragmentation. The +disadvantage is more wasted space (internal fragmentation). + +For more information, please read ["Buddy memory allocation" on Wikipedia](https://en.wikipedia.org/wiki/Buddy_memory_allocation) +or other sources that describe this concept in general. + +To use buddy allocation algorithm with a custom pool, add flag +#VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT to VmaPoolCreateInfo::flags while creating +#VmaPool object. + +Several limitations apply to pools that use buddy algorithm: + +- It is recommended to use VmaPoolCreateInfo::blockSize that is a power of two. + Otherwise, only largest power of two smaller than the size is used for + allocations. The remaining space always stays unused. +- [Margins](@ref debugging_memory_usage_margins) and + [corruption detection](@ref debugging_memory_usage_corruption_detection) + don't work in such pools. +- [Lost allocations](@ref lost_allocations) don't work in such pools. You can + use them, but they never become lost. Support may be added in the future. +- [Defragmentation](@ref defragmentation) doesn't work with allocations made from + such pool. \page defragmentation Defragmentation Interleaved allocations and deallocations of many objects of varying size can -cause fragmentation, which can lead to a situation where the library is unable +cause fragmentation over time, which can lead to a situation where the library is unable to find a continuous range of free memory for a new allocation despite there is enough free space, just scattered across many small free ranges between existing allocations. -To mitigate this problem, you can use vmaDefragment(). Given set of allocations, +To mitigate this problem, you can use defragmentation feature: +structure #VmaDefragmentationInfo2, function vmaDefragmentationBegin(), vmaDefragmentationEnd(). +Given set of allocations, this function can move them to compact used memory, ensure more continuous free -space and possibly also free some `VkDeviceMemory`. It can work only on -allocations made from memory type that is `HOST_VISIBLE`. Allocations are -modified to point to the new `VkDeviceMemory` and offset. Data in this memory is -also `memmove`-ed to the new place. However, if you have images or buffers bound -to these allocations (and you certainly do), you need to destroy, recreate, and -bind them to the new place in memory. +space and possibly also free some `VkDeviceMemory` blocks. + +What the defragmentation does is: + +- Updates #VmaAllocation objects to point to new `VkDeviceMemory` and offset. + After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or + VmaAllocationInfo::offset changes. You must query them again using + vmaGetAllocationInfo() if you need them. +- Moves actual data in memory. + +What it doesn't do, so you need to do it yourself: + +- Recreate buffers and images that were bound to allocations that were defragmented and + bind them with their new places in memory. + You must use `vkDestroyBuffer()`, `vkDestroyImage()`, + `vkCreateBuffer()`, `vkCreateImage()`, vmaBindBufferMemory(), vmaBindImageMemory() + for that purpose and NOT vmaDestroyBuffer(), + vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage(), because you don't need to + destroy or create allocation objects! +- Recreate views and update descriptors that point to these buffers and images. + +\section defragmentation_cpu Defragmenting CPU memory + +Following example demonstrates how you can run defragmentation on CPU. +Only allocations created in memory types that are `HOST_VISIBLE` can be defragmented. +Others are ignored. + +The way it works is: + +- It temporarily maps entire memory blocks when necessary. +- It moves data using `memmove()` function. + +\code +// Given following variables already initialized: +VkDevice device; +VmaAllocator allocator; +std::vector buffers; +std::vector allocations; + + +const uint32_t allocCount = (uint32_t)allocations.size(); +std::vector allocationsChanged(allocCount); + +VmaDefragmentationInfo2 defragInfo = {}; +defragInfo.allocationCount = allocCount; +defragInfo.pAllocations = allocations.data(); +defragInfo.pAllocationsChanged = allocationsChanged.data(); +defragInfo.maxCpuBytesToMove = VK_WHOLE_SIZE; // No limit. +defragInfo.maxCpuAllocationsToMove = UINT32_MAX; // No limit. + +VmaDefragmentationContext defragCtx; +vmaDefragmentationBegin(allocator, &defragInfo, nullptr, &defragCtx); +vmaDefragmentationEnd(allocator, defragCtx); + +for(uint32_t i = 0; i < allocCount; ++i) +{ + if(allocationsChanged[i]) + { + // Destroy buffer that is immutably bound to memory region which is no longer valid. + vkDestroyBuffer(device, buffers[i], nullptr); + + // Create new buffer with same parameters. + VkBufferCreateInfo bufferInfo = ...; + vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]); + + // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning. + + // Bind new buffer to new memory region. Data contained in it is already moved. + VmaAllocationInfo allocInfo; + vmaGetAllocationInfo(allocator, allocations[i], &allocInfo); + vmaBindBufferMemory(allocator, allocations[i], buffers[i]); + } +} +\endcode + +Setting VmaDefragmentationInfo2::pAllocationsChanged is optional. +This output array tells whether particular allocation in VmaDefragmentationInfo2::pAllocations at the same index +has been modified during defragmentation. +You can pass null, but you then need to query every allocation passed to defragmentation +for new parameters using vmaGetAllocationInfo() if you might need to recreate and rebind a buffer or image associated with it. + +If you use [Custom memory pools](@ref choosing_memory_type_custom_memory_pools), +you can fill VmaDefragmentationInfo2::poolCount and VmaDefragmentationInfo2::pPools +instead of VmaDefragmentationInfo2::allocationCount and VmaDefragmentationInfo2::pAllocations +to defragment all allocations in given pools. +You cannot use VmaDefragmentationInfo2::pAllocationsChanged in that case. +You can also combine both methods. + +\section defragmentation_gpu Defragmenting GPU memory + +It is also possible to defragment allocations created in memory types that are not `HOST_VISIBLE`. +To do that, you need to pass a command buffer that meets requirements as described in +VmaDefragmentationInfo2::commandBuffer. The way it works is: + +- It creates temporary buffers and binds them to entire memory blocks when necessary. +- It issues `vkCmdCopyBuffer()` to passed command buffer. + +Example: + +\code +// Given following variables already initialized: +VkDevice device; +VmaAllocator allocator; +VkCommandBuffer commandBuffer; +std::vector buffers; +std::vector allocations; + + +const uint32_t allocCount = (uint32_t)allocations.size(); +std::vector allocationsChanged(allocCount); + +VkCommandBufferBeginInfo cmdBufBeginInfo = ...; +vkBeginCommandBuffer(commandBuffer, &cmdBufBeginInfo); + +VmaDefragmentationInfo2 defragInfo = {}; +defragInfo.allocationCount = allocCount; +defragInfo.pAllocations = allocations.data(); +defragInfo.pAllocationsChanged = allocationsChanged.data(); +defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; // Notice it's "GPU" this time. +defragInfo.maxGpuAllocationsToMove = UINT32_MAX; // Notice it's "GPU" this time. +defragInfo.commandBuffer = commandBuffer; + +VmaDefragmentationContext defragCtx; +vmaDefragmentationBegin(allocator, &defragInfo, nullptr, &defragCtx); + +vkEndCommandBuffer(commandBuffer); + +// Submit commandBuffer. +// Wait for a fence that ensures commandBuffer execution finished. + +vmaDefragmentationEnd(allocator, defragCtx); + +for(uint32_t i = 0; i < allocCount; ++i) +{ + if(allocationsChanged[i]) + { + // Destroy buffer that is immutably bound to memory region which is no longer valid. + vkDestroyBuffer(device, buffers[i], nullptr); + + // Create new buffer with same parameters. + VkBufferCreateInfo bufferInfo = ...; + vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]); + + // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning. + + // Bind new buffer to new memory region. Data contained in it is already moved. + VmaAllocationInfo allocInfo; + vmaGetAllocationInfo(allocator, allocations[i], &allocInfo); + vmaBindBufferMemory(allocator, allocations[i], buffers[i]); + } +} +\endcode + +You can combine these two methods by specifying non-zero `maxGpu*` as well as `maxCpu*` parameters. +The library automatically chooses best method to defragment each memory pool. + +You may try not to block your entire program to wait until defragmentation finishes, +but do it in the background, as long as you carefully fullfill requirements described +in function vmaDefragmentationBegin(). + +\section defragmentation_additional_notes Additional notes + +It is only legal to defragment allocations bound to: + +- buffers +- images created with `VK_IMAGE_CREATE_ALIAS_BIT`, `VK_IMAGE_TILING_LINEAR`, and + being currently in `VK_IMAGE_LAYOUT_GENERAL` or `VK_IMAGE_LAYOUT_PREINITIALIZED`. + +Defragmentation of images created with `VK_IMAGE_TILING_OPTIMAL` or in any other +layout may give undefined results. + +If you defragment allocations bound to images, new images to be bound to new +memory region after defragmentation should be created with `VK_IMAGE_LAYOUT_PREINITIALIZED` +and then transitioned to their original layout from before defragmentation if +needed using an image memory barrier. + +While using defragmentation, you may experience validation layer warnings, which you just need to ignore. +See [Validation layer warnings](@ref general_considerations_validation_layer_warnings). + +Please don't expect memory to be fully compacted after defragmentation. +Algorithms inside are based on some heuristics that try to maximize number of Vulkan +memory blocks to make totally empty to release them, as well as to maximimze continuous +empty space inside remaining blocks, while minimizing the number and size of allocations that +need to be moved. Some fragmentation may still remain - this is normal. + +\section defragmentation_custom_algorithm Writing custom defragmentation algorithm + +If you want to implement your own, custom defragmentation algorithm, +there is infrastructure prepared for that, +but it is not exposed through the library API - you need to hack its source code. +Here are steps needed to do this: + +-# Main thing you need to do is to define your own class derived from base abstract + class `VmaDefragmentationAlgorithm` and implement your version of its pure virtual methods. + See definition and comments of this class for details. +-# Your code needs to interact with device memory block metadata. + If you need more access to its data than it's provided by its public interface, + declare your new class as a friend class e.g. in class `VmaBlockMetadata_Generic`. +-# If you want to create a flag that would enable your algorithm or pass some additional + flags to configure it, add them to `VmaDefragmentationFlagBits` and use them in + VmaDefragmentationInfo2::flags. +-# Modify function `VmaBlockVectorDefragmentationContext::Begin` to create object + of your new class whenever needed. -For further details and example code, see documentation of function -vmaDefragment(). \page lost_allocations Lost allocations @@ -824,6 +1378,9 @@ printf("Image name: %s\n", imageName); That string is also printed in JSON report created by vmaBuildStatsString(). +\note Passing string name to VMA allocation doesn't automatically set it to the Vulkan buffer or image created with it. +You must do it manually using an extension like VK_EXT_debug_utils, which is independent of this library. + \page debugging_memory_usage Debugging incorrect memory usage @@ -833,7 +1390,7 @@ you can use debug features of this library to verify this. \section debugging_memory_usage_initialization Memory initialization -If you experience a bug with incorrect data in your program and you suspect uninitialized memory to be used, +If you experience a bug with incorrect and nondeterministic data in your program and you suspect uninitialized memory to be used, you can enable automatic memory initialization to verify this. To do it, define macro `VMA_DEBUG_INITIALIZE_ALLOCATIONS` to 1. @@ -852,7 +1409,7 @@ read Vulkan memory that is allocated but not initialized, or already freed, resp Memory initialization works only with memory types that are `HOST_VISIBLE`. It works also with dedicated allocations. It doesn't work with allocations created with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag, -as these they cannot be mapped. +as they cannot be mapped. \section debugging_memory_usage_margins Margins @@ -886,6 +1443,7 @@ allocations, which have their own memory block of specific size. It is thus not applied to allocations made using #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT flag or those automatically decided to put into dedicated allocations, e.g. due to its large size or recommended by VK_KHR_dedicated_allocation extension. +Margins are also not active in custom pools created with #VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT flag. Margins appear in [JSON dump](@ref statistics_json_dump) as part of free space. @@ -938,6 +1496,13 @@ application. It can be useful to: \section record_and_replay_usage Usage +Recording functionality is disabled by default. +To enable it, define following macro before every include of this library: + +\code +#define VMA_RECORDING_ENABLED 1 +\endcode + To record sequence of calls to a file: Fill in VmaAllocatorCreateInfo::pRecordSettings member while creating #VmaAllocator object. File is opened and written during whole lifetime of the allocator. @@ -964,7 +1529,6 @@ It's a human-readable, text file in CSV format (Comma Separated Values). coded and tested only on Windows. Inclusion of recording code is driven by `VMA_RECORDING_ENABLED` macro. Support for other platforms should be easy to add. Contributions are welcomed. -- Currently calls to vmaDefragment() function are not recorded. \page usage_patterns Recommended usage patterns @@ -973,6 +1537,27 @@ See also slides from talk: [Sawicki, Adam. Advanced Graphics Techniques Tutorial: Memory management in Vulkan and DX12. Game Developers Conference, 2018](https://www.gdcvault.com/play/1025458/Advanced-Graphics-Techniques-Tutorial-New) +\section usage_patterns_common_mistakes Common mistakes + +Use of CPU_TO_GPU instead of CPU_ONLY memory + +#VMA_MEMORY_USAGE_CPU_TO_GPU is recommended only for resources that will be +mapped and written by the CPU, as well as read directly by the GPU - like some +buffers or textures updated every frame (dynamic). If you create a staging copy +of a resource to be written by CPU and then used as a source of transfer to +another resource placed in the GPU memory, that staging resource should be +created with #VMA_MEMORY_USAGE_CPU_ONLY. Please read the descriptions of these +enums carefully for details. + +Unnecessary use of custom pools + +\ref custom_memory_pools may be useful for special purposes - when you want to +keep certain type of resources separate e.g. to reserve minimum amount of memory +for them, limit maximum amount of memory they can occupy, or make some of them +push out the other through the mechanism of \ref lost_allocations. For most +resources this is not needed and so it is not recommended to create #VmaPool +objects and allocations out of them. Allocating from the default pool is sufficient. + \section usage_patterns_simple Simple patterns \subsection usage_patterns_simple_render_targets Render targets @@ -1027,7 +1612,8 @@ You can map it and write to it directly on CPU, as well as read from it on GPU. This is a more complex situation. Different solutions are possible, and the best one depends on specific GPU type, but you can use this simple approach for the start. Prefer to write to such resource sequentially (e.g. using `memcpy`). -Don't perform random access or any reads from it, as it may be very slow. +Don't perform random access or any reads from it on CPU, as it may be very slow. +Also note that textures written directly from the host through a mapped pointer need to be in LINEAR not OPTIMAL layout. \subsection usage_patterns_readback Readback @@ -1047,7 +1633,7 @@ You can support integrated graphics (like Intel HD Graphics, AMD APU) better by detecting it in Vulkan. To do it, call `vkGetPhysicalDeviceProperties()`, inspect `VkPhysicalDeviceProperties::deviceType` and look for `VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. -When you find it, you can assume that memory is unified and all memory types are equally fast +When you find it, you can assume that memory is unified and all memory types are comparably fast to access from GPU, regardless of `VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT`. You can then sum up sizes of all available memory heaps and treat them as useful for @@ -1060,10 +1646,10 @@ directly instead of submitting explicit transfer (see below). For resources that you frequently write on CPU and read on GPU, many solutions are possible: -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY, - second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit tranfer each time. --# Create just single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU, + second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit transfer each time. +-# Create just a single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU, read it directly on GPU. --# Create just single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU, +-# Create just a single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU, read it directly on GPU. Which solution is the most efficient depends on your resource and especially on the GPU. @@ -1071,7 +1657,7 @@ It is best to measure it and then make the decision. Some general recommendations: - On integrated graphics use (2) or (3) to avoid unnecesary time and memory overhead - related to using a second copy. + related to using a second copy and making transfer. - For small resources (e.g. constant buffers) use (2). Discrete AMD cards have special 256 MiB pool of video memory that is directly mappable. Even if the resource ends up in system memory, its data may be cached on GPU after first @@ -1091,6 +1677,10 @@ solutions are possible: You should take some measurements to decide which option is faster in case of your specific resource. +Note that textures accessed directly from the host through a mapped pointer need to be in LINEAR layout, +which may slow down their usage on the device. +Textures accessed only by the device and transfer operations can use OPTIMAL layout. + If you don't want to specialize your code for specific types of GPUs, you can still make an simple optimization for cases when your resource ends up in mappable memory to use it directly in this case instead of creating CPU-side staging copy. @@ -1106,14 +1696,38 @@ mutex, atomic etc. The library uses its own implementation of containers by default, but you can switch to using STL containers instead. +For example, define `VMA_ASSERT(expr)` before including the library to provide +custom implementation of the assertion, compatible with your project. +By default it is defined to standard C `assert(expr)` in `_DEBUG` configuration +and empty otherwise. + \section config_Vulkan_functions Pointers to Vulkan functions -The library uses Vulkan functions straight from the `vulkan.h` header by default. -If you want to provide your own pointers to these functions, e.g. fetched using -`vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`: +There are multiple ways to import pointers to Vulkan functions in the library. +In the simplest case you don't need to do anything. +If the compilation or linking of your program or the initialization of the #VmaAllocator +doesn't work for you, you can try to reconfigure it. + +First, the allocator tries to fetch pointers to Vulkan functions linked statically, +like this: + +\code +m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory; +\endcode + +If you want to disable this feature, set configuration macro: `#define VMA_STATIC_VULKAN_FUNCTIONS 0`. + +Second, you can provide the pointers yourself by setting member VmaAllocatorCreateInfo::pVulkanFunctions. +You can fetch them e.g. using functions `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` or +by using a helper library like [volk](https://github.com/zeux/volk). + +Third, VMA tries to fetch remaining pointers that are still null by calling +`vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` on its own. +If you want to disable this feature, set configuration macro: `#define VMA_DYNAMIC_VULKAN_FUNCTIONS 0`. + +Finally, all the function pointers required by the library (considering selected +Vulkan version and enabled extensions) are checked with `VMA_ASSERT` if they are not null. --# Define `VMA_STATIC_VULKAN_FUNCTIONS 0`. --# Provide valid pointers through VmaAllocatorCreateInfo::pVulkanFunctions. \section custom_memory_allocator Custom host memory allocator @@ -1132,7 +1746,16 @@ VmaAllocatorCreateInfo::pDeviceMemoryCallbacks. \section heap_memory_limit Device heap memory limit -If you want to test how your program behaves with limited amount of Vulkan device +When device memory of certain heap runs out of free space, new allocations may +fail (returning error code) or they may succeed, silently pushing some existing +memory blocks from GPU VRAM to system RAM (which degrades performance). This +behavior is implementation-dependant - it depends on GPU vendor and graphics +driver. + +On AMD cards it can be controlled while creating Vulkan device object by using +VK_AMD_memory_overallocation_behavior extension, if available. + +Alternatively, if you want to test how your program behaves with limited amount of Vulkan device memory available without switching your graphics card to one that really has smaller VRAM, you can use a feature of this library intended for this purpose. To do it, fill optional member VmaAllocatorCreateInfo::pHeapSizeLimit. @@ -1184,11 +1807,115 @@ unaware of it. To learn more about this extension, see: -- [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html#VK_KHR_dedicated_allocation) +- [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap44.html#VK_KHR_dedicated_allocation) - [VK_KHR_dedicated_allocation unofficial manual](http://asawicki.info/articles/VK_KHR_dedicated_allocation.php5) +\page vk_amd_device_coherent_memory VK_AMD_device_coherent_memory + +VK_AMD_device_coherent_memory is a device extension that enables access to +additional memory types with `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and +`VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flag. It is useful mostly for +allocation of buffers intended for writing "breadcrumb markers" in between passes +or draw calls, which in turn are useful for debugging GPU crash/hang/TDR cases. + +When the extension is available but has not been enabled, Vulkan physical device +still exposes those memory types, but their usage is forbidden. VMA automatically +takes care of that - it returns `VK_ERROR_FEATURE_NOT_PRESENT` when an attempt +to allocate memory of such type is made. + +If you want to use this extension in connection with VMA, follow these steps: + +\section vk_amd_device_coherent_memory_initialization Initialization + +1) Call `vkEnumerateDeviceExtensionProperties` for the physical device. +Check if the extension is supported - if returned array of `VkExtensionProperties` contains "VK_AMD_device_coherent_memory". + +2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`. +Attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to `VkPhysicalDeviceFeatures2::pNext` to be returned. +Check if the device feature is really supported - check if `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true. + +3) While creating device with `vkCreateDevice`, enable this extension - add "VK_AMD_device_coherent_memory" +to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`. + +4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`. +Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`. +Enable this device feature - attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to +`VkPhysicalDeviceFeatures2::pNext` and set its member `deviceCoherentMemory` to `VK_TRUE`. + +5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you +have enabled this extension and feature - add #VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT +to VmaAllocatorCreateInfo::flags. + +\section vk_amd_device_coherent_memory_usage Usage + +After following steps described above, you can create VMA allocations and custom pools +out of the special `DEVICE_COHERENT` and `DEVICE_UNCACHED` memory types on eligible +devices. There are multiple ways to do it, for example: + +- You can request or prefer to allocate out of such memory types by adding + `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` to VmaAllocationCreateInfo::requiredFlags + or VmaAllocationCreateInfo::preferredFlags. Those flags can be freely mixed with + other ways of \ref choosing_memory_type, like setting VmaAllocationCreateInfo::usage. +- If you manually found memory type index to use for this purpose, force allocation + from this specific index by setting VmaAllocationCreateInfo::memoryTypeBits `= 1u << index`. + +\section vk_amd_device_coherent_memory_more_information More information + +To learn more about this extension, see [VK_AMD_device_coherent_memory in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap44.html#VK_AMD_device_coherent_memory) + +Example use of this extension can be found in the code of the sample and test suite +accompanying this library. + + +\page enabling_buffer_device_address Enabling buffer device address + +Device extension VK_KHR_buffer_device_address +allow to fetch raw GPU pointer to a buffer and pass it for usage in a shader code. +It is promoted to core Vulkan 1.2. + +If you want to use this feature in connection with VMA, follow these steps: + +\section enabling_buffer_device_address_initialization Initialization + +1) (For Vulkan version < 1.2) Call `vkEnumerateDeviceExtensionProperties` for the physical device. +Check if the extension is supported - if returned array of `VkExtensionProperties` contains +"VK_KHR_buffer_device_address". + +2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`. +Attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to `VkPhysicalDeviceFeatures2::pNext` to be returned. +Check if the device feature is really supported - check if `VkPhysicalDeviceBufferDeviceAddressFeatures*::bufferDeviceAddress` is true. + +3) (For Vulkan version < 1.2) While creating device with `vkCreateDevice`, enable this extension - add +"VK_KHR_buffer_device_address" to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`. + +4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`. +Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`. +Enable this device feature - attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to +`VkPhysicalDeviceFeatures2::pNext` and set its member `bufferDeviceAddress` to `VK_TRUE`. + +5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you +have enabled this feature - add #VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT +to VmaAllocatorCreateInfo::flags. + +\section enabling_buffer_device_address_usage Usage + +After following steps described above, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*` using VMA. +The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT*` to +allocated memory blocks wherever it might be needed. + +Please note that the library supports only `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*`. +The second part of this functionality related to "capture and replay" is not supported, +as it is intended for usage in debugging tools like RenderDoc, not in everyday Vulkan usage. + +\section enabling_buffer_device_address_more_information More information + +To learn more about this extension, see [VK_KHR_buffer_device_address in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap46.html#VK_KHR_buffer_device_address) + +Example use of this extension can be found in the code of the sample and test suite +accompanying this library. + \page general_considerations General considerations \section general_considerations_thread_safety Thread safety @@ -1223,6 +1950,7 @@ to just ignore them. - *Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug.* - It happens when you use lost allocations, and a new image or buffer is created in place of an existing object that bacame lost. + - It may happen also when you use [defragmentation](@ref defragmentation). \section general_considerations_allocation_algorithm Allocation algorithm @@ -1244,18 +1972,105 @@ The library uses following algorithm for allocation, in order: Features deliberately excluded from the scope of this library: -- Data transfer - issuing commands that transfer data between buffers or images, any usage of - `VkCommandList` or `VkQueue` and related synchronization is responsibility of the user. +- Data transfer. Uploading (straming) and downloading data of buffers and images + between CPU and GPU memory and related synchronization is responsibility of the user. + Defining some "texture" object that would automatically stream its data from a + staging copy in CPU memory to GPU memory would rather be a feature of another, + higher-level library implemented on top of VMA. - Allocations for imported/exported external memory. They tend to require explicit memory type index and dedicated allocation anyway, so they don't interact with main features of this library. Such special purpose allocations should be made manually, using `vkCreateBuffer()` and `vkAllocateMemory()`. -- Support for any programming languages other than C/C++. - Bindings to other languages are welcomed as external projects. +- Recreation of buffers and images. Although the library has functions for + buffer and image creation (vmaCreateBuffer(), vmaCreateImage()), you need to + recreate these objects yourself after defragmentation. That's because the big + structures `VkBufferCreateInfo`, `VkImageCreateInfo` are not stored in + #VmaAllocation object. +- Handling CPU memory allocation failures. When dynamically creating small C++ + objects in CPU memory (not Vulkan memory), allocation failures are not checked + and handled gracefully, because that would complicate code significantly and + is usually not needed in desktop PC applications anyway. + Success of an allocation is just checked with an assert. +- Code free of any compiler warnings. Maintaining the library to compile and + work correctly on so many different platforms is hard enough. Being free of + any warnings, on any version of any compiler, is simply not feasible. +- This is a C++ library with C interface. + Bindings or ports to any other programming languages are welcomed as external projects and + are not going to be included into this repository. */ -#include +#if VMA_RECORDING_ENABLED + #include + #if defined(_WIN32) + #include + #else + #include + #include + #endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* +Define this macro to 0/1 to disable/enable support for recording functionality, +available through VmaAllocatorCreateInfo::pRecordSettings. +*/ +#ifndef VMA_RECORDING_ENABLED + #define VMA_RECORDING_ENABLED 0 +#endif + +#ifndef NOMINMAX + #define NOMINMAX // For windows.h +#endif + +#if defined(__ANDROID__) && defined(VK_NO_PROTOTYPES) && VMA_STATIC_VULKAN_FUNCTIONS + extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr; + extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr; + extern PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties; + extern PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties; + extern PFN_vkAllocateMemory vkAllocateMemory; + extern PFN_vkFreeMemory vkFreeMemory; + extern PFN_vkMapMemory vkMapMemory; + extern PFN_vkUnmapMemory vkUnmapMemory; + extern PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges; + extern PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges; + extern PFN_vkBindBufferMemory vkBindBufferMemory; + extern PFN_vkBindImageMemory vkBindImageMemory; + extern PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements; + extern PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements; + extern PFN_vkCreateBuffer vkCreateBuffer; + extern PFN_vkDestroyBuffer vkDestroyBuffer; + extern PFN_vkCreateImage vkCreateImage; + extern PFN_vkDestroyImage vkDestroyImage; + extern PFN_vkCmdCopyBuffer vkCmdCopyBuffer; + #if VMA_VULKAN_VERSION >= 1001000 + extern PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2; + extern PFN_vkGetImageMemoryRequirements2 vkGetImageMemoryRequirements2; + extern PFN_vkBindBufferMemory2 vkBindBufferMemory2; + extern PFN_vkBindImageMemory2 vkBindImageMemory2; + extern PFN_vkGetPhysicalDeviceMemoryProperties2 vkGetPhysicalDeviceMemoryProperties2; + #endif // #if VMA_VULKAN_VERSION >= 1001000 +#endif // #if defined(__ANDROID__) && VMA_STATIC_VULKAN_FUNCTIONS && VK_NO_PROTOTYPES + +#ifndef VULKAN_H_ + #include +#endif + +// Define this macro to declare maximum supported Vulkan version in format AAABBBCCC, +// where AAA = major, BBB = minor, CCC = patch. +// If you want to use version > 1.0, it still needs to be enabled via VmaAllocatorCreateInfo::vulkanApiVersion. +#if !defined(VMA_VULKAN_VERSION) + #if defined(VK_VERSION_1_2) + #define VMA_VULKAN_VERSION 1002000 + #elif defined(VK_VERSION_1_1) + #define VMA_VULKAN_VERSION 1001000 + #else + #define VMA_VULKAN_VERSION 1000000 + #endif +#endif #if !defined(VMA_DEDICATED_ALLOCATION) #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation @@ -1265,10 +2080,100 @@ Features deliberately excluded from the scope of this library: #endif #endif +#if !defined(VMA_BIND_MEMORY2) + #if VK_KHR_bind_memory2 + #define VMA_BIND_MEMORY2 1 + #else + #define VMA_BIND_MEMORY2 0 + #endif +#endif + +#if !defined(VMA_MEMORY_BUDGET) + #if VK_EXT_memory_budget && (VK_KHR_get_physical_device_properties2 || VMA_VULKAN_VERSION >= 1001000) + #define VMA_MEMORY_BUDGET 1 + #else + #define VMA_MEMORY_BUDGET 0 + #endif +#endif + +// Defined to 1 when VK_KHR_buffer_device_address device extension or equivalent core Vulkan 1.2 feature is defined in its headers. +#if !defined(VMA_BUFFER_DEVICE_ADDRESS) + #if VK_KHR_buffer_device_address || VMA_VULKAN_VERSION >= 1002000 + #define VMA_BUFFER_DEVICE_ADDRESS 1 + #else + #define VMA_BUFFER_DEVICE_ADDRESS 0 + #endif +#endif + +// Define these macros to decorate all public functions with additional code, +// before and after returned type, appropriately. This may be useful for +// exporting the functions when compiling VMA as a separate library. Example: +// #define VMA_CALL_PRE __declspec(dllexport) +// #define VMA_CALL_POST __cdecl +#ifndef VMA_CALL_PRE + #define VMA_CALL_PRE +#endif +#ifndef VMA_CALL_POST + #define VMA_CALL_POST +#endif + +// Define this macro to decorate pointers with an attribute specifying the +// length of the array they point to if they are not null. +// +// The length may be one of +// - The name of another parameter in the argument list where the pointer is declared +// - The name of another member in the struct where the pointer is declared +// - The name of a member of a struct type, meaning the value of that member in +// the context of the call. For example +// VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount"), +// this means the number of memory heaps available in the device associated +// with the VmaAllocator being dealt with. +#ifndef VMA_LEN_IF_NOT_NULL + #define VMA_LEN_IF_NOT_NULL(len) +#endif + +// The VMA_NULLABLE macro is defined to be _Nullable when compiling with Clang. +// see: https://clang.llvm.org/docs/AttributeReference.html#nullable +#ifndef VMA_NULLABLE + #ifdef __clang__ + #define VMA_NULLABLE _Nullable + #else + #define VMA_NULLABLE + #endif +#endif + +// The VMA_NOT_NULL macro is defined to be _Nonnull when compiling with Clang. +// see: https://clang.llvm.org/docs/AttributeReference.html#nonnull +#ifndef VMA_NOT_NULL + #ifdef __clang__ + #define VMA_NOT_NULL _Nonnull + #else + #define VMA_NOT_NULL + #endif +#endif + +// If non-dispatchable handles are represented as pointers then we can give +// then nullability annotations +#ifndef VMA_NOT_NULL_NON_DISPATCHABLE + #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__) + #define VMA_NOT_NULL_NON_DISPATCHABLE VMA_NOT_NULL + #else + #define VMA_NOT_NULL_NON_DISPATCHABLE + #endif +#endif + +#ifndef VMA_NULLABLE_NON_DISPATCHABLE + #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__) + #define VMA_NULLABLE_NON_DISPATCHABLE VMA_NULLABLE + #else + #define VMA_NULLABLE_NON_DISPATCHABLE + #endif +#endif + /** \struct VmaAllocator \brief Represents main object of this library initialized. -Fill structure VmaAllocatorCreateInfo and call function vmaCreateAllocator() to create it. +Fill structure #VmaAllocatorCreateInfo and call function vmaCreateAllocator() to create it. Call function vmaDestroyAllocator() to destroy it. It is recommended to create just one object of this type per `VkDevice` object, @@ -1278,16 +2183,18 @@ VK_DEFINE_HANDLE(VmaAllocator) /// Callback function called after successful vkAllocateMemory. typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)( - VmaAllocator allocator, - uint32_t memoryType, - VkDeviceMemory memory, - VkDeviceSize size); + VmaAllocator VMA_NOT_NULL allocator, + uint32_t memoryType, + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory, + VkDeviceSize size, + void* VMA_NULLABLE pUserData); /// Callback function called before vkFreeMemory. typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)( - VmaAllocator allocator, - uint32_t memoryType, - VkDeviceMemory memory, - VkDeviceSize size); + VmaAllocator VMA_NOT_NULL allocator, + uint32_t memoryType, + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory, + VkDeviceSize size, + void* VMA_NULLABLE pUserData); /** \brief Set of callbacks that the library will call for `vkAllocateMemory` and `vkFreeMemory`. @@ -1298,9 +2205,11 @@ Used in VmaAllocatorCreateInfo::pDeviceMemoryCallbacks. */ typedef struct VmaDeviceMemoryCallbacks { /// Optional, can be null. - PFN_vmaAllocateDeviceMemoryFunction pfnAllocate; + PFN_vmaAllocateDeviceMemoryFunction VMA_NULLABLE pfnAllocate; /// Optional, can be null. - PFN_vmaFreeDeviceMemoryFunction pfnFree; + PFN_vmaFreeDeviceMemoryFunction VMA_NULLABLE pfnFree; + /// Optional, can be null. + void* VMA_NULLABLE pUserData; } VmaDeviceMemoryCallbacks; /// Flags for created #VmaAllocator. @@ -1312,6 +2221,9 @@ typedef enum VmaAllocatorCreateFlagBits { VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001, /** \brief Enables usage of VK_KHR_dedicated_allocation extension. + The flag works only if VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_0`. + When it's `VK_API_VERSION_1_1`, the flag is ignored because the extension has been promoted to Vulkan 1.1. + Using this extenion will automatically allocate dedicated blocks of memory for some buffers and images instead of suballocating place for them out of bigger memory blocks (as if you explicitly used #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT @@ -1323,15 +2235,78 @@ typedef enum VmaAllocatorCreateFlagBits { VmaAllocatorCreateInfo::device, and you want them to be used internally by this library: - - VK_KHR_get_memory_requirements2 - - VK_KHR_dedicated_allocation + - VK_KHR_get_memory_requirements2 (device extension) + - VK_KHR_dedicated_allocation (device extension) -When this flag is set, you can experience following warnings reported by Vulkan -validation layer. You can ignore them. + When this flag is set, you can experience following warnings reported by Vulkan + validation layer. You can ignore them. -> vkBindBufferMemory(): Binding memory to buffer 0x2d but vkGetBufferMemoryRequirements() has not been called on that buffer. + > vkBindBufferMemory(): Binding memory to buffer 0x2d but vkGetBufferMemoryRequirements() has not been called on that buffer. */ VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002, + /** + Enables usage of VK_KHR_bind_memory2 extension. + + The flag works only if VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_0`. + When it's `VK_API_VERSION_1_1`, the flag is ignored because the extension has been promoted to Vulkan 1.1. + + You may set this flag only if you found out that this device extension is supported, + you enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device, + and you want it to be used internally by this library. + + The extension provides functions `vkBindBufferMemory2KHR` and `vkBindImageMemory2KHR`, + which allow to pass a chain of `pNext` structures while binding. + This flag is required if you use `pNext` parameter in vmaBindBufferMemory2() or vmaBindImageMemory2(). + */ + VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT = 0x00000004, + /** + Enables usage of VK_EXT_memory_budget extension. + + You may set this flag only if you found out that this device extension is supported, + you enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device, + and you want it to be used internally by this library, along with another instance extension + VK_KHR_get_physical_device_properties2, which is required by it (or Vulkan 1.1, where this extension is promoted). + + The extension provides query for current memory usage and budget, which will probably + be more accurate than an estimation used by the library otherwise. + */ + VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT = 0x00000008, + /** + Enables usage of VK_AMD_device_coherent_memory extension. + + You may set this flag only if you: + + - found out that this device extension is supported and enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device, + - checked that `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true and set it while creating the Vulkan device, + - want it to be used internally by this library. + + The extension and accompanying device feature provide access to memory types with + `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and `VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flags. + They are useful mostly for writing breadcrumb markers - a common method for debugging GPU crash/hang/TDR. + + When the extension is not enabled, such memory types are still enumerated, but their usage is illegal. + To protect from this error, if you don't create the allocator with this flag, it will refuse to allocate any memory or create a custom pool in such memory type, + returning `VK_ERROR_FEATURE_NOT_PRESENT`. + */ + VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT = 0x00000010, + /** + Enables usage of "buffer device address" feature, which allows you to use function + `vkGetBufferDeviceAddress*` to get raw GPU pointer to a buffer and pass it for usage inside a shader. + + You may set this flag only if you: + + 1. (For Vulkan version < 1.2) Found as available and enabled device extension + VK_KHR_buffer_device_address. + This extension is promoted to core Vulkan 1.2. + 2. Found as available and enabled device feature `VkPhysicalDeviceBufferDeviceAddressFeatures*::bufferDeviceAddress`. + + When this flag is set, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*` using VMA. + The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT*` to + allocated memory blocks wherever it might be needed. + + For more information, see documentation chapter \ref enabling_buffer_device_address. + */ + VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT = 0x00000020, VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaAllocatorCreateFlagBits; @@ -1342,25 +2317,33 @@ typedef VkFlags VmaAllocatorCreateFlags; Used in VmaAllocatorCreateInfo::pVulkanFunctions. */ typedef struct VmaVulkanFunctions { - PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties; - PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties; - PFN_vkAllocateMemory vkAllocateMemory; - PFN_vkFreeMemory vkFreeMemory; - PFN_vkMapMemory vkMapMemory; - PFN_vkUnmapMemory vkUnmapMemory; - PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges; - PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges; - PFN_vkBindBufferMemory vkBindBufferMemory; - PFN_vkBindImageMemory vkBindImageMemory; - PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements; - PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements; - PFN_vkCreateBuffer vkCreateBuffer; - PFN_vkDestroyBuffer vkDestroyBuffer; - PFN_vkCreateImage vkCreateImage; - PFN_vkDestroyImage vkDestroyImage; -#if VMA_DEDICATED_ALLOCATION - PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR; - PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR; + PFN_vkGetPhysicalDeviceProperties VMA_NULLABLE vkGetPhysicalDeviceProperties; + PFN_vkGetPhysicalDeviceMemoryProperties VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties; + PFN_vkAllocateMemory VMA_NULLABLE vkAllocateMemory; + PFN_vkFreeMemory VMA_NULLABLE vkFreeMemory; + PFN_vkMapMemory VMA_NULLABLE vkMapMemory; + PFN_vkUnmapMemory VMA_NULLABLE vkUnmapMemory; + PFN_vkFlushMappedMemoryRanges VMA_NULLABLE vkFlushMappedMemoryRanges; + PFN_vkInvalidateMappedMemoryRanges VMA_NULLABLE vkInvalidateMappedMemoryRanges; + PFN_vkBindBufferMemory VMA_NULLABLE vkBindBufferMemory; + PFN_vkBindImageMemory VMA_NULLABLE vkBindImageMemory; + PFN_vkGetBufferMemoryRequirements VMA_NULLABLE vkGetBufferMemoryRequirements; + PFN_vkGetImageMemoryRequirements VMA_NULLABLE vkGetImageMemoryRequirements; + PFN_vkCreateBuffer VMA_NULLABLE vkCreateBuffer; + PFN_vkDestroyBuffer VMA_NULLABLE vkDestroyBuffer; + PFN_vkCreateImage VMA_NULLABLE vkCreateImage; + PFN_vkDestroyImage VMA_NULLABLE vkDestroyImage; + PFN_vkCmdCopyBuffer VMA_NULLABLE vkCmdCopyBuffer; +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + PFN_vkGetBufferMemoryRequirements2KHR VMA_NULLABLE vkGetBufferMemoryRequirements2KHR; + PFN_vkGetImageMemoryRequirements2KHR VMA_NULLABLE vkGetImageMemoryRequirements2KHR; +#endif +#if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 + PFN_vkBindBufferMemory2KHR VMA_NULLABLE vkBindBufferMemory2KHR; + PFN_vkBindImageMemory2KHR VMA_NULLABLE vkBindImageMemory2KHR; +#endif +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 + PFN_vkGetPhysicalDeviceMemoryProperties2KHR VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties2KHR; #endif } VmaVulkanFunctions; @@ -1377,18 +2360,6 @@ typedef enum VmaRecordFlagBits { } VmaRecordFlagBits; typedef VkFlags VmaRecordFlags; -/* -Define this macro to 0/1 to disable/enable support for recording functionality, -available through VmaAllocatorCreateInfo::pRecordSettings. -*/ -#ifndef VMA_RECORDING_ENABLED - #ifdef _WIN32 - #define VMA_RECORDING_ENABLED 1 - #else - #define VMA_RECORDING_ENABLED 0 - #endif -#endif - /// Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings. typedef struct VmaRecordSettings { @@ -1401,7 +2372,7 @@ typedef struct VmaRecordSettings It will be opened for the whole time #VmaAllocator object is alive. If opening this file fails, creation of the whole allocator object fails. */ - const char* pFilePath; + const char* VMA_NOT_NULL pFilePath; } VmaRecordSettings; /// Description of a Allocator to be created. @@ -1411,19 +2382,19 @@ typedef struct VmaAllocatorCreateInfo VmaAllocatorCreateFlags flags; /// Vulkan physical device. /** It must be valid throughout whole lifetime of created allocator. */ - VkPhysicalDevice physicalDevice; + VkPhysicalDevice VMA_NOT_NULL physicalDevice; /// Vulkan device. /** It must be valid throughout whole lifetime of created allocator. */ - VkDevice device; + VkDevice VMA_NOT_NULL device; /// Preferred size of a single `VkDeviceMemory` block to be allocated from large heaps > 1 GiB. Optional. /** Set to 0 to use default, which is currently 256 MiB. */ VkDeviceSize preferredLargeHeapBlockSize; /// Custom CPU memory allocation callbacks. Optional. /** Optional, can be null. When specified, will also be used for all CPU-side memory allocations. */ - const VkAllocationCallbacks* pAllocationCallbacks; + const VkAllocationCallbacks* VMA_NULLABLE pAllocationCallbacks; /// Informative callbacks for `vkAllocateMemory`, `vkFreeMemory`. Optional. /** Optional, can be null. */ - const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks; + const VmaDeviceMemoryCallbacks* VMA_NULLABLE pDeviceMemoryCallbacks; /** \brief Maximum number of additional frames that are in use at the same time as current frame. This value is used only when you make allocations with @@ -1459,54 +2430,91 @@ typedef struct VmaAllocatorCreateInfo smaller amount of memory, because graphics driver doesn't necessary fail new allocations with `VK_ERROR_OUT_OF_DEVICE_MEMORY` result when memory capacity is exceeded. It may return success and just silently migrate some device memory - blocks to system RAM. + blocks to system RAM. This driver behavior can also be controlled using + VK_AMD_memory_overallocation_behavior extension. */ - const VkDeviceSize* pHeapSizeLimit; - /** \brief Pointers to Vulkan functions. Can be null if you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1`. + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount") pHeapSizeLimit; - If you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1` in configuration section, - you can pass null as this member, because the library will fetch pointers to - Vulkan functions internally in a static way, like: + /** \brief Pointers to Vulkan functions. Can be null. - vulkanFunctions.vkAllocateMemory = &vkAllocateMemory; - - Fill this member if you want to provide your own pointers to Vulkan functions, - e.g. fetched using `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`. + For details see [Pointers to Vulkan functions](@ref config_Vulkan_functions). */ - const VmaVulkanFunctions* pVulkanFunctions; + const VmaVulkanFunctions* VMA_NULLABLE pVulkanFunctions; /** \brief Parameters for recording of VMA calls. Can be null. If not null, it enables recording of calls to VMA functions to a file. If support for recording is not enabled using `VMA_RECORDING_ENABLED` macro, creation of the allocator object fails with `VK_ERROR_FEATURE_NOT_PRESENT`. */ - const VmaRecordSettings* pRecordSettings; + const VmaRecordSettings* VMA_NULLABLE pRecordSettings; + /** \brief Handle to Vulkan instance object. + + Starting from version 3.0.0 this member is no longer optional, it must be set! + */ + VkInstance VMA_NOT_NULL instance; + /** \brief Optional. The highest version of Vulkan that the application is designed to use. + + It must be a value in the format as created by macro `VK_MAKE_VERSION` or a constant like: `VK_API_VERSION_1_1`, `VK_API_VERSION_1_0`. + The patch version number specified is ignored. Only the major and minor versions are considered. + It must be less or equal (preferably equal) to value as passed to `vkCreateInstance` as `VkApplicationInfo::apiVersion`. + Only versions 1.0 and 1.1 are supported by the current implementation. + Leaving it initialized to zero is equivalent to `VK_API_VERSION_1_0`. + */ + uint32_t vulkanApiVersion; } VmaAllocatorCreateInfo; /// Creates Allocator object. -VkResult vmaCreateAllocator( - const VmaAllocatorCreateInfo* pCreateInfo, - VmaAllocator* pAllocator); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator( + const VmaAllocatorCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocator VMA_NULLABLE * VMA_NOT_NULL pAllocator); /// Destroys allocator object. -void vmaDestroyAllocator( - VmaAllocator allocator); +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator( + VmaAllocator VMA_NULLABLE allocator); + +/** \brief Information about existing #VmaAllocator object. +*/ +typedef struct VmaAllocatorInfo +{ + /** \brief Handle to Vulkan instance object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::instance. + */ + VkInstance VMA_NOT_NULL instance; + /** \brief Handle to Vulkan physical device object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::physicalDevice. + */ + VkPhysicalDevice VMA_NOT_NULL physicalDevice; + /** \brief Handle to Vulkan device object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::device. + */ + VkDevice VMA_NOT_NULL device; +} VmaAllocatorInfo; + +/** \brief Returns information about existing #VmaAllocator object - handle to Vulkan device etc. + +It might be useful if you want to keep just the #VmaAllocator handle and fetch other required handles to +`VkPhysicalDevice`, `VkDevice` etc. every time using this function. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator VMA_NOT_NULL allocator, VmaAllocatorInfo* VMA_NOT_NULL pAllocatorInfo); /** PhysicalDeviceProperties are fetched from physicalDevice by the allocator. You can access it here, without fetching it again on your own. */ -void vmaGetPhysicalDeviceProperties( - VmaAllocator allocator, - const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties); +VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties( + VmaAllocator VMA_NOT_NULL allocator, + const VkPhysicalDeviceProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceProperties); /** PhysicalDeviceMemoryProperties are fetched from physicalDevice by the allocator. You can access it here, without fetching it again on your own. */ -void vmaGetMemoryProperties( - VmaAllocator allocator, - const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties); +VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryProperties( + VmaAllocator VMA_NOT_NULL allocator, + const VkPhysicalDeviceMemoryProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceMemoryProperties); /** \brief Given Memory Type Index, returns Property Flags of this memory type. @@ -1514,10 +2522,10 @@ void vmaGetMemoryProperties( This is just a convenience function. Same information can be obtained using vmaGetMemoryProperties(). */ -void vmaGetMemoryTypeProperties( - VmaAllocator allocator, +VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryTypeProperties( + VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeIndex, - VkMemoryPropertyFlags* pFlags); + VkMemoryPropertyFlags* VMA_NOT_NULL pFlags); /** \brief Sets index of the current frame. @@ -1527,8 +2535,8 @@ This function must be used if you make allocations with when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot become lost in the current frame. */ -void vmaSetCurrentFrameIndex( - VmaAllocator allocator, +VMA_CALL_PRE void VMA_CALL_POST vmaSetCurrentFrameIndex( + VmaAllocator VMA_NOT_NULL allocator, uint32_t frameIndex); /** \brief Calculated statistics of memory usage in entire allocator. @@ -1557,26 +2565,91 @@ typedef struct VmaStats VmaStatInfo total; } VmaStats; -/// Retrieves statistics from current state of the Allocator. -void vmaCalculateStats( - VmaAllocator allocator, - VmaStats* pStats); +/** \brief Retrieves statistics from current state of the Allocator. +This function is called "calculate" not "get" because it has to traverse all +internal data structures, so it may be quite slow. For faster but more brief statistics +suitable to be called every frame or every allocation, use vmaGetBudget(). + +Note that when using allocator from multiple threads, returned information may immediately +become outdated. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaCalculateStats( + VmaAllocator VMA_NOT_NULL allocator, + VmaStats* VMA_NOT_NULL pStats); + +/** \brief Statistics of current memory usage and available budget, in bytes, for specific memory heap. +*/ +typedef struct VmaBudget +{ + /** \brief Sum size of all `VkDeviceMemory` blocks allocated from particular heap, in bytes. + */ + VkDeviceSize blockBytes; + + /** \brief Sum size of all allocations created in particular heap, in bytes. + + Usually less or equal than `blockBytes`. + Difference `blockBytes - allocationBytes` is the amount of memory allocated but unused - + available for new allocations or wasted due to fragmentation. + + It might be greater than `blockBytes` if there are some allocations in lost state, as they account + to this value as well. + */ + VkDeviceSize allocationBytes; + + /** \brief Estimated current memory usage of the program, in bytes. + + Fetched from system using `VK_EXT_memory_budget` extension if enabled. + + It might be different than `blockBytes` (usually higher) due to additional implicit objects + also occupying the memory, like swapchain, pipelines, descriptor heaps, command buffers, or + `VkDeviceMemory` blocks allocated outside of this library, if any. + */ + VkDeviceSize usage; + + /** \brief Estimated amount of memory available to the program, in bytes. + + Fetched from system using `VK_EXT_memory_budget` extension if enabled. + + It might be different (most probably smaller) than `VkMemoryHeap::size[heapIndex]` due to factors + external to the program, like other programs also consuming system resources. + Difference `budget - usage` is the amount of additional memory that can probably + be allocated without problems. Exceeding the budget may result in various problems. + */ + VkDeviceSize budget; +} VmaBudget; + +/** \brief Retrieves information about current memory budget for all memory heaps. + +\param[out] pBudget Must point to array with number of elements at least equal to number of memory heaps in physical device used. + +This function is called "get" not "calculate" because it is very fast, suitable to be called +every frame or every allocation. For more detailed statistics use vmaCalculateStats(). + +Note that when using allocator from multiple threads, returned information may immediately +become outdated. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget( + VmaAllocator VMA_NOT_NULL allocator, + VmaBudget* VMA_NOT_NULL pBudget); + +#ifndef VMA_STATS_STRING_ENABLED #define VMA_STATS_STRING_ENABLED 1 +#endif #if VMA_STATS_STRING_ENABLED /// Builds and returns statistics as string in JSON format. /** @param[out] ppStatsString Must be freed using vmaFreeStatsString() function. */ -void vmaBuildStatsString( - VmaAllocator allocator, - char** ppStatsString, +VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( + VmaAllocator VMA_NOT_NULL allocator, + char* VMA_NULLABLE * VMA_NOT_NULL ppStatsString, VkBool32 detailedMap); -void vmaFreeStatsString( - VmaAllocator allocator, - char* pStatsString); +VMA_CALL_PRE void VMA_CALL_POST vmaFreeStatsString( + VmaAllocator VMA_NOT_NULL allocator, + char* VMA_NULLABLE pStatsString); #endif // #if VMA_STATS_STRING_ENABLED @@ -1627,7 +2700,7 @@ typedef enum VmaMemoryUsage Memory that is both mappable on host (guarantees to be `HOST_VISIBLE`) and preferably fast to access by GPU. CPU access is typically uncached. Writes may be write-combined. - Usage: Resources written frequently by host (dynamic), read by device. E.g. textures, vertex buffers, uniform buffers updated every frame or every draw call. + Usage: Resources written frequently by host (dynamic), read by device. E.g. textures (with LINEAR layout), vertex buffers, uniform buffers updated every frame or every draw call. */ VMA_MEMORY_USAGE_CPU_TO_GPU = 3, /** Memory mappable on host (guarantees to be `HOST_VISIBLE`) and cached. @@ -1639,6 +2712,21 @@ typedef enum VmaMemoryUsage - Any resources read or accessed randomly on host, e.g. CPU-side copy of vertex buffer used as source of transfer, but also used for collision detection. */ VMA_MEMORY_USAGE_GPU_TO_CPU = 4, + /** CPU memory - memory that is preferably not `DEVICE_LOCAL`, but also not guaranteed to be `HOST_VISIBLE`. + + Usage: Staging copy of resources moved from GPU memory to CPU memory as part + of custom paging/residency mechanism, to be moved back to GPU memory when needed. + */ + VMA_MEMORY_USAGE_CPU_COPY = 5, + /** Lazily allocated GPU memory having `VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT`. + Exists mostly on mobile platforms. Using it on desktop PC or other GPUs with no such memory type present will fail the allocation. + + Usage: Memory for transient attachment images (color attachments, depth attachments etc.), created with `VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT`. + + Allocations with this usage are always created as dedicated - it implies #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. + */ + VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED = 6, + VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF } VmaMemoryUsage; @@ -1648,11 +2736,6 @@ typedef enum VmaAllocationCreateFlagBits { Use it for special, big resources, like fullscreen images used as attachments. - This flag must also be used for host visible resources that you want to map - simultaneously because otherwise they might end up as regions of the same - `VkDeviceMemory`, while mapping same `VkDeviceMemory` multiple times - simultaneously is illegal. - You should not use this flag if VmaAllocationCreateInfo::pool is not null. */ VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001, @@ -1671,7 +2754,7 @@ typedef enum VmaAllocationCreateFlagBits { Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. - Is it valid to use this flag for allocation made from memory type that is not + It is valid to use this flag for allocation made from memory type that is not `HOST_VISIBLE`. This flag is then ignored and memory is not mapped. This is useful if you need an allocation that is efficient to use on GPU (`DEVICE_LOCAL`) and still want to map it directly if possible on platforms that @@ -1706,6 +2789,54 @@ typedef enum VmaAllocationCreateFlagBits { freed together with the allocation. It is also used in vmaBuildStatsString(). */ VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020, + /** Allocation will be created from upper stack in a double stack pool. + + This flag is only allowed for custom pools created with #VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT flag. + */ + VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT = 0x00000040, + /** Create both buffer/image and allocation, but don't bind them together. + It is useful when you want to bind yourself to do some more advanced binding, e.g. using some extensions. + The flag is meaningful only with functions that bind by default: vmaCreateBuffer(), vmaCreateImage(). + Otherwise it is ignored. + */ + VMA_ALLOCATION_CREATE_DONT_BIND_BIT = 0x00000080, + /** Create allocation only if additional device memory required for it, if any, won't exceed + memory budget. Otherwise return `VK_ERROR_OUT_OF_DEVICE_MEMORY`. + */ + VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT = 0x00000100, + + /** Allocation strategy that chooses smallest possible free range for the + allocation. + */ + VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT = 0x00010000, + /** Allocation strategy that chooses biggest possible free range for the + allocation. + */ + VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT = 0x00020000, + /** Allocation strategy that chooses first suitable free range for the + allocation. + + "First" doesn't necessarily means the one with smallest offset in memory, + but rather the one that is easiest and fastest to find. + */ + VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT = 0x00040000, + + /** Allocation strategy that tries to minimize memory usage. + */ + VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT, + /** Allocation strategy that tries to minimize allocation time. + */ + VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT = VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT, + /** Allocation strategy that tries to minimize memory fragmentation. + */ + VMA_ALLOCATION_CREATE_STRATEGY_MIN_FRAGMENTATION_BIT = VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT, + + /** A bit mask to extract only `STRATEGY` bits from entire set of flags. + */ + VMA_ALLOCATION_CREATE_STRATEGY_MASK = + VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT | + VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT | + VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT, VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaAllocationCreateFlagBits; @@ -1744,14 +2875,14 @@ typedef struct VmaAllocationCreateInfo Leave `VK_NULL_HANDLE` to allocate from default pool. If not null, members: `usage`, `requiredFlags`, `preferredFlags`, `memoryTypeBits` are ignored. */ - VmaPool pool; + VmaPool VMA_NULLABLE pool; /** \brief Custom general-purpose pointer that will be stored in #VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData(). If #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is used, it must be either null or pointer to a null-terminated string. The string will be then copied to internal buffer, so it doesn't need to be valid after allocation call. */ - void* pUserData; + void* VMA_NULLABLE pUserData; } VmaAllocationCreateInfo; /** @@ -1770,11 +2901,11 @@ device doesn't support any memory type with requested features for the specific type of resource you want to use it for. Please check parameters of your resource, like image layout (OPTIMAL versus LINEAR) or mip level count. */ -VkResult vmaFindMemoryTypeIndex( - VmaAllocator allocator, +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex( + VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeBits, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /** \brief Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo. @@ -1788,11 +2919,11 @@ It is just a convenience function, equivalent to calling: - `vmaFindMemoryTypeIndex` - `vkDestroyBuffer` */ -VkResult vmaFindMemoryTypeIndexForBufferInfo( - VmaAllocator allocator, - const VkBufferCreateInfo* pBufferCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForBufferInfo( + VmaAllocator VMA_NOT_NULL allocator, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /** \brief Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo. @@ -1806,17 +2937,17 @@ It is just a convenience function, equivalent to calling: - `vmaFindMemoryTypeIndex` - `vkDestroyImage` */ -VkResult vmaFindMemoryTypeIndexForImageInfo( - VmaAllocator allocator, - const VkImageCreateInfo* pImageCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo( + VmaAllocator VMA_NOT_NULL allocator, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /// Flags to be passed as VmaPoolCreateInfo::flags. typedef enum VmaPoolCreateFlagBits { /** \brief Use this flag if you always allocate only buffers and linear images or only optimal images out of this pool and so Buffer-Image Granularity can be ignored. - This is na optional optimization flag. + This is an optional optimization flag. If you always allocate using vmaCreateBuffer(), vmaCreateImage(), vmaAllocateMemoryForBuffer(), then you don't need to use it because allocator @@ -1829,10 +2960,44 @@ typedef enum VmaPoolCreateFlagBits { (wasted memory). In that case, if you can make sure you always allocate only buffers and linear images or only optimal images out of this pool, use this flag to make allocator disregard Buffer-Image Granularity and so make allocations - more optimal. + faster and more optimal. */ VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002, + /** \brief Enables alternative, linear allocation algorithm in this pool. + + Specify this flag to enable linear allocation algorithm, which always creates + new allocations after last one and doesn't reuse space from allocations freed in + between. It trades memory consumption for simplified algorithm and data + structure, which has better performance and uses less memory for metadata. + + By using this flag, you can achieve behavior of free-at-once, stack, + ring buffer, and double stack. For details, see documentation chapter + \ref linear_algorithm. + + When using this flag, you must specify VmaPoolCreateInfo::maxBlockCount == 1 (or 0 for default). + + For more details, see [Linear allocation algorithm](@ref linear_algorithm). + */ + VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT = 0x00000004, + + /** \brief Enables alternative, buddy allocation algorithm in this pool. + + It operates on a tree of blocks, each having size that is a power of two and + a half of its parent's size. Comparing to default algorithm, this one provides + faster allocation and deallocation and decreased external fragmentation, + at the expense of more memory wasted (internal fragmentation). + + For more details, see [Buddy allocation algorithm](@ref buddy_algorithm). + */ + VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT = 0x00000008, + + /** Bit mask to extract only `ALGORITHM` bits from entire set of flags. + */ + VMA_POOL_CREATE_ALGORITHM_MASK = + VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT | + VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT, + VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaPoolCreateFlagBits; typedef VkFlags VmaPoolCreateFlags; @@ -1846,22 +3011,26 @@ typedef struct VmaPoolCreateInfo { /** \brief Use combination of #VmaPoolCreateFlagBits. */ VmaPoolCreateFlags flags; - /** \brief Size of a single `VkDeviceMemory` block to be allocated as part of this pool, in bytes. + /** \brief Size of a single `VkDeviceMemory` block to be allocated as part of this pool, in bytes. Optional. - Optional. Leave 0 to use default. + Specify nonzero to set explicit, constant size of memory blocks used by this + pool. + + Leave 0 to use default and let the library manage block sizes automatically. + Sizes of particular blocks may vary. */ VkDeviceSize blockSize; /** \brief Minimum number of blocks to be always allocated in this pool, even if they stay empty. - Set to 0 to have no preallocated blocks and let the pool be completely empty. + Set to 0 to have no preallocated blocks and allow the pool be completely empty. */ size_t minBlockCount; /** \brief Maximum number of blocks that can be allocated in this pool. Optional. - Optional. Set to 0 to use `SIZE_MAX`, which means no limit. + Set to 0 to use default, which is `SIZE_MAX`, which means no limit. - Set to same value as minBlockCount to have fixed amount of memory allocated - throuout whole lifetime of this pool. + Set to same value as VmaPoolCreateInfo::minBlockCount to have fixed amount of memory allocated + throughout whole lifetime of this pool. */ size_t maxBlockCount; /** \brief Maximum number of additional frames that are in use at the same time as current frame. @@ -1895,13 +3064,16 @@ typedef struct VmaPoolStats { /** \brief Number of continuous memory ranges in the pool not used by any #VmaAllocation. */ size_t unusedRangeCount; - /** \brief Size of the largest continuous free memory region. + /** \brief Size of the largest continuous free memory region available for new allocation. Making a new allocation of that size is not guaranteed to succeed because of possible additional margin required to respect alignment and buffer/image granularity. */ VkDeviceSize unusedRangeSizeMax; + /** \brief Number of `VkDeviceMemory` blocks allocated for this pool. + */ + size_t blockCount; } VmaPoolStats; /** \brief Allocates Vulkan device memory and creates #VmaPool object. @@ -1910,16 +3082,16 @@ typedef struct VmaPoolStats { @param pCreateInfo Parameters of pool to create. @param[out] pPool Handle to created pool. */ -VkResult vmaCreatePool( - VmaAllocator allocator, - const VmaPoolCreateInfo* pCreateInfo, - VmaPool* pPool); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool( + VmaAllocator VMA_NOT_NULL allocator, + const VmaPoolCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaPool VMA_NULLABLE * VMA_NOT_NULL pPool); /** \brief Destroys #VmaPool object and frees Vulkan device memory. */ -void vmaDestroyPool( - VmaAllocator allocator, - VmaPool pool); +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool( + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NULLABLE pool); /** \brief Retrieves statistics of existing #VmaPool object. @@ -1927,10 +3099,10 @@ void vmaDestroyPool( @param pool Pool object. @param[out] pPoolStats Statistics of specified pool. */ -void vmaGetPoolStats( - VmaAllocator allocator, - VmaPool pool, - VmaPoolStats* pPoolStats); +VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats( + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + VmaPoolStats* VMA_NOT_NULL pPoolStats); /** \brief Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now. @@ -1938,16 +3110,16 @@ void vmaGetPoolStats( @param pool Pool. @param[out] pLostAllocationCount Number of allocations marked as lost. Optional - pass null if you don't need this information. */ -void vmaMakePoolAllocationsLost( - VmaAllocator allocator, - VmaPool pool, - size_t* pLostAllocationCount); +VMA_CALL_PRE void VMA_CALL_POST vmaMakePoolAllocationsLost( + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + size_t* VMA_NULLABLE pLostAllocationCount); /** \brief Checks magic number in margins around all allocations in given memory pool in search for corruptions. Corruption detection is enabled only when `VMA_DEBUG_DETECT_CORRUPTION` macro is defined to nonzero, `VMA_DEBUG_MARGIN` is defined to nonzero and the pool is created in memory type that is -`HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref corruption_detection). +`HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref debugging_memory_usage_corruption_detection). Possible return values: @@ -1957,7 +3129,28 @@ Possible return values: `VMA_ASSERT` is also fired in that case. - Other value: Error returned by Vulkan, e.g. memory mapping failure. */ -VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator VMA_NOT_NULL allocator, VmaPool VMA_NOT_NULL pool); + +/** \brief Retrieves name of a custom pool. + +After the call `ppName` is either null or points to an internally-owned null-terminated string +containing name of the pool that was previously set. The pointer becomes invalid when the pool is +destroyed or its name is changed using vmaSetPoolName(). +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName( + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + const char* VMA_NULLABLE * VMA_NOT_NULL ppName); + +/** \brief Sets name of a custom pool. + +`pName` can be either null or pointer to a null-terminated string with new name for the pool. +Function makes internal copy of the string, so it can be changed or freed immediately after this call. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaSetPoolName( + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + const char* VMA_NULLABLE pName); /** \struct VmaAllocation \brief Represents single memory allocation. @@ -2001,7 +3194,7 @@ typedef struct VmaAllocationInfo { If the allocation is lost, it is equal to `VK_NULL_HANDLE`. */ - VkDeviceMemory deviceMemory; + VkDeviceMemory VMA_NULLABLE_NON_DISPATCHABLE deviceMemory; /** \brief Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation. It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost. @@ -2010,22 +3203,28 @@ typedef struct VmaAllocationInfo { /** \brief Size of this allocation, in bytes. It never changes, unless allocation is lost. + + \note Allocation size returned in this variable may be greater than the size + requested for the resource e.g. as `VkBufferCreateInfo::size`. Whole size of the + allocation is accessible for operations on memory e.g. using a pointer after + mapping with vmaMapMemory(), but operations on the resource e.g. using + `vkCmdCopyBuffer` must be limited to the size of the resource. */ VkDeviceSize size; /** \brief Pointer to the beginning of this allocation as mapped data. If the allocation hasn't been mapped using vmaMapMemory() and hasn't been - created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value null. + created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value is null. It can change after call to vmaMapMemory(), vmaUnmapMemory(). It can also change after call to vmaDefragment() if this allocation is passed to the function. */ - void* pMappedData; + void* VMA_NULLABLE pMappedData; /** \brief Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData(). It can change after call to vmaSetAllocationUserData() for this allocation. */ - void* pUserData; + void* VMA_NULLABLE pUserData; } VmaAllocationInfo; /** \brief General purpose memory allocation. @@ -2033,17 +3232,44 @@ typedef struct VmaAllocationInfo { @param[out] pAllocation Handle to allocated memory. @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo(). -You should free the memory using vmaFreeMemory(). +You should free the memory using vmaFreeMemory() or vmaFreeMemoryPages(). It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(), vmaCreateBuffer(), vmaCreateImage() instead whenever possible. */ -VkResult vmaAllocateMemory( - VmaAllocator allocator, - const VkMemoryRequirements* pVkMemoryRequirements, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory( + VmaAllocator VMA_NOT_NULL allocator, + const VkMemoryRequirements* VMA_NOT_NULL pVkMemoryRequirements, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); + +/** \brief General purpose memory allocation for multiple allocation objects at once. + +@param allocator Allocator object. +@param pVkMemoryRequirements Memory requirements for each allocation. +@param pCreateInfo Creation parameters for each alloction. +@param allocationCount Number of allocations to make. +@param[out] pAllocations Pointer to array that will be filled with handles to created allocations. +@param[out] pAllocationInfo Optional. Pointer to array that will be filled with parameters of created allocations. + +You should free the memory using vmaFreeMemory() or vmaFreeMemoryPages(). + +Word "pages" is just a suggestion to use this function to allocate pieces of memory needed for sparse binding. +It is just a general purpose allocation function able to make multiple allocations at once. +It may be internally optimized to be more efficient than calling vmaAllocateMemory() `allocationCount` times. + +All allocations are made using same parameters. All of them are created out of the same memory pool and type. +If any allocation fails, all allocations already made within this function call are also freed, so that when +returned result is not `VK_SUCCESS`, `pAllocation` array is always entirely filled with `VK_NULL_HANDLE`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( + VmaAllocator VMA_NOT_NULL allocator, + const VkMemoryRequirements* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pVkMemoryRequirements, + const VmaAllocationCreateInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pCreateInfo, + size_t allocationCount, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations, + VmaAllocationInfo* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationInfo); /** @param[out] pAllocation Handle to allocated memory. @@ -2051,25 +3277,55 @@ VkResult vmaAllocateMemory( You should free the memory using vmaFreeMemory(). */ -VkResult vmaAllocateMemoryForBuffer( - VmaAllocator allocator, - VkBuffer buffer, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( + VmaAllocator VMA_NOT_NULL allocator, + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /// Function similar to vmaAllocateMemoryForBuffer(). -VkResult vmaAllocateMemoryForImage( - VmaAllocator allocator, - VkImage image, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( + VmaAllocator VMA_NOT_NULL allocator, + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); -/// Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). -void vmaFreeMemory( - VmaAllocator allocator, - VmaAllocation allocation); +/** \brief Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). + +Passing `VK_NULL_HANDLE` as `allocation` is valid. Such function call is just skipped. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( + VmaAllocator VMA_NOT_NULL allocator, + const VmaAllocation VMA_NULLABLE allocation); + +/** \brief Frees memory and destroys multiple allocations. + +Word "pages" is just a suggestion to use this function to free pieces of memory used for sparse binding. +It is just a general purpose function to free memory and destroy allocations made using e.g. vmaAllocateMemory(), +vmaAllocateMemoryPages() and other functions. +It may be internally optimized to be more efficient than calling vmaFreeMemory() `allocationCount` times. + +Allocations in `pAllocations` array can come from any memory pools and types. +Passing `VK_NULL_HANDLE` as elements of `pAllocations` array is valid. Such entries are just skipped. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( + VmaAllocator VMA_NOT_NULL allocator, + size_t allocationCount, + const VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations); + +/** \brief Deprecated. + +\deprecated +In version 2.2.0 it used to try to change allocation's size without moving or reallocating it. +In current version it returns `VK_SUCCESS` only if `newSize` equals current allocation's size. +Otherwise returns `VK_ERROR_OUT_OF_POOL_MEMORY`, indicating that allocation's size could not be changed. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize newSize); /** \brief Returns current information about specified allocation and atomically marks it as used in current frame. @@ -2087,10 +3343,10 @@ you can avoid calling it too often. (e.g. due to defragmentation or allocation becoming lost). - If you just want to check if allocation is not lost, vmaTouchAllocation() will work faster. */ -void vmaGetAllocationInfo( - VmaAllocator allocator, - VmaAllocation allocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VmaAllocationInfo* VMA_NOT_NULL pAllocationInfo); /** \brief Returns `VK_TRUE` if allocation is not lost and atomically marks it as used in current frame. @@ -2106,9 +3362,9 @@ Lost allocation and the buffer/image still need to be destroyed. If the allocation has been created without #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag, this function always returns `VK_TRUE`. */ -VkBool32 vmaTouchAllocation( - VmaAllocator allocator, - VmaAllocation allocation); +VMA_CALL_PRE VkBool32 VMA_CALL_POST vmaTouchAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation); /** \brief Sets pUserData in given allocation to new value. @@ -2123,10 +3379,10 @@ If the flag was not used, the value of pointer `pUserData` is just copied to allocation's `pUserData`. It is opaque, so you can use it however you want - e.g. as a pointer, ordinal number or some handle to you own data. */ -void vmaSetAllocationUserData( - VmaAllocator allocator, - VmaAllocation allocation, - void* pUserData); +VMA_CALL_PRE void VMA_CALL_POST vmaSetAllocationUserData( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + void* VMA_NULLABLE pUserData); /** \brief Creates new allocation that is in lost state from the beginning. @@ -2138,9 +3394,9 @@ Returned allocation is not tied to any specific memory pool or memory type and not bound to any image or buffer. It has size = 0. It cannot be turned into a real, non-empty allocation. */ -void vmaCreateLostAllocation( - VmaAllocator allocator, - VmaAllocation* pAllocation); +VMA_CALL_PRE void VMA_CALL_POST vmaCreateLostAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation); /** \brief Maps memory represented by given allocation and returns pointer to it. @@ -2175,23 +3431,33 @@ This function fails when used on allocation made in memory type that is not This function always fails when called for allocation that was created with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocations cannot be mapped. + +This function doesn't automatically flush or invalidate caches. +If the allocation is made from a memory types that is not `HOST_COHERENT`, +you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification. */ -VkResult vmaMapMemory( - VmaAllocator allocator, - VmaAllocation allocation, - void** ppData); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaMapMemory( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + void* VMA_NULLABLE * VMA_NOT_NULL ppData); /** \brief Unmaps memory represented by given allocation, mapped previously using vmaMapMemory(). For details, see description of vmaMapMemory(). + +This function doesn't automatically flush or invalidate caches. +If the allocation is made from a memory types that is not `HOST_COHERENT`, +you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification. */ -void vmaUnmapMemory( - VmaAllocator allocator, - VmaAllocation allocation); +VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation); /** \brief Flushes memory of given allocation. Calls `vkFlushMappedMemoryRanges()` for memory associated with given range of given allocation. +It needs to be called after writing to a mapped memory for memory types that are not `HOST_COHERENT`. +Unmap operation doesn't do that automatically. - `offset` must be relative to the beginning of allocation. - `size` can be `VK_WHOLE_SIZE`. It means all memory from `offset` the the end of given allocation. @@ -2200,12 +3466,25 @@ Calls `vkFlushMappedMemoryRanges()` for memory associated with given range of gi - If `size` is 0, this call is ignored. - If memory type that the `allocation` belongs to is not `HOST_VISIBLE` or it is `HOST_COHERENT`, this call is ignored. + +Warning! `offset` and `size` are relative to the contents of given `allocation`. +If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively. +Do not pass allocation's offset as `offset`!!! + +This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. */ -void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize offset, + VkDeviceSize size); /** \brief Invalidates memory of given allocation. Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given range of given allocation. +It needs to be called before reading from a mapped memory for memory types that are not `HOST_COHERENT`. +Map operation doesn't do that automatically. - `offset` must be relative to the beginning of allocation. - `size` can be `VK_WHOLE_SIZE`. It means all memory from `offset` the the end of given allocation. @@ -2214,8 +3493,61 @@ Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given range - If `size` is 0, this call is ignored. - If memory type that the `allocation` belongs to is not `HOST_VISIBLE` or it is `HOST_COHERENT`, this call is ignored. + +Warning! `offset` and `size` are relative to the contents of given `allocation`. +If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively. +Do not pass allocation's offset as `offset`!!! + +This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if +it is called, otherwise `VK_SUCCESS`. */ -void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize offset, + VkDeviceSize size); + +/** \brief Flushes memory of given set of allocations. + +Calls `vkFlushMappedMemoryRanges()` for memory associated with given ranges of given allocations. +For more information, see documentation of vmaFlushAllocation(). + +\param allocator +\param allocationCount +\param allocations +\param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero. +\param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations. + +This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations( + VmaAllocator VMA_NOT_NULL allocator, + uint32_t allocationCount, + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes); + +/** \brief Invalidates memory of given set of allocations. + +Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given ranges of given allocations. +For more information, see documentation of vmaInvalidateAllocation(). + +\param allocator +\param allocationCount +\param allocations +\param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero. +\param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations. + +This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations( + VmaAllocator VMA_NOT_NULL allocator, + uint32_t allocationCount, + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes); /** \brief Checks magic number in margins around all allocations in given memory types (in both default and custom pools) in search for corruptions. @@ -2223,7 +3555,7 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V Corruption detection is enabled only when `VMA_DEBUG_DETECT_CORRUPTION` macro is defined to nonzero, `VMA_DEBUG_MARGIN` is defined to nonzero and only for memory types that are -`HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref corruption_detection). +`HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref debugging_memory_usage_corruption_detection). Possible return values: @@ -2233,9 +3565,118 @@ Possible return values: `VMA_ASSERT` is also fired in that case. - Other value: Error returned by Vulkan, e.g. memory mapping failure. */ -VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeBits); -/** \brief Optional configuration parameters to be passed to function vmaDefragment(). */ +/** \struct VmaDefragmentationContext +\brief Represents Opaque object that represents started defragmentation process. + +Fill structure #VmaDefragmentationInfo2 and call function vmaDefragmentationBegin() to create it. +Call function vmaDefragmentationEnd() to destroy it. +*/ +VK_DEFINE_HANDLE(VmaDefragmentationContext) + +/// Flags to be used in vmaDefragmentationBegin(). None at the moment. Reserved for future use. +typedef enum VmaDefragmentationFlagBits { + VMA_DEFRAGMENTATION_FLAG_INCREMENTAL = 0x1, + VMA_DEFRAGMENTATION_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +} VmaDefragmentationFlagBits; +typedef VkFlags VmaDefragmentationFlags; + +/** \brief Parameters for defragmentation. + +To be used with function vmaDefragmentationBegin(). +*/ +typedef struct VmaDefragmentationInfo2 { + /** \brief Reserved for future use. Should be 0. + */ + VmaDefragmentationFlags flags; + /** \brief Number of allocations in `pAllocations` array. + */ + uint32_t allocationCount; + /** \brief Pointer to array of allocations that can be defragmented. + + The array should have `allocationCount` elements. + The array should not contain nulls. + Elements in the array should be unique - same allocation cannot occur twice. + It is safe to pass allocations that are in the lost state - they are ignored. + All allocations not present in this array are considered non-moveable during this defragmentation. + */ + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations; + /** \brief Optional, output. Pointer to array that will be filled with information whether the allocation at certain index has been changed during defragmentation. + + The array should have `allocationCount` elements. + You can pass null if you are not interested in this information. + */ + VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged; + /** \brief Numer of pools in `pPools` array. + */ + uint32_t poolCount; + /** \brief Either null or pointer to array of pools to be defragmented. + + All the allocations in the specified pools can be moved during defragmentation + and there is no way to check if they were really moved as in `pAllocationsChanged`, + so you must query all the allocations in all these pools for new `VkDeviceMemory` + and offset using vmaGetAllocationInfo() if you might need to recreate buffers + and images bound to them. + + The array should have `poolCount` elements. + The array should not contain nulls. + Elements in the array should be unique - same pool cannot occur twice. + + Using this array is equivalent to specifying all allocations from the pools in `pAllocations`. + It might be more efficient. + */ + const VmaPool VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(poolCount) pPools; + /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on CPU side, like `memcpy()`, `memmove()`. + + `VK_WHOLE_SIZE` means no limit. + */ + VkDeviceSize maxCpuBytesToMove; + /** \brief Maximum number of allocations that can be moved to a different place using transfers on CPU side, like `memcpy()`, `memmove()`. + + `UINT32_MAX` means no limit. + */ + uint32_t maxCpuAllocationsToMove; + /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on GPU side, posted to `commandBuffer`. + + `VK_WHOLE_SIZE` means no limit. + */ + VkDeviceSize maxGpuBytesToMove; + /** \brief Maximum number of allocations that can be moved to a different place using transfers on GPU side, posted to `commandBuffer`. + + `UINT32_MAX` means no limit. + */ + uint32_t maxGpuAllocationsToMove; + /** \brief Optional. Command buffer where GPU copy commands will be posted. + + If not null, it must be a valid command buffer handle that supports Transfer queue type. + It must be in the recording state and outside of a render pass instance. + You need to submit it and make sure it finished execution before calling vmaDefragmentationEnd(). + + Passing null means that only CPU defragmentation will be performed. + */ + VkCommandBuffer VMA_NULLABLE commandBuffer; +} VmaDefragmentationInfo2; + +typedef struct VmaDefragmentationPassMoveInfo { + VmaAllocation VMA_NOT_NULL allocation; + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory; + VkDeviceSize offset; +} VmaDefragmentationPassMoveInfo; + +/** \brief Parameters for incremental defragmentation steps. + +To be used with function vmaBeginDefragmentationPass(). +*/ +typedef struct VmaDefragmentationPassInfo { + uint32_t moveCount; + VmaDefragmentationPassMoveInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(moveCount) pMoves; +} VmaDefragmentationPassInfo; + +/** \brief Deprecated. Optional configuration parameters to be passed to function vmaDefragment(). + +\deprecated This is a part of the old interface. It is recommended to use structure #VmaDefragmentationInfo2 and function vmaDefragmentationBegin() instead. +*/ typedef struct VmaDefragmentationInfo { /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places. @@ -2261,95 +3702,107 @@ typedef struct VmaDefragmentationStats { uint32_t deviceMemoryBlocksFreed; } VmaDefragmentationStats; -/** \brief Compacts memory by moving allocations. +/** \brief Begins defragmentation process. + +@param allocator Allocator object. +@param pInfo Structure filled with parameters of defragmentation. +@param[out] pStats Optional. Statistics of defragmentation. You can pass null if you are not interested in this information. +@param[out] pContext Context object that must be passed to vmaDefragmentationEnd() to finish defragmentation. +@return `VK_SUCCESS` and `*pContext == null` if defragmentation finished within this function call. `VK_NOT_READY` and `*pContext != null` if defragmentation has been started and you need to call vmaDefragmentationEnd() to finish it. Negative value in case of error. + +Use this function instead of old, deprecated vmaDefragment(). + +Warning! Between the call to vmaDefragmentationBegin() and vmaDefragmentationEnd(): + +- You should not use any of allocations passed as `pInfo->pAllocations` or + any allocations that belong to pools passed as `pInfo->pPools`, + including calling vmaGetAllocationInfo(), vmaTouchAllocation(), or access + their data. +- Some mutexes protecting internal data structures may be locked, so trying to + make or free any allocations, bind buffers or images, map memory, or launch + another simultaneous defragmentation in between may cause stall (when done on + another thread) or deadlock (when done on the same thread), unless you are + 100% sure that defragmented allocations are in different pools. +- Information returned via `pStats` and `pInfo->pAllocationsChanged` are undefined. + They become valid after call to vmaDefragmentationEnd(). +- If `pInfo->commandBuffer` is not null, you must submit that command buffer + and make sure it finished execution before calling vmaDefragmentationEnd(). + +For more information and important limitations regarding defragmentation, see documentation chapter: +[Defragmentation](@ref defragmentation). +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationBegin( + VmaAllocator VMA_NOT_NULL allocator, + const VmaDefragmentationInfo2* VMA_NOT_NULL pInfo, + VmaDefragmentationStats* VMA_NULLABLE pStats, + VmaDefragmentationContext VMA_NULLABLE * VMA_NOT_NULL pContext); + +/** \brief Ends defragmentation process. + +Use this function to finish defragmentation started by vmaDefragmentationBegin(). +It is safe to pass `context == null`. The function then does nothing. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd( + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context); + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass( + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context, + VmaDefragmentationPassInfo* VMA_NOT_NULL pInfo +); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass( + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context +); + +/** \brief Deprecated. Compacts memory by moving allocations. @param pAllocations Array of allocations that can be moved during this compation. @param allocationCount Number of elements in pAllocations and pAllocationsChanged arrays. @param[out] pAllocationsChanged Array of boolean values that will indicate whether matching allocation in pAllocations array has been moved. This parameter is optional. Pass null if you don't need this information. @param pDefragmentationInfo Configuration parameters. Optional - pass null to use default values. @param[out] pDefragmentationStats Statistics returned by the function. Optional - pass null if you don't need this information. -@return VK_SUCCESS if completed, VK_INCOMPLETE if succeeded but didn't make all possible optimizations because limits specified in pDefragmentationInfo have been reached, negative error code in case of error. +@return `VK_SUCCESS` if completed, negative error code in case of error. + +\deprecated This is a part of the old interface. It is recommended to use structure #VmaDefragmentationInfo2 and function vmaDefragmentationBegin() instead. This function works by moving allocations to different places (different `VkDeviceMemory` objects and/or different offsets) in order to optimize memory -usage. Only allocations that are in pAllocations array can be moved. All other +usage. Only allocations that are in `pAllocations` array can be moved. All other allocations are considered nonmovable in this call. Basic rules: - Only allocations made in memory types that have - `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag can be compacted. You may pass other - allocations but it makes no sense - these will never be moved. -- You may pass allocations made with #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT but - it makes no sense - they will never be moved. + `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` and `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` + flags can be compacted. You may pass other allocations but it makes no sense - + these will never be moved. +- Custom pools created with #VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT or + #VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT flag are not defragmented. Allocations + passed to this function that come from such pools are ignored. +- Allocations created with #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT or + created as dedicated allocations for any other reason are also ignored. - Both allocations made with or without #VMA_ALLOCATION_CREATE_MAPPED_BIT flag can be compacted. If not persistently mapped, memory will be mapped temporarily inside this function if needed. -- You must not pass same #VmaAllocation object multiple times in pAllocations array. +- You must not pass same #VmaAllocation object multiple times in `pAllocations` array. The function also frees empty `VkDeviceMemory` blocks. -After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or -VmaAllocationInfo::offset changes. You must query them again using -vmaGetAllocationInfo() if you need them. - -If an allocation has been moved, data in memory is copied to new place -automatically, but if it was bound to a buffer or an image, you must destroy -that object yourself, create new one and bind it to the new memory pointed by -the allocation. You must use `vkDestroyBuffer()`, `vkDestroyImage()`, -`vkCreateBuffer()`, `vkCreateImage()` for that purpose and NOT vmaDestroyBuffer(), -vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example: - -\code -VkDevice device = ...; -VmaAllocator allocator = ...; -std::vector buffers = ...; -std::vector allocations = ...; - -std::vector allocationsChanged(allocations.size()); -vmaDefragment(allocator, allocations.data(), allocations.size(), allocationsChanged.data(), nullptr, nullptr); - -for(size_t i = 0; i < allocations.size(); ++i) -{ - if(allocationsChanged[i]) - { - VmaAllocationInfo allocInfo; - vmaGetAllocationInfo(allocator, allocations[i], &allocInfo); - - vkDestroyBuffer(device, buffers[i], nullptr); - - VkBufferCreateInfo bufferInfo = ...; - vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]); - - // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning. - - vkBindBufferMemory(device, buffers[i], allocInfo.deviceMemory, allocInfo.offset); - } -} -\endcode - -Note: Please don't expect memory to be fully compacted after this call. -Algorithms inside are based on some heuristics that try to maximize number of Vulkan -memory blocks to make totally empty to release them, as well as to maximimze continuous -empty space inside remaining blocks, while minimizing the number and size of data that -needs to be moved. Some fragmentation still remains after this call. This is normal. - -Warning: This function is not 100% correct according to Vulkan specification. Use it -at your own risk. That's because Vulkan doesn't guarantee that memory -requirements (size and alignment) for a new buffer or image are consistent. They -may be different even for subsequent calls with the same parameters. It really -does happen on some platforms, especially with images. - Warning: This function may be time-consuming, so you shouldn't call it too often -(like every frame or after every resource creation/destruction). +(like after every resource creation/destruction). You can call it on special occasions (like when reloading a game level or -when you just destroyed a lot of objects). +when you just destroyed a lot of objects). Calling it every frame may be OK, but +you should measure that on your platform. + +For more information, see [Defragmentation](@ref defragmentation) chapter. */ -VkResult vmaDefragment( - VmaAllocator allocator, - VmaAllocation* pAllocations, +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment( + VmaAllocator VMA_NOT_NULL allocator, + const VmaAllocation VMA_NOT_NULL * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations, size_t allocationCount, - VkBool32* pAllocationsChanged, - const VmaDefragmentationInfo *pDefragmentationInfo, - VmaDefragmentationStats* pDefragmentationStats); + VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged, + const VmaDefragmentationInfo* VMA_NULLABLE pDefragmentationInfo, + VmaDefragmentationStats* VMA_NULLABLE pDefragmentationStats); /** \brief Binds buffer to allocation. @@ -2363,10 +3816,27 @@ allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from mul It is recommended to use function vmaCreateBuffer() instead of this one. */ -VkResult vmaBindBufferMemory( - VmaAllocator allocator, - VmaAllocation allocation, - VkBuffer buffer); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer); + +/** \brief Binds buffer to allocation with additional parameters. + +@param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0. +@param pNext A chain of structures to be attached to `VkBindBufferMemoryInfoKHR` structure used internally. Normally it should be null. + +This function is similar to vmaBindBufferMemory(), but it provides additional parameters. + +If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag +or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer, + const void* VMA_NULLABLE pNext); /** \brief Binds image to allocation. @@ -2380,10 +3850,27 @@ allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from mul It is recommended to use function vmaCreateImage() instead of this one. */ -VkResult vmaBindImageMemory( - VmaAllocator allocator, - VmaAllocation allocation, - VkImage image); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image); + +/** \brief Binds image to allocation with additional parameters. + +@param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0. +@param pNext A chain of structures to be attached to `VkBindImageMemoryInfoKHR` structure used internally. Normally it should be null. + +This function is similar to vmaBindImageMemory(), but it provides additional parameters. + +If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag +or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image, + const void* VMA_NULLABLE pNext); /** @param[out] pBuffer Buffer that was created. @@ -2411,13 +3898,13 @@ and VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT is not used), it creates dedicated allocation for this buffer, just like when using VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. */ -VkResult vmaCreateBuffer( - VmaAllocator allocator, - const VkBufferCreateInfo* pBufferCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - VkBuffer* pBuffer, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( + VmaAllocator VMA_NOT_NULL allocator, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pBuffer, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief Destroys Vulkan buffer and frees allocated memory. @@ -2430,19 +3917,19 @@ vmaFreeMemory(allocator, allocation); It it safe to pass null as buffer and/or allocation. */ -void vmaDestroyBuffer( - VmaAllocator allocator, - VkBuffer buffer, - VmaAllocation allocation); +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyBuffer( + VmaAllocator VMA_NOT_NULL allocator, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE buffer, + VmaAllocation VMA_NULLABLE allocation); /// Function similar to vmaCreateBuffer(). -VkResult vmaCreateImage( - VmaAllocator allocator, - const VkImageCreateInfo* pImageCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - VkImage* pImage, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( + VmaAllocator VMA_NOT_NULL allocator, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + VkImage VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pImage, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief Destroys Vulkan image and frees allocated memory. @@ -2455,10 +3942,10 @@ vmaFreeMemory(allocator, allocation); It it safe to pass null as image and/or allocation. */ -void vmaDestroyImage( - VmaAllocator allocator, - VkImage image, - VmaAllocation allocation); +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage( + VmaAllocator VMA_NOT_NULL allocator, + VkImage VMA_NULLABLE_NON_DISPATCHABLE image, + VmaAllocation VMA_NULLABLE allocation); #ifdef __cplusplus } @@ -2477,6 +3964,7 @@ void vmaDestroyImage( #include #include #include +#include /******************************************************************************* CONFIGURATION SECTION @@ -2490,12 +3978,23 @@ Define this macro to 1 to make the library fetch pointers to Vulkan functions internally, like: vulkanFunctions.vkAllocateMemory = &vkAllocateMemory; - -Define to 0 if you are going to provide you own pointers to Vulkan functions via -VmaAllocatorCreateInfo::pVulkanFunctions. */ #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES) -#define VMA_STATIC_VULKAN_FUNCTIONS 1 + #define VMA_STATIC_VULKAN_FUNCTIONS 1 +#endif + +/* +Define this macro to 1 to make the library fetch pointers to Vulkan functions +internally, like: + + vulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkGetDeviceProcAddr(m_hDevice, vkAllocateMemory); +*/ +#if !defined(VMA_DYNAMIC_VULKAN_FUNCTIONS) + #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1 + #if defined(VK_NO_PROTOTYPES) + extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr; + extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr; + #endif #endif // Define this macro to 1 to make the library use STL containers instead of its own implementation. @@ -2513,6 +4012,24 @@ the containers. #define VMA_USE_STL_LIST 1 #endif +#ifndef VMA_USE_STL_SHARED_MUTEX + // Compiler conforms to C++17. + #if __cplusplus >= 201703L + #define VMA_USE_STL_SHARED_MUTEX 1 + // Visual studio defines __cplusplus properly only when passed additional parameter: /Zc:__cplusplus + // Otherwise it's always 199711L, despite shared_mutex works since Visual Studio 2015 Update 2. + // See: https://blogs.msdn.microsoft.com/vcblog/2018/04/09/msvc-now-correctly-reports-__cplusplus/ + #elif defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 190023918 && __cplusplus == 199711L && _MSVC_LANG >= 201703L + #define VMA_USE_STL_SHARED_MUTEX 1 + #else + #define VMA_USE_STL_SHARED_MUTEX 0 + #endif +#endif + +/* +THESE INCLUDES ARE NOT ENABLED BY DEFAULT. +Library has its own container implementation. +*/ #if VMA_USE_STL_VECTOR #include #endif @@ -2531,17 +4048,16 @@ remove them if not needed. */ #include // for assert #include // for min, max -#include // for std::mutex -#include // for std::atomic +#include #ifndef VMA_NULL // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0. #define VMA_NULL nullptr #endif -#if defined(__APPLE__) || defined(__ANDROID__) +#if defined(__ANDROID_API__) && (__ANDROID_API__ < 16) #include -void *aligned_alloc(size_t alignment, size_t size) +void *vma_aligned_alloc(size_t alignment, size_t size) { // alignment must be >= sizeof(void*) if(alignment < sizeof(void*)) @@ -2549,11 +4065,50 @@ void *aligned_alloc(size_t alignment, size_t size) alignment = sizeof(void*); } + return memalign(alignment, size); +} +#elif defined(__APPLE__) || defined(__ANDROID__) || (defined(__linux__) && defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC)) +#include + +#if defined(__APPLE__) +#include +#endif + +void *vma_aligned_alloc(size_t alignment, size_t size) +{ +#if defined(__APPLE__) && (defined(MAC_OS_X_VERSION_10_16) || defined(__IPHONE_14_0)) +#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_16 || __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_14_0 + // For C++14, usr/include/malloc/_malloc.h declares aligned_alloc()) only + // with the MacOSX11.0 SDK in Xcode 12 (which is what adds + // MAC_OS_X_VERSION_10_16), even though the function is marked + // availabe for 10.15. That's why the preprocessor checks for 10.16 but + // the __builtin_available checks for 10.15. + // People who use C++17 could call aligned_alloc with the 10.15 SDK already. + if (__builtin_available(macOS 10.15, iOS 13, *)) + return aligned_alloc(alignment, size); +#endif +#endif + // alignment must be >= sizeof(void*) + if(alignment < sizeof(void*)) + { + alignment = sizeof(void*); + } + void *pointer; if(posix_memalign(&pointer, alignment, size) == 0) return pointer; return VMA_NULL; } +#elif defined(_WIN32) +void *vma_aligned_alloc(size_t alignment, size_t size) +{ + return _aligned_malloc(size, alignment); +} +#else +void *vma_aligned_alloc(size_t alignment, size_t size) +{ + return aligned_alloc(alignment, size); +} #endif // If your compiler is not compatible with C++11 and definition of @@ -2563,20 +4118,20 @@ void *aligned_alloc(size_t alignment, size_t size) // Normal assert to check for programmer's errors, especially in Debug configuration. #ifndef VMA_ASSERT - #ifdef _DEBUG - #define VMA_ASSERT(expr) assert(expr) - #else + #ifdef NDEBUG #define VMA_ASSERT(expr) + #else + #define VMA_ASSERT(expr) assert(expr) #endif #endif // Assert that will be called very often, like inside data structures e.g. operator[]. // Making it non-empty can make program slow. #ifndef VMA_HEAVY_ASSERT - #ifdef _DEBUG - #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) - #else + #ifdef NDEBUG #define VMA_HEAVY_ASSERT(expr) + #else + #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) #endif #endif @@ -2585,11 +4140,7 @@ void *aligned_alloc(size_t alignment, size_t size) #endif #ifndef VMA_SYSTEM_ALIGNED_MALLOC - #if defined(_WIN32) - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment))) - #else - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) )) - #endif + #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) vma_aligned_alloc((alignment), (size)) #endif #ifndef VMA_SYSTEM_FREE @@ -2646,42 +4197,79 @@ void *aligned_alloc(size_t alignment, size_t size) class VmaMutex { public: - VmaMutex() { } - ~VmaMutex() { } void Lock() { m_Mutex.lock(); } void Unlock() { m_Mutex.unlock(); } + bool TryLock() { return m_Mutex.try_lock(); } private: std::mutex m_Mutex; }; #define VMA_MUTEX VmaMutex #endif -/* -If providing your own implementation, you need to implement a subset of std::atomic: +// Read-write mutex, where "read" is shared access, "write" is exclusive access. +#ifndef VMA_RW_MUTEX + #if VMA_USE_STL_SHARED_MUTEX + // Use std::shared_mutex from C++17. + #include + class VmaRWMutex + { + public: + void LockRead() { m_Mutex.lock_shared(); } + void UnlockRead() { m_Mutex.unlock_shared(); } + bool TryLockRead() { return m_Mutex.try_lock_shared(); } + void LockWrite() { m_Mutex.lock(); } + void UnlockWrite() { m_Mutex.unlock(); } + bool TryLockWrite() { return m_Mutex.try_lock(); } + private: + std::shared_mutex m_Mutex; + }; + #define VMA_RW_MUTEX VmaRWMutex + #elif defined(_WIN32) && defined(WINVER) && WINVER >= 0x0600 + // Use SRWLOCK from WinAPI. + // Minimum supported client = Windows Vista, server = Windows Server 2008. + class VmaRWMutex + { + public: + VmaRWMutex() { InitializeSRWLock(&m_Lock); } + void LockRead() { AcquireSRWLockShared(&m_Lock); } + void UnlockRead() { ReleaseSRWLockShared(&m_Lock); } + bool TryLockRead() { return TryAcquireSRWLockShared(&m_Lock) != FALSE; } + void LockWrite() { AcquireSRWLockExclusive(&m_Lock); } + void UnlockWrite() { ReleaseSRWLockExclusive(&m_Lock); } + bool TryLockWrite() { return TryAcquireSRWLockExclusive(&m_Lock) != FALSE; } + private: + SRWLOCK m_Lock; + }; + #define VMA_RW_MUTEX VmaRWMutex + #else + // Less efficient fallback: Use normal mutex. + class VmaRWMutex + { + public: + void LockRead() { m_Mutex.Lock(); } + void UnlockRead() { m_Mutex.Unlock(); } + bool TryLockRead() { return m_Mutex.TryLock(); } + void LockWrite() { m_Mutex.Lock(); } + void UnlockWrite() { m_Mutex.Unlock(); } + bool TryLockWrite() { return m_Mutex.TryLock(); } + private: + VMA_MUTEX m_Mutex; + }; + #define VMA_RW_MUTEX VmaRWMutex + #endif // #if VMA_USE_STL_SHARED_MUTEX +#endif // #ifndef VMA_RW_MUTEX -- Constructor(uint32_t desired) -- uint32_t load() const -- void store(uint32_t desired) -- bool compare_exchange_weak(uint32_t& expected, uint32_t desired) +/* +If providing your own implementation, you need to implement a subset of std::atomic. */ #ifndef VMA_ATOMIC_UINT32 - #define VMA_ATOMIC_UINT32 std::atomic + #include + #define VMA_ATOMIC_UINT32 std::atomic #endif -#ifndef VMA_BEST_FIT - /** - Main parameter for function assessing how good is a free suballocation for a new - allocation request. - - - Set to 1 to use Best-Fit algorithm - prefer smaller blocks, as close to the - size of requested allocations as possible. - - Set to 0 to use Worst-Fit algorithm - prefer larger blocks, as large as - possible. - - Experiments in special testing environment showed that Best-Fit algorithm is - better. - */ - #define VMA_BEST_FIT (1) +#ifndef VMA_ATOMIC_UINT64 + #include + #define VMA_ATOMIC_UINT64 std::atomic #endif #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY @@ -2770,40 +4358,109 @@ static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF; END OF CONFIGURATION */ +// # Copy of some Vulkan definitions so we don't need to check their existence just to handle few constants. + +static const uint32_t VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY = 0x00000040; +static const uint32_t VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY = 0x00000080; +static const uint32_t VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY = 0x00020000; + +static const uint32_t VMA_ALLOCATION_INTERNAL_STRATEGY_MIN_OFFSET = 0x10000000u; + static VkAllocationCallbacks VmaEmptyAllocationCallbacks = { VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL }; // Returns number of bits set to 1 in (v). static inline uint32_t VmaCountBitsSet(uint32_t v) { - uint32_t c = v - ((v >> 1) & 0x55555555); - c = ((c >> 2) & 0x33333333) + (c & 0x33333333); - c = ((c >> 4) + c) & 0x0F0F0F0F; - c = ((c >> 8) + c) & 0x00FF00FF; - c = ((c >> 16) + c) & 0x0000FFFF; - return c; + uint32_t c = v - ((v >> 1) & 0x55555555); + c = ((c >> 2) & 0x33333333) + (c & 0x33333333); + c = ((c >> 4) + c) & 0x0F0F0F0F; + c = ((c >> 8) + c) & 0x00FF00FF; + c = ((c >> 16) + c) & 0x0000FFFF; + return c; +} + +/* +Returns true if given number is a power of two. +T must be unsigned integer number or signed integer but always nonnegative. +For 0 returns true. +*/ +template +inline bool VmaIsPow2(T x) +{ + return (x & (x-1)) == 0; } // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16. // Use types like uint32_t, uint64_t as T. template -static inline T VmaAlignUp(T val, T align) +static inline T VmaAlignUp(T val, T alignment) { - return (val + align - 1) / align * align; + VMA_HEAVY_ASSERT(VmaIsPow2(alignment)); + return (val + alignment - 1) & ~(alignment - 1); } // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8. // Use types like uint32_t, uint64_t as T. template -static inline T VmaAlignDown(T val, T align) +static inline T VmaAlignDown(T val, T alignment) { - return val / align * align; + VMA_HEAVY_ASSERT(VmaIsPow2(alignment)); + return val & ~(alignment - 1); } // Division with mathematical rounding to nearest number. template -inline T VmaRoundDiv(T x, T y) +static inline T VmaRoundDiv(T x, T y) { - return (x + (y / (T)2)) / y; + return (x + (y / (T)2)) / y; +} + +// Returns smallest power of 2 greater or equal to v. +static inline uint32_t VmaNextPow2(uint32_t v) +{ + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + return v; +} +static inline uint64_t VmaNextPow2(uint64_t v) +{ + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + v++; + return v; +} + +// Returns largest power of 2 less or equal to v. +static inline uint32_t VmaPrevPow2(uint32_t v) +{ + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v = v ^ (v >> 1); + return v; +} +static inline uint64_t VmaPrevPow2(uint64_t v) +{ + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + v = v ^ (v >> 1); + return v; } static inline bool VmaStrIsEmpty(const char* pStr) @@ -2811,6 +4468,26 @@ static inline bool VmaStrIsEmpty(const char* pStr) return pStr == VMA_NULL || *pStr == '\0'; } +#if VMA_STATS_STRING_ENABLED + +static const char* VmaAlgorithmToStr(uint32_t algorithm) +{ + switch(algorithm) + { + case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT: + return "Linear"; + case VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT: + return "Buddy"; + case 0: + return "Default"; + default: + VMA_ASSERT(0); + return ""; + } +} + +#endif // #if VMA_STATS_STRING_ENABLED + #ifndef VMA_SORT template @@ -2926,16 +4603,21 @@ static inline bool VmaIsBufferImageGranularityConflict( static void VmaWriteMagicValue(void* pData, VkDeviceSize offset) { +#if VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_DETECT_CORRUPTION uint32_t* pDst = (uint32_t*)((char*)pData + offset); const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t); for(size_t i = 0; i < numberCount; ++i, ++pDst) { *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE; } +#else + // no-op +#endif } static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset) { +#if VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_DETECT_CORRUPTION const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset); const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t); for(size_t i = 0; i < numberCount; ++i, ++pSrc) @@ -2945,35 +4627,62 @@ static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset) return false; } } +#endif return true; } +/* +Fills structure with parameters of an example buffer to be used for transfers +during GPU memory defragmentation. +*/ +static void VmaFillGpuDefragmentationBufferCreateInfo(VkBufferCreateInfo& outBufCreateInfo) +{ + memset(&outBufCreateInfo, 0, sizeof(outBufCreateInfo)); + outBufCreateInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + outBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + outBufCreateInfo.size = (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE; // Example size. +} + // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope). struct VmaMutexLock { VMA_CLASS_NO_COPY(VmaMutexLock) public: - VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) : + VmaMutexLock(VMA_MUTEX& mutex, bool useMutex = true) : m_pMutex(useMutex ? &mutex : VMA_NULL) - { - if(m_pMutex) - { - m_pMutex->Lock(); - } - } - + { if(m_pMutex) { m_pMutex->Lock(); } } ~VmaMutexLock() - { - if(m_pMutex) - { - m_pMutex->Unlock(); - } - } - + { if(m_pMutex) { m_pMutex->Unlock(); } } private: VMA_MUTEX* m_pMutex; }; +// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for reading. +struct VmaMutexLockRead +{ + VMA_CLASS_NO_COPY(VmaMutexLockRead) +public: + VmaMutexLockRead(VMA_RW_MUTEX& mutex, bool useMutex) : + m_pMutex(useMutex ? &mutex : VMA_NULL) + { if(m_pMutex) { m_pMutex->LockRead(); } } + ~VmaMutexLockRead() { if(m_pMutex) { m_pMutex->UnlockRead(); } } +private: + VMA_RW_MUTEX* m_pMutex; +}; + +// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for writing. +struct VmaMutexLockWrite +{ + VMA_CLASS_NO_COPY(VmaMutexLockWrite) +public: + VmaMutexLockWrite(VMA_RW_MUTEX& mutex, bool useMutex) : + m_pMutex(useMutex ? &mutex : VMA_NULL) + { if(m_pMutex) { m_pMutex->LockWrite(); } } + ~VmaMutexLockWrite() { if(m_pMutex) { m_pMutex->UnlockWrite(); } } +private: + VMA_RW_MUTEX* m_pMutex; +}; + #if VMA_DEBUG_GLOBAL_MUTEX static VMA_MUTEX gDebugGlobalMutex; #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true); @@ -2993,8 +4702,8 @@ Cmp should return true if first argument is less than second argument. Returned value is the found element, if present in the collection or place where new element with value (key) should be inserted. */ -template -static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp) +template +static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, const CmpLess& cmp) { size_t down = 0, up = (end - beg); while(down < up) @@ -3012,15 +4721,62 @@ static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, Cm return beg + down; } +template +IterT VmaBinaryFindSorted(const IterT& beg, const IterT& end, const KeyT& value, const CmpLess& cmp) +{ + IterT it = VmaBinaryFindFirstNotLess( + beg, end, value, cmp); + if(it == end || + (!cmp(*it, value) && !cmp(value, *it))) + { + return it; + } + return end; +} + +/* +Returns true if all pointers in the array are not-null and unique. +Warning! O(n^2) complexity. Use only inside VMA_HEAVY_ASSERT. +T must be pointer type, e.g. VmaAllocation, VmaPool. +*/ +template +static bool VmaValidatePointerArray(uint32_t count, const T* arr) +{ + for(uint32_t i = 0; i < count; ++i) + { + const T iPtr = arr[i]; + if(iPtr == VMA_NULL) + { + return false; + } + for(uint32_t j = i + 1; j < count; ++j) + { + if(iPtr == arr[j]) + { + return false; + } + } + } + return true; +} + +template +static inline void VmaPnextChainPushFront(MainT* mainStruct, NewT* newStruct) +{ + newStruct->pNext = mainStruct->pNext; + mainStruct->pNext = newStruct; +} + //////////////////////////////////////////////////////////////////////////////// // Memory allocation static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment) { + void* result = VMA_NULL; if((pAllocationCallbacks != VMA_NULL) && (pAllocationCallbacks->pfnAllocation != VMA_NULL)) { - return (*pAllocationCallbacks->pfnAllocation)( + result = (*pAllocationCallbacks->pfnAllocation)( pAllocationCallbacks->pUserData, size, alignment, @@ -3028,8 +4784,10 @@ static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t } else { - return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment); + result = VMA_SYSTEM_ALIGNED_MALLOC(size, alignment); } + VMA_ASSERT(result != VMA_NULL && "CPU memory allocation failed."); + return result; } static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr) @@ -3081,6 +4839,30 @@ static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, } } +static char* VmaCreateStringCopy(const VkAllocationCallbacks* allocs, const char* srcStr) +{ + if(srcStr != VMA_NULL) + { + const size_t len = strlen(srcStr); + char* const result = vma_new_array(allocs, char, len + 1); + memcpy(result, srcStr, len + 1); + return result; + } + else + { + return VMA_NULL; + } +} + +static void VmaFreeString(const VkAllocationCallbacks* allocs, char* str) +{ + if(str != VMA_NULL) + { + const size_t len = strlen(str); + vma_delete_array(allocs, str, len + 1); + } +} + // STL-compatible allocator. template class VmaStlAllocator @@ -3152,6 +4934,11 @@ public: { } + // This version of the constructor is here for compatibility with pre-C++14 std::vector. + // value is unused. + VmaVector(size_t count, const T& value, const AllocatorT& allocator) + : VmaVector(count, allocator) {} + VmaVector(const VmaVector& src) : m_Allocator(src.m_Allocator), m_pArray(src.m_Count ? (T*)VmaAllocateArray(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL), @@ -3377,24 +5164,174 @@ bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& return false; } -template -size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value) +//////////////////////////////////////////////////////////////////////////////// +// class VmaSmallVector + +/* +This is a vector (a variable-sized array), optimized for the case when the array is small. + +It contains some number of elements in-place, which allows it to avoid heap allocation +when the actual number of elements is below that threshold. This allows normal "small" +cases to be fast without losing generality for large inputs. +*/ + +template +class VmaSmallVector { - CmpLess comparator; - typename VectorT::iterator it = VmaBinaryFindFirstNotLess( - vector.data(), - vector.data() + vector.size(), - value, - comparator); - if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it)) +public: + typedef T value_type; + + VmaSmallVector(const AllocatorT& allocator) : + m_Count(0), + m_DynamicArray(allocator) { - return it - vector.begin(); } - else + VmaSmallVector(size_t count, const AllocatorT& allocator) : + m_Count(count), + m_DynamicArray(count > N ? count : 0, allocator) { - return vector.size(); } -} + template + VmaSmallVector(const VmaSmallVector& src) = delete; + template + VmaSmallVector& operator=(const VmaSmallVector& rhs) = delete; + + bool empty() const { return m_Count == 0; } + size_t size() const { return m_Count; } + T* data() { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; } + const T* data() const { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; } + + T& operator[](size_t index) + { + VMA_HEAVY_ASSERT(index < m_Count); + return data()[index]; + } + const T& operator[](size_t index) const + { + VMA_HEAVY_ASSERT(index < m_Count); + return data()[index]; + } + + T& front() + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[0]; + } + const T& front() const + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[0]; + } + T& back() + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[m_Count - 1]; + } + const T& back() const + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[m_Count - 1]; + } + + void resize(size_t newCount, bool freeMemory = false) + { + if(newCount > N && m_Count > N) + { + // Any direction, staying in m_DynamicArray + m_DynamicArray.resize(newCount, freeMemory); + } + else if(newCount > N && m_Count <= N) + { + // Growing, moving from m_StaticArray to m_DynamicArray + m_DynamicArray.resize(newCount, freeMemory); + if(m_Count > 0) + { + memcpy(m_DynamicArray.data(), m_StaticArray, m_Count * sizeof(T)); + } + } + else if(newCount <= N && m_Count > N) + { + // Shrinking, moving from m_DynamicArray to m_StaticArray + if(newCount > 0) + { + memcpy(m_StaticArray, m_DynamicArray.data(), newCount * sizeof(T)); + } + m_DynamicArray.resize(0, freeMemory); + } + else + { + // Any direction, staying in m_StaticArray - nothing to do here + } + m_Count = newCount; + } + + void clear(bool freeMemory = false) + { + m_DynamicArray.clear(freeMemory); + m_Count = 0; + } + + void insert(size_t index, const T& src) + { + VMA_HEAVY_ASSERT(index <= m_Count); + const size_t oldCount = size(); + resize(oldCount + 1); + T* const dataPtr = data(); + if(index < oldCount) + { + // I know, this could be more optimal for case where memmove can be memcpy directly from m_StaticArray to m_DynamicArray. + memmove(dataPtr + (index + 1), dataPtr + index, (oldCount - index) * sizeof(T)); + } + dataPtr[index] = src; + } + + void remove(size_t index) + { + VMA_HEAVY_ASSERT(index < m_Count); + const size_t oldCount = size(); + if(index < oldCount - 1) + { + // I know, this could be more optimal for case where memmove can be memcpy directly from m_DynamicArray to m_StaticArray. + T* const dataPtr = data(); + memmove(dataPtr + index, dataPtr + (index + 1), (oldCount - index - 1) * sizeof(T)); + } + resize(oldCount - 1); + } + + void push_back(const T& src) + { + const size_t newIndex = size(); + resize(newIndex + 1); + data()[newIndex] = src; + } + + void pop_back() + { + VMA_HEAVY_ASSERT(m_Count > 0); + resize(size() - 1); + } + + void push_front(const T& src) + { + insert(0, src); + } + + void pop_front() + { + VMA_HEAVY_ASSERT(m_Count > 0); + remove(0); + } + + typedef T* iterator; + + iterator begin() { return data(); } + iterator end() { return data() + m_Count; } + +private: + size_t m_Count; + T m_StaticArray[N]; // Used when m_Size <= N + VmaVector m_DynamicArray; // Used when m_Size > N +}; //////////////////////////////////////////////////////////////////////////////// // class VmaPoolAllocator @@ -3409,57 +5346,51 @@ class VmaPoolAllocator { VMA_CLASS_NO_COPY(VmaPoolAllocator) public: - VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock); + VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity); ~VmaPoolAllocator(); - void Clear(); - T* Alloc(); + template T* Alloc(Types... args); void Free(T* ptr); private: union Item { uint32_t NextFreeIndex; - T Value; + alignas(T) char Value[sizeof(T)]; }; struct ItemBlock { Item* pItems; + uint32_t Capacity; uint32_t FirstFreeIndex; }; const VkAllocationCallbacks* m_pAllocationCallbacks; - size_t m_ItemsPerBlock; + const uint32_t m_FirstBlockCapacity; VmaVector< ItemBlock, VmaStlAllocator > m_ItemBlocks; ItemBlock& CreateNewBlock(); }; template -VmaPoolAllocator::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) : +VmaPoolAllocator::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity) : m_pAllocationCallbacks(pAllocationCallbacks), - m_ItemsPerBlock(itemsPerBlock), + m_FirstBlockCapacity(firstBlockCapacity), m_ItemBlocks(VmaStlAllocator(pAllocationCallbacks)) { - VMA_ASSERT(itemsPerBlock > 0); + VMA_ASSERT(m_FirstBlockCapacity > 1); } template VmaPoolAllocator::~VmaPoolAllocator() -{ - Clear(); -} - -template -void VmaPoolAllocator::Clear() { for(size_t i = m_ItemBlocks.size(); i--; ) - vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock); + vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemBlocks[i].Capacity); m_ItemBlocks.clear(); } template -T* VmaPoolAllocator::Alloc() +template T* VmaPoolAllocator::Alloc(Types... args) { for(size_t i = m_ItemBlocks.size(); i--; ) { @@ -3469,7 +5400,9 @@ T* VmaPoolAllocator::Alloc() { Item* const pItem = &block.pItems[block.FirstFreeIndex]; block.FirstFreeIndex = pItem->NextFreeIndex; - return &pItem->Value; + T* result = (T*)&pItem->Value; + new(result)T(std::forward(args)...); // Explicit constructor call. + return result; } } @@ -3477,14 +5410,16 @@ T* VmaPoolAllocator::Alloc() ItemBlock& newBlock = CreateNewBlock(); Item* const pItem = &newBlock.pItems[0]; newBlock.FirstFreeIndex = pItem->NextFreeIndex; - return &pItem->Value; + T* result = (T*)&pItem->Value; + new(result)T(std::forward(args)...); // Explicit constructor call. + return result; } template void VmaPoolAllocator::Free(T* ptr) { // Search all memory blocks to find ptr. - for(size_t i = 0; i < m_ItemBlocks.size(); ++i) + for(size_t i = m_ItemBlocks.size(); i--; ) { ItemBlock& block = m_ItemBlocks[i]; @@ -3493,8 +5428,9 @@ void VmaPoolAllocator::Free(T* ptr) memcpy(&pItemPtr, &ptr, sizeof(pItemPtr)); // Check if pItemPtr is in address range of this block. - if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock)) + if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + block.Capacity)) { + ptr->~T(); // Explicit destructor call. const uint32_t index = static_cast(pItemPtr - block.pItems); pItemPtr->NextFreeIndex = block.FirstFreeIndex; block.FirstFreeIndex = index; @@ -3507,15 +5443,20 @@ void VmaPoolAllocator::Free(T* ptr) template typename VmaPoolAllocator::ItemBlock& VmaPoolAllocator::CreateNewBlock() { - ItemBlock newBlock = { - vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 }; + const uint32_t newBlockCapacity = m_ItemBlocks.empty() ? + m_FirstBlockCapacity : m_ItemBlocks.back().Capacity * 3 / 2; + + const ItemBlock newBlock = { + vma_new_array(m_pAllocationCallbacks, Item, newBlockCapacity), + newBlockCapacity, + 0 }; m_ItemBlocks.push_back(newBlock); // Setup singly-linked list of all free items in this block. - for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i) + for(uint32_t i = 0; i < newBlockCapacity - 1; ++i) newBlock.pItems[i].NextFreeIndex = i + 1; - newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX; + newBlock.pItems[newBlockCapacity - 1].NextFreeIndex = UINT32_MAX; return m_ItemBlocks.back(); } @@ -4101,7 +6042,6 @@ enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE }; struct VmaAllocation_T { - VMA_CLASS_NO_COPY(VmaAllocation_T) private: static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80; @@ -4118,15 +6058,20 @@ public: ALLOCATION_TYPE_DEDICATED, }; + /* + This struct is allocated using VmaPoolAllocator. + */ + VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) : - m_Alignment(1), - m_Size(0), - m_pUserData(VMA_NULL), - m_LastUseFrameIndex(currentFrameIndex), - m_Type((uint8_t)ALLOCATION_TYPE_NONE), - m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN), - m_MapCount(0), - m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0) + m_Alignment{1}, + m_Size{0}, + m_pUserData{VMA_NULL}, + m_LastUseFrameIndex{currentFrameIndex}, + m_MemoryTypeIndex{0}, + m_Type{(uint8_t)ALLOCATION_TYPE_NONE}, + m_SuballocationType{(uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN}, + m_MapCount{0}, + m_Flags{userDataString ? (uint8_t)FLAG_USER_DATA_STRING : (uint8_t)0} { #if VMA_STATS_STRING_ENABLED m_CreationFrameIndex = currentFrameIndex; @@ -4143,11 +6088,11 @@ public: } void InitBlockAllocation( - VmaPool hPool, VmaDeviceMemoryBlock* block, VkDeviceSize offset, VkDeviceSize alignment, VkDeviceSize size, + uint32_t memoryTypeIndex, VmaSuballocationType suballocationType, bool mapped, bool canBecomeLost) @@ -4157,9 +6102,9 @@ public: m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK; m_Alignment = alignment; m_Size = size; + m_MemoryTypeIndex = memoryTypeIndex; m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0; m_SuballocationType = (uint8_t)suballocationType; - m_BlockAllocation.m_hPool = hPool; m_BlockAllocation.m_Block = block; m_BlockAllocation.m_Offset = offset; m_BlockAllocation.m_CanBecomeLost = canBecomeLost; @@ -4170,7 +6115,7 @@ public: VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE); VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST); m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK; - m_BlockAllocation.m_hPool = VK_NULL_HANDLE; + m_MemoryTypeIndex = 0; m_BlockAllocation.m_Block = VMA_NULL; m_BlockAllocation.m_Offset = 0; m_BlockAllocation.m_CanBecomeLost = true; @@ -4179,7 +6124,9 @@ public: void ChangeBlockAllocation( VmaAllocator hAllocator, VmaDeviceMemoryBlock* block, - VkDeviceSize offset); + VkDeviceSize offset); + + void ChangeOffset(VkDeviceSize newOffset); // pMappedData not null means allocation is created with MAPPED flag. void InitDedicatedAllocation( @@ -4194,9 +6141,9 @@ public: m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED; m_Alignment = 0; m_Size = size; + m_MemoryTypeIndex = memoryTypeIndex; m_SuballocationType = (uint8_t)suballocationType; m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0; - m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex; m_DedicatedAllocation.m_hMemory = hMemory; m_DedicatedAllocation.m_pMappedData = pMappedData; } @@ -4216,11 +6163,10 @@ public: } VkDeviceSize GetOffset() const; VkDeviceMemory GetMemory() const; - uint32_t GetMemoryTypeIndex() const; + uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; } bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; } void* GetMappedData() const; bool CanBecomeLost() const; - VmaPool GetPool() const; uint32_t GetLastUseFrameIndex() const { @@ -4276,6 +6222,7 @@ private: VkDeviceSize m_Size; void* m_pUserData; VMA_ATOMIC_UINT32 m_LastUseFrameIndex; + uint32_t m_MemoryTypeIndex; uint8_t m_Type; // ALLOCATION_TYPE uint8_t m_SuballocationType; // VmaSuballocationType // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT. @@ -4286,7 +6233,6 @@ private: // Allocation out of VmaDeviceMemoryBlock. struct BlockAllocation { - VmaPool m_hPool; // Null if belongs to general memory. VmaDeviceMemoryBlock* m_Block; VkDeviceSize m_Offset; bool m_CanBecomeLost; @@ -4295,7 +6241,6 @@ private: // Allocation for an object that has its own private VkDeviceMemory. struct DedicatedAllocation { - uint32_t m_MemoryTypeIndex; VkDeviceMemory m_hMemory; void* m_pMappedData; // Not null means memory is mapped. }; @@ -4328,11 +6273,36 @@ struct VmaSuballocation VmaSuballocationType type; }; +// Comparator for offsets. +struct VmaSuballocationOffsetLess +{ + bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const + { + return lhs.offset < rhs.offset; + } +}; +struct VmaSuballocationOffsetGreater +{ + bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const + { + return lhs.offset > rhs.offset; + } +}; + typedef VmaList< VmaSuballocation, VmaStlAllocator > VmaSuballocationList; // Cost of one additional allocation lost, as equivalent in bytes. static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576; +enum class VmaAllocationRequestType +{ + Normal, + // Used by "Linear" algorithm. + UpperAddress, + EndOf1st, + EndOf2nd, +}; + /* Parameters of planned allocation inside a VmaDeviceMemoryBlock. @@ -4353,6 +6323,8 @@ struct VmaAllocationRequest VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation. VmaSuballocationList::iterator item; size_t itemsToMakeLostCount; + void* customData; + VmaAllocationRequestType type; VkDeviceSize CalcCost() const { @@ -4366,63 +6338,153 @@ in a single VkDeviceMemory block. */ class VmaBlockMetadata { - VMA_CLASS_NO_COPY(VmaBlockMetadata) public: VmaBlockMetadata(VmaAllocator hAllocator); - ~VmaBlockMetadata(); - void Init(VkDeviceSize size); + virtual ~VmaBlockMetadata() { } + virtual void Init(VkDeviceSize size) { m_Size = size; } // Validates all data structures inside this object. If not valid, returns false. - bool Validate() const; + virtual bool Validate() const = 0; VkDeviceSize GetSize() const { return m_Size; } - size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; } - VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; } - VkDeviceSize GetUnusedRangeSizeMax() const; + virtual size_t GetAllocationCount() const = 0; + virtual VkDeviceSize GetSumFreeSize() const = 0; + virtual VkDeviceSize GetUnusedRangeSizeMax() const = 0; // Returns true if this block is empty - contains only single free suballocation. - bool IsEmpty() const; + virtual bool IsEmpty() const = 0; - void CalcAllocationStatInfo(VmaStatInfo& outInfo) const; - void AddPoolStats(VmaPoolStats& inoutStats) const; + virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const = 0; + // Shouldn't modify blockCount. + virtual void AddPoolStats(VmaPoolStats& inoutStats) const = 0; #if VMA_STATS_STRING_ENABLED - void PrintDetailedMap(class VmaJsonWriter& json) const; + virtual void PrintDetailedMap(class VmaJsonWriter& json) const = 0; #endif // Tries to find a place for suballocation with given parameters inside this block. // If succeeded, fills pAllocationRequest and returns true. // If failed, returns false. - bool CreateAllocationRequest( + virtual bool CreateAllocationRequest( uint32_t currentFrameIndex, uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, VkDeviceSize allocSize, VkDeviceSize allocAlignment, + bool upperAddress, VmaSuballocationType allocType, bool canMakeOtherLost, + // Always one of VMA_ALLOCATION_CREATE_STRATEGY_* or VMA_ALLOCATION_INTERNAL_STRATEGY_* flags. + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest) = 0; + + virtual bool MakeRequestedAllocationsLost( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VmaAllocationRequest* pAllocationRequest) = 0; + + virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) = 0; + + virtual VkResult CheckCorruption(const void* pBlockData) = 0; + + // Makes actual allocation based on request. Request must already be checked and valid. + virtual void Alloc( + const VmaAllocationRequest& request, + VmaSuballocationType type, + VkDeviceSize allocSize, + VmaAllocation hAllocation) = 0; + + // Frees suballocation assigned to given memory region. + virtual void Free(const VmaAllocation allocation) = 0; + virtual void FreeAtOffset(VkDeviceSize offset) = 0; + +protected: + const VkAllocationCallbacks* GetAllocationCallbacks() const { return m_pAllocationCallbacks; } + +#if VMA_STATS_STRING_ENABLED + void PrintDetailedMap_Begin(class VmaJsonWriter& json, + VkDeviceSize unusedBytes, + size_t allocationCount, + size_t unusedRangeCount) const; + void PrintDetailedMap_Allocation(class VmaJsonWriter& json, + VkDeviceSize offset, + VmaAllocation hAllocation) const; + void PrintDetailedMap_UnusedRange(class VmaJsonWriter& json, + VkDeviceSize offset, + VkDeviceSize size) const; + void PrintDetailedMap_End(class VmaJsonWriter& json) const; +#endif + +private: + VkDeviceSize m_Size; + const VkAllocationCallbacks* m_pAllocationCallbacks; +}; + +#define VMA_VALIDATE(cond) do { if(!(cond)) { \ + VMA_ASSERT(0 && "Validation failed: " #cond); \ + return false; \ + } } while(false) + +class VmaBlockMetadata_Generic : public VmaBlockMetadata +{ + VMA_CLASS_NO_COPY(VmaBlockMetadata_Generic) +public: + VmaBlockMetadata_Generic(VmaAllocator hAllocator); + virtual ~VmaBlockMetadata_Generic(); + virtual void Init(VkDeviceSize size); + + virtual bool Validate() const; + virtual size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; } + virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; } + virtual VkDeviceSize GetUnusedRangeSizeMax() const; + virtual bool IsEmpty() const; + + virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const; + virtual void AddPoolStats(VmaPoolStats& inoutStats) const; + +#if VMA_STATS_STRING_ENABLED + virtual void PrintDetailedMap(class VmaJsonWriter& json) const; +#endif + + virtual bool CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + bool upperAddress, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, VmaAllocationRequest* pAllocationRequest); - bool MakeRequestedAllocationsLost( + virtual bool MakeRequestedAllocationsLost( uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest); - uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); + virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); - VkResult CheckCorruption(const void* pBlockData); + virtual VkResult CheckCorruption(const void* pBlockData); - // Makes actual allocation based on request. Request must already be checked and valid. - void Alloc( + virtual void Alloc( const VmaAllocationRequest& request, VmaSuballocationType type, VkDeviceSize allocSize, VmaAllocation hAllocation); - // Frees suballocation assigned to given memory region. - void Free(const VmaAllocation allocation); - void FreeAtOffset(VkDeviceSize offset); + virtual void Free(const VmaAllocation allocation); + virtual void FreeAtOffset(VkDeviceSize offset); + + //////////////////////////////////////////////////////////////////////////////// + // For defragmentation + + bool IsBufferImageGranularityConflictPossible( + VkDeviceSize bufferImageGranularity, + VmaSuballocationType& inOutPrevSuballocType) const; private: - VkDeviceSize m_Size; + friend class VmaDefragmentationAlgorithm_Generic; + friend class VmaDefragmentationAlgorithm_Fast; + uint32_t m_FreeCount; VkDeviceSize m_SumFreeSize; VmaSuballocationList m_Suballocations; @@ -4461,6 +6523,350 @@ private: void UnregisterFreeSuballocation(VmaSuballocationList::iterator item); }; +/* +Allocations and their references in internal data structure look like this: + +if(m_2ndVectorMode == SECOND_VECTOR_EMPTY): + + 0 +-------+ + | | + | | + | | + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount] + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount + 1] + +-------+ + | ... | + +-------+ + | Alloc | 1st[1st.size() - 1] + +-------+ + | | + | | + | | +GetSize() +-------+ + +if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER): + + 0 +-------+ + | Alloc | 2nd[0] + +-------+ + | Alloc | 2nd[1] + +-------+ + | ... | + +-------+ + | Alloc | 2nd[2nd.size() - 1] + +-------+ + | | + | | + | | + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount] + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount + 1] + +-------+ + | ... | + +-------+ + | Alloc | 1st[1st.size() - 1] + +-------+ + | | +GetSize() +-------+ + +if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK): + + 0 +-------+ + | | + | | + | | + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount] + +-------+ + | Alloc | 1st[m_1stNullItemsBeginCount + 1] + +-------+ + | ... | + +-------+ + | Alloc | 1st[1st.size() - 1] + +-------+ + | | + | | + | | + +-------+ + | Alloc | 2nd[2nd.size() - 1] + +-------+ + | ... | + +-------+ + | Alloc | 2nd[1] + +-------+ + | Alloc | 2nd[0] +GetSize() +-------+ + +*/ +class VmaBlockMetadata_Linear : public VmaBlockMetadata +{ + VMA_CLASS_NO_COPY(VmaBlockMetadata_Linear) +public: + VmaBlockMetadata_Linear(VmaAllocator hAllocator); + virtual ~VmaBlockMetadata_Linear(); + virtual void Init(VkDeviceSize size); + + virtual bool Validate() const; + virtual size_t GetAllocationCount() const; + virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; } + virtual VkDeviceSize GetUnusedRangeSizeMax() const; + virtual bool IsEmpty() const { return GetAllocationCount() == 0; } + + virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const; + virtual void AddPoolStats(VmaPoolStats& inoutStats) const; + +#if VMA_STATS_STRING_ENABLED + virtual void PrintDetailedMap(class VmaJsonWriter& json) const; +#endif + + virtual bool CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + bool upperAddress, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest); + + virtual bool MakeRequestedAllocationsLost( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VmaAllocationRequest* pAllocationRequest); + + virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); + + virtual VkResult CheckCorruption(const void* pBlockData); + + virtual void Alloc( + const VmaAllocationRequest& request, + VmaSuballocationType type, + VkDeviceSize allocSize, + VmaAllocation hAllocation); + + virtual void Free(const VmaAllocation allocation); + virtual void FreeAtOffset(VkDeviceSize offset); + +private: + /* + There are two suballocation vectors, used in ping-pong way. + The one with index m_1stVectorIndex is called 1st. + The one with index (m_1stVectorIndex ^ 1) is called 2nd. + 2nd can be non-empty only when 1st is not empty. + When 2nd is not empty, m_2ndVectorMode indicates its mode of operation. + */ + typedef VmaVector< VmaSuballocation, VmaStlAllocator > SuballocationVectorType; + + enum SECOND_VECTOR_MODE + { + SECOND_VECTOR_EMPTY, + /* + Suballocations in 2nd vector are created later than the ones in 1st, but they + all have smaller offset. + */ + SECOND_VECTOR_RING_BUFFER, + /* + Suballocations in 2nd vector are upper side of double stack. + They all have offsets higher than those in 1st vector. + Top of this stack means smaller offsets, but higher indices in this vector. + */ + SECOND_VECTOR_DOUBLE_STACK, + }; + + VkDeviceSize m_SumFreeSize; + SuballocationVectorType m_Suballocations0, m_Suballocations1; + uint32_t m_1stVectorIndex; + SECOND_VECTOR_MODE m_2ndVectorMode; + + SuballocationVectorType& AccessSuballocations1st() { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; } + SuballocationVectorType& AccessSuballocations2nd() { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; } + const SuballocationVectorType& AccessSuballocations1st() const { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; } + const SuballocationVectorType& AccessSuballocations2nd() const { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; } + + // Number of items in 1st vector with hAllocation = null at the beginning. + size_t m_1stNullItemsBeginCount; + // Number of other items in 1st vector with hAllocation = null somewhere in the middle. + size_t m_1stNullItemsMiddleCount; + // Number of items in 2nd vector with hAllocation = null. + size_t m_2ndNullItemsCount; + + bool ShouldCompact1st() const; + void CleanupAfterFree(); + + bool CreateAllocationRequest_LowerAddress( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest); + bool CreateAllocationRequest_UpperAddress( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest); +}; + +/* +- GetSize() is the original size of allocated memory block. +- m_UsableSize is this size aligned down to a power of two. + All allocations and calculations happen relative to m_UsableSize. +- GetUnusableSize() is the difference between them. + It is repoted as separate, unused range, not available for allocations. + +Node at level 0 has size = m_UsableSize. +Each next level contains nodes with size 2 times smaller than current level. +m_LevelCount is the maximum number of levels to use in the current object. +*/ +class VmaBlockMetadata_Buddy : public VmaBlockMetadata +{ + VMA_CLASS_NO_COPY(VmaBlockMetadata_Buddy) +public: + VmaBlockMetadata_Buddy(VmaAllocator hAllocator); + virtual ~VmaBlockMetadata_Buddy(); + virtual void Init(VkDeviceSize size); + + virtual bool Validate() const; + virtual size_t GetAllocationCount() const { return m_AllocationCount; } + virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize + GetUnusableSize(); } + virtual VkDeviceSize GetUnusedRangeSizeMax() const; + virtual bool IsEmpty() const { return m_Root->type == Node::TYPE_FREE; } + + virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const; + virtual void AddPoolStats(VmaPoolStats& inoutStats) const; + +#if VMA_STATS_STRING_ENABLED + virtual void PrintDetailedMap(class VmaJsonWriter& json) const; +#endif + + virtual bool CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + bool upperAddress, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest); + + virtual bool MakeRequestedAllocationsLost( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VmaAllocationRequest* pAllocationRequest); + + virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); + + virtual VkResult CheckCorruption(const void* pBlockData) { return VK_ERROR_FEATURE_NOT_PRESENT; } + + virtual void Alloc( + const VmaAllocationRequest& request, + VmaSuballocationType type, + VkDeviceSize allocSize, + VmaAllocation hAllocation); + + virtual void Free(const VmaAllocation allocation) { FreeAtOffset(allocation, allocation->GetOffset()); } + virtual void FreeAtOffset(VkDeviceSize offset) { FreeAtOffset(VMA_NULL, offset); } + +private: + static const VkDeviceSize MIN_NODE_SIZE = 32; + static const size_t MAX_LEVELS = 30; + + struct ValidationContext + { + size_t calculatedAllocationCount; + size_t calculatedFreeCount; + VkDeviceSize calculatedSumFreeSize; + + ValidationContext() : + calculatedAllocationCount(0), + calculatedFreeCount(0), + calculatedSumFreeSize(0) { } + }; + + struct Node + { + VkDeviceSize offset; + enum TYPE + { + TYPE_FREE, + TYPE_ALLOCATION, + TYPE_SPLIT, + TYPE_COUNT + } type; + Node* parent; + Node* buddy; + + union + { + struct + { + Node* prev; + Node* next; + } free; + struct + { + VmaAllocation alloc; + } allocation; + struct + { + Node* leftChild; + } split; + }; + }; + + // Size of the memory block aligned down to a power of two. + VkDeviceSize m_UsableSize; + uint32_t m_LevelCount; + + Node* m_Root; + struct { + Node* front; + Node* back; + } m_FreeList[MAX_LEVELS]; + // Number of nodes in the tree with type == TYPE_ALLOCATION. + size_t m_AllocationCount; + // Number of nodes in the tree with type == TYPE_FREE. + size_t m_FreeCount; + // This includes space wasted due to internal fragmentation. Doesn't include unusable size. + VkDeviceSize m_SumFreeSize; + + VkDeviceSize GetUnusableSize() const { return GetSize() - m_UsableSize; } + void DeleteNode(Node* node); + bool ValidateNode(ValidationContext& ctx, const Node* parent, const Node* curr, uint32_t level, VkDeviceSize levelNodeSize) const; + uint32_t AllocSizeToLevel(VkDeviceSize allocSize) const; + inline VkDeviceSize LevelToNodeSize(uint32_t level) const { return m_UsableSize >> level; } + // Alloc passed just for validation. Can be null. + void FreeAtOffset(VmaAllocation alloc, VkDeviceSize offset); + void CalcAllocationStatInfoNode(VmaStatInfo& outInfo, const Node* node, VkDeviceSize levelNodeSize) const; + // Adds node to the front of FreeList at given level. + // node->type must be FREE. + // node->free.prev, next can be undefined. + void AddToFreeListFront(uint32_t level, Node* node); + // Removes node from FreeList at given level. + // node->type must be FREE. + // node->free.prev, next stay untouched. + void RemoveFromFreeList(uint32_t level, Node* node); + +#if VMA_STATS_STRING_ENABLED + void PrintDetailedMapNode(class VmaJsonWriter& json, const Node* node, VkDeviceSize levelNodeSize) const; +#endif +}; + /* Represents a single block of device memory (`VkDeviceMemory`) with all the data about its regions (aka suballocations, #VmaAllocation), assigned and free. @@ -4471,7 +6877,7 @@ class VmaDeviceMemoryBlock { VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock) public: - VmaBlockMetadata m_Metadata; + VmaBlockMetadata* m_pMetadata; VmaDeviceMemoryBlock(VmaAllocator hAllocator); @@ -4483,13 +6889,17 @@ public: // Always call after construction. void Init( + VmaAllocator hAllocator, + VmaPool hParentPool, uint32_t newMemoryTypeIndex, VkDeviceMemory newMemory, VkDeviceSize newSize, - uint32_t id); + uint32_t id, + uint32_t algorithm); // Always call before destruction. void Destroy(VmaAllocator allocator); + VmaPool GetParentPool() const { return m_hParentPool; } VkDeviceMemory GetDeviceMemory() const { return m_hMemory; } uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; } uint32_t GetId() const { return m_Id; } @@ -4510,19 +6920,27 @@ public: VkResult BindBufferMemory( const VmaAllocator hAllocator, const VmaAllocation hAllocation, - VkBuffer hBuffer); + VkDeviceSize allocationLocalOffset, + VkBuffer hBuffer, + const void* pNext); VkResult BindImageMemory( const VmaAllocator hAllocator, const VmaAllocation hAllocation, - VkImage hImage); + VkDeviceSize allocationLocalOffset, + VkImage hImage, + const void* pNext); private: + VmaPool m_hParentPool; // VK_NULL_HANDLE if not belongs to custom pool. uint32_t m_MemoryTypeIndex; uint32_t m_Id; VkDeviceMemory m_hMemory; - // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory. - // Also protects m_MapCount, m_pMappedData. + /* + Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory. + Also protects m_MapCount, m_pMappedData. + Allocations, deallocations, any change in m_pMetadata is protected by parent's VmaBlockVector::m_Mutex. + */ VMA_MUTEX m_Mutex; uint32_t m_MapCount; void* m_pMappedData; @@ -4536,7 +6954,19 @@ struct VmaPointerLess } }; -class VmaDefragmentator; +struct VmaDefragmentationMove +{ + size_t srcBlockIndex; + size_t dstBlockIndex; + VkDeviceSize srcOffset; + VkDeviceSize dstOffset; + VkDeviceSize size; + VmaAllocation hAllocation; + VmaDeviceMemoryBlock* pSrcBlock; + VmaDeviceMemoryBlock* pDstBlock; +}; + +class VmaDefragmentationAlgorithm; /* Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific @@ -4550,38 +6980,43 @@ struct VmaBlockVector public: VmaBlockVector( VmaAllocator hAllocator, + VmaPool hParentPool, uint32_t memoryTypeIndex, VkDeviceSize preferredBlockSize, size_t minBlockCount, size_t maxBlockCount, VkDeviceSize bufferImageGranularity, uint32_t frameInUseCount, - bool isCustomPool); + bool explicitBlockSize, + uint32_t algorithm); ~VmaBlockVector(); VkResult CreateMinBlocks(); + VmaAllocator GetAllocator() const { return m_hAllocator; } + VmaPool GetParentPool() const { return m_hParentPool; } + bool IsCustomPool() const { return m_hParentPool != VMA_NULL; } uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; } VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; } VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; } uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; } + uint32_t GetAlgorithm() const { return m_Algorithm; } void GetPoolStats(VmaPoolStats* pStats); - bool IsEmpty() const { return m_Blocks.empty(); } + bool IsEmpty(); bool IsCorruptionDetectionEnabled() const; VkResult Allocate( - VmaPool hCurrentPool, uint32_t currentFrameIndex, VkDeviceSize size, VkDeviceSize alignment, const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, - VmaAllocation* pAllocation); + size_t allocationCount, + VmaAllocation* pAllocations); - void Free( - VmaAllocation hAllocation); + void Free(const VmaAllocation hAllocation); // Adds statistics of this BlockVector to pStats. void AddStats(VmaStats* pStats); @@ -4595,36 +7030,54 @@ public: size_t* pLostAllocationCount); VkResult CheckCorruption(); - VmaDefragmentator* EnsureDefragmentator( - VmaAllocator hAllocator, - uint32_t currentFrameIndex); + // Saves results in pCtx->res. + void Defragment( + class VmaBlockVectorDefragmentationContext* pCtx, + VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags, + VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove, + VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove, + VkCommandBuffer commandBuffer); + void DefragmentationEnd( + class VmaBlockVectorDefragmentationContext* pCtx, + uint32_t flags, + VmaDefragmentationStats* pStats); - VkResult Defragment( - VmaDefragmentationStats* pDefragmentationStats, - VkDeviceSize& maxBytesToMove, - uint32_t& maxAllocationsToMove); + uint32_t ProcessDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves); - void DestroyDefragmentator(); + void CommitDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationStats* pStats); + + //////////////////////////////////////////////////////////////////////////////// + // To be used only while the m_Mutex is locked. Used during defragmentation. + + size_t GetBlockCount() const { return m_Blocks.size(); } + VmaDeviceMemoryBlock* GetBlock(size_t index) const { return m_Blocks[index]; } + size_t CalcAllocationCount() const; + bool IsBufferImageGranularityConflictPossible() const; private: - friend class VmaDefragmentator; + friend class VmaDefragmentationAlgorithm_Generic; const VmaAllocator m_hAllocator; + const VmaPool m_hParentPool; const uint32_t m_MemoryTypeIndex; const VkDeviceSize m_PreferredBlockSize; const size_t m_MinBlockCount; const size_t m_MaxBlockCount; const VkDeviceSize m_BufferImageGranularity; const uint32_t m_FrameInUseCount; - const bool m_IsCustomPool; - VMA_MUTEX m_Mutex; + const bool m_ExplicitBlockSize; + const uint32_t m_Algorithm; + VMA_RW_MUTEX m_Mutex; + + /* There can be at most one allocation that is completely empty (except when minBlockCount > 0) - + a hysteresis to avoid pessimistic case of alternating creation and destruction of a VkDeviceMemory. */ + bool m_HasEmptyBlock; // Incrementally sorted by sumFreeSize, ascending. VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator > m_Blocks; - /* There can be at most one allocation that is completely empty - a - hysteresis to avoid pessimistic case of alternating creation and destruction - of a VkDeviceMemory. */ - bool m_HasEmptyBlock; - VmaDefragmentator* m_pDefragmentator; uint32_t m_NextBlockId; VkDeviceSize CalcMaxBlockSize() const; @@ -4636,7 +7089,45 @@ private: // after this call. void IncrementallySortBlocks(); + VkResult AllocatePage( + uint32_t currentFrameIndex, + VkDeviceSize size, + VkDeviceSize alignment, + const VmaAllocationCreateInfo& createInfo, + VmaSuballocationType suballocType, + VmaAllocation* pAllocation); + + // To be used only without CAN_MAKE_OTHER_LOST flag. + VkResult AllocateFromBlock( + VmaDeviceMemoryBlock* pBlock, + uint32_t currentFrameIndex, + VkDeviceSize size, + VkDeviceSize alignment, + VmaAllocationCreateFlags allocFlags, + void* pUserData, + VmaSuballocationType suballocType, + uint32_t strategy, + VmaAllocation* pAllocation); + VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex); + + // Saves result to pCtx->res. + void ApplyDefragmentationMovesCpu( + class VmaBlockVectorDefragmentationContext* pDefragCtx, + const VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves); + // Saves result to pCtx->res. + void ApplyDefragmentationMovesGpu( + class VmaBlockVectorDefragmentationContext* pDefragCtx, + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkCommandBuffer commandBuffer); + + /* + Used during defragmentation. pDefragmentationStats is optional. It's in/out + - updated with new data. + */ + void FreeEmptyBlocks(VmaDefragmentationStats* pDefragmentationStats); + + void UpdateHasEmptyBlock(); }; struct VmaPool_T @@ -4647,30 +7138,65 @@ public: VmaPool_T( VmaAllocator hAllocator, - const VmaPoolCreateInfo& createInfo); + const VmaPoolCreateInfo& createInfo, + VkDeviceSize preferredBlockSize); ~VmaPool_T(); - VmaBlockVector& GetBlockVector() { return m_BlockVector; } uint32_t GetId() const { return m_Id; } void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; } + const char* GetName() const { return m_Name; } + void SetName(const char* pName); + #if VMA_STATS_STRING_ENABLED //void PrintDetailedMap(class VmaStringBuilder& sb); #endif private: uint32_t m_Id; + char* m_Name; }; -class VmaDefragmentator +/* +Performs defragmentation: + +- Updates `pBlockVector->m_pMetadata`. +- Updates allocations by calling ChangeBlockAllocation() or ChangeOffset(). +- Does not move actual data, only returns requested moves as `moves`. +*/ +class VmaDefragmentationAlgorithm { - VMA_CLASS_NO_COPY(VmaDefragmentator) -private: - const VmaAllocator m_hAllocator; + VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm) +public: + VmaDefragmentationAlgorithm( + VmaAllocator hAllocator, + VmaBlockVector* pBlockVector, + uint32_t currentFrameIndex) : + m_hAllocator(hAllocator), + m_pBlockVector(pBlockVector), + m_CurrentFrameIndex(currentFrameIndex) + { + } + virtual ~VmaDefragmentationAlgorithm() + { + } + + virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) = 0; + virtual void AddAll() = 0; + + virtual VkResult Defragment( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) = 0; + + virtual VkDeviceSize GetBytesMoved() const = 0; + virtual uint32_t GetAllocationsMoved() const = 0; + +protected: + VmaAllocator const m_hAllocator; VmaBlockVector* const m_pBlockVector; - uint32_t m_CurrentFrameIndex; - VkDeviceSize m_BytesMoved; - uint32_t m_AllocationsMoved; + const uint32_t m_CurrentFrameIndex; struct AllocationInfo { @@ -4682,7 +7208,43 @@ private: m_pChanged(VMA_NULL) { } + AllocationInfo(VmaAllocation hAlloc, VkBool32* pChanged) : + m_hAllocation(hAlloc), + m_pChanged(pChanged) + { + } }; +}; + +class VmaDefragmentationAlgorithm_Generic : public VmaDefragmentationAlgorithm +{ + VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm_Generic) +public: + VmaDefragmentationAlgorithm_Generic( + VmaAllocator hAllocator, + VmaBlockVector* pBlockVector, + uint32_t currentFrameIndex, + bool overlappingMoveSupported); + virtual ~VmaDefragmentationAlgorithm_Generic(); + + virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged); + virtual void AddAll() { m_AllAllocations = true; } + + virtual VkResult Defragment( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags); + + virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } + virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } + +private: + uint32_t m_AllocationCount; + bool m_AllAllocations; + + VkDeviceSize m_BytesMoved; + uint32_t m_AllocationsMoved; struct AllocationInfoSizeGreater { @@ -4692,41 +7254,45 @@ private: } }; - // Used between AddAllocation and Defragment. - VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; + struct AllocationInfoOffsetGreater + { + bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const + { + return lhs.m_hAllocation->GetOffset() > rhs.m_hAllocation->GetOffset(); + } + }; struct BlockInfo { + size_t m_OriginalBlockIndex; VmaDeviceMemoryBlock* m_pBlock; bool m_HasNonMovableAllocations; VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) : + m_OriginalBlockIndex(SIZE_MAX), m_pBlock(VMA_NULL), m_HasNonMovableAllocations(true), - m_Allocations(pAllocationCallbacks), - m_pMappedDataForDefragmentation(VMA_NULL) + m_Allocations(pAllocationCallbacks) { } void CalcHasNonMovableAllocations() { - const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount(); + const size_t blockAllocCount = m_pBlock->m_pMetadata->GetAllocationCount(); const size_t defragmentAllocCount = m_Allocations.size(); m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount; } - void SortAllocationsBySizeDescecnding() + void SortAllocationsBySizeDescending() { VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater()); } - VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData); - void Unmap(VmaAllocator hAllocator); - - private: - // Not null if mapped for defragmentation only, not originally mapped. - void* m_pMappedDataForDefragmentation; + void SortAllocationsByOffsetDescending() + { + VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoOffsetGreater()); + } }; struct BlockPointerLess @@ -4755,7 +7321,7 @@ private: { return false; } - if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize()) + if(pLhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize()) { return true; } @@ -4767,29 +7333,272 @@ private: BlockInfoVector m_Blocks; VkResult DefragmentRound( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); + uint32_t maxAllocationsToMove, + bool freeOldAllocations); + + size_t CalcBlocksWithNonMovableCount() const; static bool MoveMakesSense( size_t dstBlockIndex, VkDeviceSize dstOffset, size_t srcBlockIndex, VkDeviceSize srcOffset); +}; +class VmaDefragmentationAlgorithm_Fast : public VmaDefragmentationAlgorithm +{ + VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm_Fast) public: - VmaDefragmentator( + VmaDefragmentationAlgorithm_Fast( VmaAllocator hAllocator, VmaBlockVector* pBlockVector, - uint32_t currentFrameIndex); + uint32_t currentFrameIndex, + bool overlappingMoveSupported); + virtual ~VmaDefragmentationAlgorithm_Fast(); - ~VmaDefragmentator(); + virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) { ++m_AllocationCount; } + virtual void AddAll() { m_AllAllocations = true; } - VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } - uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } + virtual VkResult Defragment( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags); + + virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } + virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } + +private: + struct BlockInfo + { + size_t origBlockIndex; + }; + + class FreeSpaceDatabase + { + public: + FreeSpaceDatabase() + { + FreeSpace s = {}; + s.blockInfoIndex = SIZE_MAX; + for(size_t i = 0; i < MAX_COUNT; ++i) + { + m_FreeSpaces[i] = s; + } + } + + void Register(size_t blockInfoIndex, VkDeviceSize offset, VkDeviceSize size) + { + if(size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + return; + } + + // Find first invalid or the smallest structure. + size_t bestIndex = SIZE_MAX; + for(size_t i = 0; i < MAX_COUNT; ++i) + { + // Empty structure. + if(m_FreeSpaces[i].blockInfoIndex == SIZE_MAX) + { + bestIndex = i; + break; + } + if(m_FreeSpaces[i].size < size && + (bestIndex == SIZE_MAX || m_FreeSpaces[bestIndex].size > m_FreeSpaces[i].size)) + { + bestIndex = i; + } + } + + if(bestIndex != SIZE_MAX) + { + m_FreeSpaces[bestIndex].blockInfoIndex = blockInfoIndex; + m_FreeSpaces[bestIndex].offset = offset; + m_FreeSpaces[bestIndex].size = size; + } + } + + bool Fetch(VkDeviceSize alignment, VkDeviceSize size, + size_t& outBlockInfoIndex, VkDeviceSize& outDstOffset) + { + size_t bestIndex = SIZE_MAX; + VkDeviceSize bestFreeSpaceAfter = 0; + for(size_t i = 0; i < MAX_COUNT; ++i) + { + // Structure is valid. + if(m_FreeSpaces[i].blockInfoIndex != SIZE_MAX) + { + const VkDeviceSize dstOffset = VmaAlignUp(m_FreeSpaces[i].offset, alignment); + // Allocation fits into this structure. + if(dstOffset + size <= m_FreeSpaces[i].offset + m_FreeSpaces[i].size) + { + const VkDeviceSize freeSpaceAfter = (m_FreeSpaces[i].offset + m_FreeSpaces[i].size) - + (dstOffset + size); + if(bestIndex == SIZE_MAX || freeSpaceAfter > bestFreeSpaceAfter) + { + bestIndex = i; + bestFreeSpaceAfter = freeSpaceAfter; + } + } + } + } + + if(bestIndex != SIZE_MAX) + { + outBlockInfoIndex = m_FreeSpaces[bestIndex].blockInfoIndex; + outDstOffset = VmaAlignUp(m_FreeSpaces[bestIndex].offset, alignment); + + if(bestFreeSpaceAfter >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + // Leave this structure for remaining empty space. + const VkDeviceSize alignmentPlusSize = (outDstOffset - m_FreeSpaces[bestIndex].offset) + size; + m_FreeSpaces[bestIndex].offset += alignmentPlusSize; + m_FreeSpaces[bestIndex].size -= alignmentPlusSize; + } + else + { + // This structure becomes invalid. + m_FreeSpaces[bestIndex].blockInfoIndex = SIZE_MAX; + } + + return true; + } + + return false; + } + + private: + static const size_t MAX_COUNT = 4; + + struct FreeSpace + { + size_t blockInfoIndex; // SIZE_MAX means this structure is invalid. + VkDeviceSize offset; + VkDeviceSize size; + } m_FreeSpaces[MAX_COUNT]; + }; + + const bool m_OverlappingMoveSupported; + + uint32_t m_AllocationCount; + bool m_AllAllocations; + + VkDeviceSize m_BytesMoved; + uint32_t m_AllocationsMoved; + + VmaVector< BlockInfo, VmaStlAllocator > m_BlockInfos; + + void PreprocessMetadata(); + void PostprocessMetadata(); + void InsertSuballoc(VmaBlockMetadata_Generic* pMetadata, const VmaSuballocation& suballoc); +}; + +struct VmaBlockDefragmentationContext +{ + enum BLOCK_FLAG + { + BLOCK_FLAG_USED = 0x00000001, + }; + uint32_t flags; + VkBuffer hBuffer; +}; + +class VmaBlockVectorDefragmentationContext +{ + VMA_CLASS_NO_COPY(VmaBlockVectorDefragmentationContext) +public: + VkResult res; + bool mutexLocked; + VmaVector< VmaBlockDefragmentationContext, VmaStlAllocator > blockContexts; + VmaVector< VmaDefragmentationMove, VmaStlAllocator > defragmentationMoves; + uint32_t defragmentationMovesProcessed; + uint32_t defragmentationMovesCommitted; + bool hasDefragmentationPlan; + + VmaBlockVectorDefragmentationContext( + VmaAllocator hAllocator, + VmaPool hCustomPool, // Optional. + VmaBlockVector* pBlockVector, + uint32_t currFrameIndex); + ~VmaBlockVectorDefragmentationContext(); + + VmaPool GetCustomPool() const { return m_hCustomPool; } + VmaBlockVector* GetBlockVector() const { return m_pBlockVector; } + VmaDefragmentationAlgorithm* GetAlgorithm() const { return m_pAlgorithm; } void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged); + void AddAll() { m_AllAllocations = true; } + void Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags); + +private: + const VmaAllocator m_hAllocator; + // Null if not from custom pool. + const VmaPool m_hCustomPool; + // Redundant, for convenience not to fetch from m_hCustomPool->m_BlockVector or m_hAllocator->m_pBlockVectors. + VmaBlockVector* const m_pBlockVector; + const uint32_t m_CurrFrameIndex; + // Owner of this object. + VmaDefragmentationAlgorithm* m_pAlgorithm; + + struct AllocInfo + { + VmaAllocation hAlloc; + VkBool32* pChanged; + }; + // Used between constructor and Begin. + VmaVector< AllocInfo, VmaStlAllocator > m_Allocations; + bool m_AllAllocations; +}; + +struct VmaDefragmentationContext_T +{ +private: + VMA_CLASS_NO_COPY(VmaDefragmentationContext_T) +public: + VmaDefragmentationContext_T( + VmaAllocator hAllocator, + uint32_t currFrameIndex, + uint32_t flags, + VmaDefragmentationStats* pStats); + ~VmaDefragmentationContext_T(); + + void AddPools(uint32_t poolCount, const VmaPool* pPools); + void AddAllocations( + uint32_t allocationCount, + const VmaAllocation* pAllocations, + VkBool32* pAllocationsChanged); + + /* + Returns: + - `VK_SUCCESS` if succeeded and object can be destroyed immediately. + - `VK_NOT_READY` if succeeded but the object must remain alive until vmaDefragmentationEnd(). + - Negative value if error occured and object can be destroyed immediately. + */ VkResult Defragment( - VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); + VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove, + VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove, + VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags); + + VkResult DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo); + VkResult DefragmentPassEnd(); + +private: + const VmaAllocator m_hAllocator; + const uint32_t m_CurrFrameIndex; + const uint32_t m_Flags; + VmaDefragmentationStats* const m_pStats; + + VkDeviceSize m_MaxCpuBytesToMove; + uint32_t m_MaxCpuAllocationsToMove; + VkDeviceSize m_MaxGpuBytesToMove; + uint32_t m_MaxGpuAllocationsToMove; + + // Owner of these objects. + VmaBlockVectorDefragmentationContext* m_DefaultPoolContexts[VK_MAX_MEMORY_TYPES]; + // Owner of these objects. + VmaVector< VmaBlockVectorDefragmentationContext*, VmaStlAllocator > m_CustomPoolContexts; }; #if VMA_RECORDING_ENABLED @@ -4802,7 +7611,11 @@ public: void WriteConfiguration( const VkPhysicalDeviceProperties& devProps, const VkPhysicalDeviceMemoryProperties& memProps, - bool dedicatedAllocationExtensionEnabled); + uint32_t vulkanApiVersion, + bool dedicatedAllocationExtensionEnabled, + bool bindMemory2ExtensionEnabled, + bool memoryBudgetExtensionEnabled, + bool deviceCoherentMemoryExtensionEnabled); ~VmaRecorder(); void RecordCreateAllocator(uint32_t frameIndex); @@ -4815,6 +7628,11 @@ public: const VkMemoryRequirements& vkMemReq, const VmaAllocationCreateInfo& createInfo, VmaAllocation allocation); + void RecordAllocateMemoryPages(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + uint64_t allocationCount, + const VmaAllocation* pAllocations); void RecordAllocateMemoryForBuffer(uint32_t frameIndex, const VkMemoryRequirements& vkMemReq, bool requiresDedicatedAllocation, @@ -4829,6 +7647,9 @@ public: VmaAllocation allocation); void RecordFreeMemory(uint32_t frameIndex, VmaAllocation allocation); + void RecordFreeMemoryPages(uint32_t frameIndex, + uint64_t allocationCount, + const VmaAllocation* pAllocations); void RecordSetAllocationUserData(uint32_t frameIndex, VmaAllocation allocation, const void* pUserData); @@ -4860,6 +7681,14 @@ public: VmaAllocation allocation); void RecordMakePoolAllocationsLost(uint32_t frameIndex, VmaPool pool); + void RecordDefragmentationBegin(uint32_t frameIndex, + const VmaDefragmentationInfo2& info, + VmaDefragmentationContext ctx); + void RecordDefragmentationEnd(uint32_t frameIndex, + VmaDefragmentationContext ctx); + void RecordSetPoolName(uint32_t frameIndex, + VmaPool pool, + const char* name); private: struct CallParams @@ -4883,30 +7712,117 @@ private: VmaRecordFlags m_Flags; FILE* m_File; VMA_MUTEX m_FileMutex; - int64_t m_Freq; - int64_t m_StartCounter; + std::chrono::time_point m_RecordingStartTime; void GetBasicParams(CallParams& outParams); + + // T must be a pointer type, e.g. VmaAllocation, VmaPool. + template + void PrintPointerList(uint64_t count, const T* pItems) + { + if(count) + { + fprintf(m_File, "%p", pItems[0]); + for(uint64_t i = 1; i < count; ++i) + { + fprintf(m_File, " %p", pItems[i]); + } + } + } + + void PrintPointerList(uint64_t count, const VmaAllocation* pItems); void Flush(); }; #endif // #if VMA_RECORDING_ENABLED +/* +Thread-safe wrapper over VmaPoolAllocator free list, for allocation of VmaAllocation_T objects. +*/ +class VmaAllocationObjectAllocator +{ + VMA_CLASS_NO_COPY(VmaAllocationObjectAllocator) +public: + VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks); + + template VmaAllocation Allocate(Types... args); + void Free(VmaAllocation hAlloc); + +private: + VMA_MUTEX m_Mutex; + VmaPoolAllocator m_Allocator; +}; + +struct VmaCurrentBudgetData +{ + VMA_ATOMIC_UINT64 m_BlockBytes[VK_MAX_MEMORY_HEAPS]; + VMA_ATOMIC_UINT64 m_AllocationBytes[VK_MAX_MEMORY_HEAPS]; + +#if VMA_MEMORY_BUDGET + VMA_ATOMIC_UINT32 m_OperationsSinceBudgetFetch; + VMA_RW_MUTEX m_BudgetMutex; + uint64_t m_VulkanUsage[VK_MAX_MEMORY_HEAPS]; + uint64_t m_VulkanBudget[VK_MAX_MEMORY_HEAPS]; + uint64_t m_BlockBytesAtBudgetFetch[VK_MAX_MEMORY_HEAPS]; +#endif // #if VMA_MEMORY_BUDGET + + VmaCurrentBudgetData() + { + for(uint32_t heapIndex = 0; heapIndex < VK_MAX_MEMORY_HEAPS; ++heapIndex) + { + m_BlockBytes[heapIndex] = 0; + m_AllocationBytes[heapIndex] = 0; +#if VMA_MEMORY_BUDGET + m_VulkanUsage[heapIndex] = 0; + m_VulkanBudget[heapIndex] = 0; + m_BlockBytesAtBudgetFetch[heapIndex] = 0; +#endif + } + +#if VMA_MEMORY_BUDGET + m_OperationsSinceBudgetFetch = 0; +#endif + } + + void AddAllocation(uint32_t heapIndex, VkDeviceSize allocationSize) + { + m_AllocationBytes[heapIndex] += allocationSize; +#if VMA_MEMORY_BUDGET + ++m_OperationsSinceBudgetFetch; +#endif + } + + void RemoveAllocation(uint32_t heapIndex, VkDeviceSize allocationSize) + { + VMA_ASSERT(m_AllocationBytes[heapIndex] >= allocationSize); // DELME + m_AllocationBytes[heapIndex] -= allocationSize; +#if VMA_MEMORY_BUDGET + ++m_OperationsSinceBudgetFetch; +#endif + } +}; + // Main allocator object. struct VmaAllocator_T { VMA_CLASS_NO_COPY(VmaAllocator_T) public: bool m_UseMutex; - bool m_UseKhrDedicatedAllocation; + uint32_t m_VulkanApiVersion; + bool m_UseKhrDedicatedAllocation; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0). + bool m_UseKhrBindMemory2; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0). + bool m_UseExtMemoryBudget; + bool m_UseAmdDeviceCoherentMemory; + bool m_UseKhrBufferDeviceAddress; VkDevice m_hDevice; + VkInstance m_hInstance; bool m_AllocationCallbacksSpecified; VkAllocationCallbacks m_AllocationCallbacks; VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks; + VmaAllocationObjectAllocator m_AllocationObjectAllocator; - // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap. - VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS]; - VMA_MUTEX m_HeapSizeLimitMutex; + // Each bit (1 << i) is set if HeapSizeLimit is enabled for that heap, so cannot allocate more than the heap size. + uint32_t m_HeapSizeLimitMask; VkPhysicalDeviceProperties m_PhysicalDeviceProperties; VkPhysicalDeviceMemoryProperties m_MemProps; @@ -4917,7 +7833,9 @@ public: // Each vector is sorted by memory (handle value). typedef VmaVector< VmaAllocation, VmaStlAllocator > AllocationVectorType; AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES]; - VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES]; + VMA_RW_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES]; + + VmaCurrentBudgetData m_Budget; VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo); VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo); @@ -4932,6 +7850,8 @@ public: return m_VulkanFunctions; } + VkPhysicalDevice GetPhysicalDevice() const { return m_PhysicalDevice; } + VkDeviceSize GetBufferImageGranularity() const { return VMA_MAX( @@ -4966,6 +7886,8 @@ public: return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; } + uint32_t GetGlobalMemoryTypeBits() const { return m_GlobalMemoryTypeBits; } + #if VMA_RECORDING_ENABLED VmaRecorder* GetRecorder() const { return m_pRecorder; } #endif @@ -4987,26 +7909,43 @@ public: bool requiresDedicatedAllocation, bool prefersDedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, // UINT32_MAX when unknown. VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, - VmaAllocation* pAllocation); + size_t allocationCount, + VmaAllocation* pAllocations); // Main deallocation function. - void FreeMemory(const VmaAllocation allocation); + void FreeMemory( + size_t allocationCount, + const VmaAllocation* pAllocations); + + VkResult ResizeAllocation( + const VmaAllocation alloc, + VkDeviceSize newSize); void CalculateStats(VmaStats* pStats); + void GetBudget( + VmaBudget* outBudget, uint32_t firstHeap, uint32_t heapCount); + #if VMA_STATS_STRING_ENABLED void PrintDetailedMap(class VmaJsonWriter& json); #endif - VkResult Defragment( - VmaAllocation* pAllocations, - size_t allocationCount, - VkBool32* pAllocationsChanged, - const VmaDefragmentationInfo* pDefragmentationInfo, - VmaDefragmentationStats* pDefragmentationStats); + VkResult DefragmentationBegin( + const VmaDefragmentationInfo2& info, + VmaDefragmentationStats* pStats, + VmaDefragmentationContext* pContext); + VkResult DefragmentationEnd( + VmaDefragmentationContext context); + + VkResult DefragmentationPassBegin( + VmaDefragmentationPassInfo* pInfo, + VmaDefragmentationContext context); + VkResult DefragmentationPassEnd( + VmaDefragmentationContext context); void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo); bool TouchAllocation(VmaAllocation hAllocation); @@ -5026,41 +7965,90 @@ public: void CreateLostAllocation(VmaAllocation* pAllocation); + // Call to Vulkan function vkAllocateMemory with accompanying bookkeeping. VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory); + // Call to Vulkan function vkFreeMemory with accompanying bookkeeping. void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory); + // Call to Vulkan function vkBindBufferMemory or vkBindBufferMemory2KHR. + VkResult BindVulkanBuffer( + VkDeviceMemory memory, + VkDeviceSize memoryOffset, + VkBuffer buffer, + const void* pNext); + // Call to Vulkan function vkBindImageMemory or vkBindImageMemory2KHR. + VkResult BindVulkanImage( + VkDeviceMemory memory, + VkDeviceSize memoryOffset, + VkImage image, + const void* pNext); VkResult Map(VmaAllocation hAllocation, void** ppData); void Unmap(VmaAllocation hAllocation); - VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer); - VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage); + VkResult BindBufferMemory( + VmaAllocation hAllocation, + VkDeviceSize allocationLocalOffset, + VkBuffer hBuffer, + const void* pNext); + VkResult BindImageMemory( + VmaAllocation hAllocation, + VkDeviceSize allocationLocalOffset, + VkImage hImage, + const void* pNext); - void FlushOrInvalidateAllocation( + VkResult FlushOrInvalidateAllocation( VmaAllocation hAllocation, VkDeviceSize offset, VkDeviceSize size, VMA_CACHE_OPERATION op); + VkResult FlushOrInvalidateAllocations( + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, const VkDeviceSize* sizes, + VMA_CACHE_OPERATION op); void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern); + /* + Returns bit mask of memory types that can support defragmentation on GPU as + they support creation of required buffer for copy operations. + */ + uint32_t GetGpuDefragmentationMemoryTypeBits(); + private: VkDeviceSize m_PreferredLargeHeapBlockSize; VkPhysicalDevice m_PhysicalDevice; VMA_ATOMIC_UINT32 m_CurrentFrameIndex; + VMA_ATOMIC_UINT32 m_GpuDefragmentationMemoryTypeBits; // UINT32_MAX means uninitialized. - VMA_MUTEX m_PoolsMutex; + VMA_RW_MUTEX m_PoolsMutex; // Protected by m_PoolsMutex. Sorted by pointer value. VmaVector > m_Pools; uint32_t m_NextPoolId; VmaVulkanFunctions m_VulkanFunctions; + // Global bit mask AND-ed with any memoryTypeBits to disallow certain memory types. + uint32_t m_GlobalMemoryTypeBits; + #if VMA_RECORDING_ENABLED VmaRecorder* m_pRecorder; #endif void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions); +#if VMA_STATIC_VULKAN_FUNCTIONS == 1 + void ImportVulkanFunctions_Static(); +#endif + + void ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions); + +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + void ImportVulkanFunctions_Dynamic(); +#endif + + void ValidateVulkanFunctions(); + VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex); VkResult AllocateMemoryOfType( @@ -5068,26 +8056,58 @@ private: VkDeviceSize alignment, bool dedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, VmaSuballocationType suballocType, + size_t allocationCount, + VmaAllocation* pAllocations); + + // Helper function only to be used inside AllocateDedicatedMemory. + VkResult AllocateDedicatedMemoryPage( + VkDeviceSize size, + VmaSuballocationType suballocType, + uint32_t memTypeIndex, + const VkMemoryAllocateInfo& allocInfo, + bool map, + bool isUserDataString, + void* pUserData, VmaAllocation* pAllocation); - // Allocates and registers new VkDeviceMemory specifically for single allocation. + // Allocates and registers new VkDeviceMemory specifically for dedicated allocations. VkResult AllocateDedicatedMemory( VkDeviceSize size, VmaSuballocationType suballocType, uint32_t memTypeIndex, + bool withinBudget, bool map, bool isUserDataString, void* pUserData, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, - VmaAllocation* pAllocation); + size_t allocationCount, + VmaAllocation* pAllocations); - // Tries to free pMemory as Dedicated Memory. Returns true if found and freed. - void FreeDedicatedMemory(VmaAllocation allocation); + void FreeDedicatedMemory(const VmaAllocation allocation); + + /* + Calculates and returns bit mask of memory types that can support defragmentation + on GPU as they support creation of required buffer for copy operations. + */ + uint32_t CalculateGpuDefragmentationMemoryTypeBits() const; + + uint32_t CalculateGlobalMemoryTypeBits() const; + + bool GetFlushOrInvalidateRange( + VmaAllocation allocation, + VkDeviceSize offset, VkDeviceSize size, + VkMappedMemoryRange& outRange) const; + +#if VMA_MEMORY_BUDGET + void UpdateVulkanBudget(); +#endif // #if VMA_MEMORY_BUDGET }; //////////////////////////////////////////////////////////////////////////////// @@ -5173,15 +8193,29 @@ void VmaStringBuilder::Add(const char* pStr) void VmaStringBuilder::AddNumber(uint32_t num) { char buf[11]; - VmaUint32ToStr(buf, sizeof(buf), num); - Add(buf); + buf[10] = '\0'; + char *p = &buf[10]; + do + { + *--p = '0' + (num % 10); + num /= 10; + } + while(num); + Add(p); } void VmaStringBuilder::AddNumber(uint64_t num) { char buf[21]; - VmaUint64ToStr(buf, sizeof(buf), num); - Add(buf); + buf[20] = '\0'; + char *p = &buf[20]; + do + { + *--p = '0' + (num % 10); + num /= 10; + } + while(num); + Add(p); } void VmaStringBuilder::AddPointer(const void* ptr) @@ -5493,11 +8527,7 @@ void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData) if(pUserData != VMA_NULL) { - const char* const newStrSrc = (char*)pUserData; - const size_t newStrLen = strlen(newStrSrc); - char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1); - memcpy(newStrDst, newStrSrc, newStrLen + 1); - m_pUserData = newStrDst; + m_pUserData = VmaCreateStringCopy(hAllocator->GetAllocationCallbacks(), (const char*)pUserData); } } else @@ -5528,6 +8558,12 @@ void VmaAllocation_T::ChangeBlockAllocation( m_BlockAllocation.m_Offset = offset; } +void VmaAllocation_T::ChangeOffset(VkDeviceSize newOffset) +{ + VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK); + m_BlockAllocation.m_Offset = newOffset; +} + VkDeviceSize VmaAllocation_T::GetOffset() const { switch(m_Type) @@ -5556,20 +8592,6 @@ VkDeviceMemory VmaAllocation_T::GetMemory() const } } -uint32_t VmaAllocation_T::GetMemoryTypeIndex() const -{ - switch(m_Type) - { - case ALLOCATION_TYPE_BLOCK: - return m_BlockAllocation.m_Block->GetMemoryTypeIndex(); - case ALLOCATION_TYPE_DEDICATED: - return m_DedicatedAllocation.m_MemoryTypeIndex; - default: - VMA_ASSERT(0); - return UINT32_MAX; - } -} - void* VmaAllocation_T::GetMappedData() const { switch(m_Type) @@ -5609,12 +8631,6 @@ bool VmaAllocation_T::CanBecomeLost() const } } -VmaPool VmaAllocation_T::GetPool() const -{ - VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK); - return m_BlockAllocation.m_hPool; -} - bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) { VMA_ASSERT(CanBecomeLost()); @@ -5700,13 +8716,8 @@ void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator) { VMA_ASSERT(IsUserDataString()); - if(m_pUserData != VMA_NULL) - { - char* const oldStr = (char*)m_pUserData; - const size_t oldStrLen = strlen(oldStr); - vma_delete_array(hAllocator, oldStr, oldStrLen + 1); - m_pUserData = VMA_NULL; - } + VmaFreeString(hAllocator->GetAllocationCallbacks(), (char*)m_pUserData); + m_pUserData = VMA_NULL; } void VmaAllocation_T::BlockAllocMap() @@ -5863,11 +8874,86 @@ struct VmaSuballocationItemSizeLess } }; + //////////////////////////////////////////////////////////////////////////////// // class VmaBlockMetadata VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) : m_Size(0), + m_pAllocationCallbacks(hAllocator->GetAllocationCallbacks()) +{ +} + +#if VMA_STATS_STRING_ENABLED + +void VmaBlockMetadata::PrintDetailedMap_Begin(class VmaJsonWriter& json, + VkDeviceSize unusedBytes, + size_t allocationCount, + size_t unusedRangeCount) const +{ + json.BeginObject(); + + json.WriteString("TotalBytes"); + json.WriteNumber(GetSize()); + + json.WriteString("UnusedBytes"); + json.WriteNumber(unusedBytes); + + json.WriteString("Allocations"); + json.WriteNumber((uint64_t)allocationCount); + + json.WriteString("UnusedRanges"); + json.WriteNumber((uint64_t)unusedRangeCount); + + json.WriteString("Suballocations"); + json.BeginArray(); +} + +void VmaBlockMetadata::PrintDetailedMap_Allocation(class VmaJsonWriter& json, + VkDeviceSize offset, + VmaAllocation hAllocation) const +{ + json.BeginObject(true); + + json.WriteString("Offset"); + json.WriteNumber(offset); + + hAllocation->PrintParameters(json); + + json.EndObject(); +} + +void VmaBlockMetadata::PrintDetailedMap_UnusedRange(class VmaJsonWriter& json, + VkDeviceSize offset, + VkDeviceSize size) const +{ + json.BeginObject(true); + + json.WriteString("Offset"); + json.WriteNumber(offset); + + json.WriteString("Type"); + json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]); + + json.WriteString("Size"); + json.WriteNumber(size); + + json.EndObject(); +} + +void VmaBlockMetadata::PrintDetailedMap_End(class VmaJsonWriter& json) const +{ + json.EndArray(); + json.EndObject(); +} + +#endif // #if VMA_STATS_STRING_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +// class VmaBlockMetadata_Generic + +VmaBlockMetadata_Generic::VmaBlockMetadata_Generic(VmaAllocator hAllocator) : + VmaBlockMetadata(hAllocator), m_FreeCount(0), m_SumFreeSize(0), m_Suballocations(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), @@ -5875,13 +8961,14 @@ VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) : { } -VmaBlockMetadata::~VmaBlockMetadata() +VmaBlockMetadata_Generic::~VmaBlockMetadata_Generic() { } -void VmaBlockMetadata::Init(VkDeviceSize size) +void VmaBlockMetadata_Generic::Init(VkDeviceSize size) { - m_Size = size; + VmaBlockMetadata::Init(size); + m_FreeCount = 1; m_SumFreeSize = size; @@ -5891,20 +8978,18 @@ void VmaBlockMetadata::Init(VkDeviceSize size) suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; suballoc.hAllocation = VK_NULL_HANDLE; + VMA_ASSERT(size > VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); m_Suballocations.push_back(suballoc); VmaSuballocationList::iterator suballocItem = m_Suballocations.end(); --suballocItem; m_FreeSuballocationsBySize.push_back(suballocItem); } -bool VmaBlockMetadata::Validate() const +bool VmaBlockMetadata_Generic::Validate() const { - if(m_Suballocations.empty()) - { - return false; - } + VMA_VALIDATE(!m_Suballocations.empty()); - // Expected offset of new suballocation as calculates from previous ones. + // Expected offset of new suballocation as calculated from previous ones. VkDeviceSize calculatedOffset = 0; // Expected number of free suballocations as calculated from traversing their list. uint32_t calculatedFreeCount = 0; @@ -5923,22 +9008,13 @@ bool VmaBlockMetadata::Validate() const const VmaSuballocation& subAlloc = *suballocItem; // Actual offset of this suballocation doesn't match expected one. - if(subAlloc.offset != calculatedOffset) - { - return false; - } + VMA_VALIDATE(subAlloc.offset == calculatedOffset); const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE); // Two adjacent free suballocations are invalid. They should be merged. - if(prevFree && currFree) - { - return false; - } + VMA_VALIDATE(!prevFree || !currFree); - if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE)) - { - return false; - } + VMA_VALIDATE(currFree == (subAlloc.hAllocation == VK_NULL_HANDLE)); if(currFree) { @@ -5950,27 +9026,15 @@ bool VmaBlockMetadata::Validate() const } // Margin required between allocations - every free space must be at least that large. - if(subAlloc.size < VMA_DEBUG_MARGIN) - { - return false; - } + VMA_VALIDATE(subAlloc.size >= VMA_DEBUG_MARGIN); } else { - if(subAlloc.hAllocation->GetOffset() != subAlloc.offset) - { - return false; - } - if(subAlloc.hAllocation->GetSize() != subAlloc.size) - { - return false; - } + VMA_VALIDATE(subAlloc.hAllocation->GetOffset() == subAlloc.offset); + VMA_VALIDATE(subAlloc.hAllocation->GetSize() == subAlloc.size); // Margin required between allocations - previous allocation must be free. - if(VMA_DEBUG_MARGIN > 0 && !prevFree) - { - return false; - } + VMA_VALIDATE(VMA_DEBUG_MARGIN == 0 || prevFree); } calculatedOffset += subAlloc.size; @@ -5979,10 +9043,7 @@ bool VmaBlockMetadata::Validate() const // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't // match expected one. - if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister) - { - return false; - } + VMA_VALIDATE(m_FreeSuballocationsBySize.size() == freeSuballocationsToRegister); VkDeviceSize lastSize = 0; for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i) @@ -5990,32 +9051,23 @@ bool VmaBlockMetadata::Validate() const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i]; // Only free suballocations can be registered in m_FreeSuballocationsBySize. - if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE) - { - return false; - } + VMA_VALIDATE(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE); // They must be sorted by size ascending. - if(suballocItem->size < lastSize) - { - return false; - } + VMA_VALIDATE(suballocItem->size >= lastSize); lastSize = suballocItem->size; } // Check if totals match calculacted values. - if(!ValidateFreeSuballocationList() || - (calculatedOffset != m_Size) || - (calculatedSumFreeSize != m_SumFreeSize) || - (calculatedFreeCount != m_FreeCount)) - { - return false; - } + VMA_VALIDATE(ValidateFreeSuballocationList()); + VMA_VALIDATE(calculatedOffset == GetSize()); + VMA_VALIDATE(calculatedSumFreeSize == m_SumFreeSize); + VMA_VALIDATE(calculatedFreeCount == m_FreeCount); return true; } -VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const +VkDeviceSize VmaBlockMetadata_Generic::GetUnusedRangeSizeMax() const { if(!m_FreeSuballocationsBySize.empty()) { @@ -6027,12 +9079,12 @@ VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const } } -bool VmaBlockMetadata::IsEmpty() const +bool VmaBlockMetadata_Generic::IsEmpty() const { return (m_Suballocations.size() == 1) && (m_FreeCount == 1); } -void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const +void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) const { outInfo.blockCount = 1; @@ -6041,7 +9093,7 @@ void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const outInfo.unusedRangeCount = m_FreeCount; outInfo.unusedBytes = m_SumFreeSize; - outInfo.usedBytes = m_Size - outInfo.unusedBytes; + outInfo.usedBytes = GetSize() - outInfo.unusedBytes; outInfo.allocationSizeMin = UINT64_MAX; outInfo.allocationSizeMax = 0; @@ -6066,11 +9118,11 @@ void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const } } -void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const +void VmaBlockMetadata_Generic::AddPoolStats(VmaPoolStats& inoutStats) const { const uint32_t rangeCount = (uint32_t)m_Suballocations.size(); - inoutStats.size += m_Size; + inoutStats.size += GetSize(); inoutStats.unusedSize += m_SumFreeSize; inoutStats.allocationCount += rangeCount - m_FreeCount; inoutStats.unusedRangeCount += m_FreeCount; @@ -6079,83 +9131,56 @@ void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const #if VMA_STATS_STRING_ENABLED -void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const +void VmaBlockMetadata_Generic::PrintDetailedMap(class VmaJsonWriter& json) const { - json.BeginObject(); + PrintDetailedMap_Begin(json, + m_SumFreeSize, // unusedBytes + m_Suballocations.size() - (size_t)m_FreeCount, // allocationCount + m_FreeCount); // unusedRangeCount - json.WriteString("TotalBytes"); - json.WriteNumber(m_Size); - - json.WriteString("UnusedBytes"); - json.WriteNumber(m_SumFreeSize); - - json.WriteString("Allocations"); - json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount); - - json.WriteString("UnusedRanges"); - json.WriteNumber(m_FreeCount); - - json.WriteString("Suballocations"); - json.BeginArray(); size_t i = 0; for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin(); suballocItem != m_Suballocations.cend(); ++suballocItem, ++i) { - json.BeginObject(true); - - json.WriteString("Offset"); - json.WriteNumber(suballocItem->offset); - if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE) { - json.WriteString("Type"); - json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]); - - json.WriteString("Size"); - json.WriteNumber(suballocItem->size); + PrintDetailedMap_UnusedRange(json, suballocItem->offset, suballocItem->size); } else { - suballocItem->hAllocation->PrintParameters(json); + PrintDetailedMap_Allocation(json, suballocItem->offset, suballocItem->hAllocation); } - - json.EndObject(); } - json.EndArray(); - json.EndObject(); + PrintDetailedMap_End(json); } #endif // #if VMA_STATS_STRING_ENABLED -/* -How many suitable free suballocations to analyze before choosing best one. -- Set to 1 to use First-Fit algorithm - first suitable free suballocation will - be chosen. -- Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free - suballocations will be analized and best one will be chosen. -- Any other value is also acceptable. -*/ -//static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8; - -bool VmaBlockMetadata::CreateAllocationRequest( +bool VmaBlockMetadata_Generic::CreateAllocationRequest( uint32_t currentFrameIndex, uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, VkDeviceSize allocSize, VkDeviceSize allocAlignment, + bool upperAddress, VmaSuballocationType allocType, bool canMakeOtherLost, + uint32_t strategy, VmaAllocationRequest* pAllocationRequest) { VMA_ASSERT(allocSize > 0); + VMA_ASSERT(!upperAddress); VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(pAllocationRequest != VMA_NULL); VMA_HEAVY_ASSERT(Validate()); + pAllocationRequest->type = VmaAllocationRequestType::Normal; + // There is not enough total free space in this block to fullfill the request: Early return. - if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN) + if(canMakeOtherLost == false && + m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN) { return false; } @@ -6164,7 +9189,7 @@ bool VmaBlockMetadata::CreateAllocationRequest( const size_t freeSuballocCount = m_FreeSuballocationsBySize.size(); if(freeSuballocCount > 0) { - if(VMA_BEST_FIT) + if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT) { // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN. VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess( @@ -6194,7 +9219,32 @@ bool VmaBlockMetadata::CreateAllocationRequest( } } } - else + else if(strategy == VMA_ALLOCATION_INTERNAL_STRATEGY_MIN_OFFSET) + { + for(VmaSuballocationList::iterator it = m_Suballocations.begin(); + it != m_Suballocations.end(); + ++it) + { + if(it->type == VMA_SUBALLOCATION_TYPE_FREE && CheckAllocation( + currentFrameIndex, + frameInUseCount, + bufferImageGranularity, + allocSize, + allocAlignment, + allocType, + it, + false, // canMakeOtherLost + &pAllocationRequest->offset, + &pAllocationRequest->itemsToMakeLostCount, + &pAllocationRequest->sumFreeSize, + &pAllocationRequest->sumItemSize)) + { + pAllocationRequest->item = it; + return true; + } + } + } + else // WORST_FIT, FIRST_FIT { // Search staring from biggest suballocations. for(size_t index = freeSuballocCount; index--; ) @@ -6224,10 +9274,9 @@ bool VmaBlockMetadata::CreateAllocationRequest( { // Brute-force algorithm. TODO: Come up with something better. - pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE; - pAllocationRequest->sumItemSize = VK_WHOLE_SIZE; - + bool found = false; VmaAllocationRequest tmpAllocRequest = {}; + tmpAllocRequest.type = VmaAllocationRequestType::Normal; for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin(); suballocIt != m_Suballocations.end(); ++suballocIt) @@ -6249,30 +9298,35 @@ bool VmaBlockMetadata::CreateAllocationRequest( &tmpAllocRequest.sumFreeSize, &tmpAllocRequest.sumItemSize)) { - tmpAllocRequest.item = suballocIt; - - if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost()) + if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT) { *pAllocationRequest = tmpAllocRequest; + pAllocationRequest->item = suballocIt; + break; + } + if(!found || tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost()) + { + *pAllocationRequest = tmpAllocRequest; + pAllocationRequest->item = suballocIt; + found = true; } } } } - if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE) - { - return true; - } + return found; } return false; } -bool VmaBlockMetadata::MakeRequestedAllocationsLost( +bool VmaBlockMetadata_Generic::MakeRequestedAllocationsLost( uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest) { + VMA_ASSERT(pAllocationRequest && pAllocationRequest->type == VmaAllocationRequestType::Normal); + while(pAllocationRequest->itemsToMakeLostCount > 0) { if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE) @@ -6300,7 +9354,7 @@ bool VmaBlockMetadata::MakeRequestedAllocationsLost( return true; } -uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) +uint32_t VmaBlockMetadata_Generic::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) { uint32_t lostAllocationCount = 0; for(VmaSuballocationList::iterator it = m_Suballocations.begin(); @@ -6318,7 +9372,7 @@ uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint3 return lostAllocationCount; } -VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData) +VkResult VmaBlockMetadata_Generic::CheckCorruption(const void* pBlockData) { for(VmaSuballocationList::iterator it = m_Suballocations.begin(); it != m_Suballocations.end(); @@ -6342,12 +9396,13 @@ VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData) return VK_SUCCESS; } -void VmaBlockMetadata::Alloc( +void VmaBlockMetadata_Generic::Alloc( const VmaAllocationRequest& request, VmaSuballocationType type, VkDeviceSize allocSize, VmaAllocation hAllocation) { + VMA_ASSERT(request.type == VmaAllocationRequestType::Normal); VMA_ASSERT(request.item != m_Suballocations.end()); VmaSuballocation& suballoc = *request.item; // Given suballocation is a free block. @@ -6406,7 +9461,7 @@ void VmaBlockMetadata::Alloc( m_SumFreeSize -= allocSize; } -void VmaBlockMetadata::Free(const VmaAllocation allocation) +void VmaBlockMetadata_Generic::Free(const VmaAllocation allocation) { for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin(); suballocItem != m_Suballocations.end(); @@ -6423,7 +9478,7 @@ void VmaBlockMetadata::Free(const VmaAllocation allocation) VMA_ASSERT(0 && "Not found!"); } -void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset) +void VmaBlockMetadata_Generic::FreeAtOffset(VkDeviceSize offset) { for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin(); suballocItem != m_Suballocations.end(); @@ -6439,35 +9494,22 @@ void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset) VMA_ASSERT(0 && "Not found!"); } -bool VmaBlockMetadata::ValidateFreeSuballocationList() const +bool VmaBlockMetadata_Generic::ValidateFreeSuballocationList() const { VkDeviceSize lastSize = 0; for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i) { const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i]; - if(it->type != VMA_SUBALLOCATION_TYPE_FREE) - { - VMA_ASSERT(0); - return false; - } - if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) - { - VMA_ASSERT(0); - return false; - } - if(it->size < lastSize) - { - VMA_ASSERT(0); - return false; - } - + VMA_VALIDATE(it->type == VMA_SUBALLOCATION_TYPE_FREE); + VMA_VALIDATE(it->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); + VMA_VALIDATE(it->size >= lastSize); lastSize = it->size; } return true; } -bool VmaBlockMetadata::CheckAllocation( +bool VmaBlockMetadata_Generic::CheckAllocation( uint32_t currentFrameIndex, uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, @@ -6511,7 +9553,7 @@ bool VmaBlockMetadata::CheckAllocation( } // Remaining size is too small for this request: Early return. - if(m_Size - suballocItem->offset < allocSize) + if(GetSize() - suballocItem->offset < allocSize) { return false; } @@ -6571,7 +9613,7 @@ bool VmaBlockMetadata::CheckAllocation( const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin; // Another early return check. - if(suballocItem->offset + totalSize > m_Size) + if(suballocItem->offset + totalSize > GetSize()) { return false; } @@ -6741,7 +9783,7 @@ bool VmaBlockMetadata::CheckAllocation( return true; } -void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item) +void VmaBlockMetadata_Generic::MergeFreeWithNext(VmaSuballocationList::iterator item) { VMA_ASSERT(item != m_Suballocations.end()); VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); @@ -6756,7 +9798,7 @@ void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item) m_Suballocations.erase(nextItem); } -VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem) +VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSuballocationList::iterator suballocItem) { // Change this suballocation to be marked as free. VmaSuballocation& suballoc = *suballocItem; @@ -6808,7 +9850,7 @@ VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocat } } -void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item) +void VmaBlockMetadata_Generic::RegisterFreeSuballocation(VmaSuballocationList::iterator item) { VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(item->size > 0); @@ -6833,7 +9875,7 @@ void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator } -void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item) +void VmaBlockMetadata_Generic::UnregisterFreeSuballocation(VmaSuballocationList::iterator item) { VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(item->size > 0); @@ -6866,11 +9908,2355 @@ void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterato //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList()); } +bool VmaBlockMetadata_Generic::IsBufferImageGranularityConflictPossible( + VkDeviceSize bufferImageGranularity, + VmaSuballocationType& inOutPrevSuballocType) const +{ + if(bufferImageGranularity == 1 || IsEmpty()) + { + return false; + } + + VkDeviceSize minAlignment = VK_WHOLE_SIZE; + bool typeConflictFound = false; + for(VmaSuballocationList::const_iterator it = m_Suballocations.cbegin(); + it != m_Suballocations.cend(); + ++it) + { + const VmaSuballocationType suballocType = it->type; + if(suballocType != VMA_SUBALLOCATION_TYPE_FREE) + { + minAlignment = VMA_MIN(minAlignment, it->hAllocation->GetAlignment()); + if(VmaIsBufferImageGranularityConflict(inOutPrevSuballocType, suballocType)) + { + typeConflictFound = true; + } + inOutPrevSuballocType = suballocType; + } + } + + return typeConflictFound || minAlignment >= bufferImageGranularity; +} + +//////////////////////////////////////////////////////////////////////////////// +// class VmaBlockMetadata_Linear + +VmaBlockMetadata_Linear::VmaBlockMetadata_Linear(VmaAllocator hAllocator) : + VmaBlockMetadata(hAllocator), + m_SumFreeSize(0), + m_Suballocations0(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + m_Suballocations1(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + m_1stVectorIndex(0), + m_2ndVectorMode(SECOND_VECTOR_EMPTY), + m_1stNullItemsBeginCount(0), + m_1stNullItemsMiddleCount(0), + m_2ndNullItemsCount(0) +{ +} + +VmaBlockMetadata_Linear::~VmaBlockMetadata_Linear() +{ +} + +void VmaBlockMetadata_Linear::Init(VkDeviceSize size) +{ + VmaBlockMetadata::Init(size); + m_SumFreeSize = size; +} + +bool VmaBlockMetadata_Linear::Validate() const +{ + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + VMA_VALIDATE(suballocations2nd.empty() == (m_2ndVectorMode == SECOND_VECTOR_EMPTY)); + VMA_VALIDATE(!suballocations1st.empty() || + suballocations2nd.empty() || + m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER); + + if(!suballocations1st.empty()) + { + // Null item at the beginning should be accounted into m_1stNullItemsBeginCount. + VMA_VALIDATE(suballocations1st[m_1stNullItemsBeginCount].hAllocation != VK_NULL_HANDLE); + // Null item at the end should be just pop_back(). + VMA_VALIDATE(suballocations1st.back().hAllocation != VK_NULL_HANDLE); + } + if(!suballocations2nd.empty()) + { + // Null item at the end should be just pop_back(). + VMA_VALIDATE(suballocations2nd.back().hAllocation != VK_NULL_HANDLE); + } + + VMA_VALIDATE(m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount <= suballocations1st.size()); + VMA_VALIDATE(m_2ndNullItemsCount <= suballocations2nd.size()); + + VkDeviceSize sumUsedSize = 0; + const size_t suballoc1stCount = suballocations1st.size(); + VkDeviceSize offset = VMA_DEBUG_MARGIN; + + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + const size_t suballoc2ndCount = suballocations2nd.size(); + size_t nullItem2ndCount = 0; + for(size_t i = 0; i < suballoc2ndCount; ++i) + { + const VmaSuballocation& suballoc = suballocations2nd[i]; + const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); + + VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE)); + VMA_VALIDATE(suballoc.offset >= offset); + + if(!currFree) + { + VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset); + VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size); + sumUsedSize += suballoc.size; + } + else + { + ++nullItem2ndCount; + } + + offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN; + } + + VMA_VALIDATE(nullItem2ndCount == m_2ndNullItemsCount); + } + + for(size_t i = 0; i < m_1stNullItemsBeginCount; ++i) + { + const VmaSuballocation& suballoc = suballocations1st[i]; + VMA_VALIDATE(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE && + suballoc.hAllocation == VK_NULL_HANDLE); + } + + size_t nullItem1stCount = m_1stNullItemsBeginCount; + + for(size_t i = m_1stNullItemsBeginCount; i < suballoc1stCount; ++i) + { + const VmaSuballocation& suballoc = suballocations1st[i]; + const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); + + VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE)); + VMA_VALIDATE(suballoc.offset >= offset); + VMA_VALIDATE(i >= m_1stNullItemsBeginCount || currFree); + + if(!currFree) + { + VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset); + VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size); + sumUsedSize += suballoc.size; + } + else + { + ++nullItem1stCount; + } + + offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN; + } + VMA_VALIDATE(nullItem1stCount == m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount); + + if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + const size_t suballoc2ndCount = suballocations2nd.size(); + size_t nullItem2ndCount = 0; + for(size_t i = suballoc2ndCount; i--; ) + { + const VmaSuballocation& suballoc = suballocations2nd[i]; + const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); + + VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE)); + VMA_VALIDATE(suballoc.offset >= offset); + + if(!currFree) + { + VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset); + VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size); + sumUsedSize += suballoc.size; + } + else + { + ++nullItem2ndCount; + } + + offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN; + } + + VMA_VALIDATE(nullItem2ndCount == m_2ndNullItemsCount); + } + + VMA_VALIDATE(offset <= GetSize()); + VMA_VALIDATE(m_SumFreeSize == GetSize() - sumUsedSize); + + return true; +} + +size_t VmaBlockMetadata_Linear::GetAllocationCount() const +{ + return AccessSuballocations1st().size() - (m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount) + + AccessSuballocations2nd().size() - m_2ndNullItemsCount; +} + +VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const +{ + const VkDeviceSize size = GetSize(); + + /* + We don't consider gaps inside allocation vectors with freed allocations because + they are not suitable for reuse in linear allocator. We consider only space that + is available for new allocations. + */ + if(IsEmpty()) + { + return size; + } + + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + + switch(m_2ndVectorMode) + { + case SECOND_VECTOR_EMPTY: + /* + Available space is after end of 1st, as well as before beginning of 1st (which + whould make it a ring buffer). + */ + { + const size_t suballocations1stCount = suballocations1st.size(); + VMA_ASSERT(suballocations1stCount > m_1stNullItemsBeginCount); + const VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount]; + const VmaSuballocation& lastSuballoc = suballocations1st[suballocations1stCount - 1]; + return VMA_MAX( + firstSuballoc.offset, + size - (lastSuballoc.offset + lastSuballoc.size)); + } + break; + + case SECOND_VECTOR_RING_BUFFER: + /* + Available space is only between end of 2nd and beginning of 1st. + */ + { + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + const VmaSuballocation& lastSuballoc2nd = suballocations2nd.back(); + const VmaSuballocation& firstSuballoc1st = suballocations1st[m_1stNullItemsBeginCount]; + return firstSuballoc1st.offset - (lastSuballoc2nd.offset + lastSuballoc2nd.size); + } + break; + + case SECOND_VECTOR_DOUBLE_STACK: + /* + Available space is only between end of 1st and top of 2nd. + */ + { + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + const VmaSuballocation& topSuballoc2nd = suballocations2nd.back(); + const VmaSuballocation& lastSuballoc1st = suballocations1st.back(); + return topSuballoc2nd.offset - (lastSuballoc1st.offset + lastSuballoc1st.size); + } + break; + + default: + VMA_ASSERT(0); + return 0; + } +} + +void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const +{ + const VkDeviceSize size = GetSize(); + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + const size_t suballoc1stCount = suballocations1st.size(); + const size_t suballoc2ndCount = suballocations2nd.size(); + + outInfo.blockCount = 1; + outInfo.allocationCount = (uint32_t)GetAllocationCount(); + outInfo.unusedRangeCount = 0; + outInfo.usedBytes = 0; + outInfo.allocationSizeMin = UINT64_MAX; + outInfo.allocationSizeMax = 0; + outInfo.unusedRangeSizeMin = UINT64_MAX; + outInfo.unusedRangeSizeMax = 0; + + VkDeviceSize lastOffset = 0; + + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset; + size_t nextAlloc2ndIndex = 0; + while(lastOffset < freeSpace2ndTo1stEnd) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc2ndIndex < suballoc2ndCount && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex < suballoc2ndCount) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + outInfo.usedBytes += suballoc.size; + outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); + outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc2ndIndex; + } + // We are at the end. + else + { + // There is free space from lastOffset to freeSpace2ndTo1stEnd. + if(lastOffset < freeSpace2ndTo1stEnd) + { + const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace2ndTo1stEnd; + } + } + } + + size_t nextAlloc1stIndex = m_1stNullItemsBeginCount; + const VkDeviceSize freeSpace1stTo2ndEnd = + m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size; + while(lastOffset < freeSpace1stTo2ndEnd) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc1stIndex < suballoc1stCount && + suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc1stIndex; + } + + // Found non-null allocation. + if(nextAlloc1stIndex < suballoc1stCount) + { + const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + outInfo.usedBytes += suballoc.size; + outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); + outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc1stIndex; + } + // We are at the end. + else + { + // There is free space from lastOffset to freeSpace1stTo2ndEnd. + if(lastOffset < freeSpace1stTo2ndEnd) + { + const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace1stTo2ndEnd; + } + } + + if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + size_t nextAlloc2ndIndex = suballocations2nd.size() - 1; + while(lastOffset < size) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc2ndIndex != SIZE_MAX && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + --nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex != SIZE_MAX) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + outInfo.usedBytes += suballoc.size; + outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); + outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + --nextAlloc2ndIndex; + } + // We are at the end. + else + { + // There is free space from lastOffset to size. + if(lastOffset < size) + { + const VkDeviceSize unusedRangeSize = size - lastOffset; + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); + outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = size; + } + } + } + + outInfo.unusedBytes = size - outInfo.usedBytes; +} + +void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const +{ + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + const VkDeviceSize size = GetSize(); + const size_t suballoc1stCount = suballocations1st.size(); + const size_t suballoc2ndCount = suballocations2nd.size(); + + inoutStats.size += size; + + VkDeviceSize lastOffset = 0; + + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset; + size_t nextAlloc2ndIndex = m_1stNullItemsBeginCount; + while(lastOffset < freeSpace2ndTo1stEnd) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex < suballoc2ndCount && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex < suballoc2ndCount) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++inoutStats.allocationCount; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < freeSpace2ndTo1stEnd) + { + // There is free space from lastOffset to freeSpace2ndTo1stEnd. + const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace2ndTo1stEnd; + } + } + } + + size_t nextAlloc1stIndex = m_1stNullItemsBeginCount; + const VkDeviceSize freeSpace1stTo2ndEnd = + m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size; + while(lastOffset < freeSpace1stTo2ndEnd) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc1stIndex < suballoc1stCount && + suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc1stIndex; + } + + // Found non-null allocation. + if(nextAlloc1stIndex < suballoc1stCount) + { + const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++inoutStats.allocationCount; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc1stIndex; + } + // We are at the end. + else + { + if(lastOffset < freeSpace1stTo2ndEnd) + { + // There is free space from lastOffset to freeSpace1stTo2ndEnd. + const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace1stTo2ndEnd; + } + } + + if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + size_t nextAlloc2ndIndex = suballocations2nd.size() - 1; + while(lastOffset < size) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex != SIZE_MAX && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + --nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex != SIZE_MAX) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++inoutStats.allocationCount; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + --nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < size) + { + // There is free space from lastOffset to size. + const VkDeviceSize unusedRangeSize = size - lastOffset; + inoutStats.unusedSize += unusedRangeSize; + ++inoutStats.unusedRangeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); + } + + // End of loop. + lastOffset = size; + } + } + } +} + +#if VMA_STATS_STRING_ENABLED +void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const +{ + const VkDeviceSize size = GetSize(); + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + const size_t suballoc1stCount = suballocations1st.size(); + const size_t suballoc2ndCount = suballocations2nd.size(); + + // FIRST PASS + + size_t unusedRangeCount = 0; + VkDeviceSize usedBytes = 0; + + VkDeviceSize lastOffset = 0; + + size_t alloc2ndCount = 0; + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset; + size_t nextAlloc2ndIndex = 0; + while(lastOffset < freeSpace2ndTo1stEnd) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex < suballoc2ndCount && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex < suballoc2ndCount) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + ++unusedRangeCount; + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++alloc2ndCount; + usedBytes += suballoc.size; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < freeSpace2ndTo1stEnd) + { + // There is free space from lastOffset to freeSpace2ndTo1stEnd. + ++unusedRangeCount; + } + + // End of loop. + lastOffset = freeSpace2ndTo1stEnd; + } + } + } + + size_t nextAlloc1stIndex = m_1stNullItemsBeginCount; + size_t alloc1stCount = 0; + const VkDeviceSize freeSpace1stTo2ndEnd = + m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size; + while(lastOffset < freeSpace1stTo2ndEnd) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc1stIndex < suballoc1stCount && + suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc1stIndex; + } + + // Found non-null allocation. + if(nextAlloc1stIndex < suballoc1stCount) + { + const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + ++unusedRangeCount; + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++alloc1stCount; + usedBytes += suballoc.size; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc1stIndex; + } + // We are at the end. + else + { + if(lastOffset < size) + { + // There is free space from lastOffset to freeSpace1stTo2ndEnd. + ++unusedRangeCount; + } + + // End of loop. + lastOffset = freeSpace1stTo2ndEnd; + } + } + + if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + size_t nextAlloc2ndIndex = suballocations2nd.size() - 1; + while(lastOffset < size) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex != SIZE_MAX && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + --nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex != SIZE_MAX) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + ++unusedRangeCount; + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + ++alloc2ndCount; + usedBytes += suballoc.size; + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + --nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < size) + { + // There is free space from lastOffset to size. + ++unusedRangeCount; + } + + // End of loop. + lastOffset = size; + } + } + } + + const VkDeviceSize unusedBytes = size - usedBytes; + PrintDetailedMap_Begin(json, unusedBytes, alloc1stCount + alloc2ndCount, unusedRangeCount); + + // SECOND PASS + lastOffset = 0; + + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset; + size_t nextAlloc2ndIndex = 0; + while(lastOffset < freeSpace2ndTo1stEnd) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex < suballoc2ndCount && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex < suballoc2ndCount) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < freeSpace2ndTo1stEnd) + { + // There is free space from lastOffset to freeSpace2ndTo1stEnd. + const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace2ndTo1stEnd; + } + } + } + + nextAlloc1stIndex = m_1stNullItemsBeginCount; + while(lastOffset < freeSpace1stTo2ndEnd) + { + // Find next non-null allocation or move nextAllocIndex to the end. + while(nextAlloc1stIndex < suballoc1stCount && + suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE) + { + ++nextAlloc1stIndex; + } + + // Found non-null allocation. + if(nextAlloc1stIndex < suballoc1stCount) + { + const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + ++nextAlloc1stIndex; + } + // We are at the end. + else + { + if(lastOffset < freeSpace1stTo2ndEnd) + { + // There is free space from lastOffset to freeSpace1stTo2ndEnd. + const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // End of loop. + lastOffset = freeSpace1stTo2ndEnd; + } + } + + if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + size_t nextAlloc2ndIndex = suballocations2nd.size() - 1; + while(lastOffset < size) + { + // Find next non-null allocation or move nextAlloc2ndIndex to the end. + while(nextAlloc2ndIndex != SIZE_MAX && + suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE) + { + --nextAlloc2ndIndex; + } + + // Found non-null allocation. + if(nextAlloc2ndIndex != SIZE_MAX) + { + const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; + + // 1. Process free space before this allocation. + if(lastOffset < suballoc.offset) + { + // There is free space from lastOffset to suballoc.offset. + const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // 2. Process this allocation. + // There is allocation with suballoc.offset, suballoc.size. + PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); + + // 3. Prepare for next iteration. + lastOffset = suballoc.offset + suballoc.size; + --nextAlloc2ndIndex; + } + // We are at the end. + else + { + if(lastOffset < size) + { + // There is free space from lastOffset to size. + const VkDeviceSize unusedRangeSize = size - lastOffset; + PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); + } + + // End of loop. + lastOffset = size; + } + } + } + + PrintDetailedMap_End(json); +} +#endif // #if VMA_STATS_STRING_ENABLED + +bool VmaBlockMetadata_Linear::CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + bool upperAddress, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest) +{ + VMA_ASSERT(allocSize > 0); + VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE); + VMA_ASSERT(pAllocationRequest != VMA_NULL); + VMA_HEAVY_ASSERT(Validate()); + return upperAddress ? + CreateAllocationRequest_UpperAddress( + currentFrameIndex, frameInUseCount, bufferImageGranularity, + allocSize, allocAlignment, allocType, canMakeOtherLost, strategy, pAllocationRequest) : + CreateAllocationRequest_LowerAddress( + currentFrameIndex, frameInUseCount, bufferImageGranularity, + allocSize, allocAlignment, allocType, canMakeOtherLost, strategy, pAllocationRequest); +} + +bool VmaBlockMetadata_Linear::CreateAllocationRequest_UpperAddress( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest) +{ + const VkDeviceSize size = GetSize(); + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + VMA_ASSERT(0 && "Trying to use pool with linear algorithm as double stack, while it is already being used as ring buffer."); + return false; + } + + // Try to allocate before 2nd.back(), or end of block if 2nd.empty(). + if(allocSize > size) + { + return false; + } + VkDeviceSize resultBaseOffset = size - allocSize; + if(!suballocations2nd.empty()) + { + const VmaSuballocation& lastSuballoc = suballocations2nd.back(); + resultBaseOffset = lastSuballoc.offset - allocSize; + if(allocSize > lastSuballoc.offset) + { + return false; + } + } + + // Start from offset equal to end of free space. + VkDeviceSize resultOffset = resultBaseOffset; + + // Apply VMA_DEBUG_MARGIN at the end. + if(VMA_DEBUG_MARGIN > 0) + { + if(resultOffset < VMA_DEBUG_MARGIN) + { + return false; + } + resultOffset -= VMA_DEBUG_MARGIN; + } + + // Apply alignment. + resultOffset = VmaAlignDown(resultOffset, allocAlignment); + + // Check next suballocations from 2nd for BufferImageGranularity conflicts. + // Make bigger alignment if necessary. + if(bufferImageGranularity > 1 && !suballocations2nd.empty()) + { + bool bufferImageGranularityConflict = false; + for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; ) + { + const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex]; + if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(nextSuballoc.type, allocType)) + { + bufferImageGranularityConflict = true; + break; + } + } + else + // Already on previous page. + break; + } + if(bufferImageGranularityConflict) + { + resultOffset = VmaAlignDown(resultOffset, bufferImageGranularity); + } + } + + // There is enough free space. + const VkDeviceSize endOf1st = !suballocations1st.empty() ? + suballocations1st.back().offset + suballocations1st.back().size : + 0; + if(endOf1st + VMA_DEBUG_MARGIN <= resultOffset) + { + // Check previous suballocations for BufferImageGranularity conflicts. + // If conflict exists, allocation cannot be made here. + if(bufferImageGranularity > 1) + { + for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; ) + { + const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex]; + if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(allocType, prevSuballoc.type)) + { + return false; + } + } + else + { + // Already on next page. + break; + } + } + } + + // All tests passed: Success. + pAllocationRequest->offset = resultOffset; + pAllocationRequest->sumFreeSize = resultBaseOffset + allocSize - endOf1st; + pAllocationRequest->sumItemSize = 0; + // pAllocationRequest->item unused. + pAllocationRequest->itemsToMakeLostCount = 0; + pAllocationRequest->type = VmaAllocationRequestType::UpperAddress; + return true; + } + + return false; +} + +bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest) +{ + const VkDeviceSize size = GetSize(); + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + // Try to allocate at the end of 1st vector. + + VkDeviceSize resultBaseOffset = 0; + if(!suballocations1st.empty()) + { + const VmaSuballocation& lastSuballoc = suballocations1st.back(); + resultBaseOffset = lastSuballoc.offset + lastSuballoc.size; + } + + // Start from offset equal to beginning of free space. + VkDeviceSize resultOffset = resultBaseOffset; + + // Apply VMA_DEBUG_MARGIN at the beginning. + if(VMA_DEBUG_MARGIN > 0) + { + resultOffset += VMA_DEBUG_MARGIN; + } + + // Apply alignment. + resultOffset = VmaAlignUp(resultOffset, allocAlignment); + + // Check previous suballocations for BufferImageGranularity conflicts. + // Make bigger alignment if necessary. + if(bufferImageGranularity > 1 && !suballocations1st.empty()) + { + bool bufferImageGranularityConflict = false; + for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; ) + { + const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex]; + if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType)) + { + bufferImageGranularityConflict = true; + break; + } + } + else + // Already on previous page. + break; + } + if(bufferImageGranularityConflict) + { + resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity); + } + } + + const VkDeviceSize freeSpaceEnd = m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? + suballocations2nd.back().offset : size; + + // There is enough free space at the end after alignment. + if(resultOffset + allocSize + VMA_DEBUG_MARGIN <= freeSpaceEnd) + { + // Check next suballocations for BufferImageGranularity conflicts. + // If conflict exists, allocation cannot be made here. + if(bufferImageGranularity > 1 && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; ) + { + const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex]; + if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type)) + { + return false; + } + } + else + { + // Already on previous page. + break; + } + } + } + + // All tests passed: Success. + pAllocationRequest->offset = resultOffset; + pAllocationRequest->sumFreeSize = freeSpaceEnd - resultBaseOffset; + pAllocationRequest->sumItemSize = 0; + // pAllocationRequest->item, customData unused. + pAllocationRequest->type = VmaAllocationRequestType::EndOf1st; + pAllocationRequest->itemsToMakeLostCount = 0; + return true; + } + } + + // Wrap-around to end of 2nd vector. Try to allocate there, watching for the + // beginning of 1st vector as the end of free space. + if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + VMA_ASSERT(!suballocations1st.empty()); + + VkDeviceSize resultBaseOffset = 0; + if(!suballocations2nd.empty()) + { + const VmaSuballocation& lastSuballoc = suballocations2nd.back(); + resultBaseOffset = lastSuballoc.offset + lastSuballoc.size; + } + + // Start from offset equal to beginning of free space. + VkDeviceSize resultOffset = resultBaseOffset; + + // Apply VMA_DEBUG_MARGIN at the beginning. + if(VMA_DEBUG_MARGIN > 0) + { + resultOffset += VMA_DEBUG_MARGIN; + } + + // Apply alignment. + resultOffset = VmaAlignUp(resultOffset, allocAlignment); + + // Check previous suballocations for BufferImageGranularity conflicts. + // Make bigger alignment if necessary. + if(bufferImageGranularity > 1 && !suballocations2nd.empty()) + { + bool bufferImageGranularityConflict = false; + for(size_t prevSuballocIndex = suballocations2nd.size(); prevSuballocIndex--; ) + { + const VmaSuballocation& prevSuballoc = suballocations2nd[prevSuballocIndex]; + if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType)) + { + bufferImageGranularityConflict = true; + break; + } + } + else + // Already on previous page. + break; + } + if(bufferImageGranularityConflict) + { + resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity); + } + } + + pAllocationRequest->itemsToMakeLostCount = 0; + pAllocationRequest->sumItemSize = 0; + size_t index1st = m_1stNullItemsBeginCount; + + if(canMakeOtherLost) + { + while(index1st < suballocations1st.size() && + resultOffset + allocSize + VMA_DEBUG_MARGIN > suballocations1st[index1st].offset) + { + // Next colliding allocation at the beginning of 1st vector found. Try to make it lost. + const VmaSuballocation& suballoc = suballocations1st[index1st]; + if(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE) + { + // No problem. + } + else + { + VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE); + if(suballoc.hAllocation->CanBecomeLost() && + suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex) + { + ++pAllocationRequest->itemsToMakeLostCount; + pAllocationRequest->sumItemSize += suballoc.size; + } + else + { + return false; + } + } + ++index1st; + } + + // Check next suballocations for BufferImageGranularity conflicts. + // If conflict exists, we must mark more allocations lost or fail. + if(bufferImageGranularity > 1) + { + while(index1st < suballocations1st.size()) + { + const VmaSuballocation& suballoc = suballocations1st[index1st]; + if(VmaBlocksOnSamePage(resultOffset, allocSize, suballoc.offset, bufferImageGranularity)) + { + if(suballoc.hAllocation != VK_NULL_HANDLE) + { + // Not checking actual VmaIsBufferImageGranularityConflict(allocType, suballoc.type). + if(suballoc.hAllocation->CanBecomeLost() && + suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex) + { + ++pAllocationRequest->itemsToMakeLostCount; + pAllocationRequest->sumItemSize += suballoc.size; + } + else + { + return false; + } + } + } + else + { + // Already on next page. + break; + } + ++index1st; + } + } + + // Special case: There is not enough room at the end for this allocation, even after making all from the 1st lost. + if(index1st == suballocations1st.size() && + resultOffset + allocSize + VMA_DEBUG_MARGIN > size) + { + // TODO: This is a known bug that it's not yet implemented and the allocation is failing. + VMA_DEBUG_LOG("Unsupported special case in custom pool with linear allocation algorithm used as ring buffer with allocations that can be lost."); + } + } + + // There is enough free space at the end after alignment. + if((index1st == suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= size) || + (index1st < suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= suballocations1st[index1st].offset)) + { + // Check next suballocations for BufferImageGranularity conflicts. + // If conflict exists, allocation cannot be made here. + if(bufferImageGranularity > 1) + { + for(size_t nextSuballocIndex = index1st; + nextSuballocIndex < suballocations1st.size(); + nextSuballocIndex++) + { + const VmaSuballocation& nextSuballoc = suballocations1st[nextSuballocIndex]; + if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type)) + { + return false; + } + } + else + { + // Already on next page. + break; + } + } + } + + // All tests passed: Success. + pAllocationRequest->offset = resultOffset; + pAllocationRequest->sumFreeSize = + (index1st < suballocations1st.size() ? suballocations1st[index1st].offset : size) + - resultBaseOffset + - pAllocationRequest->sumItemSize; + pAllocationRequest->type = VmaAllocationRequestType::EndOf2nd; + // pAllocationRequest->item, customData unused. + return true; + } + } + + return false; +} + +bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VmaAllocationRequest* pAllocationRequest) +{ + if(pAllocationRequest->itemsToMakeLostCount == 0) + { + return true; + } + + VMA_ASSERT(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER); + + // We always start from 1st. + SuballocationVectorType* suballocations = &AccessSuballocations1st(); + size_t index = m_1stNullItemsBeginCount; + size_t madeLostCount = 0; + while(madeLostCount < pAllocationRequest->itemsToMakeLostCount) + { + if(index == suballocations->size()) + { + index = 0; + // If we get to the end of 1st, we wrap around to beginning of 2nd of 1st. + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + suballocations = &AccessSuballocations2nd(); + } + // else: m_2ndVectorMode == SECOND_VECTOR_EMPTY: + // suballocations continues pointing at AccessSuballocations1st(). + VMA_ASSERT(!suballocations->empty()); + } + VmaSuballocation& suballoc = (*suballocations)[index]; + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) + { + VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE); + VMA_ASSERT(suballoc.hAllocation->CanBecomeLost()); + if(suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount)) + { + suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + suballoc.hAllocation = VK_NULL_HANDLE; + m_SumFreeSize += suballoc.size; + if(suballocations == &AccessSuballocations1st()) + { + ++m_1stNullItemsMiddleCount; + } + else + { + ++m_2ndNullItemsCount; + } + ++madeLostCount; + } + else + { + return false; + } + } + ++index; + } + + CleanupAfterFree(); + //VMA_HEAVY_ASSERT(Validate()); // Already called by ClanupAfterFree(). + + return true; +} + +uint32_t VmaBlockMetadata_Linear::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) +{ + uint32_t lostAllocationCount = 0; + + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i) + { + VmaSuballocation& suballoc = suballocations1st[i]; + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE && + suballoc.hAllocation->CanBecomeLost() && + suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount)) + { + suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + suballoc.hAllocation = VK_NULL_HANDLE; + ++m_1stNullItemsMiddleCount; + m_SumFreeSize += suballoc.size; + ++lostAllocationCount; + } + } + + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i) + { + VmaSuballocation& suballoc = suballocations2nd[i]; + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE && + suballoc.hAllocation->CanBecomeLost() && + suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount)) + { + suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + suballoc.hAllocation = VK_NULL_HANDLE; + ++m_2ndNullItemsCount; + m_SumFreeSize += suballoc.size; + ++lostAllocationCount; + } + } + + if(lostAllocationCount) + { + CleanupAfterFree(); + } + + return lostAllocationCount; +} + +VkResult VmaBlockMetadata_Linear::CheckCorruption(const void* pBlockData) +{ + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i) + { + const VmaSuballocation& suballoc = suballocations1st[i]; + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) + { + if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN)) + { + VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!"); + return VK_ERROR_VALIDATION_FAILED_EXT; + } + if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size)) + { + VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!"); + return VK_ERROR_VALIDATION_FAILED_EXT; + } + } + } + + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i) + { + const VmaSuballocation& suballoc = suballocations2nd[i]; + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) + { + if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN)) + { + VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!"); + return VK_ERROR_VALIDATION_FAILED_EXT; + } + if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size)) + { + VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!"); + return VK_ERROR_VALIDATION_FAILED_EXT; + } + } + } + + return VK_SUCCESS; +} + +void VmaBlockMetadata_Linear::Alloc( + const VmaAllocationRequest& request, + VmaSuballocationType type, + VkDeviceSize allocSize, + VmaAllocation hAllocation) +{ + const VmaSuballocation newSuballoc = { request.offset, allocSize, hAllocation, type }; + + switch(request.type) + { + case VmaAllocationRequestType::UpperAddress: + { + VMA_ASSERT(m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER && + "CRITICAL ERROR: Trying to use linear allocator as double stack while it was already used as ring buffer."); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + suballocations2nd.push_back(newSuballoc); + m_2ndVectorMode = SECOND_VECTOR_DOUBLE_STACK; + } + break; + case VmaAllocationRequestType::EndOf1st: + { + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + + VMA_ASSERT(suballocations1st.empty() || + request.offset >= suballocations1st.back().offset + suballocations1st.back().size); + // Check if it fits before the end of the block. + VMA_ASSERT(request.offset + allocSize <= GetSize()); + + suballocations1st.push_back(newSuballoc); + } + break; + case VmaAllocationRequestType::EndOf2nd: + { + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + // New allocation at the end of 2-part ring buffer, so before first allocation from 1st vector. + VMA_ASSERT(!suballocations1st.empty() && + request.offset + allocSize <= suballocations1st[m_1stNullItemsBeginCount].offset); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + switch(m_2ndVectorMode) + { + case SECOND_VECTOR_EMPTY: + // First allocation from second part ring buffer. + VMA_ASSERT(suballocations2nd.empty()); + m_2ndVectorMode = SECOND_VECTOR_RING_BUFFER; + break; + case SECOND_VECTOR_RING_BUFFER: + // 2-part ring buffer is already started. + VMA_ASSERT(!suballocations2nd.empty()); + break; + case SECOND_VECTOR_DOUBLE_STACK: + VMA_ASSERT(0 && "CRITICAL ERROR: Trying to use linear allocator as ring buffer while it was already used as double stack."); + break; + default: + VMA_ASSERT(0); + } + + suballocations2nd.push_back(newSuballoc); + } + break; + default: + VMA_ASSERT(0 && "CRITICAL INTERNAL ERROR."); + } + + m_SumFreeSize -= newSuballoc.size; +} + +void VmaBlockMetadata_Linear::Free(const VmaAllocation allocation) +{ + FreeAtOffset(allocation->GetOffset()); +} + +void VmaBlockMetadata_Linear::FreeAtOffset(VkDeviceSize offset) +{ + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + if(!suballocations1st.empty()) + { + // First allocation: Mark it as next empty at the beginning. + VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount]; + if(firstSuballoc.offset == offset) + { + firstSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + firstSuballoc.hAllocation = VK_NULL_HANDLE; + m_SumFreeSize += firstSuballoc.size; + ++m_1stNullItemsBeginCount; + CleanupAfterFree(); + return; + } + } + + // Last allocation in 2-part ring buffer or top of upper stack (same logic). + if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER || + m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + { + VmaSuballocation& lastSuballoc = suballocations2nd.back(); + if(lastSuballoc.offset == offset) + { + m_SumFreeSize += lastSuballoc.size; + suballocations2nd.pop_back(); + CleanupAfterFree(); + return; + } + } + // Last allocation in 1st vector. + else if(m_2ndVectorMode == SECOND_VECTOR_EMPTY) + { + VmaSuballocation& lastSuballoc = suballocations1st.back(); + if(lastSuballoc.offset == offset) + { + m_SumFreeSize += lastSuballoc.size; + suballocations1st.pop_back(); + CleanupAfterFree(); + return; + } + } + + // Item from the middle of 1st vector. + { + VmaSuballocation refSuballoc; + refSuballoc.offset = offset; + // Rest of members stays uninitialized intentionally for better performance. + SuballocationVectorType::iterator it = VmaBinaryFindSorted( + suballocations1st.begin() + m_1stNullItemsBeginCount, + suballocations1st.end(), + refSuballoc, + VmaSuballocationOffsetLess()); + if(it != suballocations1st.end()) + { + it->type = VMA_SUBALLOCATION_TYPE_FREE; + it->hAllocation = VK_NULL_HANDLE; + ++m_1stNullItemsMiddleCount; + m_SumFreeSize += it->size; + CleanupAfterFree(); + return; + } + } + + if(m_2ndVectorMode != SECOND_VECTOR_EMPTY) + { + // Item from the middle of 2nd vector. + VmaSuballocation refSuballoc; + refSuballoc.offset = offset; + // Rest of members stays uninitialized intentionally for better performance. + SuballocationVectorType::iterator it = m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ? + VmaBinaryFindSorted(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc, VmaSuballocationOffsetLess()) : + VmaBinaryFindSorted(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc, VmaSuballocationOffsetGreater()); + if(it != suballocations2nd.end()) + { + it->type = VMA_SUBALLOCATION_TYPE_FREE; + it->hAllocation = VK_NULL_HANDLE; + ++m_2ndNullItemsCount; + m_SumFreeSize += it->size; + CleanupAfterFree(); + return; + } + } + + VMA_ASSERT(0 && "Allocation to free not found in linear allocator!"); +} + +bool VmaBlockMetadata_Linear::ShouldCompact1st() const +{ + const size_t nullItemCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount; + const size_t suballocCount = AccessSuballocations1st().size(); + return suballocCount > 32 && nullItemCount * 2 >= (suballocCount - nullItemCount) * 3; +} + +void VmaBlockMetadata_Linear::CleanupAfterFree() +{ + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); + SuballocationVectorType& suballocations2nd = AccessSuballocations2nd(); + + if(IsEmpty()) + { + suballocations1st.clear(); + suballocations2nd.clear(); + m_1stNullItemsBeginCount = 0; + m_1stNullItemsMiddleCount = 0; + m_2ndNullItemsCount = 0; + m_2ndVectorMode = SECOND_VECTOR_EMPTY; + } + else + { + const size_t suballoc1stCount = suballocations1st.size(); + const size_t nullItem1stCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount; + VMA_ASSERT(nullItem1stCount <= suballoc1stCount); + + // Find more null items at the beginning of 1st vector. + while(m_1stNullItemsBeginCount < suballoc1stCount && + suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE) + { + ++m_1stNullItemsBeginCount; + --m_1stNullItemsMiddleCount; + } + + // Find more null items at the end of 1st vector. + while(m_1stNullItemsMiddleCount > 0 && + suballocations1st.back().hAllocation == VK_NULL_HANDLE) + { + --m_1stNullItemsMiddleCount; + suballocations1st.pop_back(); + } + + // Find more null items at the end of 2nd vector. + while(m_2ndNullItemsCount > 0 && + suballocations2nd.back().hAllocation == VK_NULL_HANDLE) + { + --m_2ndNullItemsCount; + suballocations2nd.pop_back(); + } + + // Find more null items at the beginning of 2nd vector. + while(m_2ndNullItemsCount > 0 && + suballocations2nd[0].hAllocation == VK_NULL_HANDLE) + { + --m_2ndNullItemsCount; + VmaVectorRemove(suballocations2nd, 0); + } + + if(ShouldCompact1st()) + { + const size_t nonNullItemCount = suballoc1stCount - nullItem1stCount; + size_t srcIndex = m_1stNullItemsBeginCount; + for(size_t dstIndex = 0; dstIndex < nonNullItemCount; ++dstIndex) + { + while(suballocations1st[srcIndex].hAllocation == VK_NULL_HANDLE) + { + ++srcIndex; + } + if(dstIndex != srcIndex) + { + suballocations1st[dstIndex] = suballocations1st[srcIndex]; + } + ++srcIndex; + } + suballocations1st.resize(nonNullItemCount); + m_1stNullItemsBeginCount = 0; + m_1stNullItemsMiddleCount = 0; + } + + // 2nd vector became empty. + if(suballocations2nd.empty()) + { + m_2ndVectorMode = SECOND_VECTOR_EMPTY; + } + + // 1st vector became empty. + if(suballocations1st.size() - m_1stNullItemsBeginCount == 0) + { + suballocations1st.clear(); + m_1stNullItemsBeginCount = 0; + + if(!suballocations2nd.empty() && m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER) + { + // Swap 1st with 2nd. Now 2nd is empty. + m_2ndVectorMode = SECOND_VECTOR_EMPTY; + m_1stNullItemsMiddleCount = m_2ndNullItemsCount; + while(m_1stNullItemsBeginCount < suballocations2nd.size() && + suballocations2nd[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE) + { + ++m_1stNullItemsBeginCount; + --m_1stNullItemsMiddleCount; + } + m_2ndNullItemsCount = 0; + m_1stVectorIndex ^= 1; + } + } + } + + VMA_HEAVY_ASSERT(Validate()); +} + + +//////////////////////////////////////////////////////////////////////////////// +// class VmaBlockMetadata_Buddy + +VmaBlockMetadata_Buddy::VmaBlockMetadata_Buddy(VmaAllocator hAllocator) : + VmaBlockMetadata(hAllocator), + m_Root(VMA_NULL), + m_AllocationCount(0), + m_FreeCount(1), + m_SumFreeSize(0) +{ + memset(m_FreeList, 0, sizeof(m_FreeList)); +} + +VmaBlockMetadata_Buddy::~VmaBlockMetadata_Buddy() +{ + DeleteNode(m_Root); +} + +void VmaBlockMetadata_Buddy::Init(VkDeviceSize size) +{ + VmaBlockMetadata::Init(size); + + m_UsableSize = VmaPrevPow2(size); + m_SumFreeSize = m_UsableSize; + + // Calculate m_LevelCount. + m_LevelCount = 1; + while(m_LevelCount < MAX_LEVELS && + LevelToNodeSize(m_LevelCount) >= MIN_NODE_SIZE) + { + ++m_LevelCount; + } + + Node* rootNode = vma_new(GetAllocationCallbacks(), Node)(); + rootNode->offset = 0; + rootNode->type = Node::TYPE_FREE; + rootNode->parent = VMA_NULL; + rootNode->buddy = VMA_NULL; + + m_Root = rootNode; + AddToFreeListFront(0, rootNode); +} + +bool VmaBlockMetadata_Buddy::Validate() const +{ + // Validate tree. + ValidationContext ctx; + if(!ValidateNode(ctx, VMA_NULL, m_Root, 0, LevelToNodeSize(0))) + { + VMA_VALIDATE(false && "ValidateNode failed."); + } + VMA_VALIDATE(m_AllocationCount == ctx.calculatedAllocationCount); + VMA_VALIDATE(m_SumFreeSize == ctx.calculatedSumFreeSize); + + // Validate free node lists. + for(uint32_t level = 0; level < m_LevelCount; ++level) + { + VMA_VALIDATE(m_FreeList[level].front == VMA_NULL || + m_FreeList[level].front->free.prev == VMA_NULL); + + for(Node* node = m_FreeList[level].front; + node != VMA_NULL; + node = node->free.next) + { + VMA_VALIDATE(node->type == Node::TYPE_FREE); + + if(node->free.next == VMA_NULL) + { + VMA_VALIDATE(m_FreeList[level].back == node); + } + else + { + VMA_VALIDATE(node->free.next->free.prev == node); + } + } + } + + // Validate that free lists ar higher levels are empty. + for(uint32_t level = m_LevelCount; level < MAX_LEVELS; ++level) + { + VMA_VALIDATE(m_FreeList[level].front == VMA_NULL && m_FreeList[level].back == VMA_NULL); + } + + return true; +} + +VkDeviceSize VmaBlockMetadata_Buddy::GetUnusedRangeSizeMax() const +{ + for(uint32_t level = 0; level < m_LevelCount; ++level) + { + if(m_FreeList[level].front != VMA_NULL) + { + return LevelToNodeSize(level); + } + } + return 0; +} + +void VmaBlockMetadata_Buddy::CalcAllocationStatInfo(VmaStatInfo& outInfo) const +{ + const VkDeviceSize unusableSize = GetUnusableSize(); + + outInfo.blockCount = 1; + + outInfo.allocationCount = outInfo.unusedRangeCount = 0; + outInfo.usedBytes = outInfo.unusedBytes = 0; + + outInfo.allocationSizeMax = outInfo.unusedRangeSizeMax = 0; + outInfo.allocationSizeMin = outInfo.unusedRangeSizeMin = UINT64_MAX; + outInfo.allocationSizeAvg = outInfo.unusedRangeSizeAvg = 0; // Unused. + + CalcAllocationStatInfoNode(outInfo, m_Root, LevelToNodeSize(0)); + + if(unusableSize > 0) + { + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusableSize; + outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, unusableSize); + outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusableSize); + } +} + +void VmaBlockMetadata_Buddy::AddPoolStats(VmaPoolStats& inoutStats) const +{ + const VkDeviceSize unusableSize = GetUnusableSize(); + + inoutStats.size += GetSize(); + inoutStats.unusedSize += m_SumFreeSize + unusableSize; + inoutStats.allocationCount += m_AllocationCount; + inoutStats.unusedRangeCount += m_FreeCount; + inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax()); + + if(unusableSize > 0) + { + ++inoutStats.unusedRangeCount; + // Not updating inoutStats.unusedRangeSizeMax with unusableSize because this space is not available for allocations. + } +} + +#if VMA_STATS_STRING_ENABLED + +void VmaBlockMetadata_Buddy::PrintDetailedMap(class VmaJsonWriter& json) const +{ + // TODO optimize + VmaStatInfo stat; + CalcAllocationStatInfo(stat); + + PrintDetailedMap_Begin( + json, + stat.unusedBytes, + stat.allocationCount, + stat.unusedRangeCount); + + PrintDetailedMapNode(json, m_Root, LevelToNodeSize(0)); + + const VkDeviceSize unusableSize = GetUnusableSize(); + if(unusableSize > 0) + { + PrintDetailedMap_UnusedRange(json, + m_UsableSize, // offset + unusableSize); // size + } + + PrintDetailedMap_End(json); +} + +#endif // #if VMA_STATS_STRING_ENABLED + +bool VmaBlockMetadata_Buddy::CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + bool upperAddress, + VmaSuballocationType allocType, + bool canMakeOtherLost, + uint32_t strategy, + VmaAllocationRequest* pAllocationRequest) +{ + VMA_ASSERT(!upperAddress && "VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT can be used only with linear algorithm."); + + // Simple way to respect bufferImageGranularity. May be optimized some day. + // Whenever it might be an OPTIMAL image... + if(allocType == VMA_SUBALLOCATION_TYPE_UNKNOWN || + allocType == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN || + allocType == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL) + { + allocAlignment = VMA_MAX(allocAlignment, bufferImageGranularity); + allocSize = VMA_MAX(allocSize, bufferImageGranularity); + } + + if(allocSize > m_UsableSize) + { + return false; + } + + const uint32_t targetLevel = AllocSizeToLevel(allocSize); + for(uint32_t level = targetLevel + 1; level--; ) + { + for(Node* freeNode = m_FreeList[level].front; + freeNode != VMA_NULL; + freeNode = freeNode->free.next) + { + if(freeNode->offset % allocAlignment == 0) + { + pAllocationRequest->type = VmaAllocationRequestType::Normal; + pAllocationRequest->offset = freeNode->offset; + pAllocationRequest->sumFreeSize = LevelToNodeSize(level); + pAllocationRequest->sumItemSize = 0; + pAllocationRequest->itemsToMakeLostCount = 0; + pAllocationRequest->customData = (void*)(uintptr_t)level; + return true; + } + } + } + + return false; +} + +bool VmaBlockMetadata_Buddy::MakeRequestedAllocationsLost( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VmaAllocationRequest* pAllocationRequest) +{ + /* + Lost allocations are not supported in buddy allocator at the moment. + Support might be added in the future. + */ + return pAllocationRequest->itemsToMakeLostCount == 0; +} + +uint32_t VmaBlockMetadata_Buddy::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) +{ + /* + Lost allocations are not supported in buddy allocator at the moment. + Support might be added in the future. + */ + return 0; +} + +void VmaBlockMetadata_Buddy::Alloc( + const VmaAllocationRequest& request, + VmaSuballocationType type, + VkDeviceSize allocSize, + VmaAllocation hAllocation) +{ + VMA_ASSERT(request.type == VmaAllocationRequestType::Normal); + + const uint32_t targetLevel = AllocSizeToLevel(allocSize); + uint32_t currLevel = (uint32_t)(uintptr_t)request.customData; + + Node* currNode = m_FreeList[currLevel].front; + VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE); + while(currNode->offset != request.offset) + { + currNode = currNode->free.next; + VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE); + } + + // Go down, splitting free nodes. + while(currLevel < targetLevel) + { + // currNode is already first free node at currLevel. + // Remove it from list of free nodes at this currLevel. + RemoveFromFreeList(currLevel, currNode); + + const uint32_t childrenLevel = currLevel + 1; + + // Create two free sub-nodes. + Node* leftChild = vma_new(GetAllocationCallbacks(), Node)(); + Node* rightChild = vma_new(GetAllocationCallbacks(), Node)(); + + leftChild->offset = currNode->offset; + leftChild->type = Node::TYPE_FREE; + leftChild->parent = currNode; + leftChild->buddy = rightChild; + + rightChild->offset = currNode->offset + LevelToNodeSize(childrenLevel); + rightChild->type = Node::TYPE_FREE; + rightChild->parent = currNode; + rightChild->buddy = leftChild; + + // Convert current currNode to split type. + currNode->type = Node::TYPE_SPLIT; + currNode->split.leftChild = leftChild; + + // Add child nodes to free list. Order is important! + AddToFreeListFront(childrenLevel, rightChild); + AddToFreeListFront(childrenLevel, leftChild); + + ++m_FreeCount; + //m_SumFreeSize -= LevelToNodeSize(currLevel) % 2; // Useful only when level node sizes can be non power of 2. + ++currLevel; + currNode = m_FreeList[currLevel].front; + + /* + We can be sure that currNode, as left child of node previously split, + also fullfills the alignment requirement. + */ + } + + // Remove from free list. + VMA_ASSERT(currLevel == targetLevel && + currNode != VMA_NULL && + currNode->type == Node::TYPE_FREE); + RemoveFromFreeList(currLevel, currNode); + + // Convert to allocation node. + currNode->type = Node::TYPE_ALLOCATION; + currNode->allocation.alloc = hAllocation; + + ++m_AllocationCount; + --m_FreeCount; + m_SumFreeSize -= allocSize; +} + +void VmaBlockMetadata_Buddy::DeleteNode(Node* node) +{ + if(node->type == Node::TYPE_SPLIT) + { + DeleteNode(node->split.leftChild->buddy); + DeleteNode(node->split.leftChild); + } + + vma_delete(GetAllocationCallbacks(), node); +} + +bool VmaBlockMetadata_Buddy::ValidateNode(ValidationContext& ctx, const Node* parent, const Node* curr, uint32_t level, VkDeviceSize levelNodeSize) const +{ + VMA_VALIDATE(level < m_LevelCount); + VMA_VALIDATE(curr->parent == parent); + VMA_VALIDATE((curr->buddy == VMA_NULL) == (parent == VMA_NULL)); + VMA_VALIDATE(curr->buddy == VMA_NULL || curr->buddy->buddy == curr); + switch(curr->type) + { + case Node::TYPE_FREE: + // curr->free.prev, next are validated separately. + ctx.calculatedSumFreeSize += levelNodeSize; + ++ctx.calculatedFreeCount; + break; + case Node::TYPE_ALLOCATION: + ++ctx.calculatedAllocationCount; + ctx.calculatedSumFreeSize += levelNodeSize - curr->allocation.alloc->GetSize(); + VMA_VALIDATE(curr->allocation.alloc != VK_NULL_HANDLE); + break; + case Node::TYPE_SPLIT: + { + const uint32_t childrenLevel = level + 1; + const VkDeviceSize childrenLevelNodeSize = levelNodeSize / 2; + const Node* const leftChild = curr->split.leftChild; + VMA_VALIDATE(leftChild != VMA_NULL); + VMA_VALIDATE(leftChild->offset == curr->offset); + if(!ValidateNode(ctx, curr, leftChild, childrenLevel, childrenLevelNodeSize)) + { + VMA_VALIDATE(false && "ValidateNode for left child failed."); + } + const Node* const rightChild = leftChild->buddy; + VMA_VALIDATE(rightChild->offset == curr->offset + childrenLevelNodeSize); + if(!ValidateNode(ctx, curr, rightChild, childrenLevel, childrenLevelNodeSize)) + { + VMA_VALIDATE(false && "ValidateNode for right child failed."); + } + } + break; + default: + return false; + } + + return true; +} + +uint32_t VmaBlockMetadata_Buddy::AllocSizeToLevel(VkDeviceSize allocSize) const +{ + // I know this could be optimized somehow e.g. by using std::log2p1 from C++20. + uint32_t level = 0; + VkDeviceSize currLevelNodeSize = m_UsableSize; + VkDeviceSize nextLevelNodeSize = currLevelNodeSize >> 1; + while(allocSize <= nextLevelNodeSize && level + 1 < m_LevelCount) + { + ++level; + currLevelNodeSize = nextLevelNodeSize; + nextLevelNodeSize = currLevelNodeSize >> 1; + } + return level; +} + +void VmaBlockMetadata_Buddy::FreeAtOffset(VmaAllocation alloc, VkDeviceSize offset) +{ + // Find node and level. + Node* node = m_Root; + VkDeviceSize nodeOffset = 0; + uint32_t level = 0; + VkDeviceSize levelNodeSize = LevelToNodeSize(0); + while(node->type == Node::TYPE_SPLIT) + { + const VkDeviceSize nextLevelSize = levelNodeSize >> 1; + if(offset < nodeOffset + nextLevelSize) + { + node = node->split.leftChild; + } + else + { + node = node->split.leftChild->buddy; + nodeOffset += nextLevelSize; + } + ++level; + levelNodeSize = nextLevelSize; + } + + VMA_ASSERT(node != VMA_NULL && node->type == Node::TYPE_ALLOCATION); + VMA_ASSERT(alloc == VK_NULL_HANDLE || node->allocation.alloc == alloc); + + ++m_FreeCount; + --m_AllocationCount; + m_SumFreeSize += alloc->GetSize(); + + node->type = Node::TYPE_FREE; + + // Join free nodes if possible. + while(level > 0 && node->buddy->type == Node::TYPE_FREE) + { + RemoveFromFreeList(level, node->buddy); + Node* const parent = node->parent; + + vma_delete(GetAllocationCallbacks(), node->buddy); + vma_delete(GetAllocationCallbacks(), node); + parent->type = Node::TYPE_FREE; + + node = parent; + --level; + //m_SumFreeSize += LevelToNodeSize(level) % 2; // Useful only when level node sizes can be non power of 2. + --m_FreeCount; + } + + AddToFreeListFront(level, node); +} + +void VmaBlockMetadata_Buddy::CalcAllocationStatInfoNode(VmaStatInfo& outInfo, const Node* node, VkDeviceSize levelNodeSize) const +{ + switch(node->type) + { + case Node::TYPE_FREE: + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += levelNodeSize; + outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, levelNodeSize); + outInfo.unusedRangeSizeMin = VMA_MAX(outInfo.unusedRangeSizeMin, levelNodeSize); + break; + case Node::TYPE_ALLOCATION: + { + const VkDeviceSize allocSize = node->allocation.alloc->GetSize(); + ++outInfo.allocationCount; + outInfo.usedBytes += allocSize; + outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, allocSize); + outInfo.allocationSizeMin = VMA_MAX(outInfo.allocationSizeMin, allocSize); + + const VkDeviceSize unusedRangeSize = levelNodeSize - allocSize; + if(unusedRangeSize > 0) + { + ++outInfo.unusedRangeCount; + outInfo.unusedBytes += unusedRangeSize; + outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, unusedRangeSize); + outInfo.unusedRangeSizeMin = VMA_MAX(outInfo.unusedRangeSizeMin, unusedRangeSize); + } + } + break; + case Node::TYPE_SPLIT: + { + const VkDeviceSize childrenNodeSize = levelNodeSize / 2; + const Node* const leftChild = node->split.leftChild; + CalcAllocationStatInfoNode(outInfo, leftChild, childrenNodeSize); + const Node* const rightChild = leftChild->buddy; + CalcAllocationStatInfoNode(outInfo, rightChild, childrenNodeSize); + } + break; + default: + VMA_ASSERT(0); + } +} + +void VmaBlockMetadata_Buddy::AddToFreeListFront(uint32_t level, Node* node) +{ + VMA_ASSERT(node->type == Node::TYPE_FREE); + + // List is empty. + Node* const frontNode = m_FreeList[level].front; + if(frontNode == VMA_NULL) + { + VMA_ASSERT(m_FreeList[level].back == VMA_NULL); + node->free.prev = node->free.next = VMA_NULL; + m_FreeList[level].front = m_FreeList[level].back = node; + } + else + { + VMA_ASSERT(frontNode->free.prev == VMA_NULL); + node->free.prev = VMA_NULL; + node->free.next = frontNode; + frontNode->free.prev = node; + m_FreeList[level].front = node; + } +} + +void VmaBlockMetadata_Buddy::RemoveFromFreeList(uint32_t level, Node* node) +{ + VMA_ASSERT(m_FreeList[level].front != VMA_NULL); + + // It is at the front. + if(node->free.prev == VMA_NULL) + { + VMA_ASSERT(m_FreeList[level].front == node); + m_FreeList[level].front = node->free.next; + } + else + { + Node* const prevFreeNode = node->free.prev; + VMA_ASSERT(prevFreeNode->free.next == node); + prevFreeNode->free.next = node->free.next; + } + + // It is at the back. + if(node->free.next == VMA_NULL) + { + VMA_ASSERT(m_FreeList[level].back == node); + m_FreeList[level].back = node->free.prev; + } + else + { + Node* const nextFreeNode = node->free.next; + VMA_ASSERT(nextFreeNode->free.prev == node); + nextFreeNode->free.prev = node->free.prev; + } +} + +#if VMA_STATS_STRING_ENABLED +void VmaBlockMetadata_Buddy::PrintDetailedMapNode(class VmaJsonWriter& json, const Node* node, VkDeviceSize levelNodeSize) const +{ + switch(node->type) + { + case Node::TYPE_FREE: + PrintDetailedMap_UnusedRange(json, node->offset, levelNodeSize); + break; + case Node::TYPE_ALLOCATION: + { + PrintDetailedMap_Allocation(json, node->offset, node->allocation.alloc); + const VkDeviceSize allocSize = node->allocation.alloc->GetSize(); + if(allocSize < levelNodeSize) + { + PrintDetailedMap_UnusedRange(json, node->offset + allocSize, levelNodeSize - allocSize); + } + } + break; + case Node::TYPE_SPLIT: + { + const VkDeviceSize childrenNodeSize = levelNodeSize / 2; + const Node* const leftChild = node->split.leftChild; + PrintDetailedMapNode(json, leftChild, childrenNodeSize); + const Node* const rightChild = leftChild->buddy; + PrintDetailedMapNode(json, rightChild, childrenNodeSize); + } + break; + default: + VMA_ASSERT(0); + } +} +#endif // #if VMA_STATS_STRING_ENABLED + + //////////////////////////////////////////////////////////////////////////////// // class VmaDeviceMemoryBlock VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) : - m_Metadata(hAllocator), + m_pMetadata(VMA_NULL), m_MemoryTypeIndex(UINT32_MAX), m_Id(0), m_hMemory(VK_NULL_HANDLE), @@ -6880,40 +12266,58 @@ VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) : } void VmaDeviceMemoryBlock::Init( + VmaAllocator hAllocator, + VmaPool hParentPool, uint32_t newMemoryTypeIndex, VkDeviceMemory newMemory, VkDeviceSize newSize, - uint32_t id) + uint32_t id, + uint32_t algorithm) { VMA_ASSERT(m_hMemory == VK_NULL_HANDLE); + m_hParentPool = hParentPool; m_MemoryTypeIndex = newMemoryTypeIndex; m_Id = id; m_hMemory = newMemory; - m_Metadata.Init(newSize); + switch(algorithm) + { + case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT: + m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Linear)(hAllocator); + break; + case VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT: + m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Buddy)(hAllocator); + break; + default: + VMA_ASSERT(0); + // Fall-through. + case 0: + m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Generic)(hAllocator); + } + m_pMetadata->Init(newSize); } void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator) { // This is the most important assert in the entire library. // Hitting it means you have some memory leak - unreleased VmaAllocation objects. - VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!"); - + VMA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!"); + VMA_ASSERT(m_hMemory != VK_NULL_HANDLE); - allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory); + allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_pMetadata->GetSize(), m_hMemory); m_hMemory = VK_NULL_HANDLE; + + vma_delete(allocator, m_pMetadata); + m_pMetadata = VMA_NULL; } bool VmaDeviceMemoryBlock::Validate() const { - if((m_hMemory == VK_NULL_HANDLE) || - (m_Metadata.GetSize() == 0)) - { - return false; - } + VMA_VALIDATE((m_hMemory != VK_NULL_HANDLE) && + (m_pMetadata->GetSize() != 0)); - return m_Metadata.Validate(); + return m_pMetadata->Validate(); } VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator) @@ -6925,7 +12329,7 @@ VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator) return res; } - res = m_Metadata.CheckCorruption(pData); + res = m_pMetadata->CheckCorruption(pData); Unmap(hAllocator, 1); @@ -7043,33 +12447,35 @@ VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator h VkResult VmaDeviceMemoryBlock::BindBufferMemory( const VmaAllocator hAllocator, const VmaAllocation hAllocation, - VkBuffer hBuffer) + VkDeviceSize allocationLocalOffset, + VkBuffer hBuffer, + const void* pNext) { VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK && hAllocation->GetBlock() == this); + VMA_ASSERT(allocationLocalOffset < hAllocation->GetSize() && + "Invalid allocationLocalOffset. Did you forget that this offset is relative to the beginning of the allocation, not the whole memory block?"); + const VkDeviceSize memoryOffset = hAllocation->GetOffset() + allocationLocalOffset; // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads. VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex); - return hAllocator->GetVulkanFunctions().vkBindBufferMemory( - hAllocator->m_hDevice, - hBuffer, - m_hMemory, - hAllocation->GetOffset()); + return hAllocator->BindVulkanBuffer(m_hMemory, memoryOffset, hBuffer, pNext); } VkResult VmaDeviceMemoryBlock::BindImageMemory( const VmaAllocator hAllocator, const VmaAllocation hAllocation, - VkImage hImage) + VkDeviceSize allocationLocalOffset, + VkImage hImage, + const void* pNext) { VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK && hAllocation->GetBlock() == this); + VMA_ASSERT(allocationLocalOffset < hAllocation->GetSize() && + "Invalid allocationLocalOffset. Did you forget that this offset is relative to the beginning of the allocation, not the whole memory block?"); + const VkDeviceSize memoryOffset = hAllocation->GetOffset() + allocationLocalOffset; // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads. VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex); - return hAllocator->GetVulkanFunctions().vkBindImageMemory( - hAllocator->m_hDevice, - hImage, - m_hMemory, - hAllocation->GetOffset()); + return hAllocator->BindVulkanImage(m_hMemory, memoryOffset, hImage, pNext); } static void InitStatInfo(VmaStatInfo& outInfo) @@ -7103,17 +12509,21 @@ static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo) VmaPool_T::VmaPool_T( VmaAllocator hAllocator, - const VmaPoolCreateInfo& createInfo) : + const VmaPoolCreateInfo& createInfo, + VkDeviceSize preferredBlockSize) : m_BlockVector( hAllocator, + this, // hParentPool createInfo.memoryTypeIndex, - createInfo.blockSize, + createInfo.blockSize != 0 ? createInfo.blockSize : preferredBlockSize, createInfo.minBlockCount, createInfo.maxBlockCount, (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(), createInfo.frameInUseCount, - true), // isCustomPool - m_Id(0) + createInfo.blockSize != 0, // explicitBlockSize + createInfo.flags & VMA_POOL_CREATE_ALGORITHM_MASK), // algorithm + m_Id(0), + m_Name(VMA_NULL) { } @@ -7121,38 +12531,54 @@ VmaPool_T::~VmaPool_T() { } +void VmaPool_T::SetName(const char* pName) +{ + const VkAllocationCallbacks* allocs = m_BlockVector.GetAllocator()->GetAllocationCallbacks(); + VmaFreeString(allocs, m_Name); + + if(pName != VMA_NULL) + { + m_Name = VmaCreateStringCopy(allocs, pName); + } + else + { + m_Name = VMA_NULL; + } +} + #if VMA_STATS_STRING_ENABLED #endif // #if VMA_STATS_STRING_ENABLED VmaBlockVector::VmaBlockVector( VmaAllocator hAllocator, + VmaPool hParentPool, uint32_t memoryTypeIndex, VkDeviceSize preferredBlockSize, size_t minBlockCount, size_t maxBlockCount, VkDeviceSize bufferImageGranularity, uint32_t frameInUseCount, - bool isCustomPool) : + bool explicitBlockSize, + uint32_t algorithm) : m_hAllocator(hAllocator), + m_hParentPool(hParentPool), m_MemoryTypeIndex(memoryTypeIndex), m_PreferredBlockSize(preferredBlockSize), m_MinBlockCount(minBlockCount), m_MaxBlockCount(maxBlockCount), m_BufferImageGranularity(bufferImageGranularity), m_FrameInUseCount(frameInUseCount), - m_IsCustomPool(isCustomPool), - m_Blocks(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + m_ExplicitBlockSize(explicitBlockSize), + m_Algorithm(algorithm), m_HasEmptyBlock(false), - m_pDefragmentator(VMA_NULL), + m_Blocks(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), m_NextBlockId(0) { } VmaBlockVector::~VmaBlockVector() { - VMA_ASSERT(m_pDefragmentator == VMA_NULL); - for(size_t i = m_Blocks.size(); i--; ) { m_Blocks[i]->Destroy(m_hAllocator); @@ -7175,35 +12601,93 @@ VkResult VmaBlockVector::CreateMinBlocks() void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats) { + VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex); + + const size_t blockCount = m_Blocks.size(); + pStats->size = 0; pStats->unusedSize = 0; pStats->allocationCount = 0; pStats->unusedRangeCount = 0; pStats->unusedRangeSizeMax = 0; + pStats->blockCount = blockCount; - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); - - for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + for(uint32_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) { const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; VMA_ASSERT(pBlock); VMA_HEAVY_ASSERT(pBlock->Validate()); - pBlock->m_Metadata.AddPoolStats(*pStats); + pBlock->m_pMetadata->AddPoolStats(*pStats); } } +bool VmaBlockVector::IsEmpty() +{ + VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex); + return m_Blocks.empty(); +} + bool VmaBlockVector::IsCorruptionDetectionEnabled() const { const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; return (VMA_DEBUG_DETECT_CORRUPTION != 0) && (VMA_DEBUG_MARGIN > 0) && + (m_Algorithm == 0 || m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) && (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags; } static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32; VkResult VmaBlockVector::Allocate( - VmaPool hCurrentPool, + uint32_t currentFrameIndex, + VkDeviceSize size, + VkDeviceSize alignment, + const VmaAllocationCreateInfo& createInfo, + VmaSuballocationType suballocType, + size_t allocationCount, + VmaAllocation* pAllocations) +{ + size_t allocIndex; + VkResult res = VK_SUCCESS; + + if(IsCorruptionDetectionEnabled()) + { + size = VmaAlignUp(size, sizeof(VMA_CORRUPTION_DETECTION_MAGIC_VALUE)); + alignment = VmaAlignUp(alignment, sizeof(VMA_CORRUPTION_DETECTION_MAGIC_VALUE)); + } + + { + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); + for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + res = AllocatePage( + currentFrameIndex, + size, + alignment, + createInfo, + suballocType, + pAllocations + allocIndex); + if(res != VK_SUCCESS) + { + break; + } + } + } + + if(res != VK_SUCCESS) + { + // Free all already created allocations. + while(allocIndex--) + { + Free(pAllocations[allocIndex]); + } + memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount); + } + + return res; +} + +VkResult VmaBlockVector::AllocatePage( uint32_t currentFrameIndex, VkDeviceSize size, VkDeviceSize alignment, @@ -7211,194 +12695,226 @@ VkResult VmaBlockVector::Allocate( VmaSuballocationType suballocType, VmaAllocation* pAllocation) { + const bool isUpperAddress = (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0; + bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0; + const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0; + const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0; + + VkDeviceSize freeMemory; + { + const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex); + VmaBudget heapBudget = {}; + m_hAllocator->GetBudget(&heapBudget, heapIndex, 1); + freeMemory = (heapBudget.usage < heapBudget.budget) ? (heapBudget.budget - heapBudget.usage) : 0; + } + + const bool canFallbackToDedicated = !IsCustomPool(); + const bool canCreateNewBlock = + ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) && + (m_Blocks.size() < m_MaxBlockCount) && + (freeMemory >= size || !canFallbackToDedicated); + uint32_t strategy = createInfo.flags & VMA_ALLOCATION_CREATE_STRATEGY_MASK; + + // If linearAlgorithm is used, canMakeOtherLost is available only when used as ring buffer. + // Which in turn is available only when maxBlockCount = 1. + if(m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT && m_MaxBlockCount > 1) + { + canMakeOtherLost = false; + } + + // Upper address can only be used with linear allocator and within single memory block. + if(isUpperAddress && + (m_Algorithm != VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT || m_MaxBlockCount > 1)) + { + return VK_ERROR_FEATURE_NOT_PRESENT; + } + + // Validate strategy. + switch(strategy) + { + case 0: + strategy = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT; + break; + case VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT: + case VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT: + case VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT: + break; + default: + return VK_ERROR_FEATURE_NOT_PRESENT; + } + // Early reject: requested allocation size is larger that maximum block size for this block vector. if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize) { return VK_ERROR_OUT_OF_DEVICE_MEMORY; } - const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0; - const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0; - - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); - - // 1. Search existing allocations. Try to allocate without making other allocations lost. - // Forward order in m_Blocks - prefer blocks with smallest amount of free space. - for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) + /* + Under certain condition, this whole section can be skipped for optimization, so + we move on directly to trying to allocate with canMakeOtherLost. That's the case + e.g. for custom pools with linear algorithm. + */ + if(!canMakeOtherLost || canCreateNewBlock) { - VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; - VMA_ASSERT(pCurrBlock); - VmaAllocationRequest currRequest = {}; - if(pCurrBlock->m_Metadata.CreateAllocationRequest( - currentFrameIndex, - m_FrameInUseCount, - m_BufferImageGranularity, - size, - alignment, - suballocType, - false, // canMakeOtherLost - &currRequest)) + // 1. Search existing allocations. Try to allocate without making other allocations lost. + VmaAllocationCreateFlags allocFlagsCopy = createInfo.flags; + allocFlagsCopy &= ~VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT; + + if(m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) { - // Allocate from pCurrBlock. - VMA_ASSERT(currRequest.itemsToMakeLostCount == 0); - - if(mapped) + // Use only last block. + if(!m_Blocks.empty()) { - VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL); - if(res != VK_SUCCESS) - { - return res; - } - } - - // We no longer have an empty Allocation. - if(pCurrBlock->m_Metadata.IsEmpty()) - { - m_HasEmptyBlock = false; - } - - *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString); - pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation); - (*pAllocation)->InitBlockAllocation( - hCurrentPool, - pCurrBlock, - currRequest.offset, - alignment, - size, - suballocType, - mapped, - (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); - VMA_HEAVY_ASSERT(pCurrBlock->Validate()); - VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex); - (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData); - if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) - { - m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED); - } - if(IsCorruptionDetectionEnabled()) - { - VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size); - VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value."); - } - return VK_SUCCESS; - } - } - - const bool canCreateNewBlock = - ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) && - (m_Blocks.size() < m_MaxBlockCount); - - // 2. Try to create new block. - if(canCreateNewBlock) - { - // Calculate optimal size for new block. - VkDeviceSize newBlockSize = m_PreferredBlockSize; - uint32_t newBlockSizeShift = 0; - const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3; - - // Allocating blocks of other sizes is allowed only in default pools. - // In custom pools block size is fixed. - if(m_IsCustomPool == false) - { - // Allocate 1/8, 1/4, 1/2 as first blocks. - const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize(); - for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i) - { - const VkDeviceSize smallerNewBlockSize = newBlockSize / 2; - if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2) - { - newBlockSize = smallerNewBlockSize; - ++newBlockSizeShift; - } - else - { - break; - } - } - } - - size_t newBlockIndex = 0; - VkResult res = CreateBlock(newBlockSize, &newBlockIndex); - // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize. - if(m_IsCustomPool == false) - { - while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX) - { - const VkDeviceSize smallerNewBlockSize = newBlockSize / 2; - if(smallerNewBlockSize >= size) - { - newBlockSize = smallerNewBlockSize; - ++newBlockSizeShift; - res = CreateBlock(newBlockSize, &newBlockIndex); - } - else - { - break; - } - } - } - - if(res == VK_SUCCESS) - { - VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex]; - VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size); - - if(mapped) - { - res = pBlock->Map(m_hAllocator, 1, VMA_NULL); - if(res != VK_SUCCESS) - { - return res; - } - } - - // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled. - VmaAllocationRequest allocRequest; - if(pBlock->m_Metadata.CreateAllocationRequest( - currentFrameIndex, - m_FrameInUseCount, - m_BufferImageGranularity, - size, - alignment, - suballocType, - false, // canMakeOtherLost - &allocRequest)) - { - *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString); - pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation); - (*pAllocation)->InitBlockAllocation( - hCurrentPool, - pBlock, - allocRequest.offset, - alignment, + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks.back(); + VMA_ASSERT(pCurrBlock); + VkResult res = AllocateFromBlock( + pCurrBlock, + currentFrameIndex, size, + alignment, + allocFlagsCopy, + createInfo.pUserData, suballocType, - mapped, - (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); - VMA_HEAVY_ASSERT(pBlock->Validate()); - VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize); - (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData); - if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) + strategy, + pAllocation); + if(res == VK_SUCCESS) { - m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED); + VMA_DEBUG_LOG(" Returned from last block #%u", pCurrBlock->GetId()); + return VK_SUCCESS; } - if(IsCorruptionDetectionEnabled()) - { - res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size); - VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value."); - } - return VK_SUCCESS; } - else + } + else + { + if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT) { - // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment. - return VK_ERROR_OUT_OF_DEVICE_MEMORY; + // Forward order in m_Blocks - prefer blocks with smallest amount of free space. + for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) + { + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VkResult res = AllocateFromBlock( + pCurrBlock, + currentFrameIndex, + size, + alignment, + allocFlagsCopy, + createInfo.pUserData, + suballocType, + strategy, + pAllocation); + if(res == VK_SUCCESS) + { + VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId()); + return VK_SUCCESS; + } + } + } + else // WORST_FIT, FIRST_FIT + { + // Backward order in m_Blocks - prefer blocks with largest amount of free space. + for(size_t blockIndex = m_Blocks.size(); blockIndex--; ) + { + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VkResult res = AllocateFromBlock( + pCurrBlock, + currentFrameIndex, + size, + alignment, + allocFlagsCopy, + createInfo.pUserData, + suballocType, + strategy, + pAllocation); + if(res == VK_SUCCESS) + { + VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId()); + return VK_SUCCESS; + } + } + } + } + + // 2. Try to create new block. + if(canCreateNewBlock) + { + // Calculate optimal size for new block. + VkDeviceSize newBlockSize = m_PreferredBlockSize; + uint32_t newBlockSizeShift = 0; + const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3; + + if(!m_ExplicitBlockSize) + { + // Allocate 1/8, 1/4, 1/2 as first blocks. + const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize(); + for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i) + { + const VkDeviceSize smallerNewBlockSize = newBlockSize / 2; + if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2) + { + newBlockSize = smallerNewBlockSize; + ++newBlockSizeShift; + } + else + { + break; + } + } + } + + size_t newBlockIndex = 0; + VkResult res = (newBlockSize <= freeMemory || !canFallbackToDedicated) ? + CreateBlock(newBlockSize, &newBlockIndex) : VK_ERROR_OUT_OF_DEVICE_MEMORY; + // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize. + if(!m_ExplicitBlockSize) + { + while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX) + { + const VkDeviceSize smallerNewBlockSize = newBlockSize / 2; + if(smallerNewBlockSize >= size) + { + newBlockSize = smallerNewBlockSize; + ++newBlockSizeShift; + res = (newBlockSize <= freeMemory || !canFallbackToDedicated) ? + CreateBlock(newBlockSize, &newBlockIndex) : VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + else + { + break; + } + } + } + + if(res == VK_SUCCESS) + { + VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex]; + VMA_ASSERT(pBlock->m_pMetadata->GetSize() >= size); + + res = AllocateFromBlock( + pBlock, + currentFrameIndex, + size, + alignment, + allocFlagsCopy, + createInfo.pUserData, + suballocType, + strategy, + pAllocation); + if(res == VK_SUCCESS) + { + VMA_DEBUG_LOG(" Created new block #%u Size=%llu", pBlock->GetId(), newBlockSize); + return VK_SUCCESS; + } + else + { + // Allocation from new block failed, possibly due to VMA_DEBUG_MARGIN or alignment. + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } } } } - const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0; - // 3. Try to allocate from existing blocks with making other allocations lost. if(canMakeOtherLost) { @@ -7410,33 +12926,76 @@ VkResult VmaBlockVector::Allocate( VkDeviceSize bestRequestCost = VK_WHOLE_SIZE; // 1. Search existing allocations. - // Forward order in m_Blocks - prefer blocks with smallest amount of free space. - for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) + if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT) { - VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; - VMA_ASSERT(pCurrBlock); - VmaAllocationRequest currRequest = {}; - if(pCurrBlock->m_Metadata.CreateAllocationRequest( - currentFrameIndex, - m_FrameInUseCount, - m_BufferImageGranularity, - size, - alignment, - suballocType, - canMakeOtherLost, - &currRequest)) + // Forward order in m_Blocks - prefer blocks with smallest amount of free space. + for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) { - const VkDeviceSize currRequestCost = currRequest.CalcCost(); - if(pBestRequestBlock == VMA_NULL || - currRequestCost < bestRequestCost) + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VmaAllocationRequest currRequest = {}; + if(pCurrBlock->m_pMetadata->CreateAllocationRequest( + currentFrameIndex, + m_FrameInUseCount, + m_BufferImageGranularity, + size, + alignment, + (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0, + suballocType, + canMakeOtherLost, + strategy, + &currRequest)) { - pBestRequestBlock = pCurrBlock; - bestRequest = currRequest; - bestRequestCost = currRequestCost; - - if(bestRequestCost == 0) + const VkDeviceSize currRequestCost = currRequest.CalcCost(); + if(pBestRequestBlock == VMA_NULL || + currRequestCost < bestRequestCost) { - break; + pBestRequestBlock = pCurrBlock; + bestRequest = currRequest; + bestRequestCost = currRequestCost; + + if(bestRequestCost == 0) + { + break; + } + } + } + } + } + else // WORST_FIT, FIRST_FIT + { + // Backward order in m_Blocks - prefer blocks with largest amount of free space. + for(size_t blockIndex = m_Blocks.size(); blockIndex--; ) + { + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VmaAllocationRequest currRequest = {}; + if(pCurrBlock->m_pMetadata->CreateAllocationRequest( + currentFrameIndex, + m_FrameInUseCount, + m_BufferImageGranularity, + size, + alignment, + (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0, + suballocType, + canMakeOtherLost, + strategy, + &currRequest)) + { + const VkDeviceSize currRequestCost = currRequest.CalcCost(); + if(pBestRequestBlock == VMA_NULL || + currRequestCost < bestRequestCost || + strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT) + { + pBestRequestBlock = pCurrBlock; + bestRequest = currRequest; + bestRequestCost = currRequestCost; + + if(bestRequestCost == 0 || + strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT) + { + break; + } } } } @@ -7453,31 +13012,28 @@ VkResult VmaBlockVector::Allocate( } } - if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost( + if(pBestRequestBlock->m_pMetadata->MakeRequestedAllocationsLost( currentFrameIndex, m_FrameInUseCount, &bestRequest)) { - // We no longer have an empty Allocation. - if(pBestRequestBlock->m_Metadata.IsEmpty()) - { - m_HasEmptyBlock = false; - } // Allocate from this pBlock. - *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString); - pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation); + *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString); + pBestRequestBlock->m_pMetadata->Alloc(bestRequest, suballocType, size, *pAllocation); + UpdateHasEmptyBlock(); (*pAllocation)->InitBlockAllocation( - hCurrentPool, pBestRequestBlock, bestRequest.offset, alignment, size, + m_MemoryTypeIndex, suballocType, mapped, (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); VMA_HEAVY_ASSERT(pBestRequestBlock->Validate()); - VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex); + VMA_DEBUG_LOG(" Returned from existing block"); (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData); + m_hAllocator->m_Budget.AddAllocation(m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex), size); if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) { m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED); @@ -7510,13 +13066,21 @@ VkResult VmaBlockVector::Allocate( } void VmaBlockVector::Free( - VmaAllocation hAllocation) + const VmaAllocation hAllocation) { VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL; + bool budgetExceeded = false; + { + const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex); + VmaBudget heapBudget = {}; + m_hAllocator->GetBudget(&heapBudget, heapIndex, 1); + budgetExceeded = heapBudget.usage >= heapBudget.budget; + } + // Scope for lock. { - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock(); @@ -7531,47 +13095,44 @@ void VmaBlockVector::Free( pBlock->Unmap(m_hAllocator, 1); } - pBlock->m_Metadata.Free(hAllocation); + pBlock->m_pMetadata->Free(hAllocation); VMA_HEAVY_ASSERT(pBlock->Validate()); - VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex); + VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", m_MemoryTypeIndex); + const bool canDeleteBlock = m_Blocks.size() > m_MinBlockCount; // pBlock became empty after this deallocation. - if(pBlock->m_Metadata.IsEmpty()) + if(pBlock->m_pMetadata->IsEmpty()) { - // Already has empty Allocation. We don't want to have two, so delete this one. - if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount) + // Already has empty block. We don't want to have two, so delete this one. + if((m_HasEmptyBlock || budgetExceeded) && canDeleteBlock) { pBlockToDelete = pBlock; Remove(pBlock); } - // We now have first empty block. - else - { - m_HasEmptyBlock = true; - } + // else: We now have an empty block - leave it. } // pBlock didn't become empty, but we have another empty block - find and free that one. // (This is optional, heuristics.) - else if(m_HasEmptyBlock) + else if(m_HasEmptyBlock && canDeleteBlock) { VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back(); - if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount) + if(pLastBlock->m_pMetadata->IsEmpty()) { pBlockToDelete = pLastBlock; m_Blocks.pop_back(); - m_HasEmptyBlock = false; } } + UpdateHasEmptyBlock(); IncrementallySortBlocks(); } - // Destruction of a free Allocation. Deferred until this point, outside of mutex + // Destruction of a free block. Deferred until this point, outside of mutex // lock, for performance reason. if(pBlockToDelete != VMA_NULL) { - VMA_DEBUG_LOG(" Deleted empty allocation"); + VMA_DEBUG_LOG(" Deleted empty block"); pBlockToDelete->Destroy(m_hAllocator); vma_delete(m_hAllocator, pBlockToDelete); } @@ -7582,7 +13143,7 @@ VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const VkDeviceSize result = 0; for(size_t i = m_Blocks.size(); i--; ) { - result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize()); + result = VMA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize()); if(result >= m_PreferredBlockSize) { break; @@ -7606,22 +13167,106 @@ void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock) void VmaBlockVector::IncrementallySortBlocks() { - // Bubble sort only until first swap. - for(size_t i = 1; i < m_Blocks.size(); ++i) + if(m_Algorithm != VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) { - if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize()) + // Bubble sort only until first swap. + for(size_t i = 1; i < m_Blocks.size(); ++i) { - VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]); - return; + if(m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize()) + { + VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]); + return; + } } } } +VkResult VmaBlockVector::AllocateFromBlock( + VmaDeviceMemoryBlock* pBlock, + uint32_t currentFrameIndex, + VkDeviceSize size, + VkDeviceSize alignment, + VmaAllocationCreateFlags allocFlags, + void* pUserData, + VmaSuballocationType suballocType, + uint32_t strategy, + VmaAllocation* pAllocation) +{ + VMA_ASSERT((allocFlags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) == 0); + const bool isUpperAddress = (allocFlags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0; + const bool mapped = (allocFlags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0; + const bool isUserDataString = (allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0; + + VmaAllocationRequest currRequest = {}; + if(pBlock->m_pMetadata->CreateAllocationRequest( + currentFrameIndex, + m_FrameInUseCount, + m_BufferImageGranularity, + size, + alignment, + isUpperAddress, + suballocType, + false, // canMakeOtherLost + strategy, + &currRequest)) + { + // Allocate from pCurrBlock. + VMA_ASSERT(currRequest.itemsToMakeLostCount == 0); + + if(mapped) + { + VkResult res = pBlock->Map(m_hAllocator, 1, VMA_NULL); + if(res != VK_SUCCESS) + { + return res; + } + } + + *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString); + pBlock->m_pMetadata->Alloc(currRequest, suballocType, size, *pAllocation); + UpdateHasEmptyBlock(); + (*pAllocation)->InitBlockAllocation( + pBlock, + currRequest.offset, + alignment, + size, + m_MemoryTypeIndex, + suballocType, + mapped, + (allocFlags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); + VMA_HEAVY_ASSERT(pBlock->Validate()); + (*pAllocation)->SetUserData(m_hAllocator, pUserData); + m_hAllocator->m_Budget.AddAllocation(m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex), size); + if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) + { + m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED); + } + if(IsCorruptionDetectionEnabled()) + { + VkResult res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size); + VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value."); + } + return VK_SUCCESS; + } + return VK_ERROR_OUT_OF_DEVICE_MEMORY; +} + VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex) { VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; allocInfo.memoryTypeIndex = m_MemoryTypeIndex; allocInfo.allocationSize = blockSize; + +#if VMA_BUFFER_DEVICE_ADDRESS + // Every standalone block can potentially contain a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT - always enable the feature. + VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR }; + if(m_hAllocator->m_UseKhrBufferDeviceAddress) + { + allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR; + VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo); + } +#endif // #if VMA_BUFFER_DEVICE_ADDRESS + VkDeviceMemory mem = VK_NULL_HANDLE; VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem); if(res < 0) @@ -7634,10 +13279,13 @@ VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIn // Create new Allocation for it. VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator); pBlock->Init( + m_hAllocator, + m_hParentPool, m_MemoryTypeIndex, mem, allocInfo.allocationSize, - m_NextBlockId++); + m_NextBlockId++, + m_Algorithm); m_Blocks.push_back(pBlock); if(pNewBlockIndex != VMA_NULL) @@ -7648,16 +13296,261 @@ VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIn return VK_SUCCESS; } +void VmaBlockVector::ApplyDefragmentationMovesCpu( + class VmaBlockVectorDefragmentationContext* pDefragCtx, + const VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves) +{ + const size_t blockCount = m_Blocks.size(); + const bool isNonCoherent = m_hAllocator->IsMemoryTypeNonCoherent(m_MemoryTypeIndex); + + enum BLOCK_FLAG + { + BLOCK_FLAG_USED = 0x00000001, + BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION = 0x00000002, + }; + + struct BlockInfo + { + uint32_t flags; + void* pMappedData; + }; + VmaVector< BlockInfo, VmaStlAllocator > + blockInfo(blockCount, BlockInfo(), VmaStlAllocator(m_hAllocator->GetAllocationCallbacks())); + memset(blockInfo.data(), 0, blockCount * sizeof(BlockInfo)); + + // Go over all moves. Mark blocks that are used with BLOCK_FLAG_USED. + const size_t moveCount = moves.size(); + for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex) + { + const VmaDefragmentationMove& move = moves[moveIndex]; + blockInfo[move.srcBlockIndex].flags |= BLOCK_FLAG_USED; + blockInfo[move.dstBlockIndex].flags |= BLOCK_FLAG_USED; + } + + VMA_ASSERT(pDefragCtx->res == VK_SUCCESS); + + // Go over all blocks. Get mapped pointer or map if necessary. + for(size_t blockIndex = 0; pDefragCtx->res == VK_SUCCESS && blockIndex < blockCount; ++blockIndex) + { + BlockInfo& currBlockInfo = blockInfo[blockIndex]; + VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; + if((currBlockInfo.flags & BLOCK_FLAG_USED) != 0) + { + currBlockInfo.pMappedData = pBlock->GetMappedData(); + // It is not originally mapped - map it. + if(currBlockInfo.pMappedData == VMA_NULL) + { + pDefragCtx->res = pBlock->Map(m_hAllocator, 1, &currBlockInfo.pMappedData); + if(pDefragCtx->res == VK_SUCCESS) + { + currBlockInfo.flags |= BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION; + } + } + } + } + + // Go over all moves. Do actual data transfer. + if(pDefragCtx->res == VK_SUCCESS) + { + const VkDeviceSize nonCoherentAtomSize = m_hAllocator->m_PhysicalDeviceProperties.limits.nonCoherentAtomSize; + VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE }; + + for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex) + { + const VmaDefragmentationMove& move = moves[moveIndex]; + + const BlockInfo& srcBlockInfo = blockInfo[move.srcBlockIndex]; + const BlockInfo& dstBlockInfo = blockInfo[move.dstBlockIndex]; + + VMA_ASSERT(srcBlockInfo.pMappedData && dstBlockInfo.pMappedData); + + // Invalidate source. + if(isNonCoherent) + { + VmaDeviceMemoryBlock* const pSrcBlock = m_Blocks[move.srcBlockIndex]; + memRange.memory = pSrcBlock->GetDeviceMemory(); + memRange.offset = VmaAlignDown(move.srcOffset, nonCoherentAtomSize); + memRange.size = VMA_MIN( + VmaAlignUp(move.size + (move.srcOffset - memRange.offset), nonCoherentAtomSize), + pSrcBlock->m_pMetadata->GetSize() - memRange.offset); + (*m_hAllocator->GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hAllocator->m_hDevice, 1, &memRange); + } + + // THE PLACE WHERE ACTUAL DATA COPY HAPPENS. + memmove( + reinterpret_cast(dstBlockInfo.pMappedData) + move.dstOffset, + reinterpret_cast(srcBlockInfo.pMappedData) + move.srcOffset, + static_cast(move.size)); + + if(IsCorruptionDetectionEnabled()) + { + VmaWriteMagicValue(dstBlockInfo.pMappedData, move.dstOffset - VMA_DEBUG_MARGIN); + VmaWriteMagicValue(dstBlockInfo.pMappedData, move.dstOffset + move.size); + } + + // Flush destination. + if(isNonCoherent) + { + VmaDeviceMemoryBlock* const pDstBlock = m_Blocks[move.dstBlockIndex]; + memRange.memory = pDstBlock->GetDeviceMemory(); + memRange.offset = VmaAlignDown(move.dstOffset, nonCoherentAtomSize); + memRange.size = VMA_MIN( + VmaAlignUp(move.size + (move.dstOffset - memRange.offset), nonCoherentAtomSize), + pDstBlock->m_pMetadata->GetSize() - memRange.offset); + (*m_hAllocator->GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hAllocator->m_hDevice, 1, &memRange); + } + } + } + + // Go over all blocks in reverse order. Unmap those that were mapped just for defragmentation. + // Regardless of pCtx->res == VK_SUCCESS. + for(size_t blockIndex = blockCount; blockIndex--; ) + { + const BlockInfo& currBlockInfo = blockInfo[blockIndex]; + if((currBlockInfo.flags & BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION) != 0) + { + VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; + pBlock->Unmap(m_hAllocator, 1); + } + } +} + +void VmaBlockVector::ApplyDefragmentationMovesGpu( + class VmaBlockVectorDefragmentationContext* pDefragCtx, + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkCommandBuffer commandBuffer) +{ + const size_t blockCount = m_Blocks.size(); + + pDefragCtx->blockContexts.resize(blockCount); + memset(pDefragCtx->blockContexts.data(), 0, blockCount * sizeof(VmaBlockDefragmentationContext)); + + // Go over all moves. Mark blocks that are used with BLOCK_FLAG_USED. + const size_t moveCount = moves.size(); + for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex) + { + const VmaDefragmentationMove& move = moves[moveIndex]; + + //if(move.type == VMA_ALLOCATION_TYPE_UNKNOWN) + { + // Old school move still require us to map the whole block + pDefragCtx->blockContexts[move.srcBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; + pDefragCtx->blockContexts[move.dstBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; + } + } + + VMA_ASSERT(pDefragCtx->res == VK_SUCCESS); + + // Go over all blocks. Create and bind buffer for whole block if necessary. + { + VkBufferCreateInfo bufCreateInfo; + VmaFillGpuDefragmentationBufferCreateInfo(bufCreateInfo); + + for(size_t blockIndex = 0; pDefragCtx->res == VK_SUCCESS && blockIndex < blockCount; ++blockIndex) + { + VmaBlockDefragmentationContext& currBlockCtx = pDefragCtx->blockContexts[blockIndex]; + VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; + if((currBlockCtx.flags & VmaBlockDefragmentationContext::BLOCK_FLAG_USED) != 0) + { + bufCreateInfo.size = pBlock->m_pMetadata->GetSize(); + pDefragCtx->res = (*m_hAllocator->GetVulkanFunctions().vkCreateBuffer)( + m_hAllocator->m_hDevice, &bufCreateInfo, m_hAllocator->GetAllocationCallbacks(), &currBlockCtx.hBuffer); + if(pDefragCtx->res == VK_SUCCESS) + { + pDefragCtx->res = (*m_hAllocator->GetVulkanFunctions().vkBindBufferMemory)( + m_hAllocator->m_hDevice, currBlockCtx.hBuffer, pBlock->GetDeviceMemory(), 0); + } + } + } + } + + // Go over all moves. Post data transfer commands to command buffer. + if(pDefragCtx->res == VK_SUCCESS) + { + for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex) + { + const VmaDefragmentationMove& move = moves[moveIndex]; + + const VmaBlockDefragmentationContext& srcBlockCtx = pDefragCtx->blockContexts[move.srcBlockIndex]; + const VmaBlockDefragmentationContext& dstBlockCtx = pDefragCtx->blockContexts[move.dstBlockIndex]; + + VMA_ASSERT(srcBlockCtx.hBuffer && dstBlockCtx.hBuffer); + + VkBufferCopy region = { + move.srcOffset, + move.dstOffset, + move.size }; + (*m_hAllocator->GetVulkanFunctions().vkCmdCopyBuffer)( + commandBuffer, srcBlockCtx.hBuffer, dstBlockCtx.hBuffer, 1, ®ion); + } + } + + // Save buffers to defrag context for later destruction. + if(pDefragCtx->res == VK_SUCCESS && moveCount > 0) + { + pDefragCtx->res = VK_NOT_READY; + } +} + +void VmaBlockVector::FreeEmptyBlocks(VmaDefragmentationStats* pDefragmentationStats) +{ + for(size_t blockIndex = m_Blocks.size(); blockIndex--; ) + { + VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; + if(pBlock->m_pMetadata->IsEmpty()) + { + if(m_Blocks.size() > m_MinBlockCount) + { + if(pDefragmentationStats != VMA_NULL) + { + ++pDefragmentationStats->deviceMemoryBlocksFreed; + pDefragmentationStats->bytesFreed += pBlock->m_pMetadata->GetSize(); + } + + VmaVectorRemove(m_Blocks, blockIndex); + pBlock->Destroy(m_hAllocator); + vma_delete(m_hAllocator, pBlock); + } + else + { + break; + } + } + } + UpdateHasEmptyBlock(); +} + +void VmaBlockVector::UpdateHasEmptyBlock() +{ + m_HasEmptyBlock = false; + for(size_t index = 0, count = m_Blocks.size(); index < count; ++index) + { + VmaDeviceMemoryBlock* const pBlock = m_Blocks[index]; + if(pBlock->m_pMetadata->IsEmpty()) + { + m_HasEmptyBlock = true; + break; + } + } +} + #if VMA_STATS_STRING_ENABLED void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) { - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex); json.BeginObject(); - if(m_IsCustomPool) + if(IsCustomPool()) { + const char* poolName = m_hParentPool->GetName(); + if(poolName != VMA_NULL && poolName[0] != '\0') + { + json.WriteString("Name"); + json.WriteString(poolName); + } + json.WriteString("MemoryTypeIndex"); json.WriteNumber(m_MemoryTypeIndex); @@ -7685,6 +13578,12 @@ void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) json.WriteString("FrameInUseCount"); json.WriteNumber(m_FrameInUseCount); } + + if(m_Algorithm != 0) + { + json.WriteString("Algorithm"); + json.WriteString(VmaAlgorithmToStr(m_Algorithm)); + } } else { @@ -7700,7 +13599,7 @@ void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) json.ContinueString(m_Blocks[i]->GetId()); json.EndString(); - m_Blocks[i]->m_Metadata.PrintDetailedMap(json); + m_Blocks[i]->m_pMetadata->PrintDetailedMap(json); } json.EndObject(); @@ -7709,98 +13608,238 @@ void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) #endif // #if VMA_STATS_STRING_ENABLED -VmaDefragmentator* VmaBlockVector::EnsureDefragmentator( - VmaAllocator hAllocator, - uint32_t currentFrameIndex) +void VmaBlockVector::Defragment( + class VmaBlockVectorDefragmentationContext* pCtx, + VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags, + VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove, + VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove, + VkCommandBuffer commandBuffer) { - if(m_pDefragmentator == VMA_NULL) - { - m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)( - hAllocator, - this, - currentFrameIndex); - } - - return m_pDefragmentator; -} - -VkResult VmaBlockVector::Defragment( - VmaDefragmentationStats* pDefragmentationStats, - VkDeviceSize& maxBytesToMove, - uint32_t& maxAllocationsToMove) -{ - if(m_pDefragmentator == VMA_NULL) - { - return VK_SUCCESS; - } - - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); - - // Defragment. - VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove); - - // Accumulate statistics. - if(pDefragmentationStats != VMA_NULL) - { - const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved(); - const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved(); - pDefragmentationStats->bytesMoved += bytesMoved; - pDefragmentationStats->allocationsMoved += allocationsMoved; - VMA_ASSERT(bytesMoved <= maxBytesToMove); - VMA_ASSERT(allocationsMoved <= maxAllocationsToMove); - maxBytesToMove -= bytesMoved; - maxAllocationsToMove -= allocationsMoved; - } + pCtx->res = VK_SUCCESS; - // Free empty blocks. - m_HasEmptyBlock = false; - for(size_t blockIndex = m_Blocks.size(); blockIndex--; ) - { - VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; - if(pBlock->m_Metadata.IsEmpty()) - { - if(m_Blocks.size() > m_MinBlockCount) - { - if(pDefragmentationStats != VMA_NULL) - { - ++pDefragmentationStats->deviceMemoryBlocksFreed; - pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize(); - } + const VkMemoryPropertyFlags memPropFlags = + m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags; + const bool isHostVisible = (memPropFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; - VmaVectorRemove(m_Blocks, blockIndex); - pBlock->Destroy(m_hAllocator); - vma_delete(m_hAllocator, pBlock); + const bool canDefragmentOnCpu = maxCpuBytesToMove > 0 && maxCpuAllocationsToMove > 0 && + isHostVisible; + const bool canDefragmentOnGpu = maxGpuBytesToMove > 0 && maxGpuAllocationsToMove > 0 && + !IsCorruptionDetectionEnabled() && + ((1u << m_MemoryTypeIndex) & m_hAllocator->GetGpuDefragmentationMemoryTypeBits()) != 0; + + // There are options to defragment this memory type. + if(canDefragmentOnCpu || canDefragmentOnGpu) + { + bool defragmentOnGpu; + // There is only one option to defragment this memory type. + if(canDefragmentOnGpu != canDefragmentOnCpu) + { + defragmentOnGpu = canDefragmentOnGpu; + } + // Both options are available: Heuristics to choose the best one. + else + { + defragmentOnGpu = (memPropFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 || + m_hAllocator->IsIntegratedGpu(); + } + + bool overlappingMoveSupported = !defragmentOnGpu; + + if(m_hAllocator->m_UseMutex) + { + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + if(!m_Mutex.TryLockWrite()) + { + pCtx->res = VK_ERROR_INITIALIZATION_FAILED; + return; + } } else { - m_HasEmptyBlock = true; + m_Mutex.LockWrite(); + pCtx->mutexLocked = true; + } + } + + pCtx->Begin(overlappingMoveSupported, flags); + + // Defragment. + + const VkDeviceSize maxBytesToMove = defragmentOnGpu ? maxGpuBytesToMove : maxCpuBytesToMove; + const uint32_t maxAllocationsToMove = defragmentOnGpu ? maxGpuAllocationsToMove : maxCpuAllocationsToMove; + pCtx->res = pCtx->GetAlgorithm()->Defragment(pCtx->defragmentationMoves, maxBytesToMove, maxAllocationsToMove, flags); + + // Accumulate statistics. + if(pStats != VMA_NULL) + { + const VkDeviceSize bytesMoved = pCtx->GetAlgorithm()->GetBytesMoved(); + const uint32_t allocationsMoved = pCtx->GetAlgorithm()->GetAllocationsMoved(); + pStats->bytesMoved += bytesMoved; + pStats->allocationsMoved += allocationsMoved; + VMA_ASSERT(bytesMoved <= maxBytesToMove); + VMA_ASSERT(allocationsMoved <= maxAllocationsToMove); + if(defragmentOnGpu) + { + maxGpuBytesToMove -= bytesMoved; + maxGpuAllocationsToMove -= allocationsMoved; + } + else + { + maxCpuBytesToMove -= bytesMoved; + maxCpuAllocationsToMove -= allocationsMoved; + } + } + + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + if(m_hAllocator->m_UseMutex) + m_Mutex.UnlockWrite(); + + if(pCtx->res >= VK_SUCCESS && !pCtx->defragmentationMoves.empty()) + pCtx->res = VK_NOT_READY; + + return; + } + + if(pCtx->res >= VK_SUCCESS) + { + if(defragmentOnGpu) + { + ApplyDefragmentationMovesGpu(pCtx, pCtx->defragmentationMoves, commandBuffer); + } + else + { + ApplyDefragmentationMovesCpu(pCtx, pCtx->defragmentationMoves); } } } +} +void VmaBlockVector::DefragmentationEnd( + class VmaBlockVectorDefragmentationContext* pCtx, + uint32_t flags, + VmaDefragmentationStats* pStats) +{ + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL && m_hAllocator->m_UseMutex) + { + VMA_ASSERT(pCtx->mutexLocked == false); + + // Incremental defragmentation doesn't hold the lock, so when we enter here we don't actually have any + // lock protecting us. Since we mutate state here, we have to take the lock out now + m_Mutex.LockWrite(); + pCtx->mutexLocked = true; + } + + // If the mutex isn't locked we didn't do any work and there is nothing to delete. + if(pCtx->mutexLocked || !m_hAllocator->m_UseMutex) + { + // Destroy buffers. + for(size_t blockIndex = pCtx->blockContexts.size(); blockIndex--;) + { + VmaBlockDefragmentationContext &blockCtx = pCtx->blockContexts[blockIndex]; + if(blockCtx.hBuffer) + { + (*m_hAllocator->GetVulkanFunctions().vkDestroyBuffer)(m_hAllocator->m_hDevice, blockCtx.hBuffer, m_hAllocator->GetAllocationCallbacks()); + } + } + + if(pCtx->res >= VK_SUCCESS) + { + FreeEmptyBlocks(pStats); + } + } + + if(pCtx->mutexLocked) + { + VMA_ASSERT(m_hAllocator->m_UseMutex); + m_Mutex.UnlockWrite(); + } +} + +uint32_t VmaBlockVector::ProcessDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves) +{ + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); + + const uint32_t moveCount = std::min(uint32_t(pCtx->defragmentationMoves.size()) - pCtx->defragmentationMovesProcessed, maxMoves); + + for(uint32_t i = 0; i < moveCount; ++ i) + { + VmaDefragmentationMove& move = pCtx->defragmentationMoves[pCtx->defragmentationMovesProcessed + i]; + + pMove->allocation = move.hAllocation; + pMove->memory = move.pDstBlock->GetDeviceMemory(); + pMove->offset = move.dstOffset; + + ++ pMove; + } + + pCtx->defragmentationMovesProcessed += moveCount; + + return moveCount; +} + +void VmaBlockVector::CommitDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationStats* pStats) +{ + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); + + for(uint32_t i = pCtx->defragmentationMovesCommitted; i < pCtx->defragmentationMovesProcessed; ++ i) + { + const VmaDefragmentationMove &move = pCtx->defragmentationMoves[i]; + + move.pSrcBlock->m_pMetadata->FreeAtOffset(move.srcOffset); + move.hAllocation->ChangeBlockAllocation(m_hAllocator, move.pDstBlock, move.dstOffset); + } + + pCtx->defragmentationMovesCommitted = pCtx->defragmentationMovesProcessed; + FreeEmptyBlocks(pStats); +} + +size_t VmaBlockVector::CalcAllocationCount() const +{ + size_t result = 0; + for(size_t i = 0; i < m_Blocks.size(); ++i) + { + result += m_Blocks[i]->m_pMetadata->GetAllocationCount(); + } return result; } -void VmaBlockVector::DestroyDefragmentator() +bool VmaBlockVector::IsBufferImageGranularityConflictPossible() const { - if(m_pDefragmentator != VMA_NULL) + if(m_BufferImageGranularity == 1) { - vma_delete(m_hAllocator, m_pDefragmentator); - m_pDefragmentator = VMA_NULL; + return false; } + VmaSuballocationType lastSuballocType = VMA_SUBALLOCATION_TYPE_FREE; + for(size_t i = 0, count = m_Blocks.size(); i < count; ++i) + { + VmaDeviceMemoryBlock* const pBlock = m_Blocks[i]; + VMA_ASSERT(m_Algorithm == 0); + VmaBlockMetadata_Generic* const pMetadata = (VmaBlockMetadata_Generic*)pBlock->m_pMetadata; + if(pMetadata->IsBufferImageGranularityConflictPossible(m_BufferImageGranularity, lastSuballocType)) + { + return true; + } + } + return false; } void VmaBlockVector::MakePoolAllocationsLost( uint32_t currentFrameIndex, size_t* pLostAllocationCount) { - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); size_t lostAllocationCount = 0; for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) { VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; VMA_ASSERT(pBlock); - lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount); + lostAllocationCount += pBlock->m_pMetadata->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount); } if(pLostAllocationCount != VMA_NULL) { @@ -7815,7 +13854,7 @@ VkResult VmaBlockVector::CheckCorruption() return VK_ERROR_FEATURE_NOT_PRESENT; } - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex); for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) { VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; @@ -7834,7 +13873,7 @@ void VmaBlockVector::AddStats(VmaStats* pStats) const uint32_t memTypeIndex = m_MemoryTypeIndex; const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex); - VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex); for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) { @@ -7842,7 +13881,7 @@ void VmaBlockVector::AddStats(VmaStats* pStats) VMA_ASSERT(pBlock); VMA_HEAVY_ASSERT(pBlock->Validate()); VmaStatInfo allocationStatInfo; - pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo); + pBlock->m_pMetadata->CalcAllocationStatInfo(allocationStatInfo); VmaAddStatInfo(pStats->total, allocationStatInfo); VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo); VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo); @@ -7850,23 +13889,35 @@ void VmaBlockVector::AddStats(VmaStats* pStats) } //////////////////////////////////////////////////////////////////////////////// -// VmaDefragmentator members definition +// VmaDefragmentationAlgorithm_Generic members definition -VmaDefragmentator::VmaDefragmentator( +VmaDefragmentationAlgorithm_Generic::VmaDefragmentationAlgorithm_Generic( VmaAllocator hAllocator, VmaBlockVector* pBlockVector, - uint32_t currentFrameIndex) : - m_hAllocator(hAllocator), - m_pBlockVector(pBlockVector), - m_CurrentFrameIndex(currentFrameIndex), + uint32_t currentFrameIndex, + bool overlappingMoveSupported) : + VmaDefragmentationAlgorithm(hAllocator, pBlockVector, currentFrameIndex), + m_AllocationCount(0), + m_AllAllocations(false), m_BytesMoved(0), m_AllocationsMoved(0), - m_Allocations(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), m_Blocks(VmaStlAllocator(hAllocator->GetAllocationCallbacks())) { + // Create block info for each block. + const size_t blockCount = m_pBlockVector->m_Blocks.size(); + for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) + { + BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks()); + pBlockInfo->m_OriginalBlockIndex = blockIndex; + pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex]; + m_Blocks.push_back(pBlockInfo); + } + + // Sort them by m_pBlock pointer value. + VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess()); } -VmaDefragmentator::~VmaDefragmentator() +VmaDefragmentationAlgorithm_Generic::~VmaDefragmentationAlgorithm_Generic() { for(size_t i = m_Blocks.size(); i--; ) { @@ -7874,66 +13925,72 @@ VmaDefragmentator::~VmaDefragmentator() } } -void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) +void VmaDefragmentationAlgorithm_Generic::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) { - AllocationInfo allocInfo; - allocInfo.m_hAllocation = hAlloc; - allocInfo.m_pChanged = pChanged; - m_Allocations.push_back(allocInfo); -} + // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost. + if(hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST) + { + VmaDeviceMemoryBlock* pBlock = hAlloc->GetBlock(); + BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess()); + if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock) + { + AllocationInfo allocInfo = AllocationInfo(hAlloc, pChanged); + (*it)->m_Allocations.push_back(allocInfo); + } + else + { + VMA_ASSERT(0); + } -VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData) -{ - // It has already been mapped for defragmentation. - if(m_pMappedDataForDefragmentation) - { - *ppMappedData = m_pMappedDataForDefragmentation; - return VK_SUCCESS; - } - - // It is originally mapped. - if(m_pBlock->GetMappedData()) - { - *ppMappedData = m_pBlock->GetMappedData(); - return VK_SUCCESS; - } - - // Map on first usage. - VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation); - *ppMappedData = m_pMappedDataForDefragmentation; - return res; -} - -void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator) -{ - if(m_pMappedDataForDefragmentation != VMA_NULL) - { - m_pBlock->Unmap(hAllocator, 1); + ++m_AllocationCount; } } -VkResult VmaDefragmentator::DefragmentRound( +VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) + uint32_t maxAllocationsToMove, + bool freeOldAllocations) { if(m_Blocks.empty()) { return VK_SUCCESS; } + // This is a choice based on research. + // Option 1: + uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT; + // Option 2: + //uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT; + // Option 3: + //uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_FRAGMENTATION_BIT; + + size_t srcBlockMinIndex = 0; + // When FAST_ALGORITHM, move allocations from only last out of blocks that contain non-movable allocations. + /* + if(m_AlgorithmFlags & VMA_DEFRAGMENTATION_FAST_ALGORITHM_BIT) + { + const size_t blocksWithNonMovableCount = CalcBlocksWithNonMovableCount(); + if(blocksWithNonMovableCount > 0) + { + srcBlockMinIndex = blocksWithNonMovableCount - 1; + } + } + */ + size_t srcBlockIndex = m_Blocks.size() - 1; size_t srcAllocIndex = SIZE_MAX; for(;;) { // 1. Find next allocation to move. // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source". - // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest. + // 1.2. Then start from last to first m_Allocations. while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size()) { if(m_Blocks[srcBlockIndex]->m_Allocations.empty()) { // Finished: no more allocations to process. - if(srcBlockIndex == 0) + if(srcBlockIndex == srcBlockMinIndex) { return VK_SUCCESS; } @@ -7962,14 +14019,16 @@ VkResult VmaDefragmentator::DefragmentRound( { BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex]; VmaAllocationRequest dstAllocRequest; - if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest( + if(pDstBlockInfo->m_pBlock->m_pMetadata->CreateAllocationRequest( m_CurrentFrameIndex, m_pBlockVector->GetFrameInUseCount(), m_pBlockVector->GetBufferImageGranularity(), size, alignment, + false, // upperAddress suballocType, false, // canMakeOtherLost + strategy, &dstAllocRequest) && MoveMakesSense( dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset)) @@ -7980,40 +14039,33 @@ VkResult VmaDefragmentator::DefragmentRound( if((m_AllocationsMoved + 1 > maxAllocationsToMove) || (m_BytesMoved + size > maxBytesToMove)) { - return VK_INCOMPLETE; + return VK_SUCCESS; } - void* pDstMappedData = VMA_NULL; - VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData); - if(res != VK_SUCCESS) - { - return res; - } + VmaDefragmentationMove move = {}; + move.srcBlockIndex = pSrcBlockInfo->m_OriginalBlockIndex; + move.dstBlockIndex = pDstBlockInfo->m_OriginalBlockIndex; + move.srcOffset = srcOffset; + move.dstOffset = dstAllocRequest.offset; + move.size = size; + move.hAllocation = allocInfo.m_hAllocation; + move.pSrcBlock = pSrcBlockInfo->m_pBlock; + move.pDstBlock = pDstBlockInfo->m_pBlock; - void* pSrcMappedData = VMA_NULL; - res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData); - if(res != VK_SUCCESS) + moves.push_back(move); + + pDstBlockInfo->m_pBlock->m_pMetadata->Alloc( + dstAllocRequest, + suballocType, + size, + allocInfo.m_hAllocation); + + if(freeOldAllocations) { - return res; + pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset); + allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset); } - // THE PLACE WHERE ACTUAL DATA COPY HAPPENS. - memcpy( - reinterpret_cast(pDstMappedData) + dstAllocRequest.offset, - reinterpret_cast(pSrcMappedData) + srcOffset, - static_cast(size)); - - if(VMA_DEBUG_MARGIN > 0) - { - VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN); - VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size); - } - - pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation); - pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset); - - allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset); - if(allocInfo.m_pChanged != VMA_NULL) { *allocInfo.m_pChanged = VK_TRUE; @@ -8049,75 +14101,76 @@ VkResult VmaDefragmentator::DefragmentRound( } } -VkResult VmaDefragmentator::Defragment( - VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) +size_t VmaDefragmentationAlgorithm_Generic::CalcBlocksWithNonMovableCount() const { - if(m_Allocations.empty()) + size_t result = 0; + for(size_t i = 0; i < m_Blocks.size(); ++i) + { + if(m_Blocks[i]->m_HasNonMovableAllocations) + { + ++result; + } + } + return result; +} + +VkResult VmaDefragmentationAlgorithm_Generic::Defragment( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) +{ + if(!m_AllAllocations && m_AllocationCount == 0) { return VK_SUCCESS; } - // Create block info for each block. - const size_t blockCount = m_pBlockVector->m_Blocks.size(); - for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) - { - BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks()); - pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex]; - m_Blocks.push_back(pBlockInfo); - } - - // Sort them by m_pBlock pointer value. - VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess()); - - // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations. - for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex) - { - AllocationInfo& allocInfo = m_Allocations[blockIndex]; - // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost. - if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST) - { - VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock(); - BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess()); - if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock) - { - (*it)->m_Allocations.push_back(allocInfo); - } - else - { - VMA_ASSERT(0); - } - } - } - m_Allocations.clear(); - + const size_t blockCount = m_Blocks.size(); for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) { BlockInfo* pBlockInfo = m_Blocks[blockIndex]; + + if(m_AllAllocations) + { + VmaBlockMetadata_Generic* pMetadata = (VmaBlockMetadata_Generic*)pBlockInfo->m_pBlock->m_pMetadata; + for(VmaSuballocationList::const_iterator it = pMetadata->m_Suballocations.begin(); + it != pMetadata->m_Suballocations.end(); + ++it) + { + if(it->type != VMA_SUBALLOCATION_TYPE_FREE) + { + AllocationInfo allocInfo = AllocationInfo(it->hAllocation, VMA_NULL); + pBlockInfo->m_Allocations.push_back(allocInfo); + } + } + } + pBlockInfo->CalcHasNonMovableAllocations(); - pBlockInfo->SortAllocationsBySizeDescecnding(); + + // This is a choice based on research. + // Option 1: + pBlockInfo->SortAllocationsByOffsetDescending(); + // Option 2: + //pBlockInfo->SortAllocationsBySizeDescending(); } // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks. VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination()); + // This is a choice based on research. + const uint32_t roundCount = 2; + // Execute defragmentation rounds (the main part). VkResult result = VK_SUCCESS; - for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round) + for(uint32_t round = 0; (round < roundCount) && (result == VK_SUCCESS); ++round) { - result = DefragmentRound(maxBytesToMove, maxAllocationsToMove); - } - - // Unmap blocks that were mapped for defragmentation. - for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) - { - m_Blocks[blockIndex]->Unmap(m_hAllocator); + result = DefragmentRound(moves, maxBytesToMove, maxAllocationsToMove, !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)); } return result; } -bool VmaDefragmentator::MoveMakesSense( +bool VmaDefragmentationAlgorithm_Generic::MoveMakesSense( size_t dstBlockIndex, VkDeviceSize dstOffset, size_t srcBlockIndex, VkDeviceSize srcOffset) { @@ -8136,6 +14189,809 @@ bool VmaDefragmentator::MoveMakesSense( return false; } +//////////////////////////////////////////////////////////////////////////////// +// VmaDefragmentationAlgorithm_Fast + +VmaDefragmentationAlgorithm_Fast::VmaDefragmentationAlgorithm_Fast( + VmaAllocator hAllocator, + VmaBlockVector* pBlockVector, + uint32_t currentFrameIndex, + bool overlappingMoveSupported) : + VmaDefragmentationAlgorithm(hAllocator, pBlockVector, currentFrameIndex), + m_OverlappingMoveSupported(overlappingMoveSupported), + m_AllocationCount(0), + m_AllAllocations(false), + m_BytesMoved(0), + m_AllocationsMoved(0), + m_BlockInfos(VmaStlAllocator(hAllocator->GetAllocationCallbacks())) +{ + VMA_ASSERT(VMA_DEBUG_MARGIN == 0); + +} + +VmaDefragmentationAlgorithm_Fast::~VmaDefragmentationAlgorithm_Fast() +{ +} + +VkResult VmaDefragmentationAlgorithm_Fast::Defragment( + VmaVector< VmaDefragmentationMove, VmaStlAllocator >& moves, + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) +{ + VMA_ASSERT(m_AllAllocations || m_pBlockVector->CalcAllocationCount() == m_AllocationCount); + + const size_t blockCount = m_pBlockVector->GetBlockCount(); + if(blockCount == 0 || maxBytesToMove == 0 || maxAllocationsToMove == 0) + { + return VK_SUCCESS; + } + + PreprocessMetadata(); + + // Sort blocks in order from most destination. + + m_BlockInfos.resize(blockCount); + for(size_t i = 0; i < blockCount; ++i) + { + m_BlockInfos[i].origBlockIndex = i; + } + + VMA_SORT(m_BlockInfos.begin(), m_BlockInfos.end(), [this](const BlockInfo& lhs, const BlockInfo& rhs) -> bool { + return m_pBlockVector->GetBlock(lhs.origBlockIndex)->m_pMetadata->GetSumFreeSize() < + m_pBlockVector->GetBlock(rhs.origBlockIndex)->m_pMetadata->GetSumFreeSize(); + }); + + // THE MAIN ALGORITHM + + FreeSpaceDatabase freeSpaceDb; + + size_t dstBlockInfoIndex = 0; + size_t dstOrigBlockIndex = m_BlockInfos[dstBlockInfoIndex].origBlockIndex; + VmaDeviceMemoryBlock* pDstBlock = m_pBlockVector->GetBlock(dstOrigBlockIndex); + VmaBlockMetadata_Generic* pDstMetadata = (VmaBlockMetadata_Generic*)pDstBlock->m_pMetadata; + VkDeviceSize dstBlockSize = pDstMetadata->GetSize(); + VkDeviceSize dstOffset = 0; + + bool end = false; + for(size_t srcBlockInfoIndex = 0; !end && srcBlockInfoIndex < blockCount; ++srcBlockInfoIndex) + { + const size_t srcOrigBlockIndex = m_BlockInfos[srcBlockInfoIndex].origBlockIndex; + VmaDeviceMemoryBlock* const pSrcBlock = m_pBlockVector->GetBlock(srcOrigBlockIndex); + VmaBlockMetadata_Generic* const pSrcMetadata = (VmaBlockMetadata_Generic*)pSrcBlock->m_pMetadata; + for(VmaSuballocationList::iterator srcSuballocIt = pSrcMetadata->m_Suballocations.begin(); + !end && srcSuballocIt != pSrcMetadata->m_Suballocations.end(); ) + { + VmaAllocation_T* const pAlloc = srcSuballocIt->hAllocation; + const VkDeviceSize srcAllocAlignment = pAlloc->GetAlignment(); + const VkDeviceSize srcAllocSize = srcSuballocIt->size; + if(m_AllocationsMoved == maxAllocationsToMove || + m_BytesMoved + srcAllocSize > maxBytesToMove) + { + end = true; + break; + } + const VkDeviceSize srcAllocOffset = srcSuballocIt->offset; + + VmaDefragmentationMove move = {}; + // Try to place it in one of free spaces from the database. + size_t freeSpaceInfoIndex; + VkDeviceSize dstAllocOffset; + if(freeSpaceDb.Fetch(srcAllocAlignment, srcAllocSize, + freeSpaceInfoIndex, dstAllocOffset)) + { + size_t freeSpaceOrigBlockIndex = m_BlockInfos[freeSpaceInfoIndex].origBlockIndex; + VmaDeviceMemoryBlock* pFreeSpaceBlock = m_pBlockVector->GetBlock(freeSpaceOrigBlockIndex); + VmaBlockMetadata_Generic* pFreeSpaceMetadata = (VmaBlockMetadata_Generic*)pFreeSpaceBlock->m_pMetadata; + + // Same block + if(freeSpaceInfoIndex == srcBlockInfoIndex) + { + VMA_ASSERT(dstAllocOffset <= srcAllocOffset); + + // MOVE OPTION 1: Move the allocation inside the same block by decreasing offset. + + VmaSuballocation suballoc = *srcSuballocIt; + suballoc.offset = dstAllocOffset; + suballoc.hAllocation->ChangeOffset(dstAllocOffset); + m_BytesMoved += srcAllocSize; + ++m_AllocationsMoved; + + VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt; + ++nextSuballocIt; + pSrcMetadata->m_Suballocations.erase(srcSuballocIt); + srcSuballocIt = nextSuballocIt; + + InsertSuballoc(pFreeSpaceMetadata, suballoc); + + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = freeSpaceOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + + moves.push_back(move); + } + // Different block + else + { + // MOVE OPTION 2: Move the allocation to a different block. + + VMA_ASSERT(freeSpaceInfoIndex < srcBlockInfoIndex); + + VmaSuballocation suballoc = *srcSuballocIt; + suballoc.offset = dstAllocOffset; + suballoc.hAllocation->ChangeBlockAllocation(m_hAllocator, pFreeSpaceBlock, dstAllocOffset); + m_BytesMoved += srcAllocSize; + ++m_AllocationsMoved; + + VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt; + ++nextSuballocIt; + pSrcMetadata->m_Suballocations.erase(srcSuballocIt); + srcSuballocIt = nextSuballocIt; + + InsertSuballoc(pFreeSpaceMetadata, suballoc); + + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = freeSpaceOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + + moves.push_back(move); + } + } + else + { + dstAllocOffset = VmaAlignUp(dstOffset, srcAllocAlignment); + + // If the allocation doesn't fit before the end of dstBlock, forward to next block. + while(dstBlockInfoIndex < srcBlockInfoIndex && + dstAllocOffset + srcAllocSize > dstBlockSize) + { + // But before that, register remaining free space at the end of dst block. + freeSpaceDb.Register(dstBlockInfoIndex, dstOffset, dstBlockSize - dstOffset); + + ++dstBlockInfoIndex; + dstOrigBlockIndex = m_BlockInfos[dstBlockInfoIndex].origBlockIndex; + pDstBlock = m_pBlockVector->GetBlock(dstOrigBlockIndex); + pDstMetadata = (VmaBlockMetadata_Generic*)pDstBlock->m_pMetadata; + dstBlockSize = pDstMetadata->GetSize(); + dstOffset = 0; + dstAllocOffset = 0; + } + + // Same block + if(dstBlockInfoIndex == srcBlockInfoIndex) + { + VMA_ASSERT(dstAllocOffset <= srcAllocOffset); + + const bool overlap = dstAllocOffset + srcAllocSize > srcAllocOffset; + + bool skipOver = overlap; + if(overlap && m_OverlappingMoveSupported && dstAllocOffset < srcAllocOffset) + { + // If destination and source place overlap, skip if it would move it + // by only < 1/64 of its size. + skipOver = (srcAllocOffset - dstAllocOffset) * 64 < srcAllocSize; + } + + if(skipOver) + { + freeSpaceDb.Register(dstBlockInfoIndex, dstOffset, srcAllocOffset - dstOffset); + + dstOffset = srcAllocOffset + srcAllocSize; + ++srcSuballocIt; + } + // MOVE OPTION 1: Move the allocation inside the same block by decreasing offset. + else + { + srcSuballocIt->offset = dstAllocOffset; + srcSuballocIt->hAllocation->ChangeOffset(dstAllocOffset); + dstOffset = dstAllocOffset + srcAllocSize; + m_BytesMoved += srcAllocSize; + ++m_AllocationsMoved; + ++srcSuballocIt; + + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = dstOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + + moves.push_back(move); + } + } + // Different block + else + { + // MOVE OPTION 2: Move the allocation to a different block. + + VMA_ASSERT(dstBlockInfoIndex < srcBlockInfoIndex); + VMA_ASSERT(dstAllocOffset + srcAllocSize <= dstBlockSize); + + VmaSuballocation suballoc = *srcSuballocIt; + suballoc.offset = dstAllocOffset; + suballoc.hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlock, dstAllocOffset); + dstOffset = dstAllocOffset + srcAllocSize; + m_BytesMoved += srcAllocSize; + ++m_AllocationsMoved; + + VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt; + ++nextSuballocIt; + pSrcMetadata->m_Suballocations.erase(srcSuballocIt); + srcSuballocIt = nextSuballocIt; + + pDstMetadata->m_Suballocations.push_back(suballoc); + + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = dstOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + + moves.push_back(move); + } + } + } + } + + m_BlockInfos.clear(); + + PostprocessMetadata(); + + return VK_SUCCESS; +} + +void VmaDefragmentationAlgorithm_Fast::PreprocessMetadata() +{ + const size_t blockCount = m_pBlockVector->GetBlockCount(); + for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) + { + VmaBlockMetadata_Generic* const pMetadata = + (VmaBlockMetadata_Generic*)m_pBlockVector->GetBlock(blockIndex)->m_pMetadata; + pMetadata->m_FreeCount = 0; + pMetadata->m_SumFreeSize = pMetadata->GetSize(); + pMetadata->m_FreeSuballocationsBySize.clear(); + for(VmaSuballocationList::iterator it = pMetadata->m_Suballocations.begin(); + it != pMetadata->m_Suballocations.end(); ) + { + if(it->type == VMA_SUBALLOCATION_TYPE_FREE) + { + VmaSuballocationList::iterator nextIt = it; + ++nextIt; + pMetadata->m_Suballocations.erase(it); + it = nextIt; + } + else + { + ++it; + } + } + } +} + +void VmaDefragmentationAlgorithm_Fast::PostprocessMetadata() +{ + const size_t blockCount = m_pBlockVector->GetBlockCount(); + for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) + { + VmaBlockMetadata_Generic* const pMetadata = + (VmaBlockMetadata_Generic*)m_pBlockVector->GetBlock(blockIndex)->m_pMetadata; + const VkDeviceSize blockSize = pMetadata->GetSize(); + + // No allocations in this block - entire area is free. + if(pMetadata->m_Suballocations.empty()) + { + pMetadata->m_FreeCount = 1; + //pMetadata->m_SumFreeSize is already set to blockSize. + VmaSuballocation suballoc = { + 0, // offset + blockSize, // size + VMA_NULL, // hAllocation + VMA_SUBALLOCATION_TYPE_FREE }; + pMetadata->m_Suballocations.push_back(suballoc); + pMetadata->RegisterFreeSuballocation(pMetadata->m_Suballocations.begin()); + } + // There are some allocations in this block. + else + { + VkDeviceSize offset = 0; + VmaSuballocationList::iterator it; + for(it = pMetadata->m_Suballocations.begin(); + it != pMetadata->m_Suballocations.end(); + ++it) + { + VMA_ASSERT(it->type != VMA_SUBALLOCATION_TYPE_FREE); + VMA_ASSERT(it->offset >= offset); + + // Need to insert preceding free space. + if(it->offset > offset) + { + ++pMetadata->m_FreeCount; + const VkDeviceSize freeSize = it->offset - offset; + VmaSuballocation suballoc = { + offset, // offset + freeSize, // size + VMA_NULL, // hAllocation + VMA_SUBALLOCATION_TYPE_FREE }; + VmaSuballocationList::iterator precedingFreeIt = pMetadata->m_Suballocations.insert(it, suballoc); + if(freeSize >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + pMetadata->m_FreeSuballocationsBySize.push_back(precedingFreeIt); + } + } + + pMetadata->m_SumFreeSize -= it->size; + offset = it->offset + it->size; + } + + // Need to insert trailing free space. + if(offset < blockSize) + { + ++pMetadata->m_FreeCount; + const VkDeviceSize freeSize = blockSize - offset; + VmaSuballocation suballoc = { + offset, // offset + freeSize, // size + VMA_NULL, // hAllocation + VMA_SUBALLOCATION_TYPE_FREE }; + VMA_ASSERT(it == pMetadata->m_Suballocations.end()); + VmaSuballocationList::iterator trailingFreeIt = pMetadata->m_Suballocations.insert(it, suballoc); + if(freeSize > VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + pMetadata->m_FreeSuballocationsBySize.push_back(trailingFreeIt); + } + } + + VMA_SORT( + pMetadata->m_FreeSuballocationsBySize.begin(), + pMetadata->m_FreeSuballocationsBySize.end(), + VmaSuballocationItemSizeLess()); + } + + VMA_HEAVY_ASSERT(pMetadata->Validate()); + } +} + +void VmaDefragmentationAlgorithm_Fast::InsertSuballoc(VmaBlockMetadata_Generic* pMetadata, const VmaSuballocation& suballoc) +{ + // TODO: Optimize somehow. Remember iterator instead of searching for it linearly. + VmaSuballocationList::iterator it = pMetadata->m_Suballocations.begin(); + while(it != pMetadata->m_Suballocations.end()) + { + if(it->offset < suballoc.offset) + { + ++it; + } + } + pMetadata->m_Suballocations.insert(it, suballoc); +} + +//////////////////////////////////////////////////////////////////////////////// +// VmaBlockVectorDefragmentationContext + +VmaBlockVectorDefragmentationContext::VmaBlockVectorDefragmentationContext( + VmaAllocator hAllocator, + VmaPool hCustomPool, + VmaBlockVector* pBlockVector, + uint32_t currFrameIndex) : + res(VK_SUCCESS), + mutexLocked(false), + blockContexts(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + defragmentationMoves(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + defragmentationMovesProcessed(0), + defragmentationMovesCommitted(0), + hasDefragmentationPlan(0), + m_hAllocator(hAllocator), + m_hCustomPool(hCustomPool), + m_pBlockVector(pBlockVector), + m_CurrFrameIndex(currFrameIndex), + m_pAlgorithm(VMA_NULL), + m_Allocations(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), + m_AllAllocations(false) +{ +} + +VmaBlockVectorDefragmentationContext::~VmaBlockVectorDefragmentationContext() +{ + vma_delete(m_hAllocator, m_pAlgorithm); +} + +void VmaBlockVectorDefragmentationContext::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) +{ + AllocInfo info = { hAlloc, pChanged }; + m_Allocations.push_back(info); +} + +void VmaBlockVectorDefragmentationContext::Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags) +{ + const bool allAllocations = m_AllAllocations || + m_Allocations.size() == m_pBlockVector->CalcAllocationCount(); + + /******************************** + HERE IS THE CHOICE OF DEFRAGMENTATION ALGORITHM. + ********************************/ + + /* + Fast algorithm is supported only when certain criteria are met: + - VMA_DEBUG_MARGIN is 0. + - All allocations in this block vector are moveable. + - There is no possibility of image/buffer granularity conflict. + - The defragmentation is not incremental + */ + if(VMA_DEBUG_MARGIN == 0 && + allAllocations && + !m_pBlockVector->IsBufferImageGranularityConflictPossible() && + !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)) + { + m_pAlgorithm = vma_new(m_hAllocator, VmaDefragmentationAlgorithm_Fast)( + m_hAllocator, m_pBlockVector, m_CurrFrameIndex, overlappingMoveSupported); + } + else + { + m_pAlgorithm = vma_new(m_hAllocator, VmaDefragmentationAlgorithm_Generic)( + m_hAllocator, m_pBlockVector, m_CurrFrameIndex, overlappingMoveSupported); + } + + if(allAllocations) + { + m_pAlgorithm->AddAll(); + } + else + { + for(size_t i = 0, count = m_Allocations.size(); i < count; ++i) + { + m_pAlgorithm->AddAllocation(m_Allocations[i].hAlloc, m_Allocations[i].pChanged); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// VmaDefragmentationContext + +VmaDefragmentationContext_T::VmaDefragmentationContext_T( + VmaAllocator hAllocator, + uint32_t currFrameIndex, + uint32_t flags, + VmaDefragmentationStats* pStats) : + m_hAllocator(hAllocator), + m_CurrFrameIndex(currFrameIndex), + m_Flags(flags), + m_pStats(pStats), + m_CustomPoolContexts(VmaStlAllocator(hAllocator->GetAllocationCallbacks())) +{ + memset(m_DefaultPoolContexts, 0, sizeof(m_DefaultPoolContexts)); +} + +VmaDefragmentationContext_T::~VmaDefragmentationContext_T() +{ + for(size_t i = m_CustomPoolContexts.size(); i--; ) + { + VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_CustomPoolContexts[i]; + pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats); + vma_delete(m_hAllocator, pBlockVectorCtx); + } + for(size_t i = m_hAllocator->m_MemProps.memoryTypeCount; i--; ) + { + VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_DefaultPoolContexts[i]; + if(pBlockVectorCtx) + { + pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats); + vma_delete(m_hAllocator, pBlockVectorCtx); + } + } +} + +void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, const VmaPool* pPools) +{ + for(uint32_t poolIndex = 0; poolIndex < poolCount; ++poolIndex) + { + VmaPool pool = pPools[poolIndex]; + VMA_ASSERT(pool); + // Pools with algorithm other than default are not defragmented. + if(pool->m_BlockVector.GetAlgorithm() == 0) + { + VmaBlockVectorDefragmentationContext* pBlockVectorDefragCtx = VMA_NULL; + + for(size_t i = m_CustomPoolContexts.size(); i--; ) + { + if(m_CustomPoolContexts[i]->GetCustomPool() == pool) + { + pBlockVectorDefragCtx = m_CustomPoolContexts[i]; + break; + } + } + + if(!pBlockVectorDefragCtx) + { + pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)( + m_hAllocator, + pool, + &pool->m_BlockVector, + m_CurrFrameIndex); + m_CustomPoolContexts.push_back(pBlockVectorDefragCtx); + } + + pBlockVectorDefragCtx->AddAll(); + } + } +} + +void VmaDefragmentationContext_T::AddAllocations( + uint32_t allocationCount, + const VmaAllocation* pAllocations, + VkBool32* pAllocationsChanged) +{ + // Dispatch pAllocations among defragmentators. Create them when necessary. + for(uint32_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + const VmaAllocation hAlloc = pAllocations[allocIndex]; + VMA_ASSERT(hAlloc); + // DedicatedAlloc cannot be defragmented. + if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) && + // Lost allocation cannot be defragmented. + (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)) + { + VmaBlockVectorDefragmentationContext* pBlockVectorDefragCtx = VMA_NULL; + + const VmaPool hAllocPool = hAlloc->GetBlock()->GetParentPool(); + // This allocation belongs to custom pool. + if(hAllocPool != VK_NULL_HANDLE) + { + // Pools with algorithm other than default are not defragmented. + if(hAllocPool->m_BlockVector.GetAlgorithm() == 0) + { + for(size_t i = m_CustomPoolContexts.size(); i--; ) + { + if(m_CustomPoolContexts[i]->GetCustomPool() == hAllocPool) + { + pBlockVectorDefragCtx = m_CustomPoolContexts[i]; + break; + } + } + if(!pBlockVectorDefragCtx) + { + pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)( + m_hAllocator, + hAllocPool, + &hAllocPool->m_BlockVector, + m_CurrFrameIndex); + m_CustomPoolContexts.push_back(pBlockVectorDefragCtx); + } + } + } + // This allocation belongs to default pool. + else + { + const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex(); + pBlockVectorDefragCtx = m_DefaultPoolContexts[memTypeIndex]; + if(!pBlockVectorDefragCtx) + { + pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)( + m_hAllocator, + VMA_NULL, // hCustomPool + m_hAllocator->m_pBlockVectors[memTypeIndex], + m_CurrFrameIndex); + m_DefaultPoolContexts[memTypeIndex] = pBlockVectorDefragCtx; + } + } + + if(pBlockVectorDefragCtx) + { + VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ? + &pAllocationsChanged[allocIndex] : VMA_NULL; + pBlockVectorDefragCtx->AddAllocation(hAlloc, pChanged); + } + } + } +} + +VkResult VmaDefragmentationContext_T::Defragment( + VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove, + VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove, + VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags) +{ + if(pStats) + { + memset(pStats, 0, sizeof(VmaDefragmentationStats)); + } + + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + // For incremental defragmetnations, we just earmark how much we can move + // The real meat is in the defragmentation steps + m_MaxCpuBytesToMove = maxCpuBytesToMove; + m_MaxCpuAllocationsToMove = maxCpuAllocationsToMove; + + m_MaxGpuBytesToMove = maxGpuBytesToMove; + m_MaxGpuAllocationsToMove = maxGpuAllocationsToMove; + + if(m_MaxCpuBytesToMove == 0 && m_MaxCpuAllocationsToMove == 0 && + m_MaxGpuBytesToMove == 0 && m_MaxGpuAllocationsToMove == 0) + return VK_SUCCESS; + + return VK_NOT_READY; + } + + if(commandBuffer == VK_NULL_HANDLE) + { + maxGpuBytesToMove = 0; + maxGpuAllocationsToMove = 0; + } + + VkResult res = VK_SUCCESS; + + // Process default pools. + for(uint32_t memTypeIndex = 0; + memTypeIndex < m_hAllocator->GetMemoryTypeCount() && res >= VK_SUCCESS; + ++memTypeIndex) + { + VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex]; + if(pBlockVectorCtx) + { + VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + pStats, flags, + maxCpuBytesToMove, maxCpuAllocationsToMove, + maxGpuBytesToMove, maxGpuAllocationsToMove, + commandBuffer); + if(pBlockVectorCtx->res != VK_SUCCESS) + { + res = pBlockVectorCtx->res; + } + } + } + + // Process custom pools. + for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size(); + customCtxIndex < customCtxCount && res >= VK_SUCCESS; + ++customCtxIndex) + { + VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex]; + VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + pStats, flags, + maxCpuBytesToMove, maxCpuAllocationsToMove, + maxGpuBytesToMove, maxGpuAllocationsToMove, + commandBuffer); + if(pBlockVectorCtx->res != VK_SUCCESS) + { + res = pBlockVectorCtx->res; + } + } + + return res; +} + +VkResult VmaDefragmentationContext_T::DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo) +{ + VmaDefragmentationPassMoveInfo* pCurrentMove = pInfo->pMoves; + uint32_t movesLeft = pInfo->moveCount; + + // Process default pools. + for(uint32_t memTypeIndex = 0; + memTypeIndex < m_hAllocator->GetMemoryTypeCount(); + ++memTypeIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex]; + if(pBlockVectorCtx) + { + VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + m_pStats, m_Flags, + m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove, + m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove, + VK_NULL_HANDLE); + + if(pBlockVectorCtx->res < VK_SUCCESS) + continue; + + pBlockVectorCtx->hasDefragmentationPlan = true; + } + + const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations( + pBlockVectorCtx, + pCurrentMove, movesLeft); + + movesLeft -= processed; + pCurrentMove += processed; + } + } + + // Process custom pools. + for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size(); + customCtxIndex < customCtxCount; + ++customCtxIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex]; + VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + m_pStats, m_Flags, + m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove, + m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove, + VK_NULL_HANDLE); + + if(pBlockVectorCtx->res < VK_SUCCESS) + continue; + + pBlockVectorCtx->hasDefragmentationPlan = true; + } + + const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations( + pBlockVectorCtx, + pCurrentMove, movesLeft); + + movesLeft -= processed; + pCurrentMove += processed; + } + + pInfo->moveCount = pInfo->moveCount - movesLeft; + + return VK_SUCCESS; +} +VkResult VmaDefragmentationContext_T::DefragmentPassEnd() +{ + VkResult res = VK_SUCCESS; + + // Process default pools. + for(uint32_t memTypeIndex = 0; + memTypeIndex < m_hAllocator->GetMemoryTypeCount(); + ++memTypeIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex]; + if(pBlockVectorCtx) + { + VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + res = VK_NOT_READY; + continue; + } + + pBlockVectorCtx->GetBlockVector()->CommitDefragmentations( + pBlockVectorCtx, m_pStats); + + if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted) + res = VK_NOT_READY; + } + } + + // Process custom pools. + for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size(); + customCtxIndex < customCtxCount; + ++customCtxIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex]; + VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + res = VK_NOT_READY; + continue; + } + + pBlockVectorCtx->GetBlockVector()->CommitDefragmentations( + pBlockVectorCtx, m_pStats); + + if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted) + res = VK_NOT_READY; + } + + return res; +} + //////////////////////////////////////////////////////////////////////////////// // VmaRecorder @@ -8145,8 +15001,7 @@ VmaRecorder::VmaRecorder() : m_UseMutex(true), m_Flags(0), m_File(VMA_NULL), - m_Freq(INT64_MAX), - m_StartCounter(INT64_MAX) + m_RecordingStartTime(std::chrono::high_resolution_clock::now()) { } @@ -8155,19 +15010,27 @@ VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex) m_UseMutex = useMutex; m_Flags = settings.flags; - QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq); - QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter); - +#if defined(_WIN32) // Open file for writing. errno_t err = fopen_s(&m_File, settings.pFilePath, "wb"); + if(err != 0) { return VK_ERROR_INITIALIZATION_FAILED; } +#else + // Open file for writing. + m_File = fopen(settings.pFilePath, "wb"); + + if(m_File == 0) + { + return VK_ERROR_INITIALIZATION_FAILED; + } +#endif // Write header. fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); - fprintf(m_File, "%s\n", "1,3"); + fprintf(m_File, "%s\n", "1,8"); return VK_SUCCESS; } @@ -8210,8 +15073,8 @@ void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo.memoryTypeIndex, createInfo.flags, createInfo.blockSize, - createInfo.minBlockCount, - createInfo.maxBlockCount, + (uint64_t)createInfo.minBlockCount, + (uint64_t)createInfo.maxBlockCount, createInfo.frameInUseCount, pool); Flush(); @@ -8253,6 +15116,32 @@ void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex, Flush(); } +void VmaRecorder::RecordAllocateMemoryPages(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + uint64_t allocationCount, + const VmaAllocation* pAllocations) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(createInfo.flags, createInfo.pUserData); + fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryPages,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,", callParams.threadId, callParams.time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + createInfo.flags, + createInfo.usage, + createInfo.requiredFlags, + createInfo.preferredFlags, + createInfo.memoryTypeBits, + createInfo.pool); + PrintPointerList(allocationCount, pAllocations); + fprintf(m_File, ",%s\n", userDataStr.GetString()); + Flush(); +} + void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex, const VkMemoryRequirements& vkMemReq, bool requiresDedicatedAllocation, @@ -8323,6 +15212,20 @@ void VmaRecorder::RecordFreeMemory(uint32_t frameIndex, Flush(); } +void VmaRecorder::RecordFreeMemoryPages(uint32_t frameIndex, + uint64_t allocationCount, + const VmaAllocation* pAllocations) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaFreeMemoryPages,", callParams.threadId, callParams.time, frameIndex); + PrintPointerList(allocationCount, pAllocations); + fprintf(m_File, "\n"); + Flush(); +} + void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex, VmaAllocation allocation, const void* pUserData) @@ -8525,6 +15428,54 @@ void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex, Flush(); } +void VmaRecorder::RecordDefragmentationBegin(uint32_t frameIndex, + const VmaDefragmentationInfo2& info, + VmaDefragmentationContext ctx) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDefragmentationBegin,%u,", callParams.threadId, callParams.time, frameIndex, + info.flags); + PrintPointerList(info.allocationCount, info.pAllocations); + fprintf(m_File, ","); + PrintPointerList(info.poolCount, info.pPools); + fprintf(m_File, ",%llu,%u,%llu,%u,%p,%p\n", + info.maxCpuBytesToMove, + info.maxCpuAllocationsToMove, + info.maxGpuBytesToMove, + info.maxGpuAllocationsToMove, + info.commandBuffer, + ctx); + Flush(); +} + +void VmaRecorder::RecordDefragmentationEnd(uint32_t frameIndex, + VmaDefragmentationContext ctx) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDefragmentationEnd,%p\n", callParams.threadId, callParams.time, frameIndex, + ctx); + Flush(); +} + +void VmaRecorder::RecordSetPoolName(uint32_t frameIndex, + VmaPool pool, + const char* name) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaSetPoolName,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + pool, name != VMA_NULL ? name : ""); + Flush(); +} + VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData) { if(pUserData != VMA_NULL) @@ -8535,7 +15486,8 @@ VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, } else { - sprintf_s(m_PtrStr, "%p", pUserData); + // If VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is not specified, convert the string's memory address to a string and store it. + snprintf(m_PtrStr, 17, "%p", pUserData); m_Str = m_PtrStr; } } @@ -8548,10 +15500,16 @@ VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, void VmaRecorder::WriteConfiguration( const VkPhysicalDeviceProperties& devProps, const VkPhysicalDeviceMemoryProperties& memProps, - bool dedicatedAllocationExtensionEnabled) + uint32_t vulkanApiVersion, + bool dedicatedAllocationExtensionEnabled, + bool bindMemory2ExtensionEnabled, + bool memoryBudgetExtensionEnabled, + bool deviceCoherentMemoryExtensionEnabled) { fprintf(m_File, "Config,Begin\n"); + fprintf(m_File, "VulkanApiVersion,%u,%u\n", VK_VERSION_MAJOR(vulkanApiVersion), VK_VERSION_MINOR(vulkanApiVersion)); + fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion); fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion); fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID); @@ -8577,6 +15535,9 @@ void VmaRecorder::WriteConfiguration( } fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0); + fprintf(m_File, "Extension,VK_KHR_bind_memory2,%u\n", bindMemory2ExtensionEnabled ? 1 : 0); + fprintf(m_File, "Extension,VK_EXT_memory_budget,%u\n", memoryBudgetExtensionEnabled ? 1 : 0); + fprintf(m_File, "Extension,VK_AMD_device_coherent_memory,%u\n", deviceCoherentMemoryExtensionEnabled ? 1 : 0); fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0); fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT); @@ -8593,11 +15554,34 @@ void VmaRecorder::WriteConfiguration( void VmaRecorder::GetBasicParams(CallParams& outParams) { - outParams.threadId = GetCurrentThreadId(); + #if defined(_WIN32) + outParams.threadId = GetCurrentThreadId(); + #else + // Use C++11 features to get thread id and convert it to uint32_t. + // There is room for optimization since sstream is quite slow. + // Is there a better way to convert std::this_thread::get_id() to uint32_t? + std::thread::id thread_id = std::this_thread::get_id(); + stringstream thread_id_to_string_converter; + thread_id_to_string_converter << thread_id; + string thread_id_as_string = thread_id_to_string_converter.str(); + outParams.threadId = static_cast(std::stoi(thread_id_as_string.c_str())); + #endif + + auto current_time = std::chrono::high_resolution_clock::now(); - LARGE_INTEGER counter; - QueryPerformanceCounter(&counter); - outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq; + outParams.time = std::chrono::duration(current_time - m_RecordingStartTime).count(); +} + +void VmaRecorder::PrintPointerList(uint64_t count, const VmaAllocation* pItems) +{ + if(count) + { + fprintf(m_File, "%p", pItems[0]); + for(uint64_t i = 1; i < count; ++i) + { + fprintf(m_File, " %p", pItems[i]); + } + } } void VmaRecorder::Flush() @@ -8610,37 +15594,106 @@ void VmaRecorder::Flush() #endif // #if VMA_RECORDING_ENABLED +//////////////////////////////////////////////////////////////////////////////// +// VmaAllocationObjectAllocator + +VmaAllocationObjectAllocator::VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks) : + m_Allocator(pAllocationCallbacks, 1024) +{ +} + +template VmaAllocation VmaAllocationObjectAllocator::Allocate(Types... args) +{ + VmaMutexLock mutexLock(m_Mutex); + return m_Allocator.Alloc(std::forward(args)...); +} + +void VmaAllocationObjectAllocator::Free(VmaAllocation hAlloc) +{ + VmaMutexLock mutexLock(m_Mutex); + m_Allocator.Free(hAlloc); +} + //////////////////////////////////////////////////////////////////////////////// // VmaAllocator_T VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0), + m_VulkanApiVersion(pCreateInfo->vulkanApiVersion != 0 ? pCreateInfo->vulkanApiVersion : VK_API_VERSION_1_0), m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0), + m_UseKhrBindMemory2((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT) != 0), + m_UseExtMemoryBudget((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT) != 0), + m_UseAmdDeviceCoherentMemory((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT) != 0), + m_UseKhrBufferDeviceAddress((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT) != 0), m_hDevice(pCreateInfo->device), + m_hInstance(pCreateInfo->instance), m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL), m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ? *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks), + m_AllocationObjectAllocator(&m_AllocationCallbacks), + m_HeapSizeLimitMask(0), m_PreferredLargeHeapBlockSize(0), m_PhysicalDevice(pCreateInfo->physicalDevice), m_CurrentFrameIndex(0), + m_GpuDefragmentationMemoryTypeBits(UINT32_MAX), m_Pools(VmaStlAllocator(GetAllocationCallbacks())), - m_NextPoolId(0) + m_NextPoolId(0), + m_GlobalMemoryTypeBits(UINT32_MAX) #if VMA_RECORDING_ENABLED ,m_pRecorder(VMA_NULL) #endif { + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + m_UseKhrDedicatedAllocation = false; + m_UseKhrBindMemory2 = false; + } + if(VMA_DEBUG_DETECT_CORRUPTION) { // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it. VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0); } - VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device); + VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device && pCreateInfo->instance); -#if !(VMA_DEDICATED_ALLOCATION) - if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0) + if(m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0)) { - VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros."); +#if !(VMA_DEDICATED_ALLOCATION) + if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros."); + } +#endif +#if !(VMA_BIND_MEMORY2) + if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT) != 0) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT set but required extension is disabled by preprocessor macros."); + } +#endif + } +#if !(VMA_MEMORY_BUDGET) + if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT) != 0) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT set but required extension is disabled by preprocessor macros."); + } +#endif +#if !(VMA_BUFFER_DEVICE_ADDRESS) + if(m_UseKhrBufferDeviceAddress) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT is set but required extension or Vulkan 1.2 is not available in your Vulkan header or its support in VMA has been disabled by a preprocessor macro."); + } +#endif +#if VMA_VULKAN_VERSION < 1002000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 2, 0)) + { + VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_2 but required Vulkan version is disabled by preprocessor macros."); + } +#endif +#if VMA_VULKAN_VERSION < 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_1 but required Vulkan version is disabled by preprocessor macros."); } #endif @@ -8650,14 +15703,11 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors)); memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations)); - - for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i) - { - m_HeapSizeLimit[i] = VK_WHOLE_SIZE; - } + memset(&m_VulkanFunctions, 0, sizeof(m_VulkanFunctions)); if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL) { + m_DeviceMemoryCallbacks.pUserData = pCreateInfo->pDeviceMemoryCallbacks->pUserData; m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate; m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree; } @@ -8667,9 +15717,16 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties); (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps); + VMA_ASSERT(VmaIsPow2(VMA_DEBUG_ALIGNMENT)); + VMA_ASSERT(VmaIsPow2(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY)); + VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.bufferImageGranularity)); + VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.nonCoherentAtomSize)); + m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ? pCreateInfo->preferredLargeHeapBlockSize : static_cast(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE); + m_GlobalMemoryTypeBits = CalculateGlobalMemoryTypeBits(); + if(pCreateInfo->pHeapSizeLimit != VMA_NULL) { for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex) @@ -8677,7 +15734,7 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex]; if(limit != VK_WHOLE_SIZE) { - m_HeapSizeLimit[heapIndex] = limit; + m_HeapSizeLimitMask |= 1u << heapIndex; if(limit < m_MemProps.memoryHeaps[heapIndex].size) { m_MemProps.memoryHeaps[heapIndex].size = limit; @@ -8692,13 +15749,15 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)( this, + VK_NULL_HANDLE, // hParentPool memTypeIndex, preferredBlockSize, 0, SIZE_MAX, GetBufferImageGranularity(), pCreateInfo->frameInUseCount, - false); // isCustomPool + false, // explicitBlockSize + false); // linearAlgorithm // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here, // becase minBlockCount is 0. m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator(GetAllocationCallbacks())); @@ -8723,7 +15782,11 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) m_pRecorder->WriteConfiguration( m_PhysicalDeviceProperties, m_MemProps, - m_UseKhrDedicatedAllocation); + m_VulkanApiVersion, + m_UseKhrDedicatedAllocation, + m_UseKhrBindMemory2, + m_UseExtMemoryBudget, + m_UseAmdDeviceCoherentMemory); m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); #else VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1."); @@ -8731,6 +15794,13 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) #endif } +#if VMA_MEMORY_BUDGET + if(m_UseExtMemoryBudget) + { + UpdateVulkanBudget(); + } +#endif // #if VMA_MEMORY_BUDGET + return res; } @@ -8748,6 +15818,11 @@ VmaAllocator_T::~VmaAllocator_T() for(size_t i = GetMemoryTypeCount(); i--; ) { + if(m_pDedicatedAllocations[i] != VMA_NULL && !m_pDedicatedAllocations[i]->empty()) + { + VMA_ASSERT(0 && "Unfreed dedicated allocations found."); + } + vma_delete(this, m_pDedicatedAllocations[i]); vma_delete(this, m_pBlockVectors[i]); } @@ -8756,64 +15831,174 @@ VmaAllocator_T::~VmaAllocator_T() void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions) { #if VMA_STATIC_VULKAN_FUNCTIONS == 1 - m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties; - m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties; - m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory; - m_VulkanFunctions.vkFreeMemory = &vkFreeMemory; - m_VulkanFunctions.vkMapMemory = &vkMapMemory; - m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory; - m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges; - m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges; - m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory; - m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory; - m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements; - m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements; - m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer; - m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer; - m_VulkanFunctions.vkCreateImage = &vkCreateImage; - m_VulkanFunctions.vkDestroyImage = &vkDestroyImage; -#if VMA_DEDICATED_ALLOCATION - if(m_UseKhrDedicatedAllocation) + ImportVulkanFunctions_Static(); +#endif + + if(pVulkanFunctions != VMA_NULL) { - m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = - (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR"); - m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = - (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR"); + ImportVulkanFunctions_Custom(pVulkanFunctions); } -#endif // #if VMA_DEDICATED_ALLOCATION + +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + ImportVulkanFunctions_Dynamic(); +#endif + + ValidateVulkanFunctions(); +} + +#if VMA_STATIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ImportVulkanFunctions_Static() +{ + // Vulkan 1.0 + m_VulkanFunctions.vkGetPhysicalDeviceProperties = (PFN_vkGetPhysicalDeviceProperties)vkGetPhysicalDeviceProperties; + m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = (PFN_vkGetPhysicalDeviceMemoryProperties)vkGetPhysicalDeviceMemoryProperties; + m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory; + m_VulkanFunctions.vkFreeMemory = (PFN_vkFreeMemory)vkFreeMemory; + m_VulkanFunctions.vkMapMemory = (PFN_vkMapMemory)vkMapMemory; + m_VulkanFunctions.vkUnmapMemory = (PFN_vkUnmapMemory)vkUnmapMemory; + m_VulkanFunctions.vkFlushMappedMemoryRanges = (PFN_vkFlushMappedMemoryRanges)vkFlushMappedMemoryRanges; + m_VulkanFunctions.vkInvalidateMappedMemoryRanges = (PFN_vkInvalidateMappedMemoryRanges)vkInvalidateMappedMemoryRanges; + m_VulkanFunctions.vkBindBufferMemory = (PFN_vkBindBufferMemory)vkBindBufferMemory; + m_VulkanFunctions.vkBindImageMemory = (PFN_vkBindImageMemory)vkBindImageMemory; + m_VulkanFunctions.vkGetBufferMemoryRequirements = (PFN_vkGetBufferMemoryRequirements)vkGetBufferMemoryRequirements; + m_VulkanFunctions.vkGetImageMemoryRequirements = (PFN_vkGetImageMemoryRequirements)vkGetImageMemoryRequirements; + m_VulkanFunctions.vkCreateBuffer = (PFN_vkCreateBuffer)vkCreateBuffer; + m_VulkanFunctions.vkDestroyBuffer = (PFN_vkDestroyBuffer)vkDestroyBuffer; + m_VulkanFunctions.vkCreateImage = (PFN_vkCreateImage)vkCreateImage; + m_VulkanFunctions.vkDestroyImage = (PFN_vkDestroyImage)vkDestroyImage; + m_VulkanFunctions.vkCmdCopyBuffer = (PFN_vkCmdCopyBuffer)vkCmdCopyBuffer; + + // Vulkan 1.1 +#if VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = (PFN_vkGetBufferMemoryRequirements2)vkGetBufferMemoryRequirements2; + m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = (PFN_vkGetImageMemoryRequirements2)vkGetImageMemoryRequirements2; + m_VulkanFunctions.vkBindBufferMemory2KHR = (PFN_vkBindBufferMemory2)vkBindBufferMemory2; + m_VulkanFunctions.vkBindImageMemory2KHR = (PFN_vkBindImageMemory2)vkBindImageMemory2; + m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = (PFN_vkGetPhysicalDeviceMemoryProperties2)vkGetPhysicalDeviceMemoryProperties2; + } +#endif +} + #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1 +void VmaAllocator_T::ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions) +{ + VMA_ASSERT(pVulkanFunctions != VMA_NULL); + #define VMA_COPY_IF_NOT_NULL(funcName) \ if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName; - if(pVulkanFunctions != VMA_NULL) - { - VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties); - VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties); - VMA_COPY_IF_NOT_NULL(vkAllocateMemory); - VMA_COPY_IF_NOT_NULL(vkFreeMemory); - VMA_COPY_IF_NOT_NULL(vkMapMemory); - VMA_COPY_IF_NOT_NULL(vkUnmapMemory); - VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges); - VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges); - VMA_COPY_IF_NOT_NULL(vkBindBufferMemory); - VMA_COPY_IF_NOT_NULL(vkBindImageMemory); - VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements); - VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements); - VMA_COPY_IF_NOT_NULL(vkCreateBuffer); - VMA_COPY_IF_NOT_NULL(vkDestroyBuffer); - VMA_COPY_IF_NOT_NULL(vkCreateImage); - VMA_COPY_IF_NOT_NULL(vkDestroyImage); -#if VMA_DEDICATED_ALLOCATION - VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR); - VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR); + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties); + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties); + VMA_COPY_IF_NOT_NULL(vkAllocateMemory); + VMA_COPY_IF_NOT_NULL(vkFreeMemory); + VMA_COPY_IF_NOT_NULL(vkMapMemory); + VMA_COPY_IF_NOT_NULL(vkUnmapMemory); + VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges); + VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges); + VMA_COPY_IF_NOT_NULL(vkBindBufferMemory); + VMA_COPY_IF_NOT_NULL(vkBindImageMemory); + VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements); + VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements); + VMA_COPY_IF_NOT_NULL(vkCreateBuffer); + VMA_COPY_IF_NOT_NULL(vkDestroyBuffer); + VMA_COPY_IF_NOT_NULL(vkCreateImage); + VMA_COPY_IF_NOT_NULL(vkDestroyImage); + VMA_COPY_IF_NOT_NULL(vkCmdCopyBuffer); + +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR); + VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR); +#endif + +#if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 + VMA_COPY_IF_NOT_NULL(vkBindBufferMemory2KHR); + VMA_COPY_IF_NOT_NULL(vkBindImageMemory2KHR); +#endif + +#if VMA_MEMORY_BUDGET + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties2KHR); #endif - } #undef VMA_COPY_IF_NOT_NULL +} - // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1 - // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions. +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ImportVulkanFunctions_Dynamic() +{ +#define VMA_FETCH_INSTANCE_FUNC(memberName, functionPointerType, functionNameString) \ + if(m_VulkanFunctions.memberName == VMA_NULL) \ + m_VulkanFunctions.memberName = \ + (functionPointerType)vkGetInstanceProcAddr(m_hInstance, functionNameString); +#define VMA_FETCH_DEVICE_FUNC(memberName, functionPointerType, functionNameString) \ + if(m_VulkanFunctions.memberName == VMA_NULL) \ + m_VulkanFunctions.memberName = \ + (functionPointerType)vkGetDeviceProcAddr(m_hDevice, functionNameString); + + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceProperties, PFN_vkGetPhysicalDeviceProperties, "vkGetPhysicalDeviceProperties"); + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties, PFN_vkGetPhysicalDeviceMemoryProperties, "vkGetPhysicalDeviceMemoryProperties"); + VMA_FETCH_DEVICE_FUNC(vkAllocateMemory, PFN_vkAllocateMemory, "vkAllocateMemory"); + VMA_FETCH_DEVICE_FUNC(vkFreeMemory, PFN_vkFreeMemory, "vkFreeMemory"); + VMA_FETCH_DEVICE_FUNC(vkMapMemory, PFN_vkMapMemory, "vkMapMemory"); + VMA_FETCH_DEVICE_FUNC(vkUnmapMemory, PFN_vkUnmapMemory, "vkUnmapMemory"); + VMA_FETCH_DEVICE_FUNC(vkFlushMappedMemoryRanges, PFN_vkFlushMappedMemoryRanges, "vkFlushMappedMemoryRanges"); + VMA_FETCH_DEVICE_FUNC(vkInvalidateMappedMemoryRanges, PFN_vkInvalidateMappedMemoryRanges, "vkInvalidateMappedMemoryRanges"); + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory, PFN_vkBindBufferMemory, "vkBindBufferMemory"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory, PFN_vkBindImageMemory, "vkBindImageMemory"); + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements, PFN_vkGetBufferMemoryRequirements, "vkGetBufferMemoryRequirements"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements, PFN_vkGetImageMemoryRequirements, "vkGetImageMemoryRequirements"); + VMA_FETCH_DEVICE_FUNC(vkCreateBuffer, PFN_vkCreateBuffer, "vkCreateBuffer"); + VMA_FETCH_DEVICE_FUNC(vkDestroyBuffer, PFN_vkDestroyBuffer, "vkDestroyBuffer"); + VMA_FETCH_DEVICE_FUNC(vkCreateImage, PFN_vkCreateImage, "vkCreateImage"); + VMA_FETCH_DEVICE_FUNC(vkDestroyImage, PFN_vkDestroyImage, "vkDestroyImage"); + VMA_FETCH_DEVICE_FUNC(vkCmdCopyBuffer, PFN_vkCmdCopyBuffer, "vkCmdCopyBuffer"); + +#if VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2, "vkGetBufferMemoryRequirements2"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2, "vkGetImageMemoryRequirements2"); + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2, "vkBindBufferMemory2"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2, "vkBindImageMemory2"); + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2, "vkGetPhysicalDeviceMemoryProperties2"); + } +#endif + +#if VMA_DEDICATED_ALLOCATION + if(m_UseKhrDedicatedAllocation) + { + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2KHR, "vkGetBufferMemoryRequirements2KHR"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2KHR, "vkGetImageMemoryRequirements2KHR"); + } +#endif + +#if VMA_BIND_MEMORY2 + if(m_UseKhrBindMemory2) + { + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2KHR, "vkBindBufferMemory2KHR"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2KHR, "vkBindImageMemory2KHR"); + } +#endif // #if VMA_BIND_MEMORY2 + +#if VMA_MEMORY_BUDGET + if(m_UseExtMemoryBudget) + { + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2KHR, "vkGetPhysicalDeviceMemoryProperties2KHR"); + } +#endif // #if VMA_MEMORY_BUDGET + +#undef VMA_FETCH_DEVICE_FUNC +#undef VMA_FETCH_INSTANCE_FUNC +} + +#endif // #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ValidateVulkanFunctions() +{ VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL); @@ -8830,13 +16015,30 @@ void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunc VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL); -#if VMA_DEDICATED_ALLOCATION - if(m_UseKhrDedicatedAllocation) + VMA_ASSERT(m_VulkanFunctions.vkCmdCopyBuffer != VMA_NULL); + +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrDedicatedAllocation) { VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL); } #endif + +#if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrBindMemory2) + { + VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory2KHR != VMA_NULL); + VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory2KHR != VMA_NULL); + } +#endif + +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 + if(m_UseExtMemoryBudget || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR != VMA_NULL); + } +#endif } VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex) @@ -8844,7 +16046,7 @@ VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex) const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex); const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size; const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE; - return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize; + return VmaAlignUp(isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize, (VkDeviceSize)32); } VkResult VmaAllocator_T::AllocateMemoryOfType( @@ -8852,14 +16054,16 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VkDeviceSize alignment, bool dedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, VmaSuballocationType suballocType, - VmaAllocation* pAllocation) + size_t allocationCount, + VmaAllocation* pAllocations) { - VMA_ASSERT(pAllocation != VMA_NULL); - VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size); + VMA_ASSERT(pAllocations != VMA_NULL); + VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, AllocationCount=%zu, Size=%llu", memTypeIndex, allocationCount, size); VmaAllocationCreateInfo finalCreateInfo = createInfo; @@ -8869,6 +16073,11 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( { finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT; } + // If memory is lazily allocated, it should be always dedicated. + if(finalCreateInfo.usage == VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED) + { + finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + } VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex]; VMA_ASSERT(blockVector); @@ -8899,24 +16108,27 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( size, suballocType, memTypeIndex, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0, (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0, (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0, finalCreateInfo.pUserData, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, - pAllocation); + allocationCount, + pAllocations); } } else { VkResult res = blockVector->Allocate( - VK_NULL_HANDLE, // hCurrentPool m_CurrentFrameIndex.load(), size, alignment, finalCreateInfo, suballocType, - pAllocation); + allocationCount, + pAllocations); if(res == VK_SUCCESS) { return res; @@ -8933,12 +16145,15 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( size, suballocType, memTypeIndex, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0, (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0, (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0, finalCreateInfo.pUserData, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, - pAllocation); + allocationCount, + pAllocations); if(res == VK_SUCCESS) { // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here. @@ -8959,38 +16174,147 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( VkDeviceSize size, VmaSuballocationType suballocType, uint32_t memTypeIndex, + bool withinBudget, bool map, bool isUserDataString, void* pUserData, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, - VmaAllocation* pAllocation) + size_t allocationCount, + VmaAllocation* pAllocations) { - VMA_ASSERT(pAllocation); + VMA_ASSERT(allocationCount > 0 && pAllocations); + + if(withinBudget) + { + const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex); + VmaBudget heapBudget = {}; + GetBudget(&heapBudget, heapIndex, 1); + if(heapBudget.usage + size * allocationCount > heapBudget.budget) + { + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + } VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; allocInfo.memoryTypeIndex = memTypeIndex; allocInfo.allocationSize = size; -#if VMA_DEDICATED_ALLOCATION +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR }; - if(m_UseKhrDedicatedAllocation) + if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { if(dedicatedBuffer != VK_NULL_HANDLE) { VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE); dedicatedAllocInfo.buffer = dedicatedBuffer; - allocInfo.pNext = &dedicatedAllocInfo; + VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo); } else if(dedicatedImage != VK_NULL_HANDLE) { dedicatedAllocInfo.image = dedicatedImage; - allocInfo.pNext = &dedicatedAllocInfo; + VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo); } } -#endif // #if VMA_DEDICATED_ALLOCATION +#endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 - // Allocate VkDeviceMemory. +#if VMA_BUFFER_DEVICE_ADDRESS + VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR }; + if(m_UseKhrBufferDeviceAddress) + { + bool canContainBufferWithDeviceAddress = true; + if(dedicatedBuffer != VK_NULL_HANDLE) + { + canContainBufferWithDeviceAddress = dedicatedBufferUsage == UINT32_MAX || // Usage flags unknown + (dedicatedBufferUsage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_EXT) != 0; + } + else if(dedicatedImage != VK_NULL_HANDLE) + { + canContainBufferWithDeviceAddress = false; + } + if(canContainBufferWithDeviceAddress) + { + allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR; + VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo); + } + } +#endif // #if VMA_BUFFER_DEVICE_ADDRESS + + size_t allocIndex; + VkResult res = VK_SUCCESS; + for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + res = AllocateDedicatedMemoryPage( + size, + suballocType, + memTypeIndex, + allocInfo, + map, + isUserDataString, + pUserData, + pAllocations + allocIndex); + if(res != VK_SUCCESS) + { + break; + } + } + + if(res == VK_SUCCESS) + { + // Register them in m_pDedicatedAllocations. + { + VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); + AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex]; + VMA_ASSERT(pDedicatedAllocations); + for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + VmaVectorInsertSorted(*pDedicatedAllocations, pAllocations[allocIndex]); + } + } + + VMA_DEBUG_LOG(" Allocated DedicatedMemory Count=%zu, MemoryTypeIndex=#%u", allocationCount, memTypeIndex); + } + else + { + // Free all already created allocations. + while(allocIndex--) + { + VmaAllocation currAlloc = pAllocations[allocIndex]; + VkDeviceMemory hMemory = currAlloc->GetMemory(); + + /* + There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory + before vkFreeMemory. + + if(currAlloc->GetMappedData() != VMA_NULL) + { + (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory); + } + */ + + FreeVulkanMemory(memTypeIndex, currAlloc->GetSize(), hMemory); + m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), currAlloc->GetSize()); + currAlloc->SetUserData(this, VMA_NULL); + m_AllocationObjectAllocator.Free(currAlloc); + } + + memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount); + } + + return res; +} + +VkResult VmaAllocator_T::AllocateDedicatedMemoryPage( + VkDeviceSize size, + VmaSuballocationType suballocType, + uint32_t memTypeIndex, + const VkMemoryAllocateInfo& allocInfo, + bool map, + bool isUserDataString, + void* pUserData, + VmaAllocation* pAllocation) +{ VkDeviceMemory hMemory = VK_NULL_HANDLE; VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory); if(res < 0) @@ -9017,24 +16341,15 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( } } - *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString); + *pAllocation = m_AllocationObjectAllocator.Allocate(m_CurrentFrameIndex.load(), isUserDataString); (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size); (*pAllocation)->SetUserData(this, pUserData); + m_Budget.AddAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), size); if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) { FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED); } - // Register it in m_pDedicatedAllocations. - { - VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); - AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex]; - VMA_ASSERT(pDedicatedAllocations); - VmaVectorInsertSorted(*pDedicatedAllocations, *pAllocation); - } - - VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex); - return VK_SUCCESS; } @@ -9044,8 +16359,8 @@ void VmaAllocator_T::GetBufferMemoryRequirements( bool& requiresDedicatedAllocation, bool& prefersDedicatedAllocation) const { -#if VMA_DEDICATED_ALLOCATION - if(m_UseKhrDedicatedAllocation) +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR }; memReqInfo.buffer = hBuffer; @@ -9053,7 +16368,7 @@ void VmaAllocator_T::GetBufferMemoryRequirements( VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR }; VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR }; - memReq2.pNext = &memDedicatedReq; + VmaPnextChainPushFront(&memReq2, &memDedicatedReq); (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2); @@ -9062,7 +16377,7 @@ void VmaAllocator_T::GetBufferMemoryRequirements( prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE); } else -#endif // #if VMA_DEDICATED_ALLOCATION +#endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 { (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq); requiresDedicatedAllocation = false; @@ -9076,8 +16391,8 @@ void VmaAllocator_T::GetImageMemoryRequirements( bool& requiresDedicatedAllocation, bool& prefersDedicatedAllocation) const { -#if VMA_DEDICATED_ALLOCATION - if(m_UseKhrDedicatedAllocation) +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR }; memReqInfo.image = hImage; @@ -9085,7 +16400,7 @@ void VmaAllocator_T::GetImageMemoryRequirements( VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR }; VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR }; - memReq2.pNext = &memDedicatedReq; + VmaPnextChainPushFront(&memReq2, &memDedicatedReq); (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2); @@ -9094,7 +16409,7 @@ void VmaAllocator_T::GetImageMemoryRequirements( prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE); } else -#endif // #if VMA_DEDICATED_ALLOCATION +#endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 { (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq); requiresDedicatedAllocation = false; @@ -9107,11 +16422,21 @@ VkResult VmaAllocator_T::AllocateMemory( bool requiresDedicatedAllocation, bool prefersDedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, - VmaAllocation* pAllocation) + size_t allocationCount, + VmaAllocation* pAllocations) { + memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount); + + VMA_ASSERT(VmaIsPow2(vkMemReq.alignment)); + + if(vkMemReq.size == 0) + { + return VK_ERROR_VALIDATION_FAILED_EXT; + } if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 && (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) { @@ -9149,14 +16474,23 @@ VkResult VmaAllocator_T::AllocateMemory( const VkDeviceSize alignmentForPool = VMA_MAX( vkMemReq.alignment, GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex())); + + VmaAllocationCreateInfo createInfoForPool = createInfo; + // If memory type is not HOST_VISIBLE, disable MAPPED. + if((createInfoForPool.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 && + (m_MemProps.memoryTypes[createInfo.pool->m_BlockVector.GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) + { + createInfoForPool.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT; + } + return createInfo.pool->m_BlockVector.Allocate( - createInfo.pool, m_CurrentFrameIndex.load(), vkMemReq.size, alignmentForPool, - createInfo, + createInfoForPool, suballocType, - pAllocation); + allocationCount, + pAllocations); } else { @@ -9175,11 +16509,13 @@ VkResult VmaAllocator_T::AllocateMemory( alignmentForMemType, requiresDedicatedAllocation || prefersDedicatedAllocation, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, createInfo, memTypeIndex, suballocType, - pAllocation); + allocationCount, + pAllocations); // Succeeded on first try. if(res == VK_SUCCESS) { @@ -9205,11 +16541,13 @@ VkResult VmaAllocator_T::AllocateMemory( alignmentForMemType, requiresDedicatedAllocation || prefersDedicatedAllocation, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, createInfo, memTypeIndex, suballocType, - pAllocation); + allocationCount, + pAllocations); // Allocation from this alternative memory type succeeded. if(res == VK_SUCCESS) { @@ -9232,46 +16570,73 @@ VkResult VmaAllocator_T::AllocateMemory( } } -void VmaAllocator_T::FreeMemory(const VmaAllocation allocation) +void VmaAllocator_T::FreeMemory( + size_t allocationCount, + const VmaAllocation* pAllocations) { - VMA_ASSERT(allocation); + VMA_ASSERT(pAllocations); - if(allocation->CanBecomeLost() == false || - allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST) + for(size_t allocIndex = allocationCount; allocIndex--; ) { - if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) - { - FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED); - } + VmaAllocation allocation = pAllocations[allocIndex]; - switch(allocation->GetType()) + if(allocation != VK_NULL_HANDLE) { - case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: + if(TouchAllocation(allocation)) { - VmaBlockVector* pBlockVector = VMA_NULL; - VmaPool hPool = allocation->GetPool(); - if(hPool != VK_NULL_HANDLE) + if(VMA_DEBUG_INITIALIZE_ALLOCATIONS) { - pBlockVector = &hPool->m_BlockVector; + FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED); } - else + + switch(allocation->GetType()) { - const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); - pBlockVector = m_pBlockVectors[memTypeIndex]; + case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: + { + VmaBlockVector* pBlockVector = VMA_NULL; + VmaPool hPool = allocation->GetBlock()->GetParentPool(); + if(hPool != VK_NULL_HANDLE) + { + pBlockVector = &hPool->m_BlockVector; + } + else + { + const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); + pBlockVector = m_pBlockVectors[memTypeIndex]; + } + pBlockVector->Free(allocation); + } + break; + case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: + FreeDedicatedMemory(allocation); + break; + default: + VMA_ASSERT(0); } - pBlockVector->Free(allocation); } - break; - case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: - FreeDedicatedMemory(allocation); - break; - default: - VMA_ASSERT(0); + + // Do this regardless of whether the allocation is lost. Lost allocations still account to Budget.AllocationBytes. + m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(allocation->GetMemoryTypeIndex()), allocation->GetSize()); + allocation->SetUserData(this, VMA_NULL); + m_AllocationObjectAllocator.Free(allocation); } } +} - allocation->SetUserData(this, VMA_NULL); - vma_delete(this, allocation); +VkResult VmaAllocator_T::ResizeAllocation( + const VmaAllocation alloc, + VkDeviceSize newSize) +{ + // This function is deprecated and so it does nothing. It's left for backward compatibility. + if(newSize == 0 || alloc->GetLastUseFrameIndex() == VMA_FRAME_INDEX_LOST) + { + return VK_ERROR_VALIDATION_FAILED_EXT; + } + if(newSize == alloc->GetSize()) + { + return VK_SUCCESS; + } + return VK_ERROR_OUT_OF_POOL_MEMORY; } void VmaAllocator_T::CalculateStats(VmaStats* pStats) @@ -9293,10 +16658,10 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) // Process custom pools. { - VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) { - m_Pools[poolIndex]->GetBlockVector().AddStats(pStats); + m_Pools[poolIndex]->m_BlockVector.AddStats(pStats); } } @@ -9304,7 +16669,7 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex); - VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); + VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex]; VMA_ASSERT(pDedicatedAllocVector); for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex) @@ -9325,119 +16690,109 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]); } +void VmaAllocator_T::GetBudget(VmaBudget* outBudget, uint32_t firstHeap, uint32_t heapCount) +{ +#if VMA_MEMORY_BUDGET + if(m_UseExtMemoryBudget) + { + if(m_Budget.m_OperationsSinceBudgetFetch < 30) + { + VmaMutexLockRead lockRead(m_Budget.m_BudgetMutex, m_UseMutex); + for(uint32_t i = 0; i < heapCount; ++i, ++outBudget) + { + const uint32_t heapIndex = firstHeap + i; + + outBudget->blockBytes = m_Budget.m_BlockBytes[heapIndex]; + outBudget->allocationBytes = m_Budget.m_AllocationBytes[heapIndex]; + + if(m_Budget.m_VulkanUsage[heapIndex] + outBudget->blockBytes > m_Budget.m_BlockBytesAtBudgetFetch[heapIndex]) + { + outBudget->usage = m_Budget.m_VulkanUsage[heapIndex] + + outBudget->blockBytes - m_Budget.m_BlockBytesAtBudgetFetch[heapIndex]; + } + else + { + outBudget->usage = 0; + } + + // Have to take MIN with heap size because explicit HeapSizeLimit is included in it. + outBudget->budget = VMA_MIN( + m_Budget.m_VulkanBudget[heapIndex], m_MemProps.memoryHeaps[heapIndex].size); + } + } + else + { + UpdateVulkanBudget(); // Outside of mutex lock + GetBudget(outBudget, firstHeap, heapCount); // Recursion + } + } + else +#endif + { + for(uint32_t i = 0; i < heapCount; ++i, ++outBudget) + { + const uint32_t heapIndex = firstHeap + i; + + outBudget->blockBytes = m_Budget.m_BlockBytes[heapIndex]; + outBudget->allocationBytes = m_Budget.m_AllocationBytes[heapIndex]; + + outBudget->usage = outBudget->blockBytes; + outBudget->budget = m_MemProps.memoryHeaps[heapIndex].size * 8 / 10; // 80% heuristics. + } + } +} + static const uint32_t VMA_VENDOR_ID_AMD = 4098; -VkResult VmaAllocator_T::Defragment( - VmaAllocation* pAllocations, - size_t allocationCount, - VkBool32* pAllocationsChanged, - const VmaDefragmentationInfo* pDefragmentationInfo, - VmaDefragmentationStats* pDefragmentationStats) +VkResult VmaAllocator_T::DefragmentationBegin( + const VmaDefragmentationInfo2& info, + VmaDefragmentationStats* pStats, + VmaDefragmentationContext* pContext) { - if(pAllocationsChanged != VMA_NULL) + if(info.pAllocationsChanged != VMA_NULL) { - memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged)); - } - if(pDefragmentationStats != VMA_NULL) - { - memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats)); + memset(info.pAllocationsChanged, 0, info.allocationCount * sizeof(VkBool32)); } - const uint32_t currentFrameIndex = m_CurrentFrameIndex.load(); + *pContext = vma_new(this, VmaDefragmentationContext_T)( + this, m_CurrentFrameIndex.load(), info.flags, pStats); - VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex); + (*pContext)->AddPools(info.poolCount, info.pPools); + (*pContext)->AddAllocations( + info.allocationCount, info.pAllocations, info.pAllocationsChanged); - const size_t poolCount = m_Pools.size(); + VkResult res = (*pContext)->Defragment( + info.maxCpuBytesToMove, info.maxCpuAllocationsToMove, + info.maxGpuBytesToMove, info.maxGpuAllocationsToMove, + info.commandBuffer, pStats, info.flags); - // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary. - for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + if(res != VK_NOT_READY) { - VmaAllocation hAlloc = pAllocations[allocIndex]; - VMA_ASSERT(hAlloc); - const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex(); - // DedicatedAlloc cannot be defragmented. - if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) && - // Only HOST_VISIBLE memory types can be defragmented. - ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) && - // Lost allocation cannot be defragmented. - (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)) - { - VmaBlockVector* pAllocBlockVector = VMA_NULL; - - const VmaPool hAllocPool = hAlloc->GetPool(); - // This allocation belongs to custom pool. - if(hAllocPool != VK_NULL_HANDLE) - { - pAllocBlockVector = &hAllocPool->GetBlockVector(); - } - // This allocation belongs to general pool. - else - { - pAllocBlockVector = m_pBlockVectors[memTypeIndex]; - } - - VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex); - - VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ? - &pAllocationsChanged[allocIndex] : VMA_NULL; - pDefragmentator->AddAllocation(hAlloc, pChanged); - } + vma_delete(this, *pContext); + *pContext = VMA_NULL; } - VkResult result = VK_SUCCESS; + return res; +} - // ======== Main processing. +VkResult VmaAllocator_T::DefragmentationEnd( + VmaDefragmentationContext context) +{ + vma_delete(this, context); + return VK_SUCCESS; +} - VkDeviceSize maxBytesToMove = SIZE_MAX; - uint32_t maxAllocationsToMove = UINT32_MAX; - if(pDefragmentationInfo != VMA_NULL) - { - maxBytesToMove = pDefragmentationInfo->maxBytesToMove; - maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove; - } - - // Process standard memory. - for(uint32_t memTypeIndex = 0; - (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS); - ++memTypeIndex) - { - // Only HOST_VISIBLE memory types can be defragmented. - if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) - { - result = m_pBlockVectors[memTypeIndex]->Defragment( - pDefragmentationStats, - maxBytesToMove, - maxAllocationsToMove); - } - } - - // Process custom pools. - for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex) - { - result = m_Pools[poolIndex]->GetBlockVector().Defragment( - pDefragmentationStats, - maxBytesToMove, - maxAllocationsToMove); - } - - // ======== Destroy defragmentators. - - // Process custom pools. - for(size_t poolIndex = poolCount; poolIndex--; ) - { - m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator(); - } - - // Process standard memory. - for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; ) - { - if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) - { - m_pBlockVectors[memTypeIndex]->DestroyDefragmentator(); - } - } - - return result; +VkResult VmaAllocator_T::DefragmentationPassBegin( + VmaDefragmentationPassInfo* pInfo, + VmaDefragmentationContext context) +{ + return context->DefragmentPassBegin(pInfo); +} +VkResult VmaAllocator_T::DefragmentationPassEnd( + VmaDefragmentationContext context) +{ + return context->DefragmentPassEnd(); + } void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo) @@ -9448,7 +16803,7 @@ void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationI Warning: This is a carefully designed algorithm. Do not modify unless you really know what you're doing :) */ - uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load(); + const uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load(); uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex(); for(;;) { @@ -9566,7 +16921,7 @@ bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation) VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool) { - VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex); + VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags); VmaPoolCreateInfo newCreateInfo = *pCreateInfo; @@ -9574,12 +16929,20 @@ VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPoo { newCreateInfo.maxBlockCount = SIZE_MAX; } - if(newCreateInfo.blockSize == 0) + if(newCreateInfo.minBlockCount > newCreateInfo.maxBlockCount) { - newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex); + return VK_ERROR_INITIALIZATION_FAILED; + } + // Memory type index out of range or forbidden. + if(pCreateInfo->memoryTypeIndex >= GetMemoryTypeCount() || + ((1u << pCreateInfo->memoryTypeIndex) & m_GlobalMemoryTypeBits) == 0) + { + return VK_ERROR_FEATURE_NOT_PRESENT; } - *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo); + const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex); + + *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo, preferredBlockSize); VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks(); if(res != VK_SUCCESS) @@ -9591,7 +16954,7 @@ VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPoo // Add to m_Pools. { - VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex); (*pPool)->SetId(m_NextPoolId++); VmaVectorInsertSorted(m_Pools, *pPool); } @@ -9603,7 +16966,7 @@ void VmaAllocator_T::DestroyPool(VmaPool pool) { // Remove from m_Pools. { - VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex); bool success = VmaVectorRemoveSorted(m_Pools, pool); VMA_ASSERT(success && "Pool not found in Allocator."); } @@ -9619,6 +16982,13 @@ void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats) void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex) { m_CurrentFrameIndex.store(frameIndex); + +#if VMA_MEMORY_BUDGET + if(m_UseExtMemoryBudget) + { + UpdateVulkanBudget(); + } +#endif // #if VMA_MEMORY_BUDGET } void VmaAllocator_T::MakePoolAllocationsLost( @@ -9662,12 +17032,12 @@ VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits) // Process custom pools. { - VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) { - if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0) + if(((1u << m_Pools[poolIndex]->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0) { - VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption(); + VkResult localRes = m_Pools[poolIndex]->m_BlockVector.CheckCorruption(); switch(localRes) { case VK_ERROR_FEATURE_NOT_PRESENT: @@ -9687,7 +17057,7 @@ VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits) void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation) { - *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false); + *pAllocation = m_AllocationObjectAllocator.Allocate(VMA_FRAME_INDEX_LOST, false); (*pAllocation)->InitLost(); } @@ -9695,31 +17065,47 @@ VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAlloc { const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex); - VkResult res; - if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE) + // HeapSizeLimit is in effect for this heap. + if((m_HeapSizeLimitMask & (1u << heapIndex)) != 0) { - VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex); - if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize) + const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size; + VkDeviceSize blockBytes = m_Budget.m_BlockBytes[heapIndex]; + for(;;) { - res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory); - if(res == VK_SUCCESS) + const VkDeviceSize blockBytesAfterAllocation = blockBytes + pAllocateInfo->allocationSize; + if(blockBytesAfterAllocation > heapSize) { - m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize; + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + if(m_Budget.m_BlockBytes[heapIndex].compare_exchange_strong(blockBytes, blockBytesAfterAllocation)) + { + break; } - } - else - { - res = VK_ERROR_OUT_OF_DEVICE_MEMORY; } } else { - res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory); + m_Budget.m_BlockBytes[heapIndex] += pAllocateInfo->allocationSize; } - if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL) + // VULKAN CALL vkAllocateMemory. + VkResult res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory); + + if(res == VK_SUCCESS) { - (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize); +#if VMA_MEMORY_BUDGET + ++m_Budget.m_OperationsSinceBudgetFetch; +#endif + + // Informative callback. + if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL) + { + (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize, m_DeviceMemoryCallbacks.pUserData); + } + } + else + { + m_Budget.m_BlockBytes[heapIndex] -= pAllocateInfo->allocationSize; } return res; @@ -9727,18 +17113,77 @@ VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAlloc void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory) { + // Informative callback. if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL) { - (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size); + (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size, m_DeviceMemoryCallbacks.pUserData); } + // VULKAN CALL vkFreeMemory. (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks()); - const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType); - if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE) + m_Budget.m_BlockBytes[MemoryTypeIndexToHeapIndex(memoryType)] -= size; +} + +VkResult VmaAllocator_T::BindVulkanBuffer( + VkDeviceMemory memory, + VkDeviceSize memoryOffset, + VkBuffer buffer, + const void* pNext) +{ + if(pNext != VMA_NULL) { - VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex); - m_HeapSizeLimit[heapIndex] += size; +#if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2 + if((m_UseKhrBindMemory2 || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) && + m_VulkanFunctions.vkBindBufferMemory2KHR != VMA_NULL) + { + VkBindBufferMemoryInfoKHR bindBufferMemoryInfo = { VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO_KHR }; + bindBufferMemoryInfo.pNext = pNext; + bindBufferMemoryInfo.buffer = buffer; + bindBufferMemoryInfo.memory = memory; + bindBufferMemoryInfo.memoryOffset = memoryOffset; + return (*m_VulkanFunctions.vkBindBufferMemory2KHR)(m_hDevice, 1, &bindBufferMemoryInfo); + } + else +#endif // #if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2 + { + return VK_ERROR_EXTENSION_NOT_PRESENT; + } + } + else + { + return (*m_VulkanFunctions.vkBindBufferMemory)(m_hDevice, buffer, memory, memoryOffset); + } +} + +VkResult VmaAllocator_T::BindVulkanImage( + VkDeviceMemory memory, + VkDeviceSize memoryOffset, + VkImage image, + const void* pNext) +{ + if(pNext != VMA_NULL) + { +#if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2 + if((m_UseKhrBindMemory2 || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) && + m_VulkanFunctions.vkBindImageMemory2KHR != VMA_NULL) + { + VkBindImageMemoryInfoKHR bindBufferMemoryInfo = { VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO_KHR }; + bindBufferMemoryInfo.pNext = pNext; + bindBufferMemoryInfo.image = image; + bindBufferMemoryInfo.memory = memory; + bindBufferMemoryInfo.memoryOffset = memoryOffset; + return (*m_VulkanFunctions.vkBindImageMemory2KHR)(m_hDevice, 1, &bindBufferMemoryInfo); + } + else +#endif // #if VMA_BIND_MEMORY2 + { + return VK_ERROR_EXTENSION_NOT_PRESENT; + } + } + else + { + return (*m_VulkanFunctions.vkBindImageMemory)(m_hDevice, image, memory, memoryOffset); } } @@ -9790,23 +17235,23 @@ void VmaAllocator_T::Unmap(VmaAllocation hAllocation) } } -VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer) +VkResult VmaAllocator_T::BindBufferMemory( + VmaAllocation hAllocation, + VkDeviceSize allocationLocalOffset, + VkBuffer hBuffer, + const void* pNext) { VkResult res = VK_SUCCESS; switch(hAllocation->GetType()) { case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: - res = GetVulkanFunctions().vkBindBufferMemory( - m_hDevice, - hBuffer, - hAllocation->GetMemory(), - 0); //memoryOffset + res = BindVulkanBuffer(hAllocation->GetMemory(), allocationLocalOffset, hBuffer, pNext); break; case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: { - VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock(); + VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock(); VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?"); - res = pBlock->BindBufferMemory(this, hAllocation, hBuffer); + res = pBlock->BindBufferMemory(this, hAllocation, allocationLocalOffset, hBuffer, pNext); break; } default: @@ -9815,23 +17260,23 @@ VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hB return res; } -VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage) +VkResult VmaAllocator_T::BindImageMemory( + VmaAllocation hAllocation, + VkDeviceSize allocationLocalOffset, + VkImage hImage, + const void* pNext) { VkResult res = VK_SUCCESS; switch(hAllocation->GetType()) { case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: - res = GetVulkanFunctions().vkBindImageMemory( - m_hDevice, - hImage, - hAllocation->GetMemory(), - 0); //memoryOffset + res = BindVulkanImage(hAllocation->GetMemory(), allocationLocalOffset, hImage, pNext); break; case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: { VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock(); VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?"); - res = pBlock->BindImageMemory(this, hAllocation, hImage); + res = pBlock->BindImageMemory(this, hAllocation, allocationLocalOffset, hImage, pNext); break; } default: @@ -9840,89 +17285,80 @@ VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hIma return res; } -void VmaAllocator_T::FlushOrInvalidateAllocation( +VkResult VmaAllocator_T::FlushOrInvalidateAllocation( VmaAllocation hAllocation, VkDeviceSize offset, VkDeviceSize size, VMA_CACHE_OPERATION op) { - const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex(); - if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex)) + VkResult res = VK_SUCCESS; + + VkMappedMemoryRange memRange = {}; + if(GetFlushOrInvalidateRange(hAllocation, offset, size, memRange)) { - const VkDeviceSize allocationSize = hAllocation->GetSize(); - VMA_ASSERT(offset <= allocationSize); - - const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize; - - VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE }; - memRange.memory = hAllocation->GetMemory(); - - switch(hAllocation->GetType()) - { - case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: - memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); - if(size == VK_WHOLE_SIZE) - { - memRange.size = allocationSize - memRange.offset; - } - else - { - VMA_ASSERT(offset + size <= allocationSize); - memRange.size = VMA_MIN( - VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize), - allocationSize - memRange.offset); - } - break; - - case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: - { - // 1. Still within this allocation. - memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); - if(size == VK_WHOLE_SIZE) - { - size = allocationSize - offset; - } - else - { - VMA_ASSERT(offset + size <= allocationSize); - } - memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize); - - // 2. Adjust to whole block. - const VkDeviceSize allocationOffset = hAllocation->GetOffset(); - VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0); - const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize(); - memRange.offset += allocationOffset; - memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset); - - break; - } - - default: - VMA_ASSERT(0); - } - switch(op) { case VMA_CACHE_FLUSH: - (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange); + res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange); break; case VMA_CACHE_INVALIDATE: - (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange); + res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange); break; default: VMA_ASSERT(0); } } // else: Just ignore this call. + return res; } -void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation) +VkResult VmaAllocator_T::FlushOrInvalidateAllocations( + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, const VkDeviceSize* sizes, + VMA_CACHE_OPERATION op) +{ + typedef VmaStlAllocator RangeAllocator; + typedef VmaSmallVector RangeVector; + RangeVector ranges = RangeVector(RangeAllocator(GetAllocationCallbacks())); + + for(uint32_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + const VmaAllocation alloc = allocations[allocIndex]; + const VkDeviceSize offset = offsets != VMA_NULL ? offsets[allocIndex] : 0; + const VkDeviceSize size = sizes != VMA_NULL ? sizes[allocIndex] : VK_WHOLE_SIZE; + VkMappedMemoryRange newRange; + if(GetFlushOrInvalidateRange(alloc, offset, size, newRange)) + { + ranges.push_back(newRange); + } + } + + VkResult res = VK_SUCCESS; + if(!ranges.empty()) + { + switch(op) + { + case VMA_CACHE_FLUSH: + res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data()); + break; + case VMA_CACHE_INVALIDATE: + res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data()); + break; + default: + VMA_ASSERT(0); + } + } + // else: Just ignore this call. + return res; +} + +void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation) { VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED); const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); { - VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); + VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex]; VMA_ASSERT(pDedicatedAllocations); bool success = VmaVectorRemoveSorted(*pDedicatedAllocations, allocation); @@ -9931,16 +17367,173 @@ void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation) VkDeviceMemory hMemory = allocation->GetMemory(); + /* + There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory + before vkFreeMemory. + if(allocation->GetMappedData() != VMA_NULL) { (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory); } + */ FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory); VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex); } +uint32_t VmaAllocator_T::CalculateGpuDefragmentationMemoryTypeBits() const +{ + VkBufferCreateInfo dummyBufCreateInfo; + VmaFillGpuDefragmentationBufferCreateInfo(dummyBufCreateInfo); + + uint32_t memoryTypeBits = 0; + + // Create buffer. + VkBuffer buf = VK_NULL_HANDLE; + VkResult res = (*GetVulkanFunctions().vkCreateBuffer)( + m_hDevice, &dummyBufCreateInfo, GetAllocationCallbacks(), &buf); + if(res == VK_SUCCESS) + { + // Query for supported memory types. + VkMemoryRequirements memReq; + (*GetVulkanFunctions().vkGetBufferMemoryRequirements)(m_hDevice, buf, &memReq); + memoryTypeBits = memReq.memoryTypeBits; + + // Destroy buffer. + (*GetVulkanFunctions().vkDestroyBuffer)(m_hDevice, buf, GetAllocationCallbacks()); + } + + return memoryTypeBits; +} + +uint32_t VmaAllocator_T::CalculateGlobalMemoryTypeBits() const +{ + // Make sure memory information is already fetched. + VMA_ASSERT(GetMemoryTypeCount() > 0); + + uint32_t memoryTypeBits = UINT32_MAX; + + if(!m_UseAmdDeviceCoherentMemory) + { + // Exclude memory types that have VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD. + for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) + { + if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0) + { + memoryTypeBits &= ~(1u << memTypeIndex); + } + } + } + + return memoryTypeBits; +} + +bool VmaAllocator_T::GetFlushOrInvalidateRange( + VmaAllocation allocation, + VkDeviceSize offset, VkDeviceSize size, + VkMappedMemoryRange& outRange) const +{ + const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); + if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex)) + { + const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize; + const VkDeviceSize allocationSize = allocation->GetSize(); + VMA_ASSERT(offset <= allocationSize); + + outRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + outRange.pNext = VMA_NULL; + outRange.memory = allocation->GetMemory(); + + switch(allocation->GetType()) + { + case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: + outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); + if(size == VK_WHOLE_SIZE) + { + outRange.size = allocationSize - outRange.offset; + } + else + { + VMA_ASSERT(offset + size <= allocationSize); + outRange.size = VMA_MIN( + VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize), + allocationSize - outRange.offset); + } + break; + case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: + { + // 1. Still within this allocation. + outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); + if(size == VK_WHOLE_SIZE) + { + size = allocationSize - offset; + } + else + { + VMA_ASSERT(offset + size <= allocationSize); + } + outRange.size = VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize); + + // 2. Adjust to whole block. + const VkDeviceSize allocationOffset = allocation->GetOffset(); + VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0); + const VkDeviceSize blockSize = allocation->GetBlock()->m_pMetadata->GetSize(); + outRange.offset += allocationOffset; + outRange.size = VMA_MIN(outRange.size, blockSize - outRange.offset); + + break; + } + default: + VMA_ASSERT(0); + } + return true; + } + return false; +} + +#if VMA_MEMORY_BUDGET + +void VmaAllocator_T::UpdateVulkanBudget() +{ + VMA_ASSERT(m_UseExtMemoryBudget); + + VkPhysicalDeviceMemoryProperties2KHR memProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2_KHR }; + + VkPhysicalDeviceMemoryBudgetPropertiesEXT budgetProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT }; + VmaPnextChainPushFront(&memProps, &budgetProps); + + GetVulkanFunctions().vkGetPhysicalDeviceMemoryProperties2KHR(m_PhysicalDevice, &memProps); + + { + VmaMutexLockWrite lockWrite(m_Budget.m_BudgetMutex, m_UseMutex); + + for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex) + { + m_Budget.m_VulkanUsage[heapIndex] = budgetProps.heapUsage[heapIndex]; + m_Budget.m_VulkanBudget[heapIndex] = budgetProps.heapBudget[heapIndex]; + m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] = m_Budget.m_BlockBytes[heapIndex].load(); + + // Some bugged drivers return the budget incorrectly, e.g. 0 or much bigger than heap size. + if(m_Budget.m_VulkanBudget[heapIndex] == 0) + { + m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size * 8 / 10; // 80% heuristics. + } + else if(m_Budget.m_VulkanBudget[heapIndex] > m_MemProps.memoryHeaps[heapIndex].size) + { + m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size; + } + if(m_Budget.m_VulkanUsage[heapIndex] == 0 && m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] > 0) + { + m_Budget.m_VulkanUsage[heapIndex] = m_Budget.m_BlockBytesAtBudgetFetch[heapIndex]; + } + } + m_Budget.m_OperationsSinceBudgetFetch = 0; + } +} + +#endif // #if VMA_MEMORY_BUDGET + void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern) { if(VMA_DEBUG_INITIALIZE_ALLOCATIONS && @@ -9962,6 +17555,17 @@ void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pat } } +uint32_t VmaAllocator_T::GetGpuDefragmentationMemoryTypeBits() +{ + uint32_t memoryTypeBits = m_GpuDefragmentationMemoryTypeBits.load(); + if(memoryTypeBits == UINT32_MAX) + { + memoryTypeBits = CalculateGpuDefragmentationMemoryTypeBits(); + m_GpuDefragmentationMemoryTypeBits.store(memoryTypeBits); + } + return memoryTypeBits; +} + #if VMA_STATS_STRING_ENABLED void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) @@ -9969,7 +17573,7 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) bool dedicatedAllocationsStarted = false; for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { - VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); + VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex]; VMA_ASSERT(pDedicatedAllocVector); if(pDedicatedAllocVector->empty() == false) @@ -10029,8 +17633,9 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) } } + // Custom pools { - VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); const size_t poolCount = m_Pools.size(); if(poolCount > 0) { @@ -10054,17 +17659,19 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) //////////////////////////////////////////////////////////////////////////////// // Public interface -VkResult vmaCreateAllocator( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator( const VmaAllocatorCreateInfo* pCreateInfo, VmaAllocator* pAllocator) { VMA_ASSERT(pCreateInfo && pAllocator); + VMA_ASSERT(pCreateInfo->vulkanApiVersion == 0 || + (VK_VERSION_MAJOR(pCreateInfo->vulkanApiVersion) == 1 && VK_VERSION_MINOR(pCreateInfo->vulkanApiVersion) <= 2)); VMA_DEBUG_LOG("vmaCreateAllocator"); *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo); return (*pAllocator)->Init(pCreateInfo); } -void vmaDestroyAllocator( +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator( VmaAllocator allocator) { if(allocator != VK_NULL_HANDLE) @@ -10075,7 +17682,15 @@ void vmaDestroyAllocator( } } -void vmaGetPhysicalDeviceProperties( +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator allocator, VmaAllocatorInfo* pAllocatorInfo) +{ + VMA_ASSERT(allocator && pAllocatorInfo); + pAllocatorInfo->instance = allocator->m_hInstance; + pAllocatorInfo->physicalDevice = allocator->GetPhysicalDevice(); + pAllocatorInfo->device = allocator->m_hDevice; +} + +VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties( VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties) { @@ -10083,7 +17698,7 @@ void vmaGetPhysicalDeviceProperties( *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties; } -void vmaGetMemoryProperties( +VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryProperties( VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties) { @@ -10091,7 +17706,7 @@ void vmaGetMemoryProperties( *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps; } -void vmaGetMemoryTypeProperties( +VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryTypeProperties( VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags* pFlags) @@ -10101,7 +17716,7 @@ void vmaGetMemoryTypeProperties( *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags; } -void vmaSetCurrentFrameIndex( +VMA_CALL_PRE void VMA_CALL_POST vmaSetCurrentFrameIndex( VmaAllocator allocator, uint32_t frameIndex) { @@ -10113,7 +17728,7 @@ void vmaSetCurrentFrameIndex( allocator->SetCurrentFrameIndex(frameIndex); } -void vmaCalculateStats( +VMA_CALL_PRE void VMA_CALL_POST vmaCalculateStats( VmaAllocator allocator, VmaStats* pStats) { @@ -10122,9 +17737,18 @@ void vmaCalculateStats( allocator->CalculateStats(pStats); } +VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget( + VmaAllocator allocator, + VmaBudget* pBudget) +{ + VMA_ASSERT(allocator && pBudget); + VMA_DEBUG_GLOBAL_MUTEX_LOCK + allocator->GetBudget(pBudget, 0, allocator->GetMemoryHeapCount()); +} + #if VMA_STATS_STRING_ENABLED -void vmaBuildStatsString( +VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( VmaAllocator allocator, char** ppStatsString, VkBool32 detailedMap) @@ -10137,6 +17761,9 @@ void vmaBuildStatsString( VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb); json.BeginObject(); + VmaBudget budget[VK_MAX_MEMORY_HEAPS]; + allocator->GetBudget(budget, 0, allocator->GetMemoryHeapCount()); + VmaStats stats; allocator->CalculateStats(&stats); @@ -10161,6 +17788,20 @@ void vmaBuildStatsString( } json.EndArray(); + json.WriteString("Budget"); + json.BeginObject(); + { + json.WriteString("BlockBytes"); + json.WriteNumber(budget[heapIndex].blockBytes); + json.WriteString("AllocationBytes"); + json.WriteNumber(budget[heapIndex].allocationBytes); + json.WriteString("Usage"); + json.WriteNumber(budget[heapIndex].usage); + json.WriteString("Budget"); + json.WriteNumber(budget[heapIndex].budget); + } + json.EndObject(); + if(stats.memoryHeap[heapIndex].blockCount > 0) { json.WriteString("Stats"); @@ -10200,6 +17841,18 @@ void vmaBuildStatsString( { json.WriteString("LAZILY_ALLOCATED"); } + if((flags & VK_MEMORY_PROPERTY_PROTECTED_BIT) != 0) + { + json.WriteString(" PROTECTED"); + } + if((flags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0) + { + json.WriteString(" DEVICE_COHERENT"); + } + if((flags & VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY) != 0) + { + json.WriteString(" DEVICE_UNCACHED"); + } json.EndArray(); if(stats.memoryType[typeIndex].blockCount > 0) @@ -10232,7 +17885,7 @@ void vmaBuildStatsString( *ppStatsString = pChars; } -void vmaFreeStatsString( +VMA_CALL_PRE void VMA_CALL_POST vmaFreeStatsString( VmaAllocator allocator, char* pStatsString) { @@ -10249,7 +17902,7 @@ void vmaFreeStatsString( /* This function is not protected by any mutex because it just reads immutable data. */ -VkResult vmaFindMemoryTypeIndex( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex( VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo* pAllocationCreateInfo, @@ -10259,6 +17912,8 @@ VkResult vmaFindMemoryTypeIndex( VMA_ASSERT(pAllocationCreateInfo != VMA_NULL); VMA_ASSERT(pMemoryTypeIndex != VMA_NULL); + memoryTypeBits &= allocator->GetGlobalMemoryTypeBits(); + if(pAllocationCreateInfo->memoryTypeBits != 0) { memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits; @@ -10266,12 +17921,7 @@ VkResult vmaFindMemoryTypeIndex( uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags; uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags; - - const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0; - if(mapped) - { - preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; - } + uint32_t notPreferredFlags = 0; // Convert usage to requiredFlags and preferredFlags. switch(pAllocationCreateInfo->usage) @@ -10296,12 +17946,26 @@ VkResult vmaFindMemoryTypeIndex( break; case VMA_MEMORY_USAGE_GPU_TO_CPU: requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; - preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + preferredFlags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + break; + case VMA_MEMORY_USAGE_CPU_COPY: + notPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + break; + case VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED: + requiredFlags |= VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT; break; default: + VMA_ASSERT(0); break; } + // Avoid DEVICE_COHERENT unless explicitly requested. + if(((pAllocationCreateInfo->requiredFlags | pAllocationCreateInfo->preferredFlags) & + (VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY | VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY)) == 0) + { + notPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY; + } + *pMemoryTypeIndex = UINT32_MAX; uint32_t minCost = UINT32_MAX; for(uint32_t memTypeIndex = 0, memTypeBit = 1; @@ -10317,7 +17981,8 @@ VkResult vmaFindMemoryTypeIndex( if((requiredFlags & ~currFlags) == 0) { // Calculate cost as number of bits from preferredFlags not present in this memory type. - uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags); + uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags) + + VmaCountBitsSet(currFlags & notPreferredFlags); // Remember memory type with lowest cost. if(currCost < minCost) { @@ -10334,7 +17999,7 @@ VkResult vmaFindMemoryTypeIndex( return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT; } -VkResult vmaFindMemoryTypeIndexForBufferInfo( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForBufferInfo( VmaAllocator allocator, const VkBufferCreateInfo* pBufferCreateInfo, const VmaAllocationCreateInfo* pAllocationCreateInfo, @@ -10367,7 +18032,7 @@ VkResult vmaFindMemoryTypeIndexForBufferInfo( return res; } -VkResult vmaFindMemoryTypeIndexForImageInfo( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo( VmaAllocator allocator, const VkImageCreateInfo* pImageCreateInfo, const VmaAllocationCreateInfo* pAllocationCreateInfo, @@ -10400,10 +18065,10 @@ VkResult vmaFindMemoryTypeIndexForImageInfo( return res; } -VkResult vmaCreatePool( - VmaAllocator allocator, - const VmaPoolCreateInfo* pCreateInfo, - VmaPool* pPool) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool( + VmaAllocator allocator, + const VmaPoolCreateInfo* pCreateInfo, + VmaPool* pPool) { VMA_ASSERT(allocator && pCreateInfo && pPool); @@ -10423,7 +18088,7 @@ VkResult vmaCreatePool( return res; } -void vmaDestroyPool( +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool( VmaAllocator allocator, VmaPool pool) { @@ -10448,7 +18113,7 @@ void vmaDestroyPool( allocator->DestroyPool(pool); } -void vmaGetPoolStats( +VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats( VmaAllocator allocator, VmaPool pool, VmaPoolStats* pPoolStats) @@ -10460,7 +18125,7 @@ void vmaGetPoolStats( allocator->GetPoolStats(pool, pPoolStats); } -void vmaMakePoolAllocationsLost( +VMA_CALL_PRE void VMA_CALL_POST vmaMakePoolAllocationsLost( VmaAllocator allocator, VmaPool pool, size_t* pLostAllocationCount) @@ -10479,7 +18144,7 @@ void vmaMakePoolAllocationsLost( allocator->MakePoolAllocationsLost(pool, pLostAllocationCount); } -VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool) { VMA_ASSERT(allocator && pool); @@ -10490,7 +18155,42 @@ VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool) return allocator->CheckPoolCorruption(pool); } -VkResult vmaAllocateMemory( +VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName( + VmaAllocator allocator, + VmaPool pool, + const char** ppName) +{ + VMA_ASSERT(allocator && pool && ppName); + + VMA_DEBUG_LOG("vmaGetPoolName"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + *ppName = pool->GetName(); +} + +VMA_CALL_PRE void VMA_CALL_POST vmaSetPoolName( + VmaAllocator allocator, + VmaPool pool, + const char* pName) +{ + VMA_ASSERT(allocator && pool); + + VMA_DEBUG_LOG("vmaSetPoolName"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + pool->SetName(pName); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordSetPoolName(allocator->GetCurrentFrameIndex(), pool, pName); + } +#endif +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory( VmaAllocator allocator, const VkMemoryRequirements* pVkMemoryRequirements, const VmaAllocationCreateInfo* pCreateInfo, @@ -10503,14 +18203,16 @@ VkResult vmaAllocateMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK - VkResult result = allocator->AllocateMemory( + VkResult result = allocator->AllocateMemory( *pVkMemoryRequirements, false, // requiresDedicatedAllocation false, // prefersDedicatedAllocation VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_UNKNOWN, + 1, // allocationCount pAllocation); #if VMA_RECORDING_ENABLED @@ -10529,10 +18231,64 @@ VkResult vmaAllocateMemory( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } -VkResult vmaAllocateMemoryForBuffer( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( + VmaAllocator allocator, + const VkMemoryRequirements* pVkMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, + size_t allocationCount, + VmaAllocation* pAllocations, + VmaAllocationInfo* pAllocationInfo) +{ + if(allocationCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocations); + + VMA_DEBUG_LOG("vmaAllocateMemoryPages"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + VkResult result = allocator->AllocateMemory( + *pVkMemoryRequirements, + false, // requiresDedicatedAllocation + false, // prefersDedicatedAllocation + VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage + VK_NULL_HANDLE, // dedicatedImage + *pCreateInfo, + VMA_SUBALLOCATION_TYPE_UNKNOWN, + allocationCount, + pAllocations); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordAllocateMemoryPages( + allocator->GetCurrentFrameIndex(), + *pVkMemoryRequirements, + *pCreateInfo, + (uint64_t)allocationCount, + pAllocations); + } +#endif + + if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS) + { + for(size_t i = 0; i < allocationCount; ++i) + { + allocator->GetAllocationInfo(pAllocations[i], pAllocationInfo + i); + } + } + + return result; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo* pCreateInfo, @@ -10557,9 +18313,11 @@ VkResult vmaAllocateMemoryForBuffer( requiresDedicatedAllocation, prefersDedicatedAllocation, buffer, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, + 1, // allocationCount pAllocation); #if VMA_RECORDING_ENABLED @@ -10580,10 +18338,10 @@ VkResult vmaAllocateMemoryForBuffer( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } -VkResult vmaAllocateMemoryForImage( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo* pCreateInfo, @@ -10607,9 +18365,11 @@ VkResult vmaAllocateMemoryForImage( requiresDedicatedAllocation, prefersDedicatedAllocation, VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage image, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, + 1, // allocationCount pAllocation); #if VMA_RECORDING_ENABLED @@ -10630,10 +18390,10 @@ VkResult vmaAllocateMemoryForImage( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } -void vmaFreeMemory( +VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( VmaAllocator allocator, VmaAllocation allocation) { @@ -10657,10 +18417,55 @@ void vmaFreeMemory( } #endif - allocator->FreeMemory(allocation); + allocator->FreeMemory( + 1, // allocationCount + &allocation); } -void vmaGetAllocationInfo( +VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( + VmaAllocator allocator, + size_t allocationCount, + const VmaAllocation* pAllocations) +{ + if(allocationCount == 0) + { + return; + } + + VMA_ASSERT(allocator); + + VMA_DEBUG_LOG("vmaFreeMemoryPages"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordFreeMemoryPages( + allocator->GetCurrentFrameIndex(), + (uint64_t)allocationCount, + pAllocations); + } +#endif + + allocator->FreeMemory(allocationCount, pAllocations); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation( + VmaAllocator allocator, + VmaAllocation allocation, + VkDeviceSize newSize) +{ + VMA_ASSERT(allocator && allocation); + + VMA_DEBUG_LOG("vmaResizeAllocation"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + return allocator->ResizeAllocation(allocation, newSize); +} + +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo( VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo* pAllocationInfo) @@ -10681,7 +18486,7 @@ void vmaGetAllocationInfo( allocator->GetAllocationInfo(allocation, pAllocationInfo); } -VkBool32 vmaTouchAllocation( +VMA_CALL_PRE VkBool32 VMA_CALL_POST vmaTouchAllocation( VmaAllocator allocator, VmaAllocation allocation) { @@ -10701,7 +18506,7 @@ VkBool32 vmaTouchAllocation( return allocator->TouchAllocation(allocation); } -void vmaSetAllocationUserData( +VMA_CALL_PRE void VMA_CALL_POST vmaSetAllocationUserData( VmaAllocator allocator, VmaAllocation allocation, void* pUserData) @@ -10723,7 +18528,7 @@ void vmaSetAllocationUserData( #endif } -void vmaCreateLostAllocation( +VMA_CALL_PRE void VMA_CALL_POST vmaCreateLostAllocation( VmaAllocator allocator, VmaAllocation* pAllocation) { @@ -10743,7 +18548,7 @@ void vmaCreateLostAllocation( #endif } -VkResult vmaMapMemory( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaMapMemory( VmaAllocator allocator, VmaAllocation allocation, void** ppData) @@ -10766,7 +18571,7 @@ VkResult vmaMapMemory( return res; } -void vmaUnmapMemory( +VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory( VmaAllocator allocator, VmaAllocation allocation) { @@ -10786,7 +18591,7 @@ void vmaUnmapMemory( allocator->Unmap(allocation); } -void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) { VMA_ASSERT(allocator && allocation); @@ -10794,7 +18599,7 @@ void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDevi VMA_DEBUG_GLOBAL_MUTEX_LOCK - allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); + const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) @@ -10804,9 +18609,11 @@ void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDevi allocation, offset, size); } #endif + + return res; } -void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) { VMA_ASSERT(allocator && allocation); @@ -10814,7 +18621,7 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V VMA_DEBUG_GLOBAL_MUTEX_LOCK - allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); + const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) @@ -10824,9 +18631,75 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V allocation, offset, size); } #endif + + return res; } -VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations( + VmaAllocator allocator, + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, + const VkDeviceSize* sizes) +{ + VMA_ASSERT(allocator); + + if(allocationCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(allocations); + + VMA_DEBUG_LOG("vmaFlushAllocations"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_FLUSH); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + //TODO + } +#endif + + return res; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations( + VmaAllocator allocator, + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, + const VkDeviceSize* sizes) +{ + VMA_ASSERT(allocator); + + if(allocationCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(allocations); + + VMA_DEBUG_LOG("vmaInvalidateAllocations"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_INVALIDATE); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + //TODO + } +#endif + + return res; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) { VMA_ASSERT(allocator); @@ -10837,24 +18710,142 @@ VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) return allocator->CheckCorruption(memoryTypeBits); } -VkResult vmaDefragment( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment( VmaAllocator allocator, - VmaAllocation* pAllocations, + const VmaAllocation* pAllocations, size_t allocationCount, VkBool32* pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats* pDefragmentationStats) { - VMA_ASSERT(allocator && pAllocations); + // Deprecated interface, reimplemented using new one. - VMA_DEBUG_LOG("vmaDefragment"); + VmaDefragmentationInfo2 info2 = {}; + info2.allocationCount = (uint32_t)allocationCount; + info2.pAllocations = pAllocations; + info2.pAllocationsChanged = pAllocationsChanged; + if(pDefragmentationInfo != VMA_NULL) + { + info2.maxCpuAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove; + info2.maxCpuBytesToMove = pDefragmentationInfo->maxBytesToMove; + } + else + { + info2.maxCpuAllocationsToMove = UINT32_MAX; + info2.maxCpuBytesToMove = VK_WHOLE_SIZE; + } + // info2.flags, maxGpuAllocationsToMove, maxGpuBytesToMove, commandBuffer deliberately left zero. + + VmaDefragmentationContext ctx; + VkResult res = vmaDefragmentationBegin(allocator, &info2, pDefragmentationStats, &ctx); + if(res == VK_NOT_READY) + { + res = vmaDefragmentationEnd( allocator, ctx); + } + return res; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationBegin( + VmaAllocator allocator, + const VmaDefragmentationInfo2* pInfo, + VmaDefragmentationStats* pStats, + VmaDefragmentationContext *pContext) +{ + VMA_ASSERT(allocator && pInfo && pContext); + + // Degenerate case: Nothing to defragment. + if(pInfo->allocationCount == 0 && pInfo->poolCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(pInfo->allocationCount == 0 || pInfo->pAllocations != VMA_NULL); + VMA_ASSERT(pInfo->poolCount == 0 || pInfo->pPools != VMA_NULL); + VMA_HEAVY_ASSERT(VmaValidatePointerArray(pInfo->allocationCount, pInfo->pAllocations)); + VMA_HEAVY_ASSERT(VmaValidatePointerArray(pInfo->poolCount, pInfo->pPools)); + + VMA_DEBUG_LOG("vmaDefragmentationBegin"); VMA_DEBUG_GLOBAL_MUTEX_LOCK - return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats); + VkResult res = allocator->DefragmentationBegin(*pInfo, pStats, pContext); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordDefragmentationBegin( + allocator->GetCurrentFrameIndex(), *pInfo, *pContext); + } +#endif + + return res; } -VkResult vmaBindBufferMemory( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd( + VmaAllocator allocator, + VmaDefragmentationContext context) +{ + VMA_ASSERT(allocator); + + VMA_DEBUG_LOG("vmaDefragmentationEnd"); + + if(context != VK_NULL_HANDLE) + { + VMA_DEBUG_GLOBAL_MUTEX_LOCK + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordDefragmentationEnd( + allocator->GetCurrentFrameIndex(), context); + } +#endif + + return allocator->DefragmentationEnd(context); + } + else + { + return VK_SUCCESS; + } +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass( + VmaAllocator allocator, + VmaDefragmentationContext context, + VmaDefragmentationPassInfo* pInfo + ) +{ + VMA_ASSERT(allocator); + VMA_ASSERT(pInfo); + + VMA_DEBUG_LOG("vmaBeginDefragmentationPass"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(context == VK_NULL_HANDLE) + { + pInfo->moveCount = 0; + return VK_SUCCESS; + } + + return allocator->DefragmentationPassBegin(pInfo, context); +} +VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass( + VmaAllocator allocator, + VmaDefragmentationContext context) +{ + VMA_ASSERT(allocator); + + VMA_DEBUG_LOG("vmaEndDefragmentationPass"); + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(context == VK_NULL_HANDLE) + return VK_SUCCESS; + + return allocator->DefragmentationPassEnd(context); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory( VmaAllocator allocator, VmaAllocation allocation, VkBuffer buffer) @@ -10865,10 +18856,26 @@ VkResult vmaBindBufferMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK - return allocator->BindBufferMemory(allocation, buffer); + return allocator->BindBufferMemory(allocation, 0, buffer, VMA_NULL); } -VkResult vmaBindImageMemory( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2( + VmaAllocator allocator, + VmaAllocation allocation, + VkDeviceSize allocationLocalOffset, + VkBuffer buffer, + const void* pNext) +{ + VMA_ASSERT(allocator && allocation && buffer); + + VMA_DEBUG_LOG("vmaBindBufferMemory2"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + return allocator->BindBufferMemory(allocation, allocationLocalOffset, buffer, pNext); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory( VmaAllocator allocator, VmaAllocation allocation, VkImage image) @@ -10879,10 +18886,26 @@ VkResult vmaBindImageMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK - return allocator->BindImageMemory(allocation, image); + return allocator->BindImageMemory(allocation, 0, image, VMA_NULL); } -VkResult vmaCreateBuffer( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2( + VmaAllocator allocator, + VmaAllocation allocation, + VkDeviceSize allocationLocalOffset, + VkImage image, + const void* pNext) +{ + VMA_ASSERT(allocator && allocation && image); + + VMA_DEBUG_LOG("vmaBindImageMemory2"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + return allocator->BindImageMemory(allocation, allocationLocalOffset, image, pNext); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( VmaAllocator allocator, const VkBufferCreateInfo* pBufferCreateInfo, const VmaAllocationCreateInfo* pAllocationCreateInfo, @@ -10891,6 +18914,17 @@ VkResult vmaCreateBuffer( VmaAllocationInfo* pAllocationInfo) { VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation); + + if(pBufferCreateInfo->size == 0) + { + return VK_ERROR_VALIDATION_FAILED_EXT; + } + if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY) != 0 && + !allocator->m_UseKhrBufferDeviceAddress) + { + VMA_ASSERT(0 && "Creating a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT is not valid if VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT was not used."); + return VK_ERROR_VALIDATION_FAILED_EXT; + } VMA_DEBUG_LOG("vmaCreateBuffer"); @@ -10914,33 +18948,17 @@ VkResult vmaCreateBuffer( allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq, requiresDedicatedAllocation, prefersDedicatedAllocation); - // Make sure alignment requirements for specific buffer usages reported - // in Physical Device Properties are included in alignment reported by memory requirements. - if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0) - { - VMA_ASSERT(vkMemReq.alignment % - allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0); - } - if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0) - { - VMA_ASSERT(vkMemReq.alignment % - allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0); - } - if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0) - { - VMA_ASSERT(vkMemReq.alignment % - allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0); - } - // 3. Allocate memory using allocator. res = allocator->AllocateMemory( vkMemReq, requiresDedicatedAllocation, prefersDedicatedAllocation, *pBuffer, // dedicatedBuffer + pBufferCreateInfo->usage, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pAllocationCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, + 1, // allocationCount pAllocation); #if VMA_RECORDING_ENABLED @@ -10957,7 +18975,10 @@ VkResult vmaCreateBuffer( if(res >= 0) { // 3. Bind buffer with memory. - res = allocator->BindBufferMemory(*pAllocation, *pBuffer); + if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_DONT_BIND_BIT) == 0) + { + res = allocator->BindBufferMemory(*pAllocation, 0, *pBuffer, VMA_NULL); + } if(res >= 0) { // All steps succeeded. @@ -10971,7 +18992,9 @@ VkResult vmaCreateBuffer( return VK_SUCCESS; } - allocator->FreeMemory(*pAllocation); + allocator->FreeMemory( + 1, // allocationCount + pAllocation); *pAllocation = VK_NULL_HANDLE; (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks()); *pBuffer = VK_NULL_HANDLE; @@ -10984,7 +19007,7 @@ VkResult vmaCreateBuffer( return res; } -void vmaDestroyBuffer( +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyBuffer( VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation) @@ -11016,11 +19039,13 @@ void vmaDestroyBuffer( if(allocation != VK_NULL_HANDLE) { - allocator->FreeMemory(allocation); + allocator->FreeMemory( + 1, // allocationCount + &allocation); } } -VkResult vmaCreateImage( +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( VmaAllocator allocator, const VkImageCreateInfo* pImageCreateInfo, const VmaAllocationCreateInfo* pAllocationCreateInfo, @@ -11030,6 +19055,15 @@ VkResult vmaCreateImage( { VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation); + if(pImageCreateInfo->extent.width == 0 || + pImageCreateInfo->extent.height == 0 || + pImageCreateInfo->extent.depth == 0 || + pImageCreateInfo->mipLevels == 0 || + pImageCreateInfo->arrayLayers == 0) + { + return VK_ERROR_VALIDATION_FAILED_EXT; + } + VMA_DEBUG_LOG("vmaCreateImage"); VMA_DEBUG_GLOBAL_MUTEX_LOCK @@ -11061,9 +19095,11 @@ VkResult vmaCreateImage( requiresDedicatedAllocation, prefersDedicatedAllocation, VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage *pImage, // dedicatedImage *pAllocationCreateInfo, suballocType, + 1, // allocationCount pAllocation); #if VMA_RECORDING_ENABLED @@ -11080,7 +19116,10 @@ VkResult vmaCreateImage( if(res >= 0) { // 3. Bind image with memory. - res = allocator->BindImageMemory(*pAllocation, *pImage); + if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_DONT_BIND_BIT) == 0) + { + res = allocator->BindImageMemory(*pAllocation, 0, *pImage, VMA_NULL); + } if(res >= 0) { // All steps succeeded. @@ -11094,7 +19133,9 @@ VkResult vmaCreateImage( return VK_SUCCESS; } - allocator->FreeMemory(*pAllocation); + allocator->FreeMemory( + 1, // allocationCount + pAllocation); *pAllocation = VK_NULL_HANDLE; (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks()); *pImage = VK_NULL_HANDLE; @@ -11107,7 +19148,7 @@ VkResult vmaCreateImage( return res; } -void vmaDestroyImage( +VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage( VmaAllocator allocator, VkImage image, VmaAllocation allocation) @@ -11138,7 +19179,9 @@ void vmaDestroyImage( } if(allocation != VK_NULL_HANDLE) { - allocator->FreeMemory(allocation); + allocator->FreeMemory( + 1, // allocationCount + &allocation); } } diff --git a/third_party/vkmemalloc/tnt/README b/third_party/vkmemalloc/tnt/README index 472f59d4ef..9e098bdd43 100644 --- a/third_party/vkmemalloc/tnt/README +++ b/third_party/vkmemalloc/tnt/README @@ -1,7 +1,7 @@ This folder was created as follows: -curl -L -O https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/archive/62c009.zip -unzip 62c009.zip +curl -L -O https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/archive/2193611.zip +unzip 2193611.zip mv VulkanMemoryAllocator-* vkmemalloc cd vkmemalloc rm -rf bin docs media premake third_party tools