diff --git a/bindings/c3/bgfx.c3 b/bindings/c3/bgfx.c3 index 11f767fc5..f1bf18e87 100644 --- a/bindings/c3/bgfx.c3 +++ b/bindings/c3/bgfx.c3 @@ -1853,14 +1853,21 @@ struct InitLimits { // Maximum number of encoder threads. ushort maxEncoders; - // Initial number of draw calls per frame. Rounded up to a - // multiple of 1024 (the minimum); 0 selects the default of 1024. - // The render-item buffers grow on demand up to - // `BGFX_CONFIG_MAX_DRAW_CALLS` and lazily shrink. + // Number of draw calls per frame to reserve storage for. Rounded + // up to a multiple of `BGFX_CONFIG_DRAW_CALL_BLOCK`, which is also + // the minimum. This is a reservation, not a limit: submitting more + // than this grows the storage during the frame, up to + // `BGFX_CONFIG_MAX_DRAW_CALLS`. With + // `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing grows, and + // this is a hard limit that `Caps::Limits::maxDrawCalls` reports + // back; submissions past it are dropped. See + // `Stats::numDrawCallsPeak` to size it. uint numDrawCalls; // Number of frames the draw-call peak (high-water mark) is observed - // before the render-item buffers are shrunk. Set to 0 to disable - // dynamic resizing and keep the buffers fixed at `numDrawCalls`. + // before unused storage is released. Set to 0 to keep whatever has + // been allocated for the lifetime of the context. With + // `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing per frame is + // resized at all, and this only releases unused uniform buffer space. uint numDrawCallPeakFrames; // Minimum resource command buffer size. uint minResourceCbSize; diff --git a/bindings/d/package.d b/bindings/d/package.d index 00441bf2a..8f4be71f8 100644 --- a/bindings/d/package.d +++ b/bindings/d/package.d @@ -9,7 +9,7 @@ import bindbc.common.types: c_int64, c_uint64, va_list; import bindbc.bgfx.config; static import bgfx.impl; -enum uint apiVersion = 153; +enum uint apiVersion = 154; alias ViewID = ushort; @@ -1284,17 +1284,24 @@ extern(C++, "bgfx") struct Init{ ushort maxEncoders; ///Maximum number of encoder threads. /** - Initial number of draw calls per frame. Rounded up to a - multiple of 1024 (the minimum); 0 selects the default of 1024. - The render-item buffers grow on demand up to - `BGFX_CONFIG_MAX_DRAW_CALLS` and lazily shrink. + Number of draw calls per frame to reserve storage for. Rounded + up to a multiple of `BGFX_CONFIG_DRAW_CALL_BLOCK`, which is also + the minimum. This is a reservation, not a limit: submitting more + than this grows the storage during the frame, up to + `BGFX_CONFIG_MAX_DRAW_CALLS`. With + `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing grows, and + this is a hard limit that `Caps::Limits::maxDrawCalls` reports + back; submissions past it are dropped. See + `Stats::numDrawCallsPeak` to size it. */ uint numDrawCalls; /** Number of frames the draw-call peak (high-water mark) is observed - before the render-item buffers are shrunk. Set to 0 to disable - dynamic resizing and keep the buffers fixed at `numDrawCalls`. + before unused storage is released. Set to 0 to keep whatever has + been allocated for the lifetime of the context. With + `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing per frame is + resized at all, and this only releases unused uniform buffer space. */ uint numDrawCallPeakFrames; uint minResourceCBSize; ///Minimum resource command buffer size. diff --git a/docs/internals.rst b/docs/internals.rst index 3232ea33f..138018d7c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -171,6 +171,10 @@ Resource limits ``BGFX_CONFIG_MAX_DRAW_CALLS`` - Maximum number of draw/compute calls per frame. Default is 65535 (64K - 1). +``BGFX_CONFIG_DYNAMIC_FRAME_STORAGE`` - Enable dynamic per frame storage. When enabled, storage for render items, binds, blit items and scissor rectangles is allocated in blocks, on first touch, and grows during the frame instead of dropping submissions; ``Init::Limits::numDrawCalls`` is then what is reserved up front rather than a hard limit, and ``Caps::Limits::maxDrawCalls`` always reports ``BGFX_CONFIG_MAX_DRAW_CALLS``. When disabled, all of it is allocated once, up front, at exactly the requested size, indexing has no indirection, and ``Init::Limits::numDrawCalls`` is a hard limit that submissions are dropped past. Default is 1. Disabling trades memory for a small amount of submission throughput; see ``Stats::numDrawCallsPeak`` to size ``numDrawCalls``. + +``BGFX_CONFIG_DRAW_CALL_BLOCK`` - Granularity dynamic per frame storage grows by, in items, and the multiple ``Init::Limits::numDrawCalls`` is rounded up to. Must be a power of two. Default is 64. + ``BGFX_CONFIG_MAX_BLIT_ITEMS`` - Maximum number of blit items per frame. Default is 1024. ``BGFX_CONFIG_MAX_VIEWS`` - Maximum number of views. Default is 256. Must be a power of 2. @@ -224,11 +228,11 @@ Buffer sizes ``BGFX_CONFIG_MIN_RESOURCE_COMMAND_BUFFER_SIZE`` - Minimum initial size of the resource command buffer (pre/post render commands for resource creation and updates). Default is 64 KB. The buffer grows as needed. -``BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE`` - Minimum initial size in bytes of the per-encoder uniform buffer. Default is 1 MB. This buffer will resize on demand. +``BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE`` - Minimum initial size in bytes of the per-encoder uniform buffer. Default is 128 KB. This buffer will resize on demand. Must be larger than ``BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE``, otherwise the buffer resizes on first use. -``BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE`` - Maximum amount of unused uniform buffer space (in bytes) before the buffer is shrunk. Default is 64 KB. +``BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE`` - Head room in bytes kept in the uniform buffer; it grows once less than this is left, and shrinking keeps this much above the peak. Must be at least as large as the largest single uniform record (4 + 1023*sizeof(Mat4) = 65476 bytes). Default is 64 KB. -``BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE`` - Increment size for uniform buffer resize. Default is 1 MB. +``BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE`` - Increment size for uniform buffer resize. Default is 64 KB. ``BGFX_CONFIG_CACHED_DEVICE_MEMORY_ALLOCATIONS_SIZE`` - Amount of allowed memory allocations left on device to use for recycling during later allocations. This can be beneficial in case the driver is slow allocating memory on the device. Default is 128 MB. Currently only used by the Vulkan backend. diff --git a/examples/common/debugdraw/debugdraw.cpp b/examples/common/debugdraw/debugdraw.cpp index c7f34c944..5d0587ad3 100644 --- a/examples/common/debugdraw/debugdraw.cpp +++ b/examples/common/debugdraw/debugdraw.cpp @@ -1132,9 +1132,9 @@ struct DebugDrawEncoderImpl bgfx::Transform transform; stack.mtx = m_encoder->allocTransform(&transform, _num); - stack.num = _num; + stack.num = transform.num; stack.data = transform.data; - bx::memCopy(transform.data, _mtx, _num*64); + bx::memCopy(transform.data, _mtx, transform.num*64); } void setTranslate(float _x, float _y, float _z) diff --git a/include/bgfx/bgfx.h b/include/bgfx/bgfx.h index d0091f0bb..2e6c7f6cc 100644 --- a/include/bgfx/bgfx.h +++ b/include/bgfx/bgfx.h @@ -685,13 +685,20 @@ namespace bgfx Limits(); uint16_t maxEncoders; //!< Maximum number of encoder threads. - uint32_t numDrawCalls; //!< Initial number of draw calls per frame. Rounded up to a - /// multiple of 1024 (the minimum); 0 selects the default of 1024. - /// The render-item buffers grow on demand up to - /// `BGFX_CONFIG_MAX_DRAW_CALLS` and lazily shrink. + uint32_t numDrawCalls; //!< Number of draw calls per frame to reserve storage for. Rounded + /// up to a multiple of `BGFX_CONFIG_DRAW_CALL_BLOCK`, which is also + /// the minimum. This is a reservation, not a limit: submitting more + /// than this grows the storage during the frame, up to + /// `BGFX_CONFIG_MAX_DRAW_CALLS`. With + /// `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing grows, and + /// this is a hard limit that `Caps::Limits::maxDrawCalls` reports + /// back; submissions past it are dropped. See + /// `Stats::numDrawCallsPeak` to size it. uint32_t numDrawCallPeakFrames; //!< Number of frames the draw-call peak (high-water mark) is observed - /// before the render-item buffers are shrunk. Set to 0 to disable - /// dynamic resizing and keep the buffers fixed at `numDrawCalls`. + /// before unused storage is released. Set to 0 to keep whatever has + /// been allocated for the lifetime of the context. With + /// `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing per frame is + /// resized at all, and this only releases unused uniform buffer space. uint32_t minResourceCbSize; //!< Minimum resource command buffer size. uint32_t maxTransientVbSize; //!< Maximum transient vertex buffer size. uint32_t maxTransientIbSize; //!< Maximum transient index buffer size. diff --git a/include/bgfx/c99/bgfx.h b/include/bgfx/c99/bgfx.h index d616e7514..d7cca9ca6 100644 --- a/include/bgfx/c99/bgfx.h +++ b/include/bgfx/c99/bgfx.h @@ -757,17 +757,24 @@ typedef struct bgfx_init_limits_s uint16_t maxEncoders; /** Maximum number of encoder threads. */ /** - * Initial number of draw calls per frame. Rounded up to a - * multiple of 1024 (the minimum); 0 selects the default of 1024. - * The render-item buffers grow on demand up to - * `BGFX_CONFIG_MAX_DRAW_CALLS` and lazily shrink. + * Number of draw calls per frame to reserve storage for. Rounded + * up to a multiple of `BGFX_CONFIG_DRAW_CALL_BLOCK`, which is also + * the minimum. This is a reservation, not a limit: submitting more + * than this grows the storage during the frame, up to + * `BGFX_CONFIG_MAX_DRAW_CALLS`. With + * `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing grows, and + * this is a hard limit that `Caps::Limits::maxDrawCalls` reports + * back; submissions past it are dropped. See + * `Stats::numDrawCallsPeak` to size it. */ uint32_t numDrawCalls; /** * Number of frames the draw-call peak (high-water mark) is observed - * before the render-item buffers are shrunk. Set to 0 to disable - * dynamic resizing and keep the buffers fixed at `numDrawCalls`. + * before unused storage is released. Set to 0 to keep whatever has + * been allocated for the lifetime of the context. With + * `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing per frame is + * resized at all, and this only releases unused uniform buffer space. */ uint32_t numDrawCallPeakFrames; uint32_t minResourceCbSize; /** Minimum resource command buffer size. */ diff --git a/include/bgfx/defines.h b/include/bgfx/defines.h index c86ff96fd..c1847789a 100644 --- a/include/bgfx/defines.h +++ b/include/bgfx/defines.h @@ -15,7 +15,7 @@ #ifndef BGFX_DEFINES_H_HEADER_GUARD #define BGFX_DEFINES_H_HEADER_GUARD -#define BGFX_API_VERSION UINT32_C(153) +#define BGFX_API_VERSION UINT32_C(154) /** * Color RGB/alpha/depth write. When it's not specified write will be disabled. diff --git a/scripts/bgfx.idl b/scripts/bgfx.idl index 0091260a6..e3d5e387f 100644 --- a/scripts/bgfx.idl +++ b/scripts/bgfx.idl @@ -1,7 +1,7 @@ -- vim: syntax=lua -- bgfx interface -version(153) +version(154) typedef "bool" typedef "char" @@ -927,13 +927,20 @@ struct.Resolution { ctor, section = "Initialization and Shutdown" } --- Configurable runtime limits parameters. struct.Limits { ctor, namespace = "Init" } .maxEncoders "uint16_t" --- Maximum number of encoder threads. - .numDrawCalls "uint32_t" --- Initial number of draw calls per frame. Rounded up to a - --- multiple of 1024 (the minimum); 0 selects the default of 1024. - --- The render-item buffers grow on demand up to - --- `BGFX_CONFIG_MAX_DRAW_CALLS` and lazily shrink. + .numDrawCalls "uint32_t" --- Number of draw calls per frame to reserve storage for. Rounded + --- up to a multiple of `BGFX_CONFIG_DRAW_CALL_BLOCK`, which is also + --- the minimum. This is a reservation, not a limit: submitting more + --- than this grows the storage during the frame, up to + --- `BGFX_CONFIG_MAX_DRAW_CALLS`. With + --- `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing grows, and + --- this is a hard limit that `Caps::Limits::maxDrawCalls` reports + --- back; submissions past it are dropped. See + --- `Stats::numDrawCallsPeak` to size it. .numDrawCallPeakFrames "uint32_t" --- Number of frames the draw-call peak (high-water mark) is observed - --- before the render-item buffers are shrunk. Set to 0 to disable - --- dynamic resizing and keep the buffers fixed at `numDrawCalls`. + --- before unused storage is released. Set to 0 to keep whatever has + --- been allocated for the lifetime of the context. With + --- `BGFX_CONFIG_DYNAMIC_FRAME_STORAGE` disabled nothing per frame is + --- resized at all, and this only releases unused uniform buffer space. .minResourceCbSize "uint32_t" --- Minimum resource command buffer size. .maxTransientVbSize "uint32_t" --- Maximum transient vertex buffer size. .maxTransientIbSize "uint32_t" --- Maximum transient index buffer size. diff --git a/src/bgfx.cpp b/src/bgfx.cpp index a06bb8a95..bb07fda46 100644 --- a/src/bgfx.cpp +++ b/src/bgfx.cpp @@ -1515,6 +1515,13 @@ namespace bgfx m_occlusionQuerySet.insert(_occlusionQuery.idx); } + BX_ASSERT(m_draw.m_startMatrix + m_draw.m_numMatrices <= m_frame->m_frameCache.m_matrixCache.m_max+1 + , "Draw names %d matrices from %d, but the matrix cache only holds %d." + , m_draw.m_numMatrices + , m_draw.m_startMatrix + , m_frame->m_frameCache.m_matrixCache.m_max+1 + ); + if (m_discard) { discard(_flags); @@ -1654,12 +1661,14 @@ namespace bgfx void EncoderImpl::blit(ViewId _id, TextureHandle _dst, uint8_t _dstMip, uint16_t _dstX, uint16_t _dstY, uint16_t _dstZ, TextureHandle _src, uint8_t _srcMip, uint16_t _srcX, uint16_t _srcY, uint16_t _srcZ, uint16_t _width, uint16_t _height, uint16_t _depth) { - BX_WARN(m_frame->m_numBlitItems < BGFX_CONFIG_MAX_BLIT_ITEMS - , "Exceed number of available blit items per frame. BGFX_CONFIG_MAX_BLIT_ITEMS is %d. Skipping blit." + const uint32_t blitItemIdx = bx::atomicFetchAndAddsat(&m_frame->m_numBlitItems, 1, BGFX_CONFIG_MAX_BLIT_ITEMS); + + BX_WARN(blitItemIdx < BGFX_CONFIG_MAX_BLIT_ITEMS + , "Exceeded number of available blit items per frame. BGFX_CONFIG_MAX_BLIT_ITEMS is %d. Skipping blit." , BGFX_CONFIG_MAX_BLIT_ITEMS ); - const uint32_t blitItemIdx = bx::atomicFetchAndAddsat(&m_frame->m_numBlitItems, 1, BGFX_CONFIG_MAX_BLIT_ITEMS); - if (BGFX_CONFIG_MAX_BLIT_ITEMS-1 <= blitItemIdx) + + if (blitItemIdx >= BGFX_CONFIG_MAX_BLIT_ITEMS) { return; } @@ -1678,11 +1687,7 @@ namespace bgfx bi.m_dstMip = _dstMip; bi.m_src = _src; bi.m_dst = _dst; - - BlitKey key; - key.m_view = _id; - key.m_item = bx::narrowCast(blitItemIdx); - m_frame->m_blitKeys[blitItemIdx] = key.encode(); + bi.m_view = _id; } void Frame::sort() @@ -1740,9 +1745,14 @@ namespace bgfx bx::radixSort(m_sortKeys, s_ctx->m_tempKeys, m_sortValues, s_ctx->m_tempValues, m_numRenderItems); + reserveBlitKeys(m_numBlitItems); + for (uint32_t ii = 0, num = m_numBlitItems; ii < num; ++ii) { - m_blitKeys[ii] = BlitKey::remapView(m_blitKeys[ii], m_viewOrder); + BlitKey key; + key.m_view = m_viewOrder[m_blitItem[ii].m_view]; + key.m_item = bx::narrowCast(ii); + m_blitKeys[ii] = key.encode(); } bx::radixSort(m_blitKeys, (uint32_t*)s_ctx->m_tempKeys, m_numBlitItems); @@ -2339,10 +2349,10 @@ namespace bgfx m_frameTimeLast = bx::getHPCounter(); m_flipAfterRender = !!(m_init.resolution.reset & BGFX_RESET_FLIP_AFTER_RENDER); - m_submit->create(_init.limits.minResourceCbSize, _init.limits.numDrawCalls, _init.limits.numDrawCallPeakFrames); + m_submit->create(_init.limits.minResourceCbSize, _init.limits.numDrawCalls, g_caps.limits.maxDrawCalls, _init.limits.numDrawCallPeakFrames); #if BGFX_CONFIG_MULTITHREADED - m_render->create(_init.limits.minResourceCbSize, _init.limits.numDrawCalls, _init.limits.numDrawCallPeakFrames); + m_render->create(_init.limits.minResourceCbSize, _init.limits.numDrawCalls, g_caps.limits.maxDrawCalls, _init.limits.numDrawCallPeakFrames); if (s_renderFrameCalled) { @@ -4013,8 +4023,11 @@ namespace bgfx Init::Limits::Limits() : maxEncoders(BGFX_CONFIG_DEFAULT_MAX_ENCODERS) - , numDrawCalls(BGFX_CONFIG_MAX_DRAW_CALLS) - , numDrawCallPeakFrames(0) + , numDrawCalls(BX_ENABLED(BGFX_CONFIG_DYNAMIC_FRAME_STORAGE) + ? BGFX_CONFIG_DRAW_CALL_BLOCK + : BGFX_CONFIG_MAX_DRAW_CALLS + ) + , numDrawCallPeakFrames(60) , minResourceCbSize(BGFX_CONFIG_MIN_RESOURCE_COMMAND_BUFFER_SIZE) , maxTransientVbSize(BGFX_CONFIG_MAX_TRANSIENT_VERTEX_BUFFER_SIZE) , maxTransientIbSize(BGFX_CONFIG_MAX_TRANSIENT_INDEX_BUFFER_SIZE) @@ -4093,9 +4106,9 @@ namespace bgfx } bx::memSet(&g_caps, 0, sizeof(g_caps) ); - g_caps.limits.maxDrawCalls = 0 == init.limits.numDrawCallPeakFrames - ? init.limits.numDrawCalls - : BGFX_CONFIG_MAX_DRAW_CALLS + g_caps.limits.maxDrawCalls = BX_ENABLED(BGFX_CONFIG_DYNAMIC_FRAME_STORAGE) + ? BGFX_CONFIG_MAX_DRAW_CALLS + : init.limits.numDrawCalls ; g_caps.limits.maxBlits = BGFX_CONFIG_MAX_BLIT_ITEMS; g_caps.limits.maxTextureSize = 0; @@ -5907,6 +5920,7 @@ namespace bgfx BGFX_CHECK_HANDLE("clearTexture", s_ctx->m_textureHandle, _handle); const TextureRef& ref = s_ctx->m_textureRef[_handle.idx]; + BX_UNUSED(ref); BX_ASSERT(!ref.isDepth() , "Texture (handle %d, '%S') has a depth/stencil format and can't be cleared; use a view depth clear instead." diff --git a/src/bgfx_p.h b/src/bgfx_p.h index dbff41d73..a0114207a 100644 --- a/src/bgfx_p.h +++ b/src/bgfx_p.h @@ -314,6 +314,8 @@ namespace bgfx #endif // BGFX_CONFIG_MAX_DRAW_CALLS < (64<<10) static constexpr uint32_t kDrawCallBlock = BGFX_CONFIG_DRAW_CALL_BLOCK; + static constexpr uint32_t kBlitBlock = 64; + static constexpr uint32_t kRectBlock = 64; inline uint32_t alignDrawCalls(uint32_t _num) { @@ -1526,6 +1528,10 @@ namespace bgfx ^ kItemMask ), "BlitKey: Key mask shouldn't overlap!"); + static_assert(BGFX_CONFIG_MAX_BLIT_ITEMS <= kItemMask+1 + , "BGFX_CONFIG_MAX_BLIT_ITEMS doesn't fit in the BlitKey item field." + ); + KeyT encode() { const KeyT view = (KeyT(m_view) << kViewShift) & kViewMask; @@ -1577,29 +1583,207 @@ namespace bgfx } }; + template + class FrameArenaT + { + public: + static_assert(0 == (BlockT & (BlockT-1) ), "BGFX_CONFIG_DRAW_CALL_BLOCK must be power of two."); + + ~FrameArenaT() + { + destroy(); + } + +#if BGFX_CONFIG_DYNAMIC_FRAME_STORAGE + FrameArenaT() + : m_block(NULL) + , m_reserved(NULL) + , m_numBlocks(0) + , m_numReservedBlocks(0) + { + } + + void create(uint32_t _numReserved, uint32_t _numMax) + { + destroy(); + + m_numBlocks = numBlocks(_numMax); + m_numReservedBlocks = bx::min(numBlocks(_numReserved), m_numBlocks); + + m_block = (Ty**)bx::alloc(g_allocator, sizeof(Ty*)*m_numBlocks); + bx::memSet(m_block, 0, sizeof(Ty*)*m_numBlocks); + + if (0 < m_numReservedBlocks) + { + const uint32_t size = sizeof(Ty)*BlockT*m_numReservedBlocks; + + m_reserved = (Ty*)bx::alloc(g_allocator, size, BX_ALIGNOF(Ty) ); + bx::memSet(m_reserved, 0, size); + + for (uint32_t ii = 0; ii < m_numReservedBlocks; ++ii) + { + m_block[ii] = m_reserved + ii*BlockT; + } + } + } + + void destroy() + { + shrink(0); + + if (NULL != m_reserved) + { + bx::free(g_allocator, m_reserved, BX_ALIGNOF(Ty) ); + m_reserved = NULL; + } + + bx::free(g_allocator, m_block); + m_block = NULL; + m_numBlocks = 0; + m_numReservedBlocks = 0; + } + + BX_FORCE_INLINE Ty& operator[](uint32_t _idx) + { + const uint32_t blockIdx = _idx/BlockT; + + Ty* block = load(blockIdx); + + if (NULL == block) + { + block = allocBlock(blockIdx); + } + + return block[_idx%BlockT]; + } + + BX_FORCE_INLINE const Ty& operator[](uint32_t _idx) const + { + return load(_idx/BlockT)[_idx%BlockT]; + } + + void shrink(uint32_t _numItems) + { + for (uint32_t ii = bx::max(m_numReservedBlocks, numBlocks(_numItems) ); ii < m_numBlocks; ++ii) + { + if (NULL != m_block[ii]) + { + bx::free(g_allocator, m_block[ii], BX_ALIGNOF(Ty) ); + m_block[ii] = NULL; + } + } + } + + private: + static BX_FORCE_INLINE uint32_t numBlocks(uint32_t _numItems) + { + return (_numItems + BlockT - 1)/BlockT; + } + + BX_FORCE_INLINE Ty* load(uint32_t _blockIdx) const + { + return *(Ty* volatile*)&m_block[_blockIdx]; + } + + BX_NO_INLINE Ty* allocBlock(uint32_t _blockIdx) + { + bx::MutexScope lock(m_lock); + + Ty* block = m_block[_blockIdx]; + + if (NULL == block) + { + constexpr uint32_t size = sizeof(Ty)*BlockT; + + block = (Ty*)bx::alloc(g_allocator, size, BX_ALIGNOF(Ty) ); + bx::memSet(block, 0, size); + + bx::atomicExchangePtr( (void**)&m_block[_blockIdx], block); + } + + return block; + } + + Ty** m_block; + Ty* m_reserved; + uint32_t m_numBlocks; + uint32_t m_numReservedBlocks; + bx::Mutex m_lock; +#else + FrameArenaT() + : m_data(NULL) + { + } + + void create(uint32_t /*_numReserved*/, uint32_t _numMax) + { + destroy(); + + const uint32_t size = sizeof(Ty)*_numMax; + + m_data = (Ty*)bx::alloc(g_allocator, size, BX_ALIGNOF(Ty) ); + bx::memSet(m_data, 0, size); + } + + void destroy() + { + if (NULL != m_data) + { + bx::free(g_allocator, m_data, BX_ALIGNOF(Ty) ); + m_data = NULL; + } + } + + BX_FORCE_INLINE Ty& operator[](uint32_t _idx) + { + return m_data[_idx]; + } + + BX_FORCE_INLINE const Ty& operator[](uint32_t _idx) const + { + return m_data[_idx]; + } + + void shrink(uint32_t /*_numItems*/) + { + } + + private: + Ty* m_data; +#endif // BGFX_CONFIG_DYNAMIC_FRAME_STORAGE + }; + struct MatrixCache { MatrixCache() : m_cache(NULL) , m_num(1) , m_max(0) + , m_capacity(0) , m_peak(0) , m_observe(0) , m_numPeakFrames(0) - , m_overflowed(false) + , m_overflowedBy(0) { } - void create(uint32_t _max, uint32_t _numPeakFrames) + void create(uint32_t _numReserved, uint32_t _numMax, uint32_t _numPeakFrames) { - m_max = bx::min(_max, BGFX_CONFIG_MAX_MATRIX_CACHE); - m_cache = (Matrix4*)bx::alloc(g_allocator, sizeof(Matrix4)*m_max); - m_cache[0].setIdentity(); + m_capacity = bx::min(_numMax, BGFX_CONFIG_MAX_MATRIX_CACHE); + + if (!BX_ENABLED(BGFX_CONFIG_DYNAMIC_FRAME_STORAGE) ) + { + _numReserved = m_capacity; + _numPeakFrames = 0; + } + + m_max = bx::min(_numReserved, m_capacity); + alloc(); m_num = 1; m_peak = 0; m_observe = 0; m_numPeakFrames = _numPeakFrames; - m_overflowed = false; + m_overflowedBy = 0; } void destroy() @@ -1611,50 +1795,46 @@ namespace bgfx void resize(uint32_t _max) { bx::free(g_allocator, m_cache); - m_max = _max; - m_cache = (Matrix4*)bx::alloc(g_allocator, sizeof(Matrix4)*m_max); - m_cache[0].setIdentity(); + m_max = _max; + alloc(); } - static uint32_t capacityFor(uint32_t _used) + uint32_t capacityFor(uint32_t _used) const { - return bx::min(BGFX_CONFIG_MAX_MATRIX_CACHE, uint32_t(bx::alignUp(_used-1, kDrawCallBlock) )+1); + return bx::min(m_capacity, uint32_t(bx::alignUp(_used-1, kDrawCallBlock) )+1); } void reset() { const uint32_t used = m_num; - if (0 != m_numPeakFrames) + if (0 != m_overflowedBy + && m_max < m_capacity) { - if (m_overflowed - && m_max < BGFX_CONFIG_MAX_MATRIX_CACHE) + resize(bx::min(m_capacity, capacityFor(m_max + m_overflowedBy) ) ); + m_peak = 0; + m_observe = 0; + } + else if (0 != m_numPeakFrames) + { + m_peak = bx::max(m_peak, used); + + if (++m_observe >= m_numPeakFrames) { - resize(bx::min(BGFX_CONFIG_MAX_MATRIX_CACHE, m_max*2) ); + const uint32_t want = bx::max(kDrawCallBlock+1, capacityFor(m_peak) ); + + if (want < m_max) + { + resize(want); + } + m_peak = 0; m_observe = 0; } - else - { - m_peak = bx::max(m_peak, used); - - if (++m_observe >= m_numPeakFrames) - { - const uint32_t want = bx::max(kDrawCallBlock+1, capacityFor(m_peak) ); - - if (want < m_max) - { - resize(want); - } - - m_peak = 0; - m_observe = 0; - } - } } - m_overflowed = false; - m_num = 1; + m_overflowedBy = 0; + m_num = 1; } uint32_t reserve(uint16_t* _num) @@ -1664,7 +1844,7 @@ namespace bgfx if (first+num > m_max) { - m_overflowed = true; + bx::atomicFetchAndAddsat(&m_overflowedBy, num, m_capacity); } num = bx::min(num, m_max-first); @@ -1673,21 +1853,23 @@ namespace bgfx return first; } - uint32_t add(const void* _mtx, uint16_t _num) + uint32_t add(const void* _mtx, uint16_t* _num) { if (NULL != _mtx) { - uint32_t first = reserve(&_num); - bx::memCopy(&m_cache[first], _mtx, sizeof(Matrix4)*_num); + uint32_t first = reserve(_num); + bx::memCopy(&m_cache[first], _mtx, sizeof(Matrix4)*(*_num) ); return first; } + *_num = 1; + return 0; } float* toPtr(uint32_t _cacheIdx) { - BX_ASSERT(_cacheIdx < m_max, "Matrix cache out of bounds index %d (max: %d)" + BX_ASSERT(_cacheIdx <= m_max, "Matrix cache out of bounds index %d (max: %d)" , _cacheIdx , m_max ); @@ -1699,44 +1881,83 @@ namespace bgfx return uint32_t( (const Matrix4*)_ptr - m_cache); } + void alloc() + { + m_cache = (Matrix4*)bx::alloc(g_allocator, sizeof(Matrix4)*(m_max + 1) ); + m_cache[0].setIdentity(); + m_cache[m_max].setIdentity(); + } + Matrix4* m_cache; uint32_t m_num; uint32_t m_max; + uint32_t m_capacity; uint32_t m_peak; uint32_t m_observe; uint32_t m_numPeakFrames; - bool m_overflowed; + uint32_t m_overflowedBy; }; struct RectCache { + static_assert(BGFX_CONFIG_MAX_RECT_CACHE <= UINT16_MAX + , "BGFX_CONFIG_MAX_RECT_CACHE must leave UINT16_MAX free." + ); + RectCache() : m_num(0) + , m_max(0) { } + void create(uint32_t _numMax) + { + m_max = _numMax; + m_cache.create(0, _numMax); + } + + void destroy() + { + m_cache.destroy(); + } + void reset() { m_num = 0; } + void shrink(uint32_t _numItems) + { + m_cache.shrink(bx::min(m_max, _numItems) ); + } + uint32_t add(uint16_t _x, uint16_t _y, uint16_t _width, uint16_t _height) { - const uint32_t first = bx::atomicFetchAndAddsat(&m_num, 1, BGFX_CONFIG_MAX_RECT_CACHE-1); - BX_ASSERT(first+1 < BGFX_CONFIG_MAX_RECT_CACHE, "Rect cache overflow. %d (max: %d)", first, BGFX_CONFIG_MAX_RECT_CACHE); + const uint32_t first = bx::atomicFetchAndAddsat(&m_num, 1, m_max); + + BX_WARN(first < m_max + , "Exceeded number of available scissor rectangles per frame. BGFX_CONFIG_MAX_RECT_CACHE is %d." + , BGFX_CONFIG_MAX_RECT_CACHE + ); + + if (first >= m_max) + { + return UINT16_MAX; + } Rect& rect = m_cache[first]; rect.m_x = bx::narrowCast(_x); rect.m_y = bx::narrowCast(_y); - rect.m_width = _width; + rect.m_width = _width; rect.m_height = _height; return first; } - Rect m_cache[BGFX_CONFIG_MAX_RECT_CACHE]; - uint32_t m_num; + FrameArenaT m_cache; + uint32_t m_num; + uint32_t m_max; }; static constexpr uint8_t kConstantOpcodeTypeShift = 27; @@ -1762,6 +1983,15 @@ namespace bgfx class UniformBuffer { public: + static_assert(BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE + >= sizeof(uint32_t) + (kConstantOpcodeNumMask>>kConstantOpcodeNumShift)*sizeof(float)*16 + , "BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE is too small!" + ); + + static_assert(BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE > BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE + , "BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE must be larger than BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE!" + ); + static UniformBuffer* create(uint32_t _size) { const uint32_t structSize = sizeof(UniformBuffer)-sizeof(UniformBuffer::m_buffer); @@ -1785,16 +2015,30 @@ namespace bgfx UniformBuffer* uniformBuffer = *_uniformBuffer; if (kThreshold >= uniformBuffer->m_size - uniformBuffer->m_pos) { - const uint32_t structSize = sizeof(UniformBuffer)-sizeof(UniformBuffer::m_buffer); - uint32_t size = bx::alignUp(uniformBuffer->m_size + kIncrement, 16); - void* data = bx::realloc(g_allocator, uniformBuffer, size+structSize); - uniformBuffer = reinterpret_cast(data); - uniformBuffer->m_size = size; - - *_uniformBuffer = uniformBuffer; + const uint32_t size = uniformBuffer->m_size + + bx::max(kIncrement, uniformBuffer->m_size/2) + ; + resize(_uniformBuffer, size); } } + static void shrink(UniformBuffer** _uniformBuffer, uint32_t _minSize) + { + static constexpr uint32_t kThreshold = BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE; + static constexpr uint32_t kIncrement = BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE; + + UniformBuffer* uniformBuffer = *_uniformBuffer; + + const uint32_t keep = bx::alignUp(bx::max(_minSize, uniformBuffer->m_hwm + kThreshold), 16); + + if (keep + kIncrement <= uniformBuffer->m_size) + { + resize(_uniformBuffer, keep); + } + + (*_uniformBuffer)->m_hwm = 0; + } + static uint32_t encodeOpcode(uint8_t _type, uint16_t _loc, uint16_t _num, uint16_t _copy) { const uint32_t type = _type << kConstantOpcodeTypeShift; @@ -1866,6 +2110,7 @@ namespace bgfx void finish() { write(UniformType::End); + m_hwm = bx::max(m_hwm, m_pos); m_pos = 0; } @@ -1874,9 +2119,23 @@ namespace bgfx void writeMarker(const bx::StringView& _name); private: + static void resize(UniformBuffer** _uniformBuffer, uint32_t _size) + { + const uint32_t structSize = sizeof(UniformBuffer)-sizeof(UniformBuffer::m_buffer); + + const uint32_t size = bx::alignUp(_size, 16); + void* data = bx::realloc(g_allocator, *_uniformBuffer, size+structSize); + + UniformBuffer* uniformBuffer = reinterpret_cast(data); + uniformBuffer->m_size = size; + + *_uniformBuffer = uniformBuffer; + } + UniformBuffer(uint32_t _size) : m_size(_size) , m_pos(0) + , m_hwm(0) { finish(); } @@ -1887,6 +2146,7 @@ namespace bgfx uint32_t m_size; uint32_t m_pos; + uint32_t m_hwm; char m_buffer[256<<20]; }; @@ -2246,6 +2506,7 @@ namespace bgfx uint8_t m_dstMip; Handle m_src; Handle m_dst; + ViewId m_view; }; struct IndexBuffer @@ -2566,6 +2827,13 @@ namespace bgfx , m_offset , kMaxOffset ); + BX_ASSERT(true + && 0 == (m_size & 0xf) + && 0 == (m_offset & 0xf) + , "UniformCacheKey size and offset must be 16 byte aligned (size %d, offset %d)!" + , m_size + , m_offset + ); BX_UNUSED(kMaxSize, kMaxOffset); const KeyT view = (KeyT(m_view) << kViewShift) & kViewMask; @@ -2621,8 +2889,9 @@ namespace bgfx , m_keysCapacity(kMinKeysCapacity) , m_dataCapacity(kMinDataCapacity) { - m_keys = (UniformCacheKey::KeyT*)bx::alloc(g_allocator, m_keysCapacity*sizeof(uint64_t) ); + m_keys = (UniformCacheKey::KeyT*)bx::alloc(g_allocator, (m_keysCapacity+1)*sizeof(uint64_t) ); m_data = (uint8_t*)bx::alloc(g_allocator, m_dataCapacity); + m_keys[0] = 0; } ~UniformCacheFrame() @@ -2638,7 +2907,7 @@ namespace bgfx if (newKeysCapacity != m_keysCapacity) { - m_keys = (UniformCacheKey::KeyT*)bx::realloc(g_allocator, m_keys, newKeysCapacity * sizeof(uint64_t)); + m_keys = (UniformCacheKey::KeyT*)bx::realloc(g_allocator, m_keys, (newKeysCapacity+1) * sizeof(uint64_t)); m_keysCapacity = newKeysCapacity; } } @@ -2673,14 +2942,16 @@ namespace bgfx struct FrameCache { - void create(uint32_t _maxMatrices, uint32_t _numHwmFrames) + void create(uint32_t _numReservedMatrices, uint32_t _numMaxMatrices, uint32_t _numHwmFrames) { - m_matrixCache.create(_maxMatrices, _numHwmFrames); + m_matrixCache.create(_numReservedMatrices, _numMaxMatrices, _numHwmFrames); + m_rectCache.create(BGFX_CONFIG_MAX_RECT_CACHE); } void destroy() { m_matrixCache.destroy(); + m_rectCache.destroy(); } void reset() @@ -2716,11 +2987,13 @@ namespace bgfx Frame() : m_sortKeys(NULL) , m_sortValues(NULL) - , m_renderItem(NULL) - , m_renderBind(NULL) + , m_blitKeys(NULL) + , m_blitKeysCapacity(0) , m_maxDrawCalls(0) , m_numRenderItemsRequested(0) , m_peak(0) + , m_peakBlit(0) + , m_peakRect(0) , m_observe(0) , m_numPeakFrames(0) , m_waitSubmit(0) @@ -2743,33 +3016,54 @@ namespace bgfx freeArrays(); } - void allocArrays(uint32_t _maxDrawCalls) + void allocArrays(uint32_t _numReservedDrawCalls, uint32_t _maxDrawCalls) { freeArrays(); m_maxDrawCalls = _maxDrawCalls; - const uint32_t num = m_maxDrawCalls + 1; + const uint32_t reserved = bx::min(_numReservedDrawCalls, _maxDrawCalls) + 1; + const uint32_t num = m_maxDrawCalls + 1; + m_sortKeys = (uint64_t* )bx::alloc(g_allocator, sizeof(uint64_t )*num); m_sortValues = (RenderItemCount*)bx::alloc(g_allocator, sizeof(RenderItemCount)*num); - m_renderItem = (RenderItem* )bx::alloc(g_allocator, sizeof(RenderItem )*num, BX_ALIGNOF(RenderItem) ); - m_renderBind = (RenderBind* )bx::alloc(g_allocator, sizeof(RenderBind )*num, BX_ALIGNOF(RenderBind) ); - bx::memSet(m_renderBind, 0, sizeof(RenderBind)*num); + m_renderItem.create(reserved, num); + m_renderBind.create(reserved, num); + + m_blitItem.create(0, BGFX_CONFIG_MAX_BLIT_ITEMS); + reserveBlitKeys(0); setSentinel(); } + void reserveBlitKeys(uint32_t _num) + { + const uint32_t capacity = bx::alignUp(_num+1, 64); + + if (m_blitKeysCapacity < capacity) + { + bx::free(g_allocator, m_blitKeys); + m_blitKeys = (uint32_t*)bx::alloc(g_allocator, sizeof(uint32_t)*capacity); + m_blitKeysCapacity = capacity; + } + + m_blitKeys[_num] = 0; + } + void freeArrays() { bx::free(g_allocator, m_sortKeys); bx::free(g_allocator, m_sortValues); - bx::free(g_allocator, m_renderItem, BX_ALIGNOF(RenderItem) ); - bx::free(g_allocator, m_renderBind, BX_ALIGNOF(RenderBind) ); + bx::free(g_allocator, m_blitKeys); m_sortKeys = NULL; m_sortValues = NULL; - m_renderItem = NULL; - m_renderBind = NULL; + m_blitKeys = NULL; + m_blitKeysCapacity = 0; + + m_renderItem.destroy(); + m_renderBind.destroy(); + m_blitItem.destroy(); } void setSentinel() @@ -2788,56 +3082,48 @@ namespace bgfx return; } - const uint32_t requested = m_numRenderItemsRequested; + m_peak = bx::max(m_peak, m_numRenderItemsRequested, m_numRenderBinds); + m_peakBlit = bx::max(m_peakBlit, m_numBlitItems); + m_peakRect = bx::max(m_peakRect, m_frameCache.m_rectCache.m_num); - if (requested >= m_maxDrawCalls - && m_maxDrawCalls < BGFX_CONFIG_MAX_DRAW_CALLS) + if (++m_observe >= m_numPeakFrames) { - const uint32_t want = alignDrawCalls(requested+1); - if (want > m_maxDrawCalls) + const uint32_t keep = bx::min(m_maxDrawCalls + 1, m_peak + 1 + kDrawCallBlock); + + m_renderItem.shrink(keep); + m_renderBind.shrink(keep); + m_blitItem.shrink(m_peakBlit + 1 + kBlitBlock); + m_frameCache.m_rectCache.shrink(m_peakRect + 1 + kRectBlock); + + for (uint32_t ii = 0, num = g_caps.limits.maxEncoders; ii < num; ++ii) { - allocArrays(want); + if (NULL != m_uniformBuffer[ii]) + { + UniformBuffer::shrink(&m_uniformBuffer[ii], g_caps.limits.minUniformBufferSize); + } } m_peak = 0; - m_observe = 0; - } - else - { - m_peak = bx::max(m_peak, requested); - - if (++m_observe >= m_numPeakFrames) - { - const uint32_t want = bx::max(kDrawCallBlock, alignDrawCalls(m_peak+1) ); - if (want < m_maxDrawCalls) - { - allocArrays(want); - } - - m_peak = 0; - m_observe = 0; - } + m_peakBlit = 0; + m_peakRect = 0; + m_observe = 0; } } - void create(uint32_t _minResourceCbSize, uint32_t _maxDrawCalls, uint32_t _numHwmFrames) + void create(uint32_t _minResourceCbSize, uint32_t _numReservedDrawCalls, uint32_t _maxDrawCalls, uint32_t _numHwmFrames) { m_cmdPre.init(_minResourceCbSize); m_cmdPost.init(_minResourceCbSize); m_numPeakFrames = _numHwmFrames; - allocArrays(_maxDrawCalls); - m_frameCache.create(_maxDrawCalls + 1, _numHwmFrames); + allocArrays(_numReservedDrawCalls, _maxDrawCalls); + m_frameCache.create(_numReservedDrawCalls + 1, _maxDrawCalls + 1, _numHwmFrames); { const uint32_t num = g_caps.limits.maxEncoders; m_uniformBuffer = (UniformBuffer**)bx::alloc(g_allocator, sizeof(UniformBuffer*)*num); - - for (uint32_t ii = 0; ii < num; ++ii) - { - m_uniformBuffer[ii] = UniformBuffer::create(g_caps.limits.minUniformBufferSize); - } + bx::memSet(m_uniformBuffer, 0, sizeof(UniformBuffer*)*num); } reset(); @@ -2849,13 +3135,28 @@ namespace bgfx { for (uint32_t ii = 0, num = g_caps.limits.maxEncoders; ii < num; ++ii) { - UniformBuffer::destroy(m_uniformBuffer[ii]); + if (NULL != m_uniformBuffer[ii]) + { + UniformBuffer::destroy(m_uniformBuffer[ii]); + } } bx::free(g_allocator, m_uniformBuffer); bx::deleteObject(g_allocator, m_textVideoMem); } + UniformBuffer* getUniformBuffer(uint8_t _idx) + { + UniformBuffer*& uniformBuffer = m_uniformBuffer[_idx]; + + if (NULL == uniformBuffer) + { + uniformBuffer = UniformBuffer::create(g_caps.limits.minUniformBufferSize); + } + + return uniformBuffer; + } + void reset() { start(0); @@ -2991,19 +3292,22 @@ namespace bgfx int32_t m_occlusion[BGFX_CONFIG_MAX_OCCLUSION_QUERIES]; + FrameArenaT m_renderItem; + FrameArenaT m_renderBind; + FrameArenaT m_blitItem; + uint64_t* m_sortKeys; RenderItemCount* m_sortValues; - RenderItem* m_renderItem; - RenderBind* m_renderBind; + uint32_t* m_blitKeys; + uint32_t m_blitKeysCapacity; uint32_t m_maxDrawCalls; uint32_t m_numRenderItemsRequested; uint32_t m_peak; + uint32_t m_peakBlit; + uint32_t m_peakRect; uint32_t m_observe; uint32_t m_numPeakFrames; - uint32_t m_blitKeys[BGFX_CONFIG_MAX_BLIT_ITEMS+1]; - BlitItem m_blitItem[BGFX_CONFIG_MAX_BLIT_ITEMS+1]; - UniformCacheFrame m_uniformCacheFrame; FrameCache m_frameCache; @@ -3135,7 +3439,7 @@ namespace bgfx m_uniformBegin = 0; m_uniformEnd = 0; - UniformBuffer* uniformBuffer = m_frame->m_uniformBuffer[m_uniformIdx]; + UniformBuffer* uniformBuffer = m_frame->getUniformBuffer(m_uniformIdx); uniformBuffer->reset(); m_numSubmitted = 0; @@ -3310,8 +3614,9 @@ namespace bgfx uint32_t setTransform(const void* _mtx, uint16_t _num) { - m_draw.m_startMatrix = m_frame->m_frameCache.m_matrixCache.add(_mtx, _num); - m_draw.m_numMatrices = _num; + m_draw.m_startMatrix = m_frame->m_frameCache.m_matrixCache.add(_mtx, &_num); + + m_draw.m_numMatrices = bx::max(_num, 1); return m_draw.m_startMatrix; } @@ -3327,12 +3632,18 @@ namespace bgfx void setTransform(uint32_t _cache, uint16_t _num) { - BX_ASSERT(_cache < BGFX_CONFIG_MAX_MATRIX_CACHE, "Matrix cache out of bounds index %d (max: %d)" + const MatrixCache& matrixCache = m_frame->m_frameCache.m_matrixCache; + + BX_ASSERT(_cache <= matrixCache.m_max, "Matrix cache out of bounds index %d (max: %d)" , _cache - , BGFX_CONFIG_MAX_MATRIX_CACHE + , matrixCache.m_max ); - m_draw.m_startMatrix = _cache; - m_draw.m_numMatrices = uint16_t(bx::min(_cache+_num, BGFX_CONFIG_MAX_MATRIX_CACHE-1) - _cache); + + const uint32_t first = bx::min(_cache, matrixCache.m_max); + const uint32_t last = bx::min(first + bx::max(_num, 1), matrixCache.m_max+1); + + m_draw.m_startMatrix = first; + m_draw.m_numMatrices = bx::narrowCast(last - first); } void setIndexBuffer(IndexBufferHandle _handle, const IndexBuffer& _ib, uint32_t _firstIndex, uint32_t _numIndices) @@ -3993,6 +4304,14 @@ namespace bgfx const uint32_t typeSize = g_uniformTypeSize[_type]; const uint32_t dataSize = _num * typeSize; + const uint32_t allocSize = bx::alignUp(dataSize, 16); + + BX_ASSERT(allocSize <= UINT16_MAX-15 + , "Uniform is too large for the uniform cache (%d bytes, max %d)!" + , allocSize + , UINT16_MAX-15 + ); + bx::HashMurmur3 murmur; murmur.begin(); murmur.add(_type); @@ -4039,17 +4358,26 @@ namespace bgfx } else { - const uint64_t offset = m_uniformStoreAlloc.alloc(dataSize); - BX_ASSERT(NonLocalAllocator::kInvalidBlock != offset, "UniformCache: Failed to allocate data!"); + const uint64_t offset = m_uniformStoreAlloc.alloc(allocSize); + + if (NonLocalAllocator::kInvalidBlock == offset) + { + BX_ASSERT(false, "UniformCache: Failed to allocate data (%d bytes)!", allocSize); + + m_uniformKeyHashMap.erase(m_uniformKeyHashMap.find(_uniformKey) ); + + return; + } m_uniformEntryMap.insert(stl::make_pair(hash, UniformCacheEntry { .offset = bx::narrowCast(offset), - .size = bx::narrowCast(dataSize), + .size = bx::narrowCast(allocSize), .refCount = 1 }) ); bx::memCopy(&m_data[offset], _value, dataSize); + bx::memSet(&m_data[offset + dataSize], 0, allocSize - dataSize); } } @@ -4100,6 +4428,7 @@ namespace bgfx } _outUniformCacheFrame.m_numItems = num; + _outUniformCacheFrame.m_keys[num] = 0; } void invalidate(ViewId _viewId) diff --git a/src/config.h b/src/config.h index 1bef9a124..e986734df 100644 --- a/src/config.h +++ b/src/config.h @@ -289,25 +289,69 @@ # define BGFX_CONFIG_MAX_DRAW_CALLS ( (64<<10)-1) #endif // BGFX_CONFIG_MAX_DRAW_CALLS -/// Maximum number of draw calls per block. Default is 1024. +/// Enable dynamic per frame storage. When enabled, storage for render items, +/// binds, blit items, scissor rectangles, and transform matrices are allocated +/// in blocks, on first touch, and grows during the frame instead of dropping +/// submissions. Only what is actually used is allocated, so +/// `Init::Limits::numDrawCalls` becomes the amount that is reserved up front +/// rather than a hard limit. +/// +/// When disabled, all of the above is allocated once, up front, at exactly the +/// requested size, and submissions past it are dropped. Nothing is allocated in +/// blocks, nothing is resized after init, and indexing has no indirection. +/// Default is 1. +#ifndef BGFX_CONFIG_DYNAMIC_FRAME_STORAGE +# define BGFX_CONFIG_DYNAMIC_FRAME_STORAGE 1 +#endif // BGFX_CONFIG_DYNAMIC_FRAME_STORAGE + +/// Granularity dynamic per frame storage grows by, in items. Reserving a +/// multiple of it up front keeps growing off the common path. Must be power of +/// two. Default is 1024. #ifndef BGFX_CONFIG_DRAW_CALL_BLOCK # define BGFX_CONFIG_DRAW_CALL_BLOCK 1024 #endif // BGFX_CONFIG_DRAW_CALL_BLOCK -/// Maximum number of blit items per frame. Default is 1024. +/// Maximum number of blit items per frame. +/// +/// With BGFX_CONFIG_DYNAMIC_FRAME_STORAGE enabled nothing is reserved for blit +/// items and blocks are allocated as they're used, so this is only a ceiling +/// and costs a block pointer table. It's set to what the blit sort key can +/// address. When disabled, all of it is allocated up front, so the default +/// stays at 1024. #ifndef BGFX_CONFIG_MAX_BLIT_ITEMS -# define BGFX_CONFIG_MAX_BLIT_ITEMS (1<<10) +# if BGFX_CONFIG_DYNAMIC_FRAME_STORAGE +# define BGFX_CONFIG_MAX_BLIT_ITEMS (64<<10) +# else +# define BGFX_CONFIG_MAX_BLIT_ITEMS (1<<10) +# endif // BGFX_CONFIG_DYNAMIC_FRAME_STORAGE #endif // BGFX_CONFIG_MAX_BLIT_ITEMS /// Maximum number of cached transform matrices. Default is BGFX_CONFIG_MAX_DRAW_CALLS + 1. /// Each draw call may reference a transform matrix; this cache stores them for the frame. +/// +/// A matrix cache index is handed out as a pointer that the caller writes +/// through, so unlike the rest of the per frame storage this can't be sliced +/// into blocks and has to stay one contiguous run. It is still only a ceiling +/// with BGFX_CONFIG_DYNAMIC_FRAME_STORAGE enabled, but growing into it costs a +/// reallocation and one frame of dropped transforms. When disabled, the whole +/// thing is allocated up front and never resizes. #ifndef BGFX_CONFIG_MAX_MATRIX_CACHE # define BGFX_CONFIG_MAX_MATRIX_CACHE (BGFX_CONFIG_MAX_DRAW_CALLS+1) #endif // BGFX_CONFIG_MAX_MATRIX_CACHE -/// Maximum number of cached scissor rectangles per frame. Default is 4096. +/// Maximum number of cached scissor rectangles per frame. +/// +/// With BGFX_CONFIG_DYNAMIC_FRAME_STORAGE enabled nothing is reserved for +/// scissor rectangles and blocks are allocated as they're used, so this is only +/// a ceiling. It's set to what a draw call can address, one short of UINT16_MAX +/// because that's reserved to mean no scissor. When disabled, all of it is +/// allocated up front, so the default stays at 4096. #ifndef BGFX_CONFIG_MAX_RECT_CACHE -# define BGFX_CONFIG_MAX_RECT_CACHE (4<<10) +# if BGFX_CONFIG_DYNAMIC_FRAME_STORAGE +# define BGFX_CONFIG_MAX_RECT_CACHE ( (64<<10)-1) +# else +# define BGFX_CONFIG_MAX_RECT_CACHE (4<<10) +# endif // BGFX_CONFIG_DYNAMIC_FRAME_STORAGE #endif // BGFX_CONFIG_MAX_RECT_CACHE /// Number of bits used for depth in the sort key. Default is 32. @@ -445,18 +489,25 @@ static_assert(BGFX_CONFIG_MAX_VERTEX_STREAMS < 32, "Must be less than 32!"); #endif // BGFX_CONFIG_MAX_TRANSIENT_INDEX_BUFFER_SIZE #ifndef BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE -/// Mimumum uniform buffer size. This buffer will resize on demand. -# define BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE (1<<20) +/// Mimumum uniform buffer size. This buffer will resize on demand. It's +/// allocated per encoder, so this is the price of an encoder that submits +/// only a handful of uniforms. Must be larger than +/// BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE, otherwise the buffer +/// resizes on first use. +# define BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE (128<<10) #endif // BGFX_CONFIG_MIN_UNIFORM_BUFFER_SIZE #ifndef BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE /// Max amount of unused uniform buffer space before uniform buffer resize. +/// Must be at least as large as the largest single uniform record, since +/// UniformBuffer::update reserves this much head room before every write. A +/// record is at most 4 + 1023*sizeof(Mat4) = 65476 bytes. # define BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE (64<<10) #endif // BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_THRESHOLD_SIZE #ifndef BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE /// Increment of uniform buffer resize. -# define BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE (1<<20) +# define BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE (64<<10) #endif // BGFX_CONFIG_UNIFORM_BUFFER_RESIZE_INCREMENT_SIZE #ifndef BGFX_CONFIG_CACHED_DEVICE_MEMORY_ALLOCATIONS_SIZE diff --git a/src/renderer.h b/src/renderer.h index bab0f262f..c913b260d 100644 --- a/src/renderer.h +++ b/src/renderer.h @@ -38,7 +38,7 @@ namespace bgfx const Frame* m_frame; BlitKey m_key; - uint16_t m_item; + uint32_t m_item; }; struct UniformCacheItem diff --git a/src/renderer_gl.cpp b/src/renderer_gl.cpp index 6f1ec3161..e8a3a23e7 100644 --- a/src/renderer_gl.cpp +++ b/src/renderer_gl.cpp @@ -3760,7 +3760,7 @@ namespace bgfx { namespace gl bx::free(g_allocator, m_uniforms[_handle.idx]); } - uint32_t size = g_uniformTypeSize[_type]*_num; + const uint32_t size = bx::alignUp(g_uniformTypeSize[_type]*_num, 16); void* data = bx::alloc(g_allocator, size); bx::memSet(data, 0, size); m_uniforms[_handle.idx] = data; diff --git a/src/renderer_vk.cpp b/src/renderer_vk.cpp index 5bb2dda01..545969f85 100644 --- a/src/renderer_vk.cpp +++ b/src/renderer_vk.cpp @@ -2932,6 +2932,7 @@ VK_IMPORT_DEVICE m_cmd.recycleMemory(_alloc); } + void submitBlitBatch(BlitState& _bs, uint16_t _view); void submitBlit(BlitState& _bs, uint16_t _view); void submitUniformCache(UniformCacheState& _ucs, uint16_t _view); @@ -9427,31 +9428,36 @@ VK_DESTROY BX_ASSERT(false, "Removing external texture failed!"); } - void RendererContextVK::submitBlit(BlitState& _bs, uint16_t _view) + void RendererContextVK::submitBlitBatch(BlitState& _bs, uint16_t _view) { - BGFX_PROFILER_SCOPE("RendererContextVK::submitBlit", kColorFrame); + constexpr uint32_t kMaxItems = 128; + VkImageLayout srcLayouts[kMaxItems]; + VkImageLayout dstLayouts[kMaxItems]; - VkImageLayout srcLayouts[BGFX_CONFIG_MAX_BLIT_ITEMS]; - VkImageLayout dstLayouts[BGFX_CONFIG_MAX_BLIT_ITEMS]; + uint32_t numItems = 0; BlitState bs0 = _bs; - while (bs0.hasItem(_view) ) + while (bs0.hasItem(_view) + && numItems < kMaxItems) { - uint16_t item = bs0.m_item; + const uint32_t item = numItems++; const BlitItem& blit = bs0.advance(); TextureVK& src = m_textures[blit.m_src.idx]; TextureVK& dst = m_textures[blit.m_dst.idx]; - srcLayouts[item] = VK_NULL_HANDLE != src.m_singleMsaaImage ? src.m_currentSingleMsaaImageLayout : src.m_currentImageLayout; + srcLayouts[item] = VK_NULL_HANDLE != src.m_singleMsaaImage + ? src.m_currentSingleMsaaImageLayout + : src.m_currentImageLayout + ; dstLayouts[item] = dst.m_currentImageLayout; } bs0 = _bs; - while (bs0.hasItem(_view) ) + for (uint32_t item = 0; item < numItems; ++item) { const BlitItem& blit = bs0.advance(); @@ -9550,10 +9556,8 @@ VK_DESTROY ); } - while (_bs.hasItem(_view) ) + for (uint32_t item = 0; item < numItems; ++item) { - uint16_t item = _bs.m_item; - const BlitItem& blit = _bs.advance(); TextureVK& src = m_textures[blit.m_src.idx]; @@ -9564,6 +9568,16 @@ VK_DESTROY } } + void RendererContextVK::submitBlit(BlitState& _bs, uint16_t _view) + { + BGFX_PROFILER_SCOPE("RendererContextVK::submitBlit", kColorFrame); + + while (_bs.hasItem(_view) ) + { + submitBlitBatch(_bs, _view); + } + } + void RendererContextVK::submitUniformCache(UniformCacheState& _ucs, uint16_t _view) { while (_ucs.hasItem(_view) ) diff --git a/src/renderer_webgpu.cpp b/src/renderer_webgpu.cpp index 6ee456b99..81a29dfaa 100644 --- a/src/renderer_webgpu.cpp +++ b/src/renderer_webgpu.cpp @@ -1943,7 +1943,7 @@ WGPU_IMPORT bx::free(g_allocator, m_uniforms[_handle.idx]); } - uint32_t size = g_uniformTypeSize[_type]*_num; + const uint32_t size = bx::alignUp(g_uniformTypeSize[_type]*_num, 16); void* data = bx::alloc(g_allocator, size); bx::memSet(data, 0, size); m_uniforms[_handle.idx] = data;