diff --git a/filament/src/Allocators.h b/filament/src/Allocators.h index eb354b8d32..84962e30c0 100644 --- a/filament/src/Allocators.h +++ b/filament/src/Allocators.h @@ -54,7 +54,7 @@ using LinearAllocatorArena = utils::Arena< #endif -using ArenaScope = utils::ArenaScope; +using RootArenaScope = utils::ArenaScope; } // namespace filament diff --git a/filament/src/Froxelizer.cpp b/filament/src/Froxelizer.cpp index c469932c25..47bd0d343d 100644 --- a/filament/src/Froxelizer.cpp +++ b/filament/src/Froxelizer.cpp @@ -168,7 +168,8 @@ void Froxelizer::setProjection(const mat4f& projection, } bool Froxelizer::prepare( - FEngine::DriverApi& driverApi, ArenaScope& arena, filament::Viewport const& viewport, + FEngine::DriverApi& driverApi, RootArenaScope& rootArenaScope, + filament::Viewport const& viewport, const mat4f& projection, float projectionNear, float projectionFar) noexcept { setViewport(viewport); setProjection(projection, projectionNear, projectionFar); @@ -199,12 +200,12 @@ bool Froxelizer::prepare( // light records per froxel (~256 KiB) mLightRecords = { - arena.allocate(getFroxelBufferEntryCount(), CACHELINE_SIZE), + rootArenaScope.allocate(getFroxelBufferEntryCount(), CACHELINE_SIZE), getFroxelBufferEntryCount() }; // froxel thread data (~256 KiB) mFroxelShardedData = { - arena.allocate(GROUP_COUNT, CACHELINE_SIZE), + rootArenaScope.allocate(GROUP_COUNT, CACHELINE_SIZE), uint32_t(GROUP_COUNT) }; diff --git a/filament/src/Froxelizer.h b/filament/src/Froxelizer.h index 27885e24bc..27ba3c5764 100644 --- a/filament/src/Froxelizer.h +++ b/filament/src/Froxelizer.h @@ -110,7 +110,7 @@ public: * * return true if updateUniforms() needs to be called */ - bool prepare(backend::DriverApi& driverApi, ArenaScope& arena, Viewport const& viewport, + bool prepare(backend::DriverApi& driverApi, RootArenaScope& rootArenaScope, Viewport const& viewport, const math::mat4f& projection, float projectionNear, float projectionFar) noexcept; Froxel getFroxelAt(size_t x, size_t y, size_t z) const noexcept; diff --git a/filament/src/RenderPass.cpp b/filament/src/RenderPass.cpp index 788ca9d432..392b69395d 100644 --- a/filament/src/RenderPass.cpp +++ b/filament/src/RenderPass.cpp @@ -60,10 +60,9 @@ namespace filament { using namespace backend; -RenderPass::RenderPass(FEngine& engine, - RenderPass::Arena& arena) noexcept +RenderPass::RenderPass(FEngine& engine, RenderPass::Arena& arena) noexcept : mCommandArena(arena), - mCustomCommands(engine.getPerRenderPassAllocator()) { + mCustomCommands(engine.getPerRenderPassArena()) { } RenderPass::RenderPass(RenderPass const& rhs) = default; diff --git a/filament/src/RenderPass.h b/filament/src/RenderPass.h index 5df9589ac6..da92d2abf9 100644 --- a/filament/src/RenderPass.h +++ b/filament/src/RenderPass.h @@ -31,14 +31,21 @@ #include #include +#include #include #include #include +#include + #include #include +#include #include +#include +#include + namespace filament { namespace backend { @@ -244,7 +251,6 @@ public: uint32_t skinningOffset = 0; // 4 bytes uint16_t instanceCount; // 2 bytes [MSb: user] Variant materialVariant; // 1 byte -// uint8_t reserved[0] = {}; // 0 bytes static const uint16_t USER_INSTANCE_MASK = 0x8000u; static const uint16_t INSTANCE_COUNT_MASK = 0x7fffu; @@ -257,7 +263,7 @@ public: uint64_t reserved[1] = {}; // 8 bytes bool operator < (Command const& rhs) const noexcept { return key < rhs.key; } // placement new declared as "throw" to avoid the compiler's null-check - inline void* operator new (std::size_t, void* ptr) { + inline void* operator new (size_t, void* ptr) { assert_invariant(ptr); return ptr; } @@ -298,7 +304,7 @@ public: void setGeometry(FScene::RenderableSoa const& soa, utils::Range vr, backend::Handle uboHandle) noexcept; - // specifies camera information (e.g. used for sorting commands) + // Specifies camera information (e.g. used for sorting commands) void setCamera(const CameraInfo& camera) noexcept; // flags controlling how commands are generated diff --git a/filament/src/details/Engine.cpp b/filament/src/details/Engine.cpp index c09711afa1..82b9399ec6 100644 --- a/filament/src/details/Engine.cpp +++ b/filament/src/details/Engine.cpp @@ -198,7 +198,7 @@ FEngine::FEngine(Engine::Builder const& builder) : mCommandBufferQueue( builder->mConfig.minCommandBufferSizeMB * MiB, builder->mConfig.commandBufferSizeMB * MiB), - mPerRenderPassAllocator( + mPerRenderPassArena( "FEngine::mPerRenderPassAllocator", builder->mConfig.perRenderPassArenaSizeMB * MiB), mHeapAllocator("FEngine::mHeapAllocator", AreaPolicy::NullArea{}), diff --git a/filament/src/details/Engine.h b/filament/src/details/Engine.h index 03889f1393..03de2ce841 100644 --- a/filament/src/details/Engine.h +++ b/filament/src/details/Engine.h @@ -58,17 +58,6 @@ #include #include -#if FILAMENT_ENABLE_MATDBG -#include -#else -namespace filament { -namespace matdbg { -class DebugServer; -using MaterialKey = uint32_t; -} // namespace matdbg -} // namespace filament -#endif - #include #include #include @@ -78,8 +67,19 @@ using MaterialKey = uint32_t; #include #include #include +#include +#include #include +#if FILAMENT_ENABLE_MATDBG +#include +#else +namespace filament::matdbg { +class DebugServer; +using MaterialKey = uint32_t; +} // namespace filament::matdbg +#endif + namespace filament { class Renderer; @@ -142,7 +142,7 @@ public: // the per-frame Area is used by all Renderer, so they must run in sequence and // have freed all allocated memory when done. If this needs to change in the future, // we'll simply have to use separate Areas (for instance). - LinearAllocatorArena& getPerRenderPassAllocator() noexcept { return mPerRenderPassAllocator; } + LinearAllocatorArena& getPerRenderPassArena() noexcept { return mPerRenderPassArena; } // Material IDs... uint32_t getMaterialId() const noexcept { return mMaterialId++; } @@ -508,7 +508,7 @@ private: uint32_t mFlushCounter = 0; - LinearAllocatorArena mPerRenderPassAllocator; + RootArenaScope::Arena mPerRenderPassArena; HeapAllocatorArena mHeapAllocator; utils::JobSystem mJobSystem; diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp index b29cf1563e..7170cc9467 100644 --- a/filament/src/details/Renderer.cpp +++ b/filament/src/details/Renderer.cpp @@ -16,6 +16,9 @@ #include "details/Renderer.h" +#include "Allocators.h" +#include "DebugRegistry.h" +#include "FrameHistory.h" #include "PostProcessManager.h" #include "RendererUtils.h" #include "RenderPass.h" @@ -28,21 +31,40 @@ #include "details/Texture.h" #include "details/View.h" +#include +#include +#include #include +#include +#include +#include #include #include "fg/FrameGraph.h" #include "fg/FrameGraphId.h" #include "fg/FrameGraphResources.h" +#include "fg/FrameGraphTexture.h" + +#include +#include +#include #include #include +#include +#include #include #include -#include #include +#include +#include +#include + +#include +#include + // this helps visualize what dynamic-scaling is doing #define DEBUG_DYNAMIC_SCALING false @@ -62,8 +84,7 @@ FRenderer::FRenderer(FEngine& engine) : mHdrQualityMedium(TextureFormat::R11F_G11F_B10F), mHdrQualityHigh(TextureFormat::RGB16F), mIsRGB8Supported(false), - mUserEpoch(engine.getEngineEpoch()), - mPerRenderPassArena(engine.getPerRenderPassAllocator()) + mUserEpoch(engine.getEngineEpoch()) { FDebugRegistry& debugRegistry = engine.getDebugRegistry(); debugRegistry.registerProperty("d.renderer.doFrameCapture", @@ -452,17 +473,17 @@ void FRenderer::render(FView const* view) { } void FRenderer::renderInternal(FView const* view) { - // per-renderpass data - ArenaScope rootArena(mPerRenderPassArena); - FEngine& engine = mEngine; - JobSystem& js = engine.getJobSystem(); + + // per-renderpass data + RootArenaScope rootArenaScope(engine.getPerRenderPassArena()); // create a root job so no other job can escape + JobSystem& js = engine.getJobSystem(); auto *rootJob = js.setRootJob(js.createJob()); // execute the render pass - renderJob(rootArena, const_cast(*view)); + renderJob(rootArenaScope, const_cast(*view)); // make sure to flush the command buffer engine.flush(); @@ -471,7 +492,7 @@ void FRenderer::renderInternal(FView const* view) { js.runAndWait(rootJob); } -void FRenderer::renderJob(ArenaScope& arena, FView& view) { +void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) { FEngine& engine = mEngine; JobSystem& js = engine.getJobSystem(); FEngine::DriverApi& driver = engine.getDriverApi(); @@ -636,7 +657,7 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) { xvp.bottom = int32_t(guardBand); } - view.prepare(engine, driver, arena, svp, cameraInfo, getShaderUserTime(), needsAlphaChannel); + view.prepare(engine, driver, rootArenaScope, svp, cameraInfo, getShaderUserTime(), needsAlphaChannel); view.prepareUpscaler(scale, taaOptions, dsrOptions); @@ -649,7 +670,7 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) { // Allocate some space for our commands in the per-frame Arena, and use that space as // an Arena for commands. All this space is released when we exit this method. size_t const perFrameCommandsSize = engine.getPerFrameCommandsSize(); - void* const arenaBegin = arena.allocate(perFrameCommandsSize, CACHELINE_SIZE); + void* const arenaBegin = rootArenaScope.allocate(perFrameCommandsSize, CACHELINE_SIZE); void* const arenaEnd = pointermath::add(arenaBegin, perFrameCommandsSize); RenderPass::Arena commandArena("Command Arena", { arenaBegin, arenaEnd }); diff --git a/filament/src/details/Renderer.h b/filament/src/details/Renderer.h index 2d08b6cfe0..056d559977 100644 --- a/filament/src/details/Renderer.h +++ b/filament/src/details/Renderer.h @@ -163,7 +163,7 @@ private: } void renderInternal(FView const* view); - void renderJob(ArenaScope& arena, FView& view); + void renderJob(RootArenaScope& rootArenaScope, FView& view); // keep a reference to our engine FEngine& mEngine; @@ -187,9 +187,6 @@ private: backend::TargetBufferFlags mClearFlags{}; tsl::robin_set mPreviousRenderTargets; std::function mBeginFrameInternal; - - // per-frame arena for this Renderer - LinearAllocatorArena& mPerRenderPassArena; }; FILAMENT_DOWNCAST(Renderer) diff --git a/filament/src/details/Scene.cpp b/filament/src/details/Scene.cpp index 942fb1e24f..21840fa60c 100644 --- a/filament/src/details/Scene.cpp +++ b/filament/src/details/Scene.cpp @@ -53,7 +53,7 @@ FScene::~FScene() noexcept = default; void FScene::prepare(utils::JobSystem& js, - LinearAllocatorArena& allocator, + RootArenaScope& rootArenaScope, mat4 const& worldTransform, bool shadowReceiversAreCasters) noexcept { // TODO: can we skip this in most cases? Since we rely on indices staying the same, @@ -64,7 +64,7 @@ void FScene::prepare(utils::JobSystem& js, SYSTRACE_CONTEXT(); // This will reset the allocator upon exiting - ArenaScope const arena(allocator); + ArenaScope localArenaScope(rootArenaScope.getArena()); FEngine& engine = mEngine; EntityManager const& em = engine.getEntityManager(); @@ -85,10 +85,10 @@ void FScene::prepare(utils::JobSystem& js, utils::STLAllocator< LightContainerData, LinearAllocatorArena >, false>; RenderableInstanceContainer renderableInstances{ - RenderableInstanceContainer::with_capacity(entities.size(), allocator) }; + RenderableInstanceContainer::with_capacity(entities.size(), localArenaScope.getArena()) }; LightInstanceContainer lightInstances{ - LightInstanceContainer::with_capacity(entities.size(), allocator) }; + LightInstanceContainer::with_capacity(entities.size(), localArenaScope.getArena()) }; SYSTRACE_NAME_BEGIN("InstanceLoop"); @@ -454,7 +454,7 @@ void FScene::terminate(FEngine&) { mRenderableViewUbh.clear(); } -void FScene::prepareDynamicLights(const CameraInfo& camera, ArenaScope&, +void FScene::prepareDynamicLights(const CameraInfo& camera, Handle lightUbh) noexcept { FEngine::DriverApi& driver = mEngine.getDriverApi(); FLightManager const& lcm = mEngine.getLightManager(); diff --git a/filament/src/details/Scene.h b/filament/src/details/Scene.h index 1882bb4dc3..490d115af3 100644 --- a/filament/src/details/Scene.h +++ b/filament/src/details/Scene.h @@ -31,6 +31,8 @@ #include #include +#include + #include #include #include @@ -70,12 +72,12 @@ public: ~FScene() noexcept; void terminate(FEngine& engine); - void prepare(utils::JobSystem& js, LinearAllocatorArena& allocator, + void prepare(utils::JobSystem& js, RootArenaScope& rootArenaScope, math::mat4 const& worldTransform, bool shadowReceiversAreCasters) noexcept; void prepareVisibleRenderables(utils::Range visibleRenderables) noexcept; - void prepareDynamicLights(const CameraInfo& camera, ArenaScope& arena, + void prepareDynamicLights(const CameraInfo& camera, backend::Handle lightUbh) noexcept; backend::Handle getRenderableUBO() const noexcept { diff --git a/filament/src/details/View.cpp b/filament/src/details/View.cpp index 7cc641de09..af61cfb698 100644 --- a/filament/src/details/View.cpp +++ b/filament/src/details/View.cpp @@ -341,8 +341,7 @@ void FView::prepareShadowing(FEngine& engine, FScene::RenderableSoa& renderableD mNeedsShadowMap = any(shadowTechnique & ShadowMapManager::ShadowTechnique::SHADOW_MAP); } -void FView::prepareLighting(FEngine& engine, ArenaScope& arena, - CameraInfo const& cameraInfo) noexcept { +void FView::prepareLighting(FEngine& engine, CameraInfo const& cameraInfo) noexcept { SYSTRACE_CALL(); SYSTRACE_CONTEXT(); @@ -354,7 +353,7 @@ void FView::prepareLighting(FEngine& engine, ArenaScope& arena, */ if (hasDynamicLighting()) { - scene->prepareDynamicLights(cameraInfo, arena, mLightUbh); + scene->prepareDynamicLights(cameraInfo, mLightUbh); } // here the array of visible lights has been shrunk to CONFIG_MAX_LIGHT_COUNT @@ -427,7 +426,7 @@ CameraInfo FView::computeCameraInfo(FEngine& engine) const noexcept { return { *camera, mat4{ rotation } * mat4::translation(translation) }; } -void FView::prepare(FEngine& engine, DriverApi& driver, ArenaScope& arena, +void FView::prepare(FEngine& engine, DriverApi& driver, RootArenaScope& rootArenaScope, filament::Viewport viewport, CameraInfo cameraInfo, float4 const& userTime, bool needsAlphaChannel) noexcept { @@ -465,7 +464,7 @@ void FView::prepare(FEngine& engine, DriverApi& driver, ArenaScope& arena, * Gather all information needed to render this scene. Apply the world origin to all * objects in the scene. */ - scene->prepare(js, arena.getAllocator(), + scene->prepare(js, rootArenaScope, cameraInfo.worldTransform, hasVSM()); @@ -475,14 +474,22 @@ void FView::prepare(FEngine& engine, DriverApi& driver, ArenaScope& arena, JobSystem::Job* froxelizeLightsJob = nullptr; JobSystem::Job* prepareVisibleLightsJob = nullptr; - if (scene->getLightData().size() > FScene::DIRECTIONAL_LIGHTS_COUNT) { + size_t const lightCount = scene->getLightData().size(); + if (lightCount > FScene::DIRECTIONAL_LIGHTS_COUNT) { // create and start the prepareVisibleLights job // note: this job updates LightData (non const) + // allocate a scratch buffer for distances outside the job below, so we don't need + // to use a locked allocator; the downside is that we need to account for the worst case. + size_t const positionalLightCount = lightCount - FScene::DIRECTIONAL_LIGHTS_COUNT; + float* const distances = rootArenaScope.allocate( + (positionalLightCount + 3u) & ~3u, CACHELINE_SIZE); + prepareVisibleLightsJob = js.runAndRetain(js.createJob(nullptr, - [&engine, &arena, &viewMatrix = cameraInfo.view, &cullingFrustum, + [&engine, distances, positionalLightCount, &viewMatrix = cameraInfo.view, &cullingFrustum, &lightData = scene->getLightData()] (JobSystem&, JobSystem::Job*) { - FView::prepareVisibleLights(engine.getLightManager(), arena, + FView::prepareVisibleLights(engine.getLightManager(), + { distances, distances + positionalLightCount }, viewMatrix, cullingFrustum, lightData); })); } @@ -530,7 +537,7 @@ void FView::prepare(FEngine& engine, DriverApi& driver, ArenaScope& arena, // As soon as prepareVisibleLight finishes, we can kick-off the froxelization if (hasDynamicLighting()) { auto& froxelizer = mFroxelizer; - if (froxelizer.prepare(driver, arena, viewport, + if (froxelizer.prepare(driver, rootArenaScope, viewport, cameraInfo.projection, cameraInfo.zn, cameraInfo.zf)) { // TODO: might be more consistent to do this in prepareLighting(), but it's not // strictly necessary @@ -645,7 +652,7 @@ void FView::prepare(FEngine& engine, DriverApi& driver, ArenaScope& arena, * Relies on FScene::prepare() and prepareVisibleLights() */ - prepareLighting(engine, arena, cameraInfo); + prepareLighting(engine, cameraInfo); /* * Update driver state @@ -850,7 +857,8 @@ void FView::cullRenderables(JobSystem&, functor(0, renderableData.size()); } -void FView::prepareVisibleLights(FLightManager const& lcm, ArenaScope& rootArena, +void FView::prepareVisibleLights(FLightManager const& lcm, + utils::Slice scratch, mat4f const& viewMatrix, Frustum const& frustum, FScene::LightSoa& lightData) noexcept { SYSTRACE_CALL(); @@ -918,28 +926,25 @@ void FView::prepareVisibleLights(FLightManager const& lcm, ArenaScope& rootArena * - This helps our limited numbers of spot-shadow as well. */ - ArenaScope arena(rootArena.getAllocator()); - size_t const size = visibleLightCount; // number of point/spotlights - size_t const positionalLightCount = size - FScene::DIRECTIONAL_LIGHTS_COUNT; + size_t const positionalLightCount = visibleLightCount - FScene::DIRECTIONAL_LIGHTS_COUNT; if (positionalLightCount) { - // always allocate at least 4 entries, because the vectorized loops below rely on that - float* const UTILS_RESTRICT distances = - arena.allocate((size + 3u) & ~3u, CACHELINE_SIZE); - + assert_invariant(positionalLightCount <= scratch.size()); // pre-compute the lights' distance to the camera, for sorting below // - we don't skip the directional light, because we don't care, it's ignored during sorting + float* const UTILS_RESTRICT distances = scratch.data(); float4 const* const UTILS_RESTRICT spheres = lightData.data(); - computeLightCameraDistances(distances, viewMatrix, spheres, size); + computeLightCameraDistances(distances, viewMatrix, spheres, visibleLightCount); // skip directional light Zip2Iterator b = { lightData.begin(), distances }; - std::sort(b + FScene::DIRECTIONAL_LIGHTS_COUNT, b + size, + std::sort(b + FScene::DIRECTIONAL_LIGHTS_COUNT, b + visibleLightCount, [](auto const& lhs, auto const& rhs) { return lhs.second < rhs.second; }); } // drop excess lights - lightData.resize(std::min(size, CONFIG_MAX_LIGHT_COUNT + FScene::DIRECTIONAL_LIGHTS_COUNT)); + lightData.resize(std::min(visibleLightCount, + CONFIG_MAX_LIGHT_COUNT + FScene::DIRECTIONAL_LIGHTS_COUNT)); } // These methods need to exist so clang honors the __restrict__ keyword, which in turn diff --git a/filament/src/details/View.h b/filament/src/details/View.h index 204f11b9d5..64db9fcce8 100644 --- a/filament/src/details/View.h +++ b/filament/src/details/View.h @@ -88,7 +88,7 @@ public: // note: viewport/cameraInfo are passed by value to make it clear that prepare cannot // keep references on them that would outlive the scope of prepare() (e.g. with JobSystem). - void prepare(FEngine& engine, backend::DriverApi& driver, ArenaScope& arena, + void prepare(FEngine& engine, backend::DriverApi& driver, RootArenaScope& rootArenaScope, filament::Viewport viewport, CameraInfo cameraInfo, math::float4 const& userTime, bool needsAlphaChannel) noexcept; @@ -144,7 +144,7 @@ public: void prepareShadowing(FEngine& engine, FScene::RenderableSoa& renderableData, FScene::LightSoa const& lightData, CameraInfo const& cameraInfo) noexcept; - void prepareLighting(FEngine& engine, ArenaScope& arena, CameraInfo const& cameraInfo) noexcept; + void prepareLighting(FEngine& engine, CameraInfo const& cameraInfo) noexcept; void prepareSSAO(backend::Handle ssao) const noexcept; void prepareSSR(backend::Handle ssr, bool disableSSR, @@ -460,7 +460,8 @@ private: void prepareVisibleRenderables(utils::JobSystem& js, Frustum const& frustum, FScene::RenderableSoa& renderableData) const noexcept; - static void prepareVisibleLights(FLightManager const& lcm, ArenaScope& rootArena, + static void prepareVisibleLights(FLightManager const& lcm, + utils::Slice scratch, math::mat4f const& viewMatrix, Frustum const& frustum, FScene::LightSoa& lightData) noexcept; diff --git a/libs/utils/include/utils/Allocator.h b/libs/utils/include/utils/Allocator.h index c726ac5ffe..a52fe18a02 100644 --- a/libs/utils/include/utils/Allocator.h +++ b/libs/utils/include/utils/Allocator.h @@ -720,6 +720,8 @@ class ArenaScope { } public: + using Arena = ARENA; + explicit ArenaScope(ARENA& allocator) : mArena(allocator), mRewind(allocator.getCurrent()) { } @@ -771,7 +773,7 @@ public: } // use with caution - ARENA& getAllocator() noexcept { return mArena; } + ARENA& getArena() noexcept { return mArena; } private: ARENA& mArena;