Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New froxelization algorithm that uses much less memory

Browse Source 
Instead of building a list of froxel per light,
we now build a bitfield of lights per froxel.
To keep the code efficient we have to split
the bitfield array in 4, so that we can use
multiple threads without having them or’ing
each other’s cache-lines.

This data structure is also what we want in 
froxelizeAssignRecords(), and the conversion from
one format to the other is much faster.

Our internal data structure is now 256 KiB
instead of 2 MiB. It would be 1 MiB for 1024 lights
(down from 16 MiB)

Additionally, because the code is simpler, we get
more vectorization, especially with spot-lights.

One draw-back of the current version is that
multi-threading happens only every 64 lights.
This will be fixed later.
This commit is contained in:
Mathias Agopian
2018-08-10 00:50:46 -07:00
committed by Mathias Agopian
parent 1f380a6ec6
commit 7b7bd85def
4 changed files with 85 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
filament/src/Froxelizer.cpp
View File

@@ -49,16 +49,6 @@ static constexpr bool SUPPORTS_NON_SQUARE_FROXELS = false;
// With n limited by the supported texture dimension, which is guaranteed to be at least 2048
// in all version of GLES.
// Max number of froxels limited by:
// - max texture size [min 2048]
// - chosen texture width [64]
// - size of CPU-side indices [16 bits]
// Also, increasing the number of froxels adds more pressure on the "record buffer" which stores
// the light indices per froxel. The record buffer is limited to 65536 entries, so with
// 8192 froxels, we can store 8 lights per froxels assuming they're all used. In practice, some
// froxels are not used, so we can store more.
constexpr size_t FROXEL_BUFFER_ENTRY_COUNT_MAX = 8192;
// Make sure this matches the same constants in shading_lit.fs
constexpr size_t FROXEL_BUFFER_WIDTH_SHIFT = 6u;
constexpr size_t FROXEL_BUFFER_WIDTH = 1u << FROXEL_BUFFER_WIDTH_SHIFT;
@@ -174,29 +164,21 @@ bool Froxelizer::prepare(
arena.allocate<LightRecord>(FROXEL_BUFFER_ENTRY_COUNT_MAX, CACHELINE_SIZE),
FROXEL_BUFFER_ENTRY_COUNT_MAX };
// indices to the per-light froxel list (~2 KiB)
mFroxelListIndices = {
arena.allocate<FroxelRunEntry>(CONFIG_MAX_LIGHT_COUNT, CACHELINE_SIZE),
CONFIG_MAX_LIGHT_COUNT };
// per-light froxel list (~4 MiB)
constexpr uint32_t EXTRA_DATA = 1; // to store the index/type of each light
const uint32_t froxelListSize = uint32_t(CONFIG_MAX_LIGHT_COUNT) * (maxFroxelCount + EXTRA_DATA);
mFroxelList = {
arena.allocate<uint16_t>(froxelListSize, CACHELINE_SIZE),
froxelListSize };
// froxel thread data (~256 KiB)
mFroxelThreadData = {
arena.allocate<ThreadData>((CONFIG_MAX_LIGHT_COUNT + 63) / 64, CACHELINE_SIZE),
(CONFIG_MAX_LIGHT_COUNT + 63) / 64
};
assert(mFroxelBufferUser.begin());
assert(mRecordBufferUser.begin());
assert(mLightRecords.begin());
assert(mFroxelListIndices.begin());
assert(mFroxelList.begin());
assert(mFroxelThreadData.begin());
#ifndef NDEBUG
memset(mFroxelBufferUser.data(), 0x55, mFroxelBufferUser.sizeInBytes());
memset(mRecordBufferUser.data(), 0xEB, mRecordBufferUser.sizeInBytes());
memset(mFroxelListIndices.data(), 0xAA, mFroxelListIndices.sizeInBytes());
memset(mFroxelList.data(), 0xEF, mFroxelList.sizeInBytes());
memset(mFroxelThreadData.data(), 0xAA, mFroxelThreadData.sizeInBytes());
#endif
return uniformsNeedUpdating;
@@ -519,23 +501,20 @@ void Froxelizer::commit(driver::DriverApi& driverApi) {
#ifndef NDEBUG
mFroxelBufferUser.clear();
mRecordBufferUser.clear();
mFroxelListIndices.clear();
mFroxelList.clear();
mFroxelThreadData.clear();
#endif
}
void Froxelizer::froxelizeLights(FEngine& engine,
math::mat4f const& viewMatrix, const FScene::LightSoa& lightData) noexcept {
math::mat4f const& UTILS_RESTRICT viewMatrix,
const FScene::LightSoa& UTILS_RESTRICT lightData) noexcept {
// note: this is called asynchronously
// now that we know the light count, we can shrink the size of mFroxelListIndices
// (account for the extra directional light)
mFroxelListIndices.set(mFroxelListIndices.data(),
uint32_t(lightData.size() - FScene::DIRECTIONAL_LIGHTS_COUNT));
memset(mFroxelThreadData.data(), 0, mFroxelThreadData.sizeInBytes());
froxelizeLoop(engine, mFroxelList, mFroxelListIndices, viewMatrix, lightData);
froxelizeAssignRecordsCompress(mFroxelList, mFroxelListIndices);
froxelizeLoop(engine, mFroxelThreadData, viewMatrix, lightData);
froxelizeAssignRecordsCompress(mFroxelThreadData);
#ifndef NDEBUG
if (lightData.size()) {
@@ -559,10 +538,10 @@ void Froxelizer::froxelizeLights(FEngine& engine,
#endif
}
void Froxelizer::froxelizeLoop(FEngine& engine, utils::Slice<uint16_t>& froxelsList,
utils::Slice<FroxelRunEntry> froxelsListIndices, mat4f const& viewMatrix,
const FScene::LightSoa& lightData) noexcept
{
void Froxelizer::froxelizeLoop(FEngine& engine,
Slice<ThreadData>& UTILS_RESTRICT froxelThreadData,
mat4f const& UTILS_RESTRICT viewMatrix,
const FScene::LightSoa& UTILS_RESTRICT lightData) noexcept {
SYSTRACE_CALL();
constexpr bool SINGLE_THREADED = false;
@@ -571,18 +550,14 @@ void Froxelizer::froxelizeLoop(FEngine& engine, utils::Slice<uint16_t>& froxelsL
auto const* UTILS_RESTRICT directions = lightData.data<FScene::DIRECTION>();
auto const* UTILS_RESTRICT instances = lightData.data<FScene::LIGHT_INSTANCE>();
auto process = [ this,
spheres, directions, instances,
&froxelsList, &froxelsListIndices, &viewMatrix, &lcm ]
auto process = [ this, &froxelThreadData,
spheres, directions, instances, &viewMatrix, &lcm ]
(uint32_t s, uint32_t c) {
const uint32_t froxelCount = mFroxelCountX * mFroxelCountY * mFroxelCountZ;
constexpr uint32_t EXTRA_DATA = 1; // to store the index/type of each light
const mat4f& projection = mProjection;
const mat3f& vn = viewMatrix.upperLeft();
FroxelRunEntry* const UTILS_RESTRICT indices = froxelsListIndices.begin();
for (size_t i = s; i < s + c; ++i) {
size_t gpuIndex = i - FScene::DIRECTIONAL_LIGHTS_COUNT;
FLightManager::Instance li = instances[i];
LightParams light = {
.position = (viewMatrix * float4{ spheres[i].xyz, 1 }).xyz, // to view-space
@@ -592,48 +567,46 @@ void Froxelizer::froxelizeLoop(FEngine& engine, utils::Slice<uint16_t>& froxelsL
.radius = spheres[i].w,
};
// find froxels affected by this light and record them in the list
// also record the index of this light as the first entry
FLightManager::Type type = lcm.getType(li);
size_t gpuIndex = i - FScene::DIRECTIONAL_LIGHTS_COUNT;
assert(gpuIndex < (1 << 15));
uint16_t extra = (uint16_t(gpuIndex) << 1) | uint16_t(type == FLightManager::Type::POINT ? 0 : 1);
uint32_t index = uint32_t(gpuIndex * (froxelCount + EXTRA_DATA));
GrowingSlice<uint16_t> localFroxelsList(
froxelsList.begin() + index, froxelCount + EXTRA_DATA);
localFroxelsList.push_back(extra);
froxelizePointAndSpotLight(localFroxelsList, projection, light);
indices[gpuIndex] = { index, localFroxelsList.size() };
ThreadData& threadData = froxelThreadData[gpuIndex / 64];
const size_t bit = gpuIndex % 64;
const bool isSpot = light.invSin != std::numeric_limits<float>::infinity();
threadData[0] |= isSpot << bit;
froxelizePointAndSpotLight(threadData, bit, projection, light);
}
};
// let's do at least 4 lights per Job.
// we do 64 lights per job
JobSystem& js = engine.getJobSystem();
auto job = jobs::parallel_for(js, nullptr,
1, uint32_t(lightData.size() - FScene::DIRECTIONAL_LIGHTS_COUNT),
std::cref(process), jobs::CountSplitter<4, SINGLE_THREADED ? 0 : 8>());
std::cref(process), jobs::CountSplitter<64, SINGLE_THREADED ? 0 : 8>());
js.runAndWait(job);
}
void Froxelizer::froxelizeAssignRecordsCompress(
const utils::Slice<uint16_t>& froxelsList,
const utils::Slice<FroxelRunEntry>& froxelsListIndices) noexcept {
Slice<ThreadData> const& UTILS_RESTRICT froxelThreadData) noexcept {
SYSTRACE_CALL();
Slice<FroxelEntry> gpuFroxelEntries(mFroxelBufferUser);
utils::Slice<LightRecord> records(mLightRecords);
memset(records.data(), 0, records.sizeInBytes());
const size_t threadDataSize = (CONFIG_MAX_LIGHT_COUNT + 63) / 64;
assert(records.size() == FROXEL_BUFFER_ENTRY_COUNT_MAX);
assert(froxelThreadData.size() == threadDataSize);
LightRecord::bitset spotLights;
for (size_t i = 0, c = froxelsListIndices.size(); i < c; ++i) {
// (skip first entry, which is the light's index)
const int isSpotLight = froxelsList[froxelsListIndices[i].index] & 1;
spotLights.set(i, (bool) isSpotLight);
for (size_t j = froxelsListIndices[i].index + 1,
m = froxelsListIndices[i].index + froxelsListIndices[i].count; j < m; ++j) {
records[froxelsList[j]].lights.set(i);
for (size_t i = 0; i < threadDataSize; i++) {
spotLights.getBitsAt(i) = froxelThreadData[i][0];
}
// TODO: could we do this in-place? Both data structures have exactly the same size.
// this gets very well vectorized...
for (size_t j = 1, jc = FROXEL_BUFFER_ENTRY_COUNT_MAX; j < jc; j++) {
for (size_t i = 0; i < threadDataSize; i++) {
records[j].lights.getBitsAt(i) = froxelThreadData[i][j];
}
}
@@ -645,6 +618,7 @@ void Froxelizer::froxelizeAssignRecordsCompress(
// todo: pld in main loop (not easy because variable size steps)
uint16_t offset = 0;
Slice<FroxelEntry> gpuFroxelEntries(mFroxelBufferUser);
FroxelEntry* const UTILS_RESTRICT froxels = gpuFroxelEntries.data();
RecordBufferType* UTILS_RESTRICT froxelRecords = mRecordBufferUser.data();
@@ -663,7 +637,6 @@ void Froxelizer::froxelizeAssignRecordsCompress(
.spotLightCount = (uint8_t)std::min(size_t(255), (b.lights & spotLights).count())
};
const size_t lightCount = entry.count[0] + entry.count[1];
assert(lightCount <= froxelsListIndices.size());
if (UTILS_UNLIKELY(offset + lightCount >= RECORD_BUFFER_ENTRY_COUNT)) {
#ifndef NDEBUG
@@ -732,7 +705,7 @@ static inline float2 project(mat4f const& p, float3 const& v) noexcept {
}
void Froxelizer::froxelizePointAndSpotLight(
GrowingSlice<uint16_t>& UTILS_RESTRICT froxels,
ThreadData& froxelThread, size_t bit,
mat4f const& UTILS_RESTRICT p,
const Froxelizer::LightParams& UTILS_RESTRICT light) const noexcept {
@@ -836,40 +809,27 @@ void Froxelizer::froxelizePointAndSpotLight(
assert(bx < mFroxelCountX && ex <= mFroxelCountX);
// Gross hack (but correct). For some reason (maybe it's right) the compiler
// isn't able to take froxels's attribute outside of the loops below
// (I'm guessing it thinks it is aliasing something), so we do this
// manually, we write our data at the end of the slice and we grow the
// slice later. We can't run past the end of the slice by construction.
uint16_t* pf = froxels.end();
uint16_t fi = getFroxelIndex(bx, iy, iz);
// The first entry reserved for type of light, i.e. point/spot
size_t fi = getFroxelIndex(bx, iy, iz) + 1;
if (light.invSin != std::numeric_limits<float>::infinity()) {
// This is a spotlight (common case)
// this loops gets vectorized (x2 on arm64) w/ clang
while (bx != ex) {
// Here we always write the froxel, but we only increment the pointer
// if there was an intersection (i.e. it will be overwritten if not).
// This allows us to avoid a branch.
*pf = fi;
if (sphereConeIntersectionFast(boundingSpheres[fi],
light.position, light.axis, light.invSin, light.cosSqr)) {
// see if this froxel intersects the cone
pf++;
}
// see if this froxel intersects the cone
bool intersect = sphereConeIntersectionFast(boundingSpheres[fi],
light.position, light.axis, light.invSin, light.cosSqr);
froxelThread[fi] |= uint64_t(intersect) << bit;
++fi;
++bx;
}
} else {
// this loops gets vectorized (x8 on arm64) w/ clang
// this loops gets vectorized (x2 on arm64) w/ clang
while (bx != ex) {
*pf++ = fi;
froxelThread[fi] |= uint64_t(1) << bit;
++fi;
++bx;
}
}
// fix-up our slice's size
froxels.grow(uint32_t(pf - froxels.end()));
}
}
}

4
filament/src/details/Allocators.h
View File

@@ -23,8 +23,8 @@ namespace filament {
namespace details {
// per render pass allocations
// Froxelization needs about 5 MiB. Command buffer needs about 1 MiB.
static constexpr size_t CONFIG_PER_RENDER_PASS_ARENA_SIZE = 6 * 1024 * 1024;
// Froxelization needs about 1 MiB. Command buffer needs about 1 MiB.
static constexpr size_t CONFIG_PER_RENDER_PASS_ARENA_SIZE = 2 * 1024 * 1024;
// size of the high-level draw commands buffer (comes from the per-render pass allocator)
static constexpr size_t CONFIG_PER_FRAME_COMMANDS_SIZE = 1 * 1024 * 1024;

45
filament/src/details/Froxelizer.h
View File

@@ -73,6 +73,16 @@ public:
// 256 lights max
//
// Max number of froxels limited by:
// - max texture size [min 2048]
// - chosen texture width [64]
// - size of CPU-side indices [16 bits]
// Also, increasing the number of froxels adds more pressure on the "record buffer" which stores
// the light indices per froxel. The record buffer is limited to 65536 entries, so with
// 8192 froxels, we can store 8 lights per froxels assuming they're all used. In practice, some
// froxels are not used, so we can store more.
static constexpr size_t FROXEL_BUFFER_ENTRY_COUNT_MAX = 8192;
class Froxelizer {
public:
explicit Froxelizer(FEngine& engine);
@@ -149,11 +159,6 @@ public:
const utils::Slice<RecordBufferType>& getRecordBufferUser() const { return mRecordBufferUser; }
private:
struct FroxelRunEntry {
uint32_t index;
uint32_t count;
};
struct LightRecord {
using bitset = utils::bitset<uint64_t, (CONFIG_MAX_LIGHT_COUNT + 63) / 64>;
bitset lights;
@@ -169,21 +174,24 @@ private:
float radius;
};
// The first entry always encodes the type of light, i.e. point/spot
using ThreadData = std::array<uint64_t, FROXEL_BUFFER_ENTRY_COUNT_MAX + 1>;
void setViewport(Viewport const& viewport) noexcept;
void setProjection(const math::mat4f& projection, float near, float far) noexcept;
bool update() noexcept;
void froxelizeLoop(FEngine& engine, utils::Slice<uint16_t>& froxelsList,
utils::Slice<FroxelRunEntry> froxelsListIndices, const math::mat4f& viewMatrix,
const FScene::LightSoa& lightData) noexcept;
void froxelizePointAndSpotLight(
utils::GrowingSlice<uint16_t>& froxels,
math::mat4f const& projection, const LightParams& light) const noexcept;
void froxelizeLoop(FEngine& engine,
utils::Slice<ThreadData>& froxelThreadData,
const math::mat4f& viewMatrix, const FScene::LightSoa& lightData) noexcept;
void froxelizeAssignRecordsCompress(
const utils::Slice<uint16_t>& froxelsList,
const utils::Slice<FroxelRunEntry>& froxelsListIndices) noexcept;
utils::Slice<ThreadData> const& froxelThreadData) noexcept;
void froxelizePointAndSpotLight(
ThreadData& froxelThread, size_t bit,
math::mat4f const& projection, const LightParams& light) const noexcept;
uint16_t getFroxelIndex(size_t ix, size_t iy, size_t iz) const noexcept {
return uint16_t(ix + (iy * mFroxelCountX) + (iz * mFroxelCountX * mFroxelCountY));
@@ -205,13 +213,12 @@ private:
math::float4* mPlanesY = nullptr;
math::float4* mBoundingSpheres = nullptr;
utils::Slice<FroxelRunEntry> mFroxelListIndices; // ~2 KiB
utils::Slice<uint16_t> mFroxelList; // ~4 MiB + 510 B
utils::Slice<FroxelEntry> mFroxelBufferUser;
utils::Slice<ThreadData> mFroxelThreadData; // 256 KiB w/ 256 lights
utils::Slice<FroxelEntry> mFroxelBufferUser; // 32 KiB w/ 8192 froxels
// max 32 KiB (actual: resolution dependant)
utils::Slice<RecordBufferType> mRecordBufferUser; // max 64 KiB
utils::Slice<LightRecord> mLightRecords; // 256 KiB
utils::Slice<RecordBufferType> mRecordBufferUser; // 64 KiB
utils::Slice<LightRecord> mLightRecords; // 256 KiB w/ 256 lights
uint16_t mFroxelCountX = 0;
uint16_t mFroxelCountY = 0;

4
libs/utils/include/utils/bitset.h
View File

@@ -58,6 +58,10 @@ public:
return storage[n];
}
T& getBitsAt(size_t n) noexcept {
return storage[n];
}
T getValue() const noexcept {
static_assert(N == 1, "bitfield must only have one storage word");
return storage[0];