Testing ankerl containers

* improve parallel_for a bit

We get about 40% performance increase. The gain comes from not having
to copy the JobData structure each time we create a job, by using a
new emplaceJob() method, we can create the structure directly into
its destination.

* avoid calling wakeAll() when possible

wakeAll() is very expensive and not always needed when a job finishes
because there may not be anyone waiting on that job.

We now maintain a waiter count per job, and use that to determine if
we need to notify or not.

And now that the JobSystem overhead is lower, we can decrease the size
of the jobs, which improves the load balancing.

* mActiveJobs fixes

some comments claimed mActiveJobs needed to be modified before or after
accessing the WorkQueue; this couldn't be correct because there were no
guaranteed global ordering with the workQueue.

CMakeLists.txt

@@ -746,6 +746,7 @@ add_subdirectory(${EXTERNAL}/draco/tnt)
 add_subdirectory(${EXTERNAL}/jsmn/tnt)
 add_subdirectory(${EXTERNAL}/stb/tnt)
 add_subdirectory(${EXTERNAL}/getopt)
+add_subdirectory(${EXTERNAL}/unordered_dense)
 
 # Note that this has to be placed after mikktspace in order for combine_static_libs to work.
 add_subdirectory(${LIBRARIES}/geometry)

android/filament-android/CMakeLists.txt

@@ -128,7 +128,6 @@ target_include_directories(filament-jni PRIVATE
         ..
         ${FILAMENT_DIR}/include
         ../../filament/backend/include
-        ../../third_party/robin-map
         ../../libs/utils/include)
 
 # Force a relink when the version script is changed:

filament/backend/CMakeLists.txt

@@ -313,6 +313,7 @@ endif()
 
 target_link_libraries(${TARGET} PUBLIC math)
 target_link_libraries(${TARGET} PUBLIC utils)
+target_link_libraries(${TARGET} PRIVATE unordered_dense)
 
 # Android, iOS, and WebGL do not use bluegl.
 if(FILAMENT_SUPPORTS_OPENGL AND NOT IOS AND NOT ANDROID AND NOT WEBGL)

filament/backend/src/vulkan/VulkanPipelineCache.cpp

@@ -53,7 +53,7 @@ VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::getOrCreatePipelin
     // If a cached object exists, re-use it, otherwise create a new one.
     if (PipelineMap::iterator pipelineIter = mPipelines.find(mPipelineRequirements);
             pipelineIter != mPipelines.end()) {
-        auto& pipeline = pipelineIter.value();
+        auto& pipeline = pipelineIter->second;
         pipeline.lastUsed = mCurrentTime;
         return &pipeline;
     }
@@ -242,7 +242,7 @@ VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::createPipeline() n
         return nullptr;
     }
 
-    return &mPipelines.emplace(mPipelineRequirements, cacheEntry).first.value();
+    return &mPipelines.emplace(mPipelineRequirements, cacheEntry).first->second;
 }
 
 void VulkanPipelineCache::bindProgram(VulkanProgram* program) noexcept {
@@ -310,7 +310,7 @@ void VulkanPipelineCache::gc() noexcept {
     // Any pipeline older than FVK_MAX_COMMAND_BUFFERS can be safely destroyed.
    using ConstPipeIterator = decltype(mPipelines)::const_iterator;
    for (ConstPipeIterator iter = mPipelines.begin(); iter != mPipelines.end();) {
-       const PipelineCacheEntry& cacheEntry = iter.value();
+       const PipelineCacheEntry& cacheEntry = iter->second;
        if (cacheEntry.lastUsed + FVK_MAX_PIPELINE_AGE < mCurrentTime) {
            vkDestroyPipeline(mDevice, iter->second.handle, VKALLOC);
            iter = mPipelines.erase(iter);

filament/backend/src/vulkan/VulkanPipelineCache.h

@@ -33,11 +33,11 @@
 #include <utils/compiler.h>
 #include <utils/Hash.h>
 
+#include <ankerl/unordered_dense.h>
+
 #include <list>
-#include <tsl/robin_map.h>
 #include <type_traits>
 #include <vector>
-#include <unordered_map>
 
 namespace filament::backend {
 
@@ -238,7 +238,10 @@ private:
     // CACHE CONTAINERS
     // ----------------
 
-    using PipelineMap = tsl::robin_map<PipelineKey, PipelineCacheEntry,
+//    using PipelineMap = tsl::robin_map<PipelineKey, PipelineCacheEntry,
+//            PipelineHashFn, PipelineEqual>;
+
+    using PipelineMap = ankerl::unordered_dense::map<PipelineKey, PipelineCacheEntry,
             PipelineHashFn, PipelineEqual>;
 
 private:

filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.cpp

@@ -53,7 +53,7 @@ VkPipelineLayout VulkanPipelineLayoutCache::getLayout(
     }
 
     if (PipelineLayoutMap::iterator iter = mPipelineLayouts.find(key); iter != mPipelineLayouts.end()) {
-        PipelineLayoutCacheEntry& entry = iter.value();
+        PipelineLayoutCacheEntry& entry = iter->second;
         entry.lastUsed = mTimestamp++;
         return entry.handle;
     }

filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.h

@@ -23,6 +23,7 @@
 #include <utils/Hash.h>
 
 #include <tsl/robin_map.h>
+#include <ankerl/unordered_dense.h>
 
 namespace filament::backend {
 
@@ -73,7 +74,10 @@ private:
         }
     };
 
-    using PipelineLayoutMap = tsl::robin_map<PipelineLayoutKey, PipelineLayoutCacheEntry,
+//    using PipelineLayoutMap = tsl::robin_map<PipelineLayoutKey, PipelineLayoutCacheEntry,
+//            PipelineLayoutKeyHashFn, PipelineLayoutKeyEqual>;
+
+    using PipelineLayoutMap = ankerl::unordered_dense::map<PipelineLayoutKey, PipelineLayoutCacheEntry,
             PipelineLayoutKeyHashFn, PipelineLayoutKeyEqual>;
 
     VkDevice mDevice;

filament/src/PostProcessManager.cpp

@@ -1965,8 +1965,8 @@ PostProcessManager::BloomPassOutput PostProcessManager::bloom(FrameGraph& fg,
         // - visible bloom size changes with dynamic resolution in non-homogenous mode
         // This allows us to use the 9 sample downsampling filter (instead of 13)
         // for at least 4 levels.
-        uint32_t width  = std::max(1u, uint32_t(std::floor(bloomWidth)));
-        uint32_t height = std::max(1u, uint32_t(std::floor(bloomHeight)));
+        uint32_t width  = std::max(16u, uint32_t(std::floor(bloomWidth)));
+        uint32_t height = std::max(16u, uint32_t(std::floor(bloomHeight)));
         width  &= ~((1 << 4) - 1);  // at least 4 levels
         height &= ~((1 << 4) - 1);
         bloomWidth  = float(width);
@@ -1978,6 +1978,8 @@ PostProcessManager::BloomPassOutput PostProcessManager::bloom(FrameGraph& fg,
     // we don't need to do the fireflies reduction if we have TAA (it already does it)
     bool fireflies = threshold && !taaOptions.enabled;
 
+    assert_invariant(bloomWidth && bloomHeight);
+
     while (2 * bloomWidth < float(desc.width) || 2 * bloomHeight < float(desc.height)) {
         if (inoutBloomOptions.quality == QualityLevel::LOW ||
             inoutBloomOptions.quality == QualityLevel::MEDIUM) {

filament/src/RenderPass.cpp

@@ -221,7 +221,7 @@ void RenderPass::appendCommands(FEngine& engine,
         work(vr.first, vr.size());
     } else {
         auto* jobCommandsParallel = jobs::parallel_for(js, nullptr, vr.first, (uint32_t)vr.size(),
-                std::cref(work), jobs::CountSplitter<JOBS_PARALLEL_FOR_COMMANDS_COUNT, 5>());
+                std::cref(work), jobs::CountSplitter<JOBS_PARALLEL_FOR_COMMANDS_COUNT>());
         js.runAndWait(jobCommandsParallel);
     }
 

filament/src/RenderPass.h

@@ -417,7 +417,7 @@ private:
     void instanceify(FEngine& engine, Arena& arena, int32_t eyeCount) noexcept;
 
     // We choose the command count per job to minimize JobSystem overhead.
-    static constexpr size_t JOBS_PARALLEL_FOR_COMMANDS_COUNT = 1024;
+    static constexpr size_t JOBS_PARALLEL_FOR_COMMANDS_COUNT = 128;
     static constexpr size_t JOBS_PARALLEL_FOR_COMMANDS_SIZE  =
             sizeof(Command) * JOBS_PARALLEL_FOR_COMMANDS_COUNT;
 

filament/src/details/Scene.cpp

@@ -248,7 +248,7 @@ void FScene::prepare(utils::JobSystem& js,
 
     auto* renderableJob = jobs::parallel_for(js, rootJob,
             renderableInstances.data(), renderableInstances.size(),
-            std::cref(renderableWork), jobs::CountSplitter<128, 5>());
+            std::cref(renderableWork), jobs::CountSplitter<64>());
 
     auto* lightJob = jobs::parallel_for(js, rootJob,
             lightInstances.data(), lightInstances.size(),

libs/utils/include/utils/JobSystem.h

@@ -44,7 +44,9 @@
 namespace utils {
 
 class JobSystem {
-    static constexpr size_t MAX_JOB_COUNT = 16384;
+    static constexpr size_t MAX_JOB_COUNT = 1 << 14; // 16384
+    static constexpr uint32_t JOB_COUNT_MASK = MAX_JOB_COUNT - 1;
+    static constexpr uint32_t WAITER_COUNT_SHIFT = 24;
     static_assert(MAX_JOB_COUNT <= 0x7FFE, "MAX_JOB_COUNT must be <= 0x7FFE");
     using WorkQueue = WorkStealingDequeue<uint16_t, MAX_JOB_COUNT>;
     using Mutex = utils::Mutex;
@@ -81,14 +83,18 @@ public:
         void* storage[JOB_STORAGE_SIZE_WORDS];                  // 48 | 48
         JobFunc function;                                       //  4 |  8
         uint16_t parent;                                        //  2 |  2
-        std::atomic<uint16_t> runningJobCount = { 1 };          //  2 |  2
-        mutable std::atomic<uint16_t> refCount = { 1 };         //  2 |  2
         mutable ThreadId id = invalidThreadId;                  //  1 |  1
-                                                                //  1 |  1 (padding)
+        mutable std::atomic<uint8_t> refCount = { 1 };          //  1 |  1
+        std::atomic<uint32_t> runningJobCount = { 1 };          //  4 |  4
                                                                 //  4 |  0 (padding)
                                                                 // 64 | 64
     };
 
+#ifndef WIN32
+    // on windows std::function<void()> is bigger and forces the whole structure to be larger
+    static_assert(sizeof(Job) == 64);
+#endif
+
     explicit JobSystem(size_t threadCount = 0, size_t adoptableThreadsCount = 1) noexcept;
 
     ~JobSystem();
@@ -207,6 +213,21 @@ public:
         return job;
     }
 
+    // creates a job from a KNOWN method pointer w/ object passed by value
+    template<typename T, void(T::*method)(JobSystem&, Job*), typename ... ARGS>
+    Job* emplaceJob(Job* parent, ARGS&& ... args) noexcept {
+        static_assert(sizeof(T) <= sizeof(Job::storage), "user data too large");
+        Job* job = create(parent, [](void* storage, JobSystem& js, Job* job) {
+            T* const that = static_cast<T*>(storage);
+            (that->*method)(js, job);
+            that->~T();
+        });
+        if (job) {
+            new(job->storage) T(std::forward<ARGS>(args)...);
+        }
+        return job;
+    }
+
     // creates a job from a functor passed by value
     template<typename T>
     Job* createJob(Job* parent, T functor) noexcept {
@@ -222,6 +243,21 @@ public:
         return job;
     }
 
+    // creates a job from a functor passed by value
+    template<typename T, typename ... ARGS>
+    Job* emplaceJob(Job* parent, ARGS&& ... args) noexcept {
+        static_assert(sizeof(T) <= sizeof(Job::storage), "functor too large");
+        Job* job = create(parent, [](void* storage, JobSystem& js, Job* job){
+            T* const that = static_cast<T*>(storage);
+            that->operator()(js, job);
+            that->~T();
+        });
+        if (job) {
+            new(job->storage) T(std::forward<ARGS>(args)...);
+        }
+        return job;
+    }
+
 
     /*
      * Jobs are normally finished automatically, this can be used to cancel a job before it is run.
@@ -345,6 +381,16 @@ private:
         static constexpr uint32_t m = 0x7fffffffu;
         uint32_t mState; // must be 0 < seed < 0x7fffffff
     public:
+        using result_type = uint32_t;
+
+        static constexpr result_type min() noexcept {
+            return 1;
+        }
+
+        static constexpr result_type max() noexcept {
+            return m - 1;
+        }
+
         inline constexpr explicit default_random_engine(uint32_t seed = 1u) noexcept
                 : mState(((seed % m) == 0u) ? 1u : seed % m) {
         }
@@ -389,7 +435,9 @@ private:
     Job* pop(WorkQueue& workQueue) noexcept;
     Job* steal(WorkQueue& workQueue) noexcept;
 
-    void wait(std::unique_lock<Mutex>& lock, Job* job = nullptr) noexcept;
+    [[nodiscard]]
+    uint32_t wait(std::unique_lock<Mutex>& lock, Job* job) noexcept;
+    void wait(std::unique_lock<Mutex>& lock) noexcept;
     void wakeAll() noexcept;
     void wakeOne() noexcept;
 
@@ -418,7 +466,7 @@ private:
     uint8_t mParallelSplitCount = 0;                    // # of split allowable in parallel_for
     Job* mRootJob = nullptr;
 
-    utils::Mutex mThreadMapLock; // this should have very little contention
+    Mutex mThreadMapLock; // this should have very little contention
     tsl::robin_map<std::thread::id, ThreadState *> mThreadMap;
 };
 
@@ -436,12 +484,13 @@ template<typename CALLABLE, typename ... ARGS>
 JobSystem::Job* createJob(JobSystem& js, JobSystem::Job* parent,
         CALLABLE&& func, ARGS&&... args) noexcept {
     struct Data {
+        explicit Data(std::function<void()> f) noexcept: f(std::move(f)) {}
         std::function<void()> f;
         // Renaming the method below could cause an Arrested Development.
         void gob(JobSystem&, JobSystem::Job*) noexcept { f(); }
-    } user{ std::bind(std::forward<CALLABLE>(func),
-            std::forward<ARGS>(args)...) };
-    return js.createJob<Data, &Data::gob>(parent, std::move(user));
+    };
+    return js.emplaceJob<Data, &Data::gob>(parent,
+            std::bind(std::forward<CALLABLE>(func), std::forward<ARGS>(args)...));
 }
 
 template<typename CALLABLE, typename T, typename ... ARGS,
@@ -452,12 +501,13 @@ template<typename CALLABLE, typename T, typename ... ARGS,
 JobSystem::Job* createJob(JobSystem& js, JobSystem::Job* parent,
         CALLABLE&& func, T&& o, ARGS&&... args) noexcept {
     struct Data {
+        explicit Data(std::function<void()> f) noexcept: f(std::move(f)) {}
         std::function<void()> f;
         // Renaming the method below could cause an Arrested Development.
         void gob(JobSystem&, JobSystem::Job*) noexcept { f(); }
-    } user{ std::bind(std::forward<CALLABLE>(func), std::forward<T>(o),
-            std::forward<ARGS>(args)...) };
-    return js.createJob<Data, &Data::gob>(parent, std::move(user));
+    };
+    return js.emplaceJob<Data, &Data::gob>(parent,
+            std::bind(std::forward<CALLABLE>(func), std::forward<T>(o), std::forward<ARGS>(args)...));
 }
 
 
@@ -486,8 +536,8 @@ struct ParallelForJobData {
 right_side:
         if (splitter.split(splits, count)) {
             const size_type lc = count / 2;
-            JobData ld(start, lc, splits + uint8_t(1), functor, splitter);
-            JobSystem::Job* l = js.createJob<JobData, &JobData::parallelWithJobs>(parent, std::move(ld));
+            JobSystem::Job* l = js.emplaceJob<JobData, &JobData::parallelWithJobs>(parent,
+                    start, lc, splits + uint8_t(1), functor, splitter);
             if (UTILS_UNLIKELY(l == nullptr)) {
                 // couldn't create a job, just pretend we're done splitting
                 goto execute;
@@ -527,8 +577,8 @@ template<typename S, typename F>
 JobSystem::Job* parallel_for(JobSystem& js, JobSystem::Job* parent,
         uint32_t start, uint32_t count, F functor, const S& splitter) noexcept {
     using JobData = details::ParallelForJobData<S, F>;
-    JobData jobData(start, count, 0, std::move(functor), splitter);
-    return js.createJob<JobData, &JobData::parallelWithJobs>(parent, std::move(jobData));
+    return js.emplaceJob<JobData, &JobData::parallelWithJobs>(parent,
+            start, count, 0, std::move(functor), splitter);
 }
 
 // parallel jobs with pointer/count
@@ -539,8 +589,8 @@ JobSystem::Job* parallel_for(JobSystem& js, JobSystem::Job* parent,
         f(data + s, c);
     };
     using JobData = details::ParallelForJobData<S, decltype(user)>;
-    JobData jobData(0, count, 0, std::move(user), splitter);
-    return js.createJob<JobData, &JobData::parallelWithJobs>(parent, std::move(jobData));
+    return js.emplaceJob<JobData, &JobData::parallelWithJobs>(parent,
+            0, count, 0, std::move(user), splitter);
 }
 
 // parallel jobs on a Slice<>

libs/utils/include/utils/WorkStealingDequeue.h

@@ -35,7 +35,13 @@ namespace utils {
  *    steal()                      push(), pop()
  *  any thread                     main thread
  *
- *
+ * References:
+ * - This code is largely inspired from
+ *   https://blog.molecular-matters.com/2015/09/25/job-system-2-0-lock-free-work-stealing-part-3-going-lock-free/
+ * - other implementations
+ *   https://github.com/ConorWilliams/ConcurrentDeque/blob/main/include/riften/deque.hpp
+ *   https://github.com/ssbl/concurrent-deque/blob/master/include/deque.hpp
+ *   https://github.com/taskflow/work-stealing-queue/blob/master/wsq.hpp
  */
 template <typename TYPE, size_t COUNT>
 class WorkStealingDequeue {
@@ -117,7 +123,7 @@ TYPE WorkStealingDequeue<TYPE, COUNT>::pop() noexcept {
     index_t top = mTop.load(std::memory_order_seq_cst);
 
     if (top < bottom) {
-        // Queue isn't empty and it's not the last item, just return it, this is the common case.
+        // Queue isn't empty, and it's not the last item, just return it, this is the common case.
         return getItemAt(bottom);
     }
 
@@ -132,13 +138,13 @@ TYPE WorkStealingDequeue<TYPE, COUNT>::pop() noexcept {
         if (mTop.compare_exchange_strong(top, top + 1,
                 std::memory_order_seq_cst,
                 std::memory_order_relaxed)) {
-            // success: we stole our last item from ourself, meaning that a concurrent steal()
+            // Success: we stole our last item from ourselves, meaning that a concurrent steal()
             //          would have failed.
             // mTop now equals top + 1, we adjust top to make the queue empty.
             top++;
         } else {
-            // failure: mTop was not equal to top, which means the item was stolen under our feet.
-            // top now equals to mTop. Simply discard the item we just popped.
+            // Failure: mTop was not equal to top, which means the item was stolen under our feet.
+            // `top` now equals to mTop. Simply discard the item we just popped.
             // The queue is now empty.
             item = TYPE();
         }
@@ -149,7 +155,7 @@ TYPE WorkStealingDequeue<TYPE, COUNT>::pop() noexcept {
     }
 
     // std::memory_order_relaxed used because we're not publishing any data.
-    // no concurrent writes to mBottom possible, it's always safe to write mBottom.
+    // No concurrent writes to mBottom possible, it's always safe to write mBottom.
     mBottom.store(top, std::memory_order_relaxed);
     return item;
 }
@@ -194,6 +200,8 @@ TYPE WorkStealingDequeue<TYPE, COUNT>::steal() noexcept {
         }
         // failure: the item we just tried to steal was pop()'ed under our feet,
         // simply discard it; nothing to do -- it's okay to try again.
+        // However, item might be corrupted, so it must be trivially destructible
+        static_assert(std::is_trivially_destructible_v<TYPE>);
     }
 }
 

libs/utils/src/JobSystem.cpp

@@ -23,9 +23,6 @@
 // when SYSTRACE_TAG_JOBSYSTEM is used, enables even heavier systraces
 #define HEAVY_SYSTRACE  0
 
-// enable for catching hangs waiting on a job to finish
-static constexpr bool DEBUG_FINISH_HANGS = false;
-
 #include <utils/JobSystem.h>
 
 #include <utils/compiler.h>
@@ -249,53 +246,44 @@ inline bool JobSystem::hasActiveJobs() const noexcept {
 }
 
 inline bool JobSystem::hasJobCompleted(JobSystem::Job const* job) noexcept {
-    return job->runningJobCount.load(std::memory_order_acquire) <= 0;
+    return (job->runningJobCount.load(std::memory_order_acquire) & JOB_COUNT_MASK) == 0;
 }
 
-void JobSystem::wait(std::unique_lock<Mutex>& lock, Job* job) noexcept {
+inline void JobSystem::wait(std::unique_lock<Mutex>& lock) noexcept {
     HEAVY_SYSTRACE_CALL();
-    if constexpr (!DEBUG_FINISH_HANGS) {
-        mWaiterCondition.wait(lock);
-    } else {
-        do {
-            // we use a pretty long timeout (4s) so we're very confident that the system is hung
-            // and nothing else is happening.
-            std::cv_status status = mWaiterCondition.wait_for(lock,
-                    std::chrono::milliseconds(4000));
-            if (status == std::cv_status::no_timeout) {
-                break;
-            }
-
-            // hang debugging...
-
-            // We check of we had active jobs or if the job we're waiting on had completed already.
-            // There is the possibility of a race condition, but our long timeout gives us some
-            // confidence that we're in an incorrect state.
-
-            size_t const id = std::distance(mThreadStates.data(), &getState());
-            auto activeJobs = mActiveJobs.load();
-
-            if (job) {
-                auto runningJobCount = job->runningJobCount.load();
-                FILAMENT_CHECK_POSTCONDITION(runningJobCount > 0)
-                        << "JobSystem(" << this << ", " << unsigned(id) << "): waiting while job "
-                        << job << " has completed and " << activeJobs << " jobs are active!";
-            }
-
-            FILAMENT_CHECK_POSTCONDITION(activeJobs <= 0)
-                    << "JobSystem(" << this << ", " << unsigned(id) << "): waiting while "
-                    << activeJobs << " jobs are active!";
-
-        } while (true);
-    }
+    mWaiterCondition.wait(lock);
 }
 
+inline uint32_t JobSystem::wait(std::unique_lock<Mutex>& lock, Job* const job) noexcept {
+    HEAVY_SYSTRACE_CALL();
+    // signal we are waiting
+
+    if (hasActiveJobs() || exitRequested()) {
+        return job->runningJobCount.load(std::memory_order_acquire);
+    }
+
+    uint32_t runningJobCount =
+            job->runningJobCount.fetch_add(1 << WAITER_COUNT_SHIFT, std::memory_order_relaxed);
+
+    if (runningJobCount & JOB_COUNT_MASK) {
+        mWaiterCondition.wait(lock);
+    }
+
+    runningJobCount =
+            job->runningJobCount.fetch_sub(1 << WAITER_COUNT_SHIFT, std::memory_order_acquire);
+
+    assert_invariant((runningJobCount >> WAITER_COUNT_SHIFT) >= 1);
+
+    return runningJobCount;
+}
+
+UTILS_NOINLINE
 void JobSystem::wakeAll() noexcept {
     // wakeAll() is called when a job finishes (to wake up any thread that might be waiting on it)
-    HEAVY_SYSTRACE_CALL();
+    SYSTRACE_CALL();
     mWaiterLock.lock();
     // this empty critical section is needed -- it guarantees that notify_all() happens
-    // after the condition's variables are set.
+    // either before the condition is checked, or after the condition variable sleeps.
     mWaiterLock.unlock();
     // notify_all() can be pretty slow, and it doesn't need to be inside the lock.
     mWaiterCondition.notify_all();
@@ -306,7 +294,7 @@ void JobSystem::wakeOne() noexcept {
     HEAVY_SYSTRACE_CALL();
     mWaiterLock.lock();
     // this empty critical section is needed -- it guarantees that notify_one() happens
-    // after the condition's variables are set.
+    // either before the condition is checked, or after the condition variable sleeps.
     mWaiterLock.unlock();
     // notify_one() can be pretty slow, and it doesn't need to be inside the lock.
     mWaiterCondition.notify_one();
@@ -328,50 +316,37 @@ void JobSystem::put(WorkQueue& workQueue, Job* job) noexcept {
     size_t const index = job - mJobStorageBase;
     assert(index >= 0 && index < MAX_JOB_COUNT);
 
-    // put the job into the queue first
+    // put the job into the queue
     workQueue.push(uint16_t(index + 1));
-    // then increase our active job count
-    int32_t const oldActiveJobs = mActiveJobs.fetch_add(1, std::memory_order_relaxed);
-    // But it's possible that the job has already been picked-up, so oldActiveJobs could be
-    // negative for instance. We signal only if that's not the case.
-    if (oldActiveJobs >= 0) {
-        wakeOne(); // wake-up a thread if needed...
-    }
+
+    // increase our active job count (the order in which we're doing this must not matter
+    // because we're not using std::memory_order_seq_cst (here or in WorkQueue::push()).
+    mActiveJobs.fetch_add(1, std::memory_order_relaxed);
+
+    // Note: it's absolutely possible for mActiveJobs to be 0 here, because the job could have
+    // been handled by a zealous worker already. In that case we could avoid calling wakeOne(),
+    // but that is not the common case.
+
+    wakeOne();
 }
 
 JobSystem::Job* JobSystem::pop(WorkQueue& workQueue) noexcept {
-    // decrement mActiveJobs first, this is to ensure that if there is only a single job left
-    // (and we're about to pick it up), other threads don't loop trying to do the same.
-    mActiveJobs.fetch_sub(1, std::memory_order_relaxed);
-
     size_t const index = workQueue.pop();
     assert(index <= MAX_JOB_COUNT);
     Job* const job = !index ? nullptr : &mJobStorageBase[index - 1];
-
-    // If our guess was wrong, i.e. we couldn't pick up a job (b/c our queue was empty), we
-    // need to correct mActiveJobs.
-    if (!job) {
-        // no need to wake someone else up because, we will go into job-stealing mode
-        // immediately after this
-        mActiveJobs.fetch_add(1, std::memory_order_relaxed);
+    if (UTILS_LIKELY(job)) {
+        mActiveJobs.fetch_sub(1, std::memory_order_relaxed);
     }
     return job;
 }
 
 JobSystem::Job* JobSystem::steal(WorkQueue& workQueue) noexcept {
-    // decrement mActiveJobs first, this is to ensure that if there is only a single job left
-    // (and we're about to pick it up), other threads don't loop trying to do the same.
-    mActiveJobs.fetch_sub(1, std::memory_order_relaxed);
-
     size_t const index = workQueue.steal();
     assert_invariant(index <= MAX_JOB_COUNT);
     Job* const job = !index ? nullptr : &mJobStorageBase[index - 1];
-
-    if (!job) {
-        // If we failed taking a job, we need to correct mActiveJobs.
-        mActiveJobs.fetch_add(1, std::memory_order_relaxed);
+    if (UTILS_LIKELY(job)) {
+        mActiveJobs.fetch_sub(1, std::memory_order_relaxed);
     }
-
     return job;
 }
 
@@ -402,7 +377,7 @@ JobSystem::Job* JobSystem::steal(JobSystem::ThreadState& state) noexcept {
     Job* job = nullptr;
     do {
         ThreadState* const stateToStealFrom = getStateToStealFrom(state);
-        if (UTILS_LIKELY(stateToStealFrom)) {
+        if (stateToStealFrom) {
             job = steal(stateToStealFrom->workQueue);
         }
         // nullptr -> nothing to steal in that queue either, if there are active jobs,
@@ -415,14 +390,18 @@ bool JobSystem::execute(JobSystem::ThreadState& state) noexcept {
     HEAVY_SYSTRACE_CALL();
 
     Job* job = pop(state.workQueue);
-    if (UTILS_UNLIKELY(job == nullptr)) {
+
+    // It is beneficial for some benchmarks to poll on steal() for a bit, because going back to
+    // sleep and waking up is pretty expensive. However, it is unclear it helps in practice with
+    // larger jobs or when parallel_for is used.
+    constexpr size_t const STEAL_TRY_COUNT = 1;
+    for (size_t i = 0; UTILS_UNLIKELY(!job && i < STEAL_TRY_COUNT); i++) {
         // our queue is empty, try to steal a job
         job = steal(state);
     }
 
-    if (job) {
-        assert(job->runningJobCount.load(std::memory_order_relaxed) >= 1);
-
+    if (UTILS_LIKELY(job)) {
+        assert((job->runningJobCount.load(std::memory_order_relaxed) & JOB_COUNT_MASK) >= 1);
         if (UTILS_LIKELY(job->function)) {
             HEAVY_SYSTRACE_NAME("job->function");
             job->id = std::distance(mThreadStates.data(), &state);
@@ -467,11 +446,16 @@ void JobSystem::finish(Job* job) noexcept {
     do {
         // std::memory_order_release here is needed to synchronize with JobSystem::wait()
         // which needs to "see" all changes that happened before the job terminated.
-        auto runningJobCount = job->runningJobCount.fetch_sub(1, std::memory_order_acq_rel);
+        uint32_t const v = job->runningJobCount.fetch_sub(1, std::memory_order_acq_rel);
+        uint32_t const runningJobCount = v & JOB_COUNT_MASK;
         assert(runningJobCount > 0);
+
         if (runningJobCount == 1) {
             // no more work, destroy this job and notify its parent
-            notify = true;
+            uint32_t const waiters = v >> WAITER_COUNT_SHIFT;
+            if (waiters) {
+                notify = true;
+            }
             Job* const parent = job->parent == 0x7FFF ? nullptr : &storage[job->parent];
             decRef(job);
             job = parent;
@@ -482,7 +466,8 @@ void JobSystem::finish(Job* job) noexcept {
     } while (job);
 
     // wake-up all threads that could potentially be waiting on this job finishing
-    if (notify) {
+    if (UTILS_UNLIKELY(notify)) {
+        // but avoid calling notify_all() at all cost, because it's always expensive
         wakeAll();
     }
 }
@@ -500,10 +485,11 @@ JobSystem::Job* JobSystem::create(JobSystem::Job* parent, JobFunc func) noexcept
             // add a reference to the parent to make sure it can't be terminated.
             // memory_order_relaxed is safe because no action is taken at this point
             // (the job is not started yet).
-            auto parentJobCount = parent->runningJobCount.fetch_add(1, std::memory_order_relaxed);
+            UTILS_UNUSED_IN_RELEASE auto const parentJobCount =
+                    parent->runningJobCount.fetch_add(1, std::memory_order_relaxed);
 
             // can't create a child job of a terminated parent
-            assert(parentJobCount > 0);
+            assert((parentJobCount & JOB_COUNT_MASK) > 0);
 
             index = parent - mJobStorageBase;
             assert(index < MAX_JOB_COUNT);
@@ -567,7 +553,7 @@ void JobSystem::waitAndRelease(Job*& job) noexcept {
 
     ThreadState& state(getState());
     do {
-        if (!execute(state)) {
+        if (UTILS_UNLIKELY(!execute(state))) {
             // test if job has completed first, to possibly avoid taking the lock
             if (hasJobCompleted(job)) {
                 break;
@@ -583,11 +569,23 @@ void JobSystem::waitAndRelease(Job*& job) noexcept {
             // continue to handle more jobs, as they get added.
 
             std::unique_lock<Mutex> lock(mWaiterLock);
-            if (!hasJobCompleted(job) && !hasActiveJobs() && !exitRequested()) {
-                wait(lock, job);
+            uint32_t const runningJobCount = wait(lock, job);
+            // we could be waking up because either:
+            // - the job we're waiting on has completed
+            // - more jobs where added to the JobSystem
+            // - we're asked to exit
+            if ((runningJobCount & JOB_COUNT_MASK) == 0 || exitRequested()) {
+                break;
             }
+
+            // if we get here, it means that
+            // - the job we're waiting on is still running, and
+            // - we're not asked to exit, and
+            // - there were some active jobs
+            // So we try to handle one.
+            continue;
         }
-    } while (!hasJobCompleted(job) && !exitRequested());
+    } while (UTILS_LIKELY(!hasJobCompleted(job) && !exitRequested()));
 
     if (job == mRootJob) {
         mRootJob = nullptr;

third_party/unordered_dense/.gitignore

@@ -0,0 +1,14 @@
+build
+builddir
+.cache
+.vscode
+compile_commands.json
+
+# ignore all in subprojects except the .wrap files
+/subprojects/*
+!/subprojects/*.wrap
+
+# c++ modules
+*.pcm
+a.out
+*.o

third_party/unordered_dense/CMakeLists.txt

@@ -0,0 +1,61 @@
+cmake_minimum_required(VERSION 3.12)
+project("unordered_dense"
+    VERSION 4.4.0
+    DESCRIPTION "A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion"
+    HOMEPAGE_URL "https://github.com/martinus/unordered_dense")
+
+include(GNUInstallDirs)
+
+# determine whether this is a standalone project or included by other projects
+if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
+    set(_unordered_dense_is_toplevel_project TRUE)
+else()
+    set(_unordered_dense_is_toplevel_project FALSE)
+endif()
+
+add_library(unordered_dense INTERFACE)
+add_library(unordered_dense::unordered_dense ALIAS unordered_dense)
+
+target_include_directories(
+    unordered_dense
+    INTERFACE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+
+target_compile_features(unordered_dense INTERFACE cxx_std_17)
+
+if(_unordered_dense_is_toplevel_project)
+    # locations are provided by GNUInstallDirs
+    install(
+        TARGETS unordered_dense
+        EXPORT unordered_dense_Targets
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+    include(CMakePackageConfigHelpers)
+    write_basic_package_version_file(
+        "unordered_denseConfigVersion.cmake"
+        VERSION ${PROJECT_VERSION}
+        COMPATIBILITY SameMajorVersion)
+
+    configure_package_config_file(
+        "${PROJECT_SOURCE_DIR}/cmake/unordered_denseConfig.cmake.in"
+        "${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake"
+        INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+    install(
+        EXPORT unordered_dense_Targets
+        FILE unordered_denseTargets.cmake
+        NAMESPACE unordered_dense::
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+    install(
+        FILES "${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake"
+        "${PROJECT_BINARY_DIR}/unordered_denseConfigVersion.cmake"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+    install(
+        DIRECTORY ${PROJECT_SOURCE_DIR}/include/ankerl
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+endif()

third_party/unordered_dense/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at martin.ankerl@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

third_party/unordered_dense/CONTRIBUTING.md

@@ -0,0 +1,3 @@
+* Coding style should be consistent with the code around you.
+* Use automatic formatting with clang-format.
+* One feature per pull request

third_party/unordered_dense/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Martin Leitner-Ankerl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

third_party/unordered_dense/README.md

@@ -0,0 +1,374 @@
+<a id="top"></a>
+
+[![Release](https://img.shields.io/github/release/martinus/unordered_dense.svg)](https://github.com/martinus/unordered_dense/releases)
+[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/martinus/unordered_dense/main/LICENSE)
+[![meson_build_test](https://github.com/martinus/unordered_dense/actions/workflows/main.yml/badge.svg)](https://github.com/martinus/unordered_dense/actions)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6220/badge)](https://bestpractices.coreinfrastructure.org/projects/6220)
+[![Sponsors](https://img.shields.io/github/sponsors/martinus?style=social)](https://github.com/sponsors/martinus)
+
+# 🚀 ankerl::unordered_dense::{map, set} <!-- omit in toc -->
+
+A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion for C++17 and later.
+
+The classes `ankerl::unordered_dense::map` and `ankerl::unordered_dense::set` are (almost) drop-in replacements of `std::unordered_map` and `std::unordered_set`. While they don't have as strong iterator / reference stability guaranties, they are typically *much* faster.
+
+Additionally, there are `ankerl::unordered_dense::segmented_map` and `ankerl::unordered_dense::segmented_set` with lower peak memory usage. and stable iterator/references on insert.
+
+- [1. Overview](#1-overview)
+- [2. Installation](#2-installation)
+  - [2.1. Installing using cmake](#21-installing-using-cmake)
+- [3. Usage](#3-usage)
+  - [3.1. Modules](#31-modules)
+  - [3.2. Hash](#32-hash)
+    - [3.2.1. Simple Hash](#321-simple-hash)
+    - [3.2.2. High Quality Hash](#322-high-quality-hash)
+    - [3.2.3. Specialize `ankerl::unordered_dense::hash`](#323-specialize-ankerlunordered_densehash)
+    - [3.2.4. Heterogeneous Overloads using `is_transparent`](#324-heterogeneous-overloads-using-is_transparent)
+    - [3.2.5. Automatic Fallback to `std::hash`](#325-automatic-fallback-to-stdhash)
+    - [3.2.6. Hash the Whole Memory](#326-hash-the-whole-memory)
+  - [3.3. Container API](#33-container-api)
+    - [3.3.1. `auto extract() && -> value_container_type`](#331-auto-extract----value_container_type)
+    - [3.3.2. `extract()` single Elements](#332-extract-single-elements)
+    - [3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&`](#333-nodiscard-auto-values-const-noexcept---value_container_type-const)
+    - [3.3.4. `auto replace(value_container_type&& container)`](#334-auto-replacevalue_container_type-container)
+  - [3.4. Custom Container Types](#34-custom-container-types)
+  - [3.5. Custom Bucket Types](#35-custom-bucket-types)
+    - [3.5.1. `ankerl::unordered_dense::bucket_type::standard`](#351-ankerlunordered_densebucket_typestandard)
+    - [3.5.2. `ankerl::unordered_dense::bucket_type::big`](#352-ankerlunordered_densebucket_typebig)
+- [4. `segmented_map` and `segmented_set`](#4-segmented_map-and-segmented_set)
+- [5. Design](#5-design)
+  - [5.1. Inserts](#51-inserts)
+  - [5.2. Lookups](#52-lookups)
+  - [5.3. Removals](#53-removals)
+- [6. Real World Usage](#6-real-world-usage)
+
+## 1. Overview
+
+The chosen design has a few advantages over `std::unordered_map`: 
+
+* Perfect iteration speed - Data is stored in a `std::vector`, all data is contiguous!
+* Very fast insertion & lookup speed, in the same ballpark as [`absl::flat_hash_map`](https://abseil.io/docs/cpp/guides/container`)
+* Low memory usage
+* Full support for `std::allocators`, and [polymorphic allocators](https://en.cppreference.com/w/cpp/memory/polymorphic_allocator). There are `ankerl::unordered_dense::pmr` typedefs available
+* Customizeable storage type: with a template parameter you can e.g. switch from `std::vector` to `boost::interprocess::vector` or any other compatible random-access container.
+* Better debugging: the underlying data can be easily seen in any debugger that can show an `std::vector`.
+
+There's no free lunch, so there are a few disadvantages:
+
+* Deletion speed is relatively slow. This needs two lookups: one for the element to delete, and one for the element that is moved onto the newly empty spot.
+* no `const Key` in `std::pair<Key, Value>`
+* Iterators and references are not stable on insert or erase.
+
+## 2. Installation
+
+<!-- See https://github.com/bernedom/SI/blob/main/doc/installation-guide.md -->
+The default installation location is `/usr/local`.
+
+### 2.1. Installing using cmake 
+
+Clone the repository and run these commands in the cloned folder:
+
+```sh
+mkdir build && cd build
+cmake ..
+cmake --build . --target install
+```
+
+Consider setting an install prefix if you do not want to install `unordered_dense` system wide, like so:
+
+```sh
+mkdir build && cd build
+cmake -DCMAKE_INSTALL_PREFIX:PATH=${HOME}/unordered_dense_install ..
+cmake --build . --target install
+```
+
+To make use of the installed library, add this to your project:
+
+```cmake
+find_package(unordered_dense CONFIG REQUIRED)
+target_link_libraries(your_project_name unordered_dense::unordered_dense)
+```
+
+## 3. Usage
+
+### 3.1. Modules
+
+`ankerl::unordered_dense` supports c++20 modules. Simply compile `src/ankerl.unordered_dense.cpp` and use the resulting module, e.g. like so:
+
+```sh
+clang++ -std=c++20 -I include --precompile -x c++-module src/ankerl.unordered_dense.cpp
+clang++ -std=c++20 -c ankerl.unordered_dense.pcm
+```
+
+To use the module with e.g. in `module_test.cpp`, use 
+
+```cpp
+import ankerl.unordered_dense;
+```
+
+and compile with e.g.
+
+```sh
+clang++ -std=c++20 -fprebuilt-module-path=. ankerl.unordered_dense.o module_test.cpp -o main
+```
+
+A simple demo script can be found in `test/modules`.
+
+### 3.2. Hash
+
+`ankerl::unordered_dense::hash` is a fast and high quality hash, based on [wyhash](https://github.com/wangyi-fudan/wyhash). The `ankerl::unordered_dense` map/set differentiates between hashes of high quality (good [avalanching effect](https://en.wikipedia.org/wiki/Avalanche_effect)) and bad quality. Hashes with good quality contain a special marker:
+
+```cpp
+using is_avalanching = void;
+```
+
+This is the cases for the specializations `bool`, `char`, `signed char`, `unsigned char`, `char8_t`, `char16_t`, `char32_t`, `wchar_t`, `short`, `unsigned short`, `int`, `unsigned int`, `long`, `long long`, `unsigned long`, `unsigned long long`, `T*`, `std::unique_ptr<T>`, `std::shared_ptr<T>`, `enum`, `std::basic_string<C>`, and `std::basic_string_view<C>`.
+
+Hashes that do not contain such a marker are assumed to be of bad quality and receive an additional mixing step inside the map/set implementation.
+
+#### 3.2.1. Simple Hash
+
+Consider a simple custom key type:
+
+```cpp
+struct id {
+    uint64_t value{};
+
+    auto operator==(id const& other) const -> bool {
+        return value == other.value;
+    }
+};
+```
+
+The simplest implementation of a hash is this:
+
+```cpp
+struct custom_hash_simple {
+    auto operator()(id const& x) const noexcept -> uint64_t {
+        return x.value;
+    }
+};
+```
+This can be used e.g. with 
+
+```cpp
+auto ids = ankerl::unordered_dense::set<id, custom_hash_simple>();
+```
+
+Since `custom_hash_simple` doesn't have a `using is_avalanching = void;` marker it is considered to be of bad quality and additional mixing of `x.value` is automatically provided inside the set.
+
+#### 3.2.2. High Quality Hash
+
+Back to the `id` example, we can easily implement a higher quality hash:
+
+```cpp
+struct custom_hash_avalanching {
+    using is_avalanching = void;
+
+    auto operator()(id const& x) const noexcept -> uint64_t {
+        return ankerl::unordered_dense::detail::wyhash::hash(x.value);
+    }
+};
+```
+
+We know `wyhash::hash` is of high quality, so we can add `using is_avalanching = void;` which makes the map/set directly use the returned value.
+
+
+#### 3.2.3. Specialize `ankerl::unordered_dense::hash`
+
+Instead of creating a new class you can also specialize `ankerl::unordered_dense::hash`:
+
+```cpp
+template <>
+struct ankerl::unordered_dense::hash<id> {
+    using is_avalanching = void;
+
+    [[nodiscard]] auto operator()(id const& x) const noexcept -> uint64_t {
+        return detail::wyhash::hash(x.value);
+    }
+};
+```
+
+#### 3.2.4. Heterogeneous Overloads using `is_transparent`
+
+This map/set supports heterogeneous overloads as described in [P2363 Extending associative containers with the remaining heterogeneous overloads](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html) which is [targeted for C++26](https://wg21.link/p2077r2). This has overloads for `find`, `count`, `contains`, `equal_range` (see [P0919R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0919r3.html)), `erase` (see [P2077R2](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2077r2.html)), and  `try_emplace`, `insert_or_assign`, `operator[]`, `at`, and `insert` & `emplace` for sets (see [P2363R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html)).
+
+For heterogeneous overloads to take affect, both `hasher` and `key_equal` need to have the attribute `is_transparent` set.
+
+Here is an example implementation that's usable with any string types that is convertible to `std::string_view` (e.g. `char const*` and `std::string`):
+
+```cpp
+struct string_hash {
+    using is_transparent = void; // enable heterogeneous overloads
+    using is_avalanching = void; // mark class as high quality avalanching hash
+
+    [[nodiscard]] auto operator()(std::string_view str) const noexcept -> uint64_t {
+        return ankerl::unordered_dense::hash<std::string_view>{}(str);
+    }
+};
+```
+
+To make use of this hash you'll need to specify it as a type, and also a `key_equal` with `is_transparent` like [std::equal_to<>](https://en.cppreference.com/w/cpp/utility/functional/equal_to_void):
+
+```cpp
+auto map = ankerl::unordered_dense::map<std::string, size_t, string_hash, std::equal_to<>>();
+```
+
+For more information see the examples in `test/unit/transparent.cpp`.
+
+
+#### 3.2.5. Automatic Fallback to `std::hash`
+
+When an implementation for `std::hash` of a custom type is available, this is automatically used and assumed to be of bad quality (thus `std::hash` is used, but an additional mixing step is performed).
+
+
+#### 3.2.6. Hash the Whole Memory
+
+When the type [has a unique object representation](https://en.cppreference.com/w/cpp/types/has_unique_object_representations) (no padding, trivially copyable), one can just hash the object's memory. Consider a simple class
+
+```cpp
+struct point {
+    int x{};
+    int y{};
+
+    auto operator==(point const& other) const -> bool {
+        return x == other.x && y == other.y;
+    }
+};
+```
+
+A fast and high quality hash can be easily provided like so:
+
+```cpp
+struct custom_hash_unique_object_representation {
+    using is_avalanching = void;
+
+    [[nodiscard]] auto operator()(point const& f) const noexcept -> uint64_t {
+        static_assert(std::has_unique_object_representations_v<point>);
+        return ankerl::unordered_dense::detail::wyhash::hash(&f, sizeof(f));
+    }
+};
+```
+
+### 3.3. Container API
+
+In addition to the standard `std::unordered_map` API (see https://en.cppreference.com/w/cpp/container/unordered_map) we have additional API that is somewhat similar to the node API, but leverages the fact that we're using a random access container internally:
+
+#### 3.3.1. `auto extract() && -> value_container_type`
+
+Extracts the internally used container. `*this` is emptied.
+
+#### 3.3.2. `extract()` single Elements
+
+Similar to `erase()` I have an API call `extract()`. It behaves exactly the same as `erase`, except that the return value is the moved element that is removed from the container:
+
+* `auto extract(const_iterator it) -> value_type`
+* `auto extract(Key const& key) -> std::optional<value_type>`
+* `template <class K> auto extract(K&& key) -> std::optional<value_type>`
+
+Note that the `extract(key)` API returns an `std::optional<value_type>` that is empty when the key is not found.
+
+#### 3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&`
+
+Exposes the underlying values container.
+
+#### 3.3.4. `auto replace(value_container_type&& container)`
+
+Discards the internally held container and replaces it with the one passed. Non-unique elements are
+removed, and the container will be partly reordered when non-unique elements are found.
+
+### 3.4. Custom Container Types
+
+`unordered_dense` accepts a custom allocator, but you can also specify a custom container for that template argument. That way it is possible to replace the internally used `std::vector` with e.g. `std::deque` or any other container like `boost::interprocess::vector`. This supports fancy pointers (e.g. [offset_ptr](https://www.boost.org/doc/libs/1_80_0/doc/html/interprocess/offset_ptr.html)), so the container can be used with e.g. shared memory provided by `boost::interprocess`.
+
+### 3.5. Custom Bucket Types
+
+The map/set supports two different bucket types. The default should be good for pretty much everyone.
+
+#### 3.5.1. `ankerl::unordered_dense::bucket_type::standard`
+
+* Up to 2^32 = 4.29 billion elements.
+* 8 bytes overhead per bucket.
+
+#### 3.5.2. `ankerl::unordered_dense::bucket_type::big`
+
+* up to 2^63 = 9223372036854775808 elements.
+* 12 bytes overhead per bucket.
+
+## 4. `segmented_map` and `segmented_set`
+
+`ankerl::unordered_dense` provides a custom container implementation that has lower memory requirements than the default `std::vector`. Memory is not contiguous, but it can allocate segments without having to reallocate and move all the elements. In summary, this leads to
+
+* Much smoother memory usage, memory usage increases continuously.
+* No high peak memory usage.
+* Faster insertion because elements never need to be moved to new allocated blocks
+* Slightly slower indexing compared to `std::vector` because an additional indirection is needed.
+
+Here is a comparison against `absl::flat_hash_map` and the `ankerl::unordered_dense::map` when inserting 10 million entries
+![allocated memory](doc/allocated_memory.png)
+
+Abseil is fastest for this simple inserting test, taking a bit over 0.8 seconds. It's peak memory usage is about 430 MB. Note how the memory usage goes down after the last peak; when it goes down to ~290MB it has finished rehashing and could free the previously used memory block.
+
+`ankerl::unordered_dense::segmented_map` doesn't have these peaks, and instead has a smooth increase of memory usage. Note there are still sudden drops & increases in memory because the indexing data structure needs still needs to increase by a fixed factor. But due to holding the data in a separate container we are able to first free the old data structure, and then allocate a new, bigger indexing structure; thus we do not have peaks.
+
+## 5. Design
+
+The map/set has two data structures:
+* `std::vector<value_type>` which holds all data. map/set iterators are just `std::vector<value_type>::iterator`!
+* An indexing structure (bucket array), which is a flat array with 8-byte buckets.
+
+### 5.1. Inserts
+
+Whenever an element is added it is `emplace_back` to the vector. The key is hashed, and an entry (bucket) is added at the
+corresponding location in the bucket array. The bucket has this structure:
+
+```cpp
+struct Bucket {
+    uint32_t dist_and_fingerprint;
+    uint32_t value_idx;
+};
+```
+
+Each bucket stores 3 things:
+* The distance of that value from the original hashed location (3 most significant bytes in `dist_and_fingerprint`)
+* A fingerprint; 1 byte of the hash (lowest significant byte in `dist_and_fingerprint`)
+* An index where in the vector the actual data is stored.
+
+This structure is especially designed for the collision resolution strategy robin-hood hashing with backward shift
+deletion.
+
+### 5.2. Lookups
+
+The key is hashed and the bucket array is searched if it has an entry at that location with that fingerprint. When found,
+the key in the data vector is compared, and when equal the value is returned.
+
+### 5.3. Removals
+
+Since all data is stored in a vector, removals are a bit more complicated:
+
+1. First, lookup the element to delete in the index array.
+2. When found, replace that element in the vector with the last element in the vector. 
+3. Update *two* locations in the bucket array: First remove the bucket for the removed element
+4. Then, update the `value_idx` of the moved element. This requires another lookup.
+
+
+## 6. Real World Usage
+
+On 2023-09-10 I did a quick search on github to see if this map is used in any popular open source projects. Here are some of the projects
+I found. Please send me a note if you want on that list!
+
+* [PruaSlicer](https://github.com/prusa3d/PrusaSlicer) -  G-code generator for 3D printers (RepRap, Makerbot, Ultimaker etc.) 
+* [Kismet](https://github.com/kismetwireless/kismet): Wi-Fi, Bluetooth, RF, and more. Kismet is a sniffer, WIDS, and wardriving tool for Wi-Fi, Bluetooth, Zigbee, RF, and more, which runs on Linux and macOS
+* [Rspamd](https://github.com/rspamd/rspamd) - Fast, free and open-source spam filtering system.
+* [kallisto](https://github.com/pachterlab/kallisto) -  Near-optimal RNA-Seq quantification
+* [Slang](https://github.com/shader-slang/slang) - Slang is a shading language that makes it easier to build and maintain large shader codebases in a modular and extensible fashion.
+* [CyberFSR2](https://github.com/PotatoOfDoom/CyberFSR2) - Drop-in DLSS replacement with FSR 2.0 for various games such as Cyberpunk 2077.
+* [ossia score](https://github.com/ossia/score) - A free, open-source, cross-platform intermedia sequencer for precise and flexible scripting of interactive scenarios. 
+* [HiveWE](https://github.com/stijnherfst/HiveWE) - A Warcraft III World Editor (WE) that focusses on speed and ease of use.
+* [opentxs](https://github.com/Open-Transactions/opentxs) - The Open-Transactions project is a collaborative effort to develop a robust, commercial-grade, fully-featured, free-software toolkit implementing the OTX protocol as well as a full-strength financial cryptography library, API, GUI, command-line interface, and prototype notary server.
+* [LuisaCompute](https://github.com/LuisaGroup/LuisaCompute) - High-Performance Rendering Framework on Stream Architectures
+* [Lethe](https://github.com/lethe-cfd/lethe) - Lethe (pronounced /ˈliːθiː/) is open-source computational fluid dynamics (CFD) software which uses high-order continuous Galerkin formulations to solve the incompressible Navier–Stokes equations (among others).
+* [PECOS](https://github.com/amzn/pecos) - PECOS is a versatile and modular machine learning (ML) framework for fast learning and inference on problems with large output spaces, such as extreme multi-label ranking (XMR) and large-scale retrieval.
+* [Operon](https://github.com/heal-research/operon) - A modern C++ framework for symbolic regression that uses genetic programming to explore a hypothesis space of possible mathematical expressions in order to find the best-fitting model for a given regression target.
+* [MashMap](https://github.com/marbl/MashMap) - A fast approximate aligner for long DNA sequences
+* [minigpt4.cpp](https://github.com/Maknee/minigpt4.cpp) - Port of MiniGPT4 in C++ (4bit, 5bit, 6bit, 8bit, 16bit CPU inference with GGML)

third_party/unordered_dense/cmake/unordered_denseConfig.cmake.in

@@ -0,0 +1,4 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+check_required_components("@PROJECT_NAME@")

third_party/unordered_dense/data/fuzz/fuzz_api/00b120066609a5d8f34a0faab4008a1a.00000010.honggfuzz.cov

@@ -0,0 +1 @@
+§ount-aftpuu=o=d

third_party/unordered_dense/data/fuzz/fuzz_api/00d4e4e4e4e4e4774c54e4e4e4e4e4e4.00000010.honggfuzz.cov

@@ -0,0 +1 @@
+=<3D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

third_party/unordered_dense/data/fuzz/fuzz_api/02a00000e38104353fe0716870000000.00000010.honggfuzz.cov

@@ -0,0 +1 @@
+nU
€UUUUUUUUUUUU

third_party/unordered_dense/data/fuzz/fuzz_api/02b1224a7a165d6f0642336bb63347dc.00000020.honggfuzz.cov

@@ -0,0 +1 @@
+dt-q=fu{
€W<$
=×test-case=nX¶)u

third_party/unordered_dense/data/fuzz/fuzz_api/059102b7b7648f18da3c744c7b7c1253.00000020.honggfuzz.cov

@@ -0,0 +1 @@
+dt-nddt-qàÿÿÿÿÿÿno-line-number

third_party/unordered_dense/data/fuzz/fuzz_api/060d4caf04d0e6b409d104c643a025ba.00000020.honggfuzz.cov

@@ -0,0 +1 @@
+ødt-exitnlist-reporterso-tkipped

third_party/unordered_dense/data/fuzz/fuzz_api/06d2184ff06fa75dd490c7ac07bee262.0000002e.honggfuzz.cov

@@ -0,0 +1 @@
+dt-exnÿÿÿÿÿÿàno-exitcode=path-filenamesitlr"

third_party/unordered_dense/data/fuzz/fuzz_api/07b5f62901ef073dbc9220e69a4b1dab.00000040.honggfuzz.cov

@@ -0,0 +1 @@
+å?fordt-ablisssssssssssssssssssssst-reportergfl=ors=-cas-caseeee

third_party/unordered_dense/data/fuzz/fuzz_api/07f839f695a0d6f7df49f5f91abbe3e2.00000020.honggfuzz.cov

@@ -0,0 +1 @@
+dt-ndt-tce=Ăcou~˙˙˙˙nt˙řncfctĂ

third_party/unordered_dense/data/fuzz/fuzz_api/086393821cb79cf5033c87fca2639382.00000008.honggfuzz.cov

@@ -0,0 +1 @@
+"dt-nssn

third_party/unordered_dense/data/fuzz/fuzz_api/095a5abbeb8288ec80b1f771a14a5aae.00000010.honggfuzz.cov

@@ -0,0 +1 @@
+ntlht££££££££££e

third_party/unordered_dense/data/fuzz/fuzz_api/0b051de918719c6d95a3864a54e9e7ae.00000040.honggfuzz.cov

@@ -0,0 +1 @@
+•j£«ŽVfÃHminimal=H‘ P*<ë"@qÙBÅpäCúŒ<C3BA>ï@înrhash_stc=bench_find_