diff --git a/NEW_RELEASE_NOTES.md b/NEW_RELEASE_NOTES.md index f408ba3462..3e018ece46 100644 --- a/NEW_RELEASE_NOTES.md +++ b/NEW_RELEASE_NOTES.md @@ -6,3 +6,5 @@ appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md). ## Release notes for next branch cut + +- matc: better material compression using a Multi-Base Variable-Length Interleaved Token Stream [⚠️ **New Material Version**] diff --git a/filament/test/test_material.filamat b/filament/test/test_material.filamat index 5b7f135b6e..4a20b0f46e 100644 Binary files a/filament/test/test_material.filamat and b/filament/test/test_material.filamat differ diff --git a/libs/filabridge/include/filament/MaterialEnums.h b/libs/filabridge/include/filament/MaterialEnums.h index 6e71b366bf..bc6c8c0b50 100644 --- a/libs/filabridge/include/filament/MaterialEnums.h +++ b/libs/filabridge/include/filament/MaterialEnums.h @@ -28,7 +28,7 @@ namespace filament { // update this when a new version of filament wouldn't work with older materials -static constexpr size_t MATERIAL_VERSION = 70; +static constexpr size_t MATERIAL_VERSION = 71; // Those are the api levels that are used in the source material file (.mat) // diff --git a/libs/filabridge/include/private/filament/LineDictionaryUtils.h b/libs/filabridge/include/private/filament/LineDictionaryUtils.h new file mode 100644 index 0000000000..bd2e2f03bd --- /dev/null +++ b/libs/filabridge/include/private/filament/LineDictionaryUtils.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file law. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_PRIVATE_LINEDICTIONARYUTILS_H +#define TNT_FILAMENT_PRIVATE_LINEDICTIONARYUTILS_H + +#include +#include + +namespace filament { + +class LineDictionaryUtils { +public: + // Base markers + static constexpr uint8_t DICTIONARY_1_BYTE_ID_MAX = 240; + static constexpr uint8_t DICTIONARY_ESCAPE_BASE = 254; + + static constexpr uint8_t DICTIONARY_NUMERIC_ID = DICTIONARY_ESCAPE_BASE; + static constexpr uint8_t DICTIONARY_3_BYTE_ID = DICTIONARY_ESCAPE_BASE + 1; // 255 + + // Calculated boundaries + static constexpr size_t DICTIONARY_2_BYTE_ID_PREFIX_COUNT = DICTIONARY_ESCAPE_BASE - DICTIONARY_1_BYTE_ID_MAX; // 14 + static constexpr size_t DICTIONARY_1_BYTE_ID_CAPACITY = DICTIONARY_1_BYTE_ID_MAX; // 240 + static constexpr size_t DICTIONARY_2_BYTE_ID_CAPACITY = DICTIONARY_2_BYTE_ID_PREFIX_COUNT << 8; // 3584 + static constexpr size_t DICTIONARY_2_BYTE_ID_MAX = DICTIONARY_1_BYTE_ID_CAPACITY + DICTIONARY_2_BYTE_ID_CAPACITY; // 3824 + + // Numerical Mapping Flag + static constexpr uint32_t DICTIONARY_NUMERIC_FLAG = 0x40000000; + + // Structured Outputs + struct Pack2ByteResult { + uint8_t prefix; + uint8_t ext; + }; + + struct Pack3ByteResult { + uint8_t extb0; + uint8_t extb1; + }; + + // ------------------------------------------------------------------------------------------------ + // Inline Encoders & Decoders + // ------------------------------------------------------------------------------------------------ + + static inline uint32_t unpack2ByteDictionaryId(uint8_t prefixb8, uint8_t extb0) noexcept { + return static_cast(DICTIONARY_1_BYTE_ID_CAPACITY + (((prefixb8 - DICTIONARY_1_BYTE_ID_MAX) << 8) | extb0)); + } + + static inline uint32_t unpack3ByteDictionaryId(uint8_t extb0, uint8_t extb1) noexcept { + return static_cast(DICTIONARY_2_BYTE_ID_MAX + (extb0 | (extb1 << 8))); + } + + static inline Pack2ByteResult pack2ByteDictionaryId(uint32_t global_index) noexcept { + uint32_t const rel = global_index - DICTIONARY_1_BYTE_ID_CAPACITY; + return { + static_cast(DICTIONARY_1_BYTE_ID_MAX + (rel >> 8)), + static_cast(rel & 0xFF) + }; + } + + static inline Pack3ByteResult pack3ByteDictionaryId(uint32_t global_index) noexcept { + uint32_t const rel = global_index - DICTIONARY_2_BYTE_ID_MAX; + return { + static_cast(rel & 0xFF), + static_cast((rel >> 8) & 0xFF) + }; + } +}; + +} // namespace filament + +#endif // TNT_FILAMENT_PRIVATE_LINEDICTIONARYUTILS_H diff --git a/libs/filaflat/include/filaflat/MaterialChunk.h b/libs/filaflat/include/filaflat/MaterialChunk.h index 0f46b62a06..41a670c0f3 100644 --- a/libs/filaflat/include/filaflat/MaterialChunk.h +++ b/libs/filaflat/include/filaflat/MaterialChunk.h @@ -56,6 +56,10 @@ public: bool hasShader(ShaderModel model, Variant variant, ShaderStage stage) const noexcept; + // Populates a pre-sized vector (matching dictionary size) with the frequency of each index + // Returns the total exact byte length of the variable-length indices stream. + size_t getDictionaryOccurrences(std::vector& outOccurrences) const; + // These methods are for debugging purposes only (matdbg) // @{ static void decodeKey(uint32_t key, @@ -70,6 +74,11 @@ private: const uint8_t* mBase = nullptr; tsl::robin_map mOffsets; + uint16_t mSharedStrings = 0; + uint16_t mVertexStrings = 0; + uint16_t mFragmentStrings = 0; + uint16_t mComputeStrings = 0; + bool getTextShader(Unflattener unflattener, BlobDictionary const& dictionary, ShaderContent& shaderContent, ShaderModel shaderModel, filament::Variant variant, ShaderStage shaderStage) const; diff --git a/libs/filaflat/src/MaterialChunk.cpp b/libs/filaflat/src/MaterialChunk.cpp index 03746b188a..2f5944fa3c 100644 --- a/libs/filaflat/src/MaterialChunk.cpp +++ b/libs/filaflat/src/MaterialChunk.cpp @@ -15,26 +15,32 @@ */ #include + +#include + + +#include "private/filament/Variant.h" + #include #include -#include - -#include - #include -#include #include +#include #include -#include -#include -#include +#include +#include +#include + +#include +#include +#include namespace filaflat { -static uint32_t makeKey( +static inline uint32_t makeKey( MaterialChunk::ShaderModel shaderModel, MaterialChunk::Variant const variant, MaterialChunk::ShaderStage stage) noexcept { @@ -71,10 +77,24 @@ bool MaterialChunk::initialize(filamat::ChunkType const materialTag) { Unflattener unflattener(start, end); - mUnflattener = unflattener; mMaterialTag = materialTag; mBase = unflattener.getCursor(); + bool const isTextChunk = ( + mMaterialTag == filamat::ChunkType::MaterialGlsl || + mMaterialTag == filamat::ChunkType::MaterialEssl1 || + mMaterialTag == filamat::ChunkType::MaterialWgsl || + mMaterialTag == filamat::ChunkType::MaterialMetal); + + if (isTextChunk) { + if (!unflattener.read(&mSharedStrings)) return false; + if (!unflattener.read(&mVertexStrings)) return false; + if (!unflattener.read(&mFragmentStrings)) return false; + if (!unflattener.read(&mComputeStrings)) return false; + } + + mUnflattener = unflattener; + // Read how many shaders we have in the chunk. uint64_t numShaders; if (!unflattener.read(&numShaders) || numShaders == 0) { @@ -110,6 +130,36 @@ bool MaterialChunk::initialize(filamat::ChunkType const materialTag) { return true; } +static bool readDictionaryId(Unflattener& base, Unflattener& ext, uint32_t& outId, size_t& outBytesRead) noexcept { + uint8_t b8; + if (!base.read(&b8)) { + return false; + } + + if (b8 < filament::LineDictionaryUtils::DICTIONARY_1_BYTE_ID_CAPACITY) { + outId = b8; + outBytesRead = 1; + return true; + } + if (b8 == filament::LineDictionaryUtils::DICTIONARY_NUMERIC_ID) { + outId = filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG; + outBytesRead = 1; + return true; + } + if (b8 < filament::LineDictionaryUtils::DICTIONARY_3_BYTE_ID) { + uint8_t e; + if (!ext.read(&e)) return false; + outId = filament::LineDictionaryUtils::unpack2ByteDictionaryId(b8, e); + outBytesRead = 2; + return true; + } + uint8_t e0, e1; + if (!ext.read(&e0) || !ext.read(&e1)) return false; + outId = filament::LineDictionaryUtils::unpack3ByteDictionaryId(e0, e1); + outBytesRead = 3; + return true; +} + bool MaterialChunk::getTextShader(Unflattener unflattener, BlobDictionary const& dictionary, ShaderContent& shaderContent, ShaderModel const shaderModel, Variant const variant, ShaderStage const shaderStage) const { @@ -143,22 +193,74 @@ bool MaterialChunk::getTextShader(Unflattener unflattener, return false; } + uint32_t extLength = 0; + if (!unflattener.read(&extLength)) { + return false; + } + + uint32_t baseLength = 0; + if (!unflattener.read(&baseLength)) { + return false; + } + + uint32_t numericLength = 0; + if (!unflattener.read(&numericLength)) { return false; } + shaderContent.reserve(shaderSize); shaderContent.resize(shaderSize); size_t cursor = 0; - // Read all lines. - for(int32_t i = 0 ; i < lineCount; i++) { - uint16_t lineIndex; - if (!unflattener.read(&lineIndex)) { + Unflattener extUnflattener(unflattener); + extUnflattener.setCursor(unflattener.getCursor() + baseLength); + + Unflattener numericUnflattener(unflattener); + numericUnflattener.setCursor(unflattener.getCursor() + baseLength + extLength); + + auto readNumericLiteral = [](Unflattener& stream) -> uint32_t { + uint8_t e0; + if (!stream.read(&e0)) { + return 0; + } + if (e0 < 128) { + return e0; + } + uint8_t e1; + if (!stream.read(&e1)) { + return 0; + } + return (e0 & 0x7F) | (e1 << 7); + }; + + for (size_t i = 0; i < lineCount; ++i) { + uint32_t lineIndex = 0; + size_t bytesRead = 0; + if (!readDictionaryId(unflattener, extUnflattener, lineIndex, bytesRead)) { return false; } - if (UTILS_UNLIKELY(lineIndex >= dictionary.size())) { + if (lineIndex == filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG) { + uint32_t const numericLiteral = readNumericLiteral(numericUnflattener); + char buf[16]; + buf[0] = '_'; + auto const [ptr, ec] = std::to_chars(buf + 1, buf + 16, numericLiteral); + size_t const len = ptr - buf; + memcpy(&shaderContent[cursor], buf, len); + cursor += len; + continue; + } + + uint32_t globalIndex = lineIndex; + if (shaderStage == ShaderStage::FRAGMENT && lineIndex >= mSharedStrings) { + globalIndex += mVertexStrings; + } else if (shaderStage == ShaderStage::COMPUTE && lineIndex >= mSharedStrings) { + globalIndex += mVertexStrings + mFragmentStrings; + } + + if (UTILS_UNLIKELY(globalIndex >= dictionary.size())) { return false; } - const auto& content = dictionary[lineIndex]; + const auto& content = dictionary[globalIndex]; // Ensure string is correctly formed and doesn't exceed reserved shader space. if (UTILS_UNLIKELY(content.size() == 0 || cursor + content.size() - 1 > shaderSize)) { @@ -166,10 +268,15 @@ bool MaterialChunk::getTextShader(Unflattener unflattener, } // remove the terminating null character. - memcpy(&shaderContent[cursor], content.data(), content.size() - 1); - cursor += content.size() - 1; + size_t const length = content.size() - 1; + memcpy(&shaderContent[cursor], content.data(), length); + cursor += length; } + // Explicitly leapfrog the native stream reader past the isolated Extension stream + // to preserve unflatten sync consistency natively across chunks. + unflattener.setCursor(numericUnflattener.getCursor()); + // Write the terminating null character. shaderContent[cursor++] = 0; assert_invariant(cursor == shaderSize); @@ -178,7 +285,8 @@ bool MaterialChunk::getTextShader(Unflattener unflattener, } bool MaterialChunk::getBinaryShader(BlobDictionary const& dictionary, - ShaderContent& shaderContent, ShaderModel const shaderModel, filament::Variant const variant, ShaderStage const shaderStage) const { + ShaderContent& shaderContent, ShaderModel const shaderModel, + filament::Variant const variant, ShaderStage const shaderStage) const { if (mBase == nullptr) { return false; @@ -257,4 +365,73 @@ void MaterialChunk::visitShaders( } } +size_t MaterialChunk::getDictionaryOccurrences(std::vector& outOccurrences) const { + size_t totalIndicesSize = 0; + + if (mBase == nullptr || ( + mMaterialTag != filamat::ChunkType::MaterialGlsl && + mMaterialTag != filamat::ChunkType::MaterialEssl1 && + mMaterialTag != filamat::ChunkType::MaterialWgsl && + mMaterialTag != filamat::ChunkType::MaterialMetal)) { + return 0; + } + + for (auto const& chunk : mOffsets) { + ShaderModel model; + Variant variant; + ShaderStage stage; + decodeKey(chunk.first, &model, &variant, &stage); + + Unflattener unflattener(mBase + chunk.second, mContainer.getChunkRange(mMaterialTag).second); + + uint32_t shaderSize = 0; + if (!unflattener.read(&shaderSize)) { + continue; + } + + uint32_t lineCount = 0; + if (!unflattener.read(&lineCount)) { + continue; + } + + uint32_t extLength = 0; + if (!unflattener.read(&extLength)) { + continue; + } + + uint32_t baseLength = 0; + if (!unflattener.read(&baseLength)) { + continue; + } + + Unflattener extUnflattener(unflattener); + extUnflattener.setCursor(unflattener.getCursor() + baseLength); + + for (size_t i = 0; i < lineCount; ++i) { + uint32_t lineIndex = 0; + size_t bytesRead = 0; + if (!readDictionaryId(unflattener, extUnflattener, lineIndex, bytesRead)) { + break; + } + totalIndicesSize += bytesRead; + + if (lineIndex == filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG) { + continue; + } + + uint32_t globalIndex = lineIndex; + if (stage == ShaderStage::FRAGMENT && lineIndex >= mSharedStrings) { + globalIndex += mVertexStrings; + } else if (stage == ShaderStage::COMPUTE && lineIndex >= mSharedStrings) { + globalIndex += mVertexStrings + mFragmentStrings; + } + + if (globalIndex < outOccurrences.size()) { + outOccurrences[globalIndex]++; + } + } + } + return totalIndicesSize; +} + } // namespace filaflat diff --git a/libs/filamat/src/GLSLPostProcessor.cpp b/libs/filamat/src/GLSLPostProcessor.cpp index c6323cc2e0..c887967b41 100644 --- a/libs/filamat/src/GLSLPostProcessor.cpp +++ b/libs/filamat/src/GLSLPostProcessor.cpp @@ -752,15 +752,11 @@ bool GLSLPostProcessor::process(const std::string& inputShader, Config const& co if (internalConfig.glslOutput) { if (!mGenerateDebugInfo) { - *internalConfig.glslOutput = - internalConfig.minifier.removeWhitespace( - *internalConfig.glslOutput, - mOptimization == MaterialBuilder::Optimization::SIZE); - + *internalConfig.glslOutput = internalConfig.minifier.removeWhitespace( *internalConfig.glslOutput, + mOptimization == MaterialBuilder::Optimization::SIZE); // In theory this should only be enabled for SIZE, but in practice we often use PERFORMANCE. if (mOptimization != MaterialBuilder::Optimization::NONE) { - *internalConfig.glslOutput = - internalConfig.minifier.renameStructFields(*internalConfig.glslOutput); + *internalConfig.glslOutput = internalConfig.minifier.renameStructFields(*internalConfig.glslOutput); } } if (mPrintShaders) { diff --git a/libs/filamat/src/MaterialBuilder.cpp b/libs/filamat/src/MaterialBuilder.cpp index 67da4cc4b0..ebeafda643 100644 --- a/libs/filamat/src/MaterialBuilder.cpp +++ b/libs/filamat/src/MaterialBuilder.cpp @@ -1135,22 +1135,24 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector( std::move(textDictionary), DictionaryText); diff --git a/libs/filamat/src/ShaderMinifier.cpp b/libs/filamat/src/ShaderMinifier.cpp index d60d5dcad3..1d99bd5fd9 100644 --- a/libs/filamat/src/ShaderMinifier.cpp +++ b/libs/filamat/src/ShaderMinifier.cpp @@ -147,16 +147,18 @@ std::string ShaderMinifier::removeWhitespace(const std::string& s, bool mergeBra size_t pos = cur; size_t len = 0; - while (s[cur] != '\n') { + while (cur < s.length() && s[cur] != '\n') { cur++; len++; } size_t newPos = s.find_first_not_of(" \t", pos); - if (newPos == std::string::npos) newPos = pos; + if (newPos == std::string::npos || newPos >= pos + len) { + newPos = pos + len; + } // If we have a single { or } on a line, move it to the previous line instead - size_t subLen = len - (newPos - pos); + size_t subLen = (pos + len) - newPos; if (mergeBraces && subLen == 1 && (s[newPos] == '{' || s[newPos] == '}')) { r.replace(r.size() - 1, 1, 1, s[newPos]); } else { @@ -164,11 +166,36 @@ std::string ShaderMinifier::removeWhitespace(const std::string& s, bool mergeBra } r += '\n'; - while (s[cur] == '\n') { + while (cur < s.length() && s[cur] == '\n') { cur++; } } + // Safely strip explicitly padded spacing surrounding standard punctuation that + // spirv-cross systematically injects, to optimally pack our generated uncompressed outputs. + auto replaceAll = [](std::string& str, const char* from, const char* to) { + size_t start_pos = 0; + size_t from_len = std::strlen(from); + size_t to_len = std::strlen(to); + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from_len, to); + start_pos += to_len; + } + }; + + replaceAll(r, " = ", "="); + replaceAll(r, ", ", ","); + replaceAll(r, " + ", "+"); + replaceAll(r, " - ", "-"); + replaceAll(r, " * ", "*"); + replaceAll(r, " / ", "/"); + replaceAll(r, " == ", "=="); + replaceAll(r, " != ", "!="); + replaceAll(r, " >= ", ">="); + replaceAll(r, " <= ", "<="); + replaceAll(r, " > ", ">"); + replaceAll(r, " < ", "<"); + return r; } diff --git a/libs/filamat/src/eiff/LineDictionary.cpp b/libs/filamat/src/eiff/LineDictionary.cpp index 2cd0ba9d35..ebec02370a 100644 --- a/libs/filamat/src/eiff/LineDictionary.cpp +++ b/libs/filamat/src/eiff/LineDictionary.cpp @@ -14,10 +14,78 @@ * limitations under the License. */ +/* + * SHADER DICTIONARY ENCODING ARCHITECTURE + * --------------------------------------- + * The LineDictionary compresses raw shader source text into an optimized + * dictionary-based token stream. It specifically targets the redundancy found in + * generated materials, such as monolithic ubershaders (e.g. GLTFIO), where + * recurring tokens and numbered suffix identifiers (like 'param_1', 'param_2') + * dominate the text length. + * + * Tokenization and Splitting Algorithm: + * To maximize dictionary reusability without polluting the global token pool with + * thousands of unique permutations, the parser automatically splits pure numerical + * digit strings out of the text (e.g. "param_112" -> "param_" + "112"). + * + * Triple-Stream Topology (Base + String Extension + Numeric Literal): + * Achieving the lowest uncompressed RAM footprint requires + * aggressively splitting strings based on a broad set of keywords. However, ZLib + * (used in Android `.aar` packaging) inherently struggles with fragmented token + * layouts. When strings are highly fragmented, ZLib loses predictive "sliding + * window" sequence continuity. Generating unique String Dictionary ID hashes for + * every dynamic numerical variable prevents cross-shader Zlib ZIP deduplication. + * + * To balance both targets, the String splitting algorithm pushes extension + * and numeric data out of the core base representation. Zstandard compresses + * the remaining high-entropy base bytes cleanly. + * + * [Base Stream] (High entropy identifiers, densely packed 1-byte variables) + * ┌────┬────┬────┬──────┬──────┬────┬─────┐ + * │ 43 │ 12 │ 08 │ ESC1 │ 0xFF │ 15 │ NUM │ => (43, 12, 08, ESC1+01, 0xFF+031A, 15, NUM:152) + * └────┴────┴────┴──────┴──────┴────┴─────┘ + * │ │ │ ESC1 = [240-253] (1-byte String Ext) + * ▼ ▼ │ NUM = [254] (Numeric Escape) + * [Ext Stream] ┌────┐ ┌────┬────┐ │ 0xFF = [255] (2-byte String Ext) + * │ 01 │ │ 03 │ 1A │ ▼ + * └────┘ └────┴────┘ + * [Num Stream] ┌─────────┐ + * │ 152 [N] │ (Integers 15-bit LEB128 layout) + * └─────────┘ N < 128: 1 byte, N >= 128: 2 bytes + * + * By isolating integers into a static array, LZ77 can match identical values + * like `var_1024` and `other_var_1024` efficiently across the entire archive. + * + * Stage-Partitioned Bucket Encoding (1-Byte Collapse): + * ---------------------------------------------------- + * To hyper-optimize the `0 to 239` limits of the 1-byte base representation, the global + * string dictionary is partitioned into 4 distinct stage buckets: + * + * [ Dictionary Array Mappings ] + * ┌──────────────────┐ ◀─── 0: (S) Shared Strings (multi-stage) + * │ Shared │ + * ├──────────────────┤ ◀─── S: (V) Vertex Strings + * │ Vertex │ + * ├──────────────────┤ ◀─── S+V: (F) Fragment Strings + * │ Fragment │ + * ├──────────────────┤ ◀─── S+V+F: (C) Compute Strings + * │ Compute │ + * └──────────────────┘ + * + * During Material encoding, any string isolated to a single pipeline + * collapses its offset value by bypassing preceding buckets: + * + * [ Fragment Shader Encoding Example ]: + * Local_Index = (Global_Index >= S) ? (Global_Index - V) : Global_Index + * + * This guarantees the majority of isolated strings drop back into + * the dense 1-byte target limits without duplicating memory! + */ + #include "LineDictionary.h" -#include -#include +#include + #include #include @@ -28,12 +96,41 @@ #include #include #include +#include +#include +#include #include #include namespace filamat { +/* + * Note on String Splitting & Dictionary Optimization + * -------------------------------------------------- + * The array below defines heuristics to split generated variables from their trailing digits + * (e.g. `hp_copy_1` -> `hp_copy_`, `1`). We mined the full compiled corpus of + * gltfio/filament materials to find the most common variable prefixes ending in digits that + * break otherwise identical lines in spirv-cross output. The optimal top results are: + * + * "SPIRV_CROSS_CONSTANT_ID_", "VARIABLE_CUSTOM", "spvDescriptorSet", "HAS_ATTRIBUTE_UV", + * "mesh_custom", "dynReserved", "material_", "vertex_uv", "hp_copy_", "mp_copy_", "normal_", + * "normal", "pixel_", "param_", "mesh_uv", "Arr_", "uv_", "n_", "x_", "i_", "uv", "f", + * "s", "i", "r", "p", "u", "a", "n", "m", "x", "_" + * + * While hardcoding all 32 discovered prefixes reduces uncompressed binaries by ~33 KB across the library, + * the aggressive fragmentation of tiny, highly localized variables (like `s1` or `f2`) + * destructs contiguous literal sequences. This causes Zlib/Deflate (LZ77) compressed `.aar` and `.bin` + * archive sizes to balloon by ~2-3 KB. + * + * Therefore, we restrict this list to an optimal subset, maximizing ZIP synergy. + * Patterns must be ordered from longest to shortest to ensure correct prefix matching. + */ +static constexpr std::string_view kSplittingPatterns[] = {"hp_copy_", "mp_copy_", "_"}; + +using namespace ::filament::backend; + namespace { + bool isWordChar(char const c) { // Note: isalnum is locale-dependent, which can be problematic. // For our purpose, we define word characters as ASCII alphanumeric characters plus underscore. @@ -53,53 +150,70 @@ std::string const& LineDictionary::getString(index_t const index) const noexcept return *mStrings[index]; } -std::vector LineDictionary::getIndices( - std::string_view const& line) const noexcept { - std::vector result; - std::vector const sublines = splitString(line); - for (std::string_view const& subline : sublines) { - if (auto iter = mLineIndices.find(subline); iter != mLineIndices.end()) { - result.push_back(iter->second.index); - } else { - return {}; - } +std::pair, std::vector> LineDictionary::tokenize( + std::string_view const text) const noexcept { + auto const it = mFinalTokenizedShadersMap.find(text); + if (it != mFinalTokenizedShadersMap.end()) { + return it->second; } - return result; + return {}; } -void LineDictionary::addText(std::string_view const text) noexcept { - size_t cur = 0; - size_t const len = text.length(); - const char* s = text.data(); - while (cur < len) { - // Start of the current line - size_t const pos = cur; - // Find the end of the current line or end of text - while (cur < len && s[cur] != '\n') { - cur++; - } - // If we found a newline, advance past it for the next iteration, ensuring '\n' is included - if (cur < len) { - cur++; - } - addLine({ s + pos, cur - pos }); +void LineDictionary::addText(ShaderStage stage, std::string_view const text) noexcept { + mTokenizedShaders.push_back({stage, text, {}, {}}); + + size_t pos = 0; + while (pos < text.length()) { + size_t const start = pos; + while (pos < text.length() && text[pos] != '\n') pos++; + size_t const len = (pos < text.length()) ? (pos - start + 1) : (text.length() - start); + addLine(stage, text.substr(start, len), mTokenizedShaders.back().tokens, mTokenizedShaders.back().numericTokens); + if (pos < text.length()) pos++; } } -void LineDictionary::addLine(std::string_view const line) noexcept { + +void LineDictionary::addLine(ShaderStage stage, std::string_view const line, std::vector& ids, std::vector& numerics) noexcept { auto const lines = splitString(line); for (std::string_view const& subline : lines) { - // Never add a line twice. - auto pos = mLineIndices.find(subline); - if (pos != mLineIndices.end()) { - pos->second.count++; + bool isNumeric = !subline.empty() && std::all_of(subline.begin(), subline.end(), ::isdigit); + if (isNumeric) { + if (subline.data() > line.data() && *(subline.data() - 1) == '_') { + // keep true + } else { + isNumeric = false; + } + } + + uint32_t numValue = 0; + bool parsedNumeric = false; + if (isNumeric) { + auto const result = std::from_chars(subline.data(), subline.data() + subline.size(), numValue); + if (result.ec == std::errc() && numValue <= 32767) { + parsedNumeric = true; + } + } + + if (parsedNumeric) { + ids.push_back(filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); + numerics.push_back(numValue); continue; } - mStrings.emplace_back(std::make_unique(subline)); - mLineIndices.emplace(*mStrings.back(), - LineInfo{ - .index = index_t(mStrings.size() - 1), - .count = 1 }); + + index_t id; + auto pos = mLineIndices.find(subline); + if (pos != mLineIndices.end()) { + pos->second.count[size_t(stage)]++; + id = pos->second.index; + } else { + mStrings.emplace_back(std::make_unique(subline)); + LineInfo info = { .index = index_t(mStrings.size() - 1) }; + info.count[size_t(stage)] = 1; + mLineIndices.emplace(*mStrings.back(), info); + id = info.index; + } + + ids.push_back(id); } } @@ -111,9 +225,6 @@ std::string_view LineDictionary::ltrim(std::string_view s) { std::pair LineDictionary::findPattern( std::string_view const line, size_t const offset) { - // Patterns are ordered from longest to shortest to ensure correct prefix matching. - static constexpr std::string_view kPatterns[] = { "hp_copy_", "mp_copy_", "_" }; - const size_t line_len = line.length(); for (size_t i = offset; i < line_len; ++i) { // A pattern must be a whole word (or at the start of the string). @@ -121,7 +232,7 @@ std::pair LineDictionary::findPattern( continue; } - for (const auto& prefix : kPatterns) { + for (const auto& prefix : kSplittingPatterns) { if (line.size() - i >= prefix.size() && line.substr(i, prefix.size()) == prefix) { // A known prefix has been matched. Now, check for a sequence of digits. size_t const startOfDigits = i + prefix.size(); @@ -172,8 +283,17 @@ std::vector LineDictionary::splitString(std::string_view const result.push_back(line.substr(current_pos, match_pos - current_pos)); } - // Add the match itself. - result.push_back(line.substr(match_pos, match_len)); + // Add the match itself, but split prefix from numbers. + for (const auto& prefix : kSplittingPatterns) { + if (match_len >= prefix.size() && line.substr(match_pos, prefix.size()) == prefix) { + std::string_view const prefixWithoutUnderscore = line.substr(match_pos, prefix.size() - 1); + if (!prefixWithoutUnderscore.empty()) { + result.push_back(prefixWithoutUnderscore); + } + result.push_back(line.substr(match_pos + prefix.size(), match_len - prefix.size())); + break; + } + } // Move cursor past the match. current_pos = match_pos + match_len; @@ -182,6 +302,8 @@ std::vector LineDictionary::splitString(std::string_view const return result; } + + void LineDictionary::printStatistics(utils::io::ostream& stream) const noexcept { std::vector> info; for (auto const& pair : mLineIndices) { @@ -191,8 +313,10 @@ void LineDictionary::printStatistics(utils::io::ostream& stream) const noexcept // Sort by count, then by index. std::sort(info.begin(), info.end(), [](auto const& lhs, auto const& rhs) { - if (lhs.second.count != rhs.second.count) { - return lhs.second.count > rhs.second.count; + uint32_t const lhsTotal = lhs.second.count[0] + lhs.second.count[1] + lhs.second.count[2]; + uint32_t const rhsTotal = rhs.second.count[0] + rhs.second.count[1] + rhs.second.count[2]; + if (lhsTotal != rhsTotal) { + return lhsTotal > rhsTotal; } return lhs.second.index < rhs.second.index; }); @@ -208,22 +332,23 @@ void LineDictionary::printStatistics(utils::io::ostream& stream) const noexcept // Print the dictionary. stream << "Line dictionary:" << io::endl; for (auto const& pair : info) { + uint32_t const totalCount = pair.second.count[0] + pair.second.count[1] + pair.second.count[2]; compressed_size += pair.first.length(); - total_size += pair.first.length() * pair.second.count; - total_lines += pair.second.count; - indices_size += sizeof(uint16_t) * pair.second.count; + total_size += pair.first.length() * totalCount; + total_lines += totalCount; + indices_size += sizeof(uint16_t) * totalCount; if (pair.second.index <= 127) { - indices_size_if_varlen += sizeof(uint8_t) * pair.second.count; + indices_size_if_varlen += sizeof(uint8_t) * totalCount; } else { - indices_size_if_varlen += sizeof(uint16_t) * pair.second.count; + indices_size_if_varlen += sizeof(uint16_t) * totalCount; } if (i <= 128) { - indices_size_if_varlen_sorted += sizeof(uint8_t) * pair.second.count; + indices_size_if_varlen_sorted += sizeof(uint8_t) * totalCount; } else { - indices_size_if_varlen_sorted += sizeof(uint16_t) * pair.second.count; + indices_size_if_varlen_sorted += sizeof(uint16_t) * totalCount; } i++; - stream << " " << pair.second.count << ": " << pair.first << io::endl; + stream << " " << totalCount << ": " << pair.first << io::endl; } stream << "Total size: " << total_size << ", compressed size: " << compressed_size << io::endl; stream << "Saved size: " << total_size - compressed_size << io::endl; @@ -235,25 +360,156 @@ void LineDictionary::printStatistics(utils::io::ostream& stream) const noexcept stream << "Indices size: " << indices_size << io::endl; stream << "Indices size (if varlen): " << indices_size_if_varlen << io::endl; stream << "Indices size (if varlen, sorted): " << indices_size_if_varlen_sorted << io::endl; +} - // some data we gathered +// ------------------------------------------------------------------------------------------------ - // Total size: 751161, compressed size: 59818 - // Saved size: 691343 - // Unique lines: 3659 - // Total lines: 61686 - // Compression ratio: 12.557440904075696 - // Average line length (total): 12.177171481373406 - // Average line length (compressed): 16.34818256354195 +// ------------------------------------------------------------------------------------------------ +// calculateAndFilterFrequencies() +// Analysis to determine the exact isolated occurrences of each surviving String token. +// +// Input/Dependencies: +// - `mTokenizedShaders`: Read to tally exact occurrences across stages. +// - `mLineIndices`: LineInfo counts are zeroed and then overwritten. +// +// Output: +// - Returns `std::vector` containing ONLY the strings with `total > 0` hits. +// Returns this vector rather than mutating state directly to prepare for downstream sorting. +// ------------------------------------------------------------------------------------------------ +std::vector> LineDictionary::calculateAndFilterFrequencies() noexcept { + // Rebuild exact occurrences across the final optimized token arrays + for (auto& entry : mLineIndices) { + entry.second.count[0] = 0; + entry.second.count[1] = 0; + entry.second.count[2] = 0; + } + for (const auto& shader : mTokenizedShaders) { + ShaderStage const st = shader.stage; + size_t st_idx = 0; + if (st == ShaderStage::VERTEX) st_idx = 0; + else if (st == ShaderStage::FRAGMENT) st_idx = 1; + else if (st == ShaderStage::COMPUTE) st_idx = 2; - // Total size: 751161, compressed size: 263215 - // Saved size: 487946 - // Unique lines: 4672 - // Total lines: 23258 - // Compression ratio: 2.8537925270216364 - // Average line length (total): 32.296887092613296 - // Average line length (compressed): 56.338827054794521 + for (uint32_t const id : shader.tokens) { + if (id & filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG) continue; + const auto& str = *mStrings[id]; + auto it = mLineIndices.find(str); + if (it != mLineIndices.end()) { + it->second.count[st_idx]++; + } + } + } + + std::vector> info; + info.reserve(mLineIndices.size()); + for (auto const& pair : mLineIndices) { + uint32_t const total = pair.second.count[0] + pair.second.count[1] + pair.second.count[2]; + if (total > 0) { + info.push_back(pair); + } + } + return info; +} + +// ------------------------------------------------------------------------------------------------ +// finalizeDictionaryBuckets(info) +// Maps the final sorted tokens into their stage-partitioned subsets (Shared, Vertex, Fragment, Compute), +// resolving the absolute 1-byte schema boundaries for downstream chunk encoding. +// +// Input/Dependencies: +// - `info`: The filtered and occurrence-populated vector from `calculateAndFilterFrequencies`. +// - `mStrings`: Will be stripped of abandoned subsumed tokens and densely repacked. +// - `mLineIndices`: Updates the index mappings for runtime lookup scaling. +// +// Output/Modifications: +// - `mStageStringCounts`: Records the exact boundary limits per pipeline stage constraint. +// - `mFinalTokenizedShadersMap`: Populates the resolved binary integer sequence mapping for the encoder. +// ------------------------------------------------------------------------------------------------ +void LineDictionary::finalizeDictionaryBuckets(std::vector>& info) noexcept { + // Step A: Define the closure that categorizes Strings into 4 distinct groups: + // [0] Shared: Appears in > 1 Pipeline Stage + // [1] Vertex, [2] Fragment, [3] Compute (Isolated locally to that explicit pipeline) + auto getBucketIndex = [](const uint32_t count[3]) -> uint8_t { + uint32_t const v = count[size_t(ShaderStage::VERTEX)]; + uint32_t const f = count[size_t(ShaderStage::FRAGMENT)]; + uint32_t const c = count[size_t(ShaderStage::COMPUTE)]; + int const sharedClasses = (v > 0) + (f > 0) + (c > 0); + if (sharedClasses > 1) return 0; + if (v > 0) return 1; + if (f > 0) return 2; + if (c > 0) return 3; + return 0; // fallback + }; + + // Step B: Sort the surviving vocabulary tokens. + // 1st Priority: Stage Bucket (Shared strings go first, then Vertex, Fragment, Compute) + // This ensures we can offset isolated indices. + // 2nd Priority: Occurrence count globally. The most common tokens must map to + // 0-239 to compress into a single variable-length byte. + // 3rd Priority: Original parse token allocation order to enforce stable deterministic builds. + std::sort(info.begin(), info.end(), + [&getBucketIndex](auto const& lhs, auto const& rhs) { + uint8_t const lhsBucket = getBucketIndex(lhs.second.count); + uint8_t const rhsBucket = getBucketIndex(rhs.second.count); + if (lhsBucket != rhsBucket) { + return lhsBucket < rhsBucket; + } + + uint32_t const lhsTotal = lhs.second.count[0] + lhs.second.count[1] + lhs.second.count[2]; + uint32_t const rhsTotal = rhs.second.count[0] + rhs.second.count[1] + rhs.second.count[2]; + + if (lhsTotal != rhsTotal) { + return lhsTotal > rhsTotal; + } + return lhs.second.index < rhs.second.index; + }); + + for (int i = 0; i < 4; i++) { + mStageStringCounts[i] = 0; + } + + // Step C: Repackage the physical mStrings layout! + // We map every old string ID layout sequentially into the new partitioned layout mappings. + // Discarded (0-occurrence) substrings are bypassed here and cleared off RAM matrix. + std::vector initialToFinal(mStrings.size(), 0xFFFFFFFF); + std::vector> newStrings; + newStrings.reserve(info.size()); + + for (index_t i = 0; i < info.size(); i++) { + uint8_t const bucket = getBucketIndex(info[i].second.count); + mStageStringCounts[bucket]++; // Log boundary bounds for `MaterialTextChunk` metadata block + + auto& entry = mLineIndices[info[i].first]; + initialToFinal[entry.index] = newStrings.size(); // Map Old ID -> New ID + newStrings.push_back(std::move(mStrings[entry.index])); + entry.index = newStrings.size() - 1; + } + mStrings = std::move(newStrings); + + // Step D: Encode the final uncompressed Sequence index Array. + // Loop through our shader text blocks one last time and map the optimized tokens tightly. + for (auto& shader : mTokenizedShaders) { + std::vector final_sequence; + final_sequence.reserve(shader.tokens.size()); + + for (uint32_t const init_id : shader.tokens) { + if (init_id & filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG) { + final_sequence.push_back(init_id); + } else { + uint32_t const final_id = initialToFinal[init_id]; + if (final_id != 0xFFFFFFFF) { + final_sequence.push_back(final_id); + } + } + } + mFinalTokenizedShadersMap.emplace(shader.text, std::make_pair(std::move(final_sequence), std::move(shader.numericTokens))); + } +} + +void LineDictionary::resolve() noexcept { + auto info = calculateAndFilterFrequencies(); + finalizeDictionaryBuckets(info); } } // namespace filamat diff --git a/libs/filamat/src/eiff/LineDictionary.h b/libs/filamat/src/eiff/LineDictionary.h index 7423897462..a18f6afdac 100644 --- a/libs/filamat/src/eiff/LineDictionary.h +++ b/libs/filamat/src/eiff/LineDictionary.h @@ -17,6 +17,8 @@ #ifndef TNT_FILAMAT_LINEDICTIONARY_H #define TNT_FILAMAT_LINEDICTIONARY_H +#include + #include #include #include @@ -25,12 +27,60 @@ #include #include +#include + namespace utils::io { class ostream; } namespace filamat { +using ShaderStage = filament::backend::ShaderStage; + +/** + * LineDictionary parses and deduplicates a text shader into a minimal set of optimal string tokens. + * + * Encoding Schema (Triple-Stream Interleaved Token Stream): + * ---------------------------------------------------------------------- + * Once the dictionary determines the global frequency of tokens, MaterialTextChunk encodes the shader + * text into a binary stream according to the following layout, designed to maximize + * LZ77 sliding window deduplication across archives: + * + * Tokens with frequency-sorted dictionary ID < 240: + * [ 1 byte ]: The exact dictionary ID representing the token (0 to 239). + * + * Tokens with frequency-sorted dictionary ID < 3824: + * [ 2 bytes ]: `<240 to 253> `. + * The first byte dedicates 14 statically bound permutation slots to provide 4 bits + * (nibble) of spatial MSB data. + * + * Numeric Literals (Bypassing Dictionary): + * [ 2-3 bytes ]: `<254> `. + * Integers up to 32767 are stored as inline LEB128 values rather than dictionary IDs. + * This ensures uniform byte sequences across all shaders for optimal LZ77 compression. + * + * Tokens with frequency-sorted dictionary ID >= 3824: + * [ 3 bytes ]: `<255> `. + * The `255` byte acts as an escape for long-tail string dictionary indexing. + * + * + * Stage-Partitioned Sliding Overlaps: + * ---------------------------------------------------------------------- + * To hyper-optimize the `0 to 239` subset mappings native to 1-byte encoding, the LineDictionary + * partitions strings by their respective occurrence boundaries into four sorted buckets: + * [0]: Shared (Strings required by MULTIPLE shader stages). + * [1]: Vertex (Strings required exclusively by vertex shaders). + * [2]: Fragment (Strings required exclusively by fragment shaders). + * [3]: Compute (Strings required exclusively by compute shaders). + * + * During Binary Encoder/Decoder passes (`MaterialTextChunk` & `MaterialChunk`), indices mapped outside + * the [0] Shared partition are shifted back towards 0 using simple offsets. + * For example, a Fragment string at global dictionary index 500 might be referenced as + * index `500 - [Vertex String Count]` during Fragment mapping block evaluations, pushing it + * down into a 1-byte schema bounds without risking literal memory duplication. + * + * The matching decoder loop reverses this inside `MaterialChunk::getTextShader()`. + */ class LineDictionary { public: using index_t = uint32_t; @@ -43,13 +93,20 @@ public: LineDictionary(LineDictionary&&) = default; // Adds text to the dictionary, parsing it into lines. - void addText(std::string_view text) noexcept; + void addText(filament::backend::ShaderStage stage, std::string_view text) noexcept; + + // Sorts the dictionary entries by frequency and reassigns their indices + void resolve() noexcept; // Returns the total number of unique lines stored in the dictionary. size_t getDictionaryLineCount() const { return mStrings.size(); } + uint32_t getStageStringCount(size_t stageIndex) const noexcept { + return mStageStringCounts[stageIndex]; + } + // Checks if the dictionary is empty. bool isEmpty() const noexcept { return mStrings.empty(); @@ -58,8 +115,10 @@ public: // Retrieves a string by its index. std::string const& getString(index_t index) const noexcept; - // Retrieves the indices of lines that match the given string view. - std::vector getIndices(std::string_view const& line) const noexcept; + + + // Gets the indices and numeric stream for a given registered text block. + std::pair, std::vector> tokenize(std::string_view text) const noexcept; // Prints statistics about the dictionary to the given output stream. void printStatistics(utils::io::ostream& stream) const noexcept; @@ -79,7 +138,7 @@ public: private: // Adds a single line to the dictionary. - void addLine(std::string_view line) noexcept; + void addLine(ShaderStage stage, std::string_view line, std::vector& ids, std::vector& numerics) noexcept; // Trims leading whitespace from a string view. static std::string_view ltrim(std::string_view s); @@ -92,11 +151,28 @@ private: struct LineInfo { index_t index; - uint32_t count; + uint32_t count[3] = {0, 0, 0}; + }; + + // Recalculates exact occurrences for each token across stages and discards unused tokens. + std::vector> calculateAndFilterFrequencies() noexcept; + + // Sorts the surviving tokens into buckets and maps them into their final stage-partitioned offsets. + void finalizeDictionaryBuckets(std::vector>& info) noexcept; + + struct TokenizedShader { + ShaderStage stage; + std::string_view text; + std::vector tokens; + std::vector numericTokens; }; std::unordered_map mLineIndices; std::vector> mStrings; + uint32_t mStageStringCounts[4] = {0, 0, 0, 0}; // Shared, Vertex, Fragment, Compute + std::vector mTokenizedShaders; + std::unordered_map, std::vector>> mFinalTokenizedShadersMap; + }; } // namespace filamat diff --git a/libs/filamat/src/eiff/MaterialTextChunk.cpp b/libs/filamat/src/eiff/MaterialTextChunk.cpp index 66d3c1aabc..78c2dc6a0a 100644 --- a/libs/filamat/src/eiff/MaterialTextChunk.cpp +++ b/libs/filamat/src/eiff/MaterialTextChunk.cpp @@ -19,16 +19,20 @@ #include "LineDictionary.h" #include "ShaderEntry.h" -#include +#include + #include #include +#include #include #include #include +#include +#include #include #include -#include +#include namespace filamat { @@ -39,45 +43,85 @@ void MaterialTextChunk::writeEntryAttributes(size_t const entryIndex, Flattener& f.writeUint8(uint8_t(entry.stage)); } -void compressShader(std::string_view const src, Flattener &f, const LineDictionary& dictionary) { +void compressShader(const std::string& src, ShaderStage const stage, Flattener& f, const LineDictionary& dictionary) { if (dictionary.getDictionaryLineCount() > 65536) { slog.e << "Dictionary is too large!" << io::endl; std::terminate(); } f.writeUint32(static_cast(src.size() + 1)); - f.writeValuePlaceholder(); + f.writeValuePlaceholder(); // Num Lines + + // F.writeValue resolves backwards matching the LIFO queue of Placeholders! + f.writeValuePlaceholder(); // Ext Stream Size + f.writeValuePlaceholder(); // Base Stream Size + f.writeValuePlaceholder(); // Numeric Stream Size size_t numLines = 0; + std::vector base_stream; + std::vector ext_stream; - size_t cur = 0; - size_t const len = src.length(); - const char* s = src.data(); - while (cur < len) { - // Start of the current line - size_t const pos = cur; - // Find the end of the current line or end of text - while (cur < len && s[cur] != '\n') { - cur++; - } - // If we found a newline, advance past it for the next iteration, ensuring '\n' is included - if (cur < len) { - cur++; - } - std::string_view const newLine{ s + pos, cur - pos }; + uint32_t const S = dictionary.getStageStringCount(0); + uint32_t const V = dictionary.getStageStringCount(1); + uint32_t const F = dictionary.getStageStringCount(2); - auto const indices = dictionary.getIndices(newLine); - if (indices.empty()) { - slog.e << "Line not found in dictionary!" << io::endl; - std::terminate(); + auto const [indices, numerics] = dictionary.tokenize(src); + if (indices.empty() && !src.empty()) { + slog.e << "Shader completely failed to tokenize!" << io::endl; + slog.e << "Shader size: " << src.size() << " | src substring: " << src.substr(0, std::min(size_t(50), src.size())) << io::endl; + slog.e << "Indices map size: " << dictionary.size() << io::endl; + std::terminate(); + } + + numLines = indices.size(); + for (auto const index : indices) { + if (index == filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG) { + base_stream.push_back(filament::LineDictionaryUtils::DICTIONARY_NUMERIC_ID); + continue; } - numLines += indices.size(); - for (auto const index : indices) { - f.writeUint16(static_cast(index)); + + + uint32_t local_index = index; + if (stage == ShaderStage::FRAGMENT && index >= S) { + local_index -= V; + } else if (stage == ShaderStage::COMPUTE && index >= S) { + local_index -= V + F; + } + + if (local_index < filament::LineDictionaryUtils::DICTIONARY_1_BYTE_ID_CAPACITY) { + base_stream.push_back(static_cast(local_index)); + } else if (local_index < filament::LineDictionaryUtils::DICTIONARY_2_BYTE_ID_MAX) { + auto const [prefix, ext] = filament::LineDictionaryUtils::pack2ByteDictionaryId(local_index); + base_stream.push_back(prefix); + ext_stream.push_back(ext); + } else { + base_stream.push_back(filament::LineDictionaryUtils::DICTIONARY_3_BYTE_ID); + auto const [extb0, extb1] = filament::LineDictionaryUtils::pack3ByteDictionaryId(local_index); + ext_stream.push_back(extb0); + ext_stream.push_back(extb1); } } - f.writeValue(numLines); + + std::vector numeric_stream; + + for (auto const value : numerics) { + if (value < 128) { + numeric_stream.push_back(static_cast(value)); + } else { + numeric_stream.push_back(static_cast((value & 0x7F) | 0x80)); + numeric_stream.push_back(static_cast((value >> 7) & 0xFF)); + } + } + + f.writeRaw(reinterpret_cast(base_stream.data()), base_stream.size()); + f.writeRaw(reinterpret_cast(ext_stream.data()), ext_stream.size()); + f.writeRaw(reinterpret_cast(numeric_stream.data()), numeric_stream.size()); + + f.writeValue(static_cast(numeric_stream.size())); + f.writeValue(static_cast(base_stream.size())); + f.writeValue(static_cast(ext_stream.size())); + f.writeValue(static_cast(numLines)); } void MaterialTextChunk::flatten(Flattener& f) { @@ -103,6 +147,12 @@ void MaterialTextChunk::flatten(Flattener& f) { // All offsets expressed later will start at the current flattener cursor position f.markOffsetBase(); + // Write stage partition counts for decoding restitution + f.writeUint16(static_cast(mDictionary.getStageStringCount(0))); + f.writeUint16(static_cast(mDictionary.getStageStringCount(1))); + f.writeUint16(static_cast(mDictionary.getStageStringCount(2))); + f.writeUint16(static_cast(mDictionary.getStageStringCount(3))); + // Write how many shaders we have f.writeUint64(mEntries.size()); @@ -119,7 +169,7 @@ void MaterialTextChunk::flatten(Flattener& f) { continue; } f.writeOffsets(i); - compressShader(mEntries.at(i).shader, f, mDictionary); + compressShader(mEntries.at(i).shader, mEntries.at(i).stage, f, mDictionary); } } diff --git a/libs/filamat/src/eiff/MaterialTextChunk.h b/libs/filamat/src/eiff/MaterialTextChunk.h index 812f72c7da..be0a13e1b0 100644 --- a/libs/filamat/src/eiff/MaterialTextChunk.h +++ b/libs/filamat/src/eiff/MaterialTextChunk.h @@ -27,8 +27,8 @@ namespace filamat { class MaterialTextChunk final : public Chunk { public: - MaterialTextChunk(const std::vector&& entries, const LineDictionary& dictionary, - ChunkType type) : Chunk(type), mEntries(entries), mDictionary(dictionary) { + MaterialTextChunk(std::vector&& entries, const LineDictionary& dictionary, + ChunkType type) : Chunk(type), mEntries(std::move(entries)), mDictionary(dictionary) { } ~MaterialTextChunk() override = default; diff --git a/libs/filamat/tests/test_line_dictionary.cpp b/libs/filamat/tests/test_line_dictionary.cpp index 767620a238..1ed3d1f84a 100644 --- a/libs/filamat/tests/test_line_dictionary.cpp +++ b/libs/filamat/tests/test_line_dictionary.cpp @@ -17,19 +17,22 @@ #include #include "eiff/LineDictionary.h" +#include #include using namespace filamat; +using namespace ::filament::backend; TEST(LineDictionary, splitString) { LineDictionary dictionary; const std::string text = "first line hp_copy_123456 second line"; - dictionary.addText(text); - EXPECT_EQ(dictionary.size(), 3); + dictionary.addText(ShaderStage::FRAGMENT, text); + EXPECT_EQ(dictionary.size(), 4); EXPECT_EQ(dictionary[0], "first line "); - EXPECT_EQ(dictionary[1], "hp_copy_123456"); - EXPECT_EQ(dictionary[2], " second line"); + EXPECT_EQ(dictionary[1], "hp_copy"); + EXPECT_EQ(dictionary[2], "123456"); // Bypassing fails because 123456 > 16383 + EXPECT_EQ(dictionary[3], " second line"); } TEST(LineDictionary, Empty) { @@ -40,7 +43,7 @@ TEST(LineDictionary, Empty) { TEST(LineDictionary, AddTextSimple) { LineDictionary dictionary; - dictionary.addText("Hello world\n"); + dictionary.addText(ShaderStage::FRAGMENT, "Hello world\n"); EXPECT_FALSE(dictionary.empty()); EXPECT_EQ(dictionary.size(), 1); EXPECT_EQ(dictionary[0], "Hello world\n"); @@ -48,7 +51,7 @@ TEST(LineDictionary, AddTextSimple) { TEST(LineDictionary, AddTextMultipleLines) { LineDictionary dictionary; - dictionary.addText("First line\nSecond line\n"); + dictionary.addText(ShaderStage::FRAGMENT, "First line\nSecond line\n"); EXPECT_EQ(dictionary.size(), 2); EXPECT_EQ(dictionary[0], "First line\n"); EXPECT_EQ(dictionary[1], "Second line\n"); @@ -56,169 +59,156 @@ TEST(LineDictionary, AddTextMultipleLines) { TEST(LineDictionary, AddTextDuplicateLines) { LineDictionary dictionary; - dictionary.addText("Same line\nSame line\n"); + dictionary.addText(ShaderStage::FRAGMENT, "Same line\nSame line\n"); EXPECT_EQ(dictionary.size(), 1); EXPECT_EQ(dictionary[0], "Same line\n"); } -TEST(LineDictionary, GetIndices) { - LineDictionary dictionary; - dictionary.addText("Line one\nLine two\nLine one\n"); - auto const indicesOne = dictionary.getIndices("Line one\n"); - ASSERT_EQ(indicesOne.size(), 1); - EXPECT_EQ(indicesOne[0], 0); - - auto const indicesTwo = dictionary.getIndices("Line two\n"); - ASSERT_EQ(indicesTwo.size(), 1); - EXPECT_EQ(indicesTwo[0], 1); -} - TEST(LineDictionary, SplitLogicNoPattern) { LineDictionary dictionary; - dictionary.addText("A simple line with no patterns."); + dictionary.addText(ShaderStage::FRAGMENT, "A simple line with no patterns."); EXPECT_EQ(dictionary.size(), 1); EXPECT_EQ(dictionary[0], "A simple line with no patterns."); } TEST(LineDictionary, SplitLogicHpPattern) { LineDictionary dictionary; - dictionary.addText("some_var = hp_copy_123;"); + dictionary.addText(ShaderStage::FRAGMENT, "some_var = hp_copy_123;"); EXPECT_EQ(dictionary.size(), 3); EXPECT_EQ(dictionary[0], "some_var = "); - EXPECT_EQ(dictionary[1], "hp_copy_123"); - EXPECT_EQ(dictionary[2], ";"); + EXPECT_EQ(dictionary[1], "hp_copy"); + EXPECT_EQ(dictionary[2], ";"); // 123 is bypassed! } TEST(LineDictionary, SplitLogicMpPattern) { LineDictionary dictionary; - dictionary.addText("another_var = mp_copy_4567;"); + dictionary.addText(ShaderStage::FRAGMENT, "another_var = mp_copy_4567;"); EXPECT_EQ(dictionary.size(), 3); EXPECT_EQ(dictionary[0], "another_var = "); - EXPECT_EQ(dictionary[1], "mp_copy_4567"); + EXPECT_EQ(dictionary[1], "mp_copy"); EXPECT_EQ(dictionary[2], ";"); } TEST(LineDictionary, SplitLogicUnderscorePattern) { LineDictionary dictionary; - dictionary.addText("var_1 = 0;"); + dictionary.addText(ShaderStage::FRAGMENT, "var_1 = 0;"); EXPECT_EQ(dictionary.size(), 1); - EXPECT_EQ(dictionary[0], "var_1 = 0;"); + EXPECT_EQ(dictionary[0], "var_1 = 0;"); // Underscore preceded by a word char is not considered a pattern boundary } TEST(LineDictionary, SplitLogicMultiplePatterns) { LineDictionary dictionary; - dictionary.addText("hp_copy_1 mp_copy_2 _3"); - EXPECT_EQ(dictionary.size(), 4); - EXPECT_EQ(dictionary[0], "hp_copy_1"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1 mp_copy_2 _3"); + EXPECT_EQ(dictionary.size(), 3); + EXPECT_EQ(dictionary[0], "hp_copy"); EXPECT_EQ(dictionary[1], " "); - EXPECT_EQ(dictionary[2], "mp_copy_2"); - EXPECT_EQ(dictionary[3], "_3"); + EXPECT_EQ(dictionary[2], "mp_copy"); } TEST(LineDictionary, SplitLogicInvalidPattern) { LineDictionary dictionary; - dictionary.addText("hp_copy_ a_b_c"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_ a_b_c"); EXPECT_EQ(dictionary.size(), 1); EXPECT_EQ(dictionary[0], "hp_copy_ a_b_c"); } TEST(LineDictionary, SplitLogicPatternFollowedByWordChar) { LineDictionary dictionary; - dictionary.addText("hp_copy_99rest"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_99rest"); EXPECT_EQ(dictionary.size(), 1); - EXPECT_EQ(dictionary[0], "hp_copy_99rest"); + EXPECT_EQ(dictionary[0], "hp_copy_99rest"); // Invalid word boundary on right side } TEST(LineDictionary, SplitLogicPatternPrecededByWordChar) { LineDictionary dictionary; - dictionary.addText("rest_of_it_hp_copy_99"); + dictionary.addText(ShaderStage::FRAGMENT, "rest_of_it_hp_copy_99"); EXPECT_EQ(dictionary.size(), 1); - EXPECT_EQ(dictionary[0], "rest_of_it_hp_copy_99"); + EXPECT_EQ(dictionary[0], "rest_of_it_hp_copy_99"); // Preceded by word char } TEST(LineDictionary, SplitLogicPatternNotFollowedByWordChar) { LineDictionary dictionary; - dictionary.addText("hp_copy_99;"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_99;"); EXPECT_EQ(dictionary.size(), 2); - EXPECT_EQ(dictionary[0], "hp_copy_99"); + EXPECT_EQ(dictionary[0], "hp_copy"); EXPECT_EQ(dictionary[1], ";"); } TEST(LineDictionary, AddEmptyText) { LineDictionary dictionary; - dictionary.addText(""); + dictionary.addText(ShaderStage::FRAGMENT, ""); EXPECT_TRUE(dictionary.empty()); } TEST(LineDictionary, GetIndicesMultiple) { LineDictionary dictionary; - dictionary.addText("A _1 B _2"); - auto const indices = dictionary.getIndices("A _1"); + dictionary.addText(ShaderStage::FRAGMENT, "A _1 B _2"); + dictionary.addText(ShaderStage::FRAGMENT, "A _1"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("A _1"); // String is in dictionary ASSERT_EQ(indices.size(), 2); - EXPECT_EQ(indices[0], 0); - EXPECT_EQ(indices[1], 1); + EXPECT_EQ(indices[0], 0); // "A " + EXPECT_EQ(indices[1], filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); // 1 + ASSERT_EQ(numerics.size(), 1); + EXPECT_EQ(numerics[0], 1); } TEST(LineDictionary, GetIndicesMultiplePatternsInARow) { LineDictionary dictionary; - dictionary.addText("hp_copy_1 hp_copy_2"); - auto const indices = dictionary.getIndices("hp_copy_1 hp_copy_2"); - ASSERT_EQ(indices.size(), 3); - EXPECT_EQ(indices[0], 0); - EXPECT_EQ(indices[1], 1); - EXPECT_EQ(indices[2], 2); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1 hp_copy_2"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("hp_copy_1 hp_copy_2"); + ASSERT_EQ(indices.size(), 5); + EXPECT_EQ(indices[0], 0); // hp_copy + EXPECT_EQ(indices[1], filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); + EXPECT_EQ(indices[2], 1); // " " + EXPECT_EQ(indices[3], 0); // hp_copy + EXPECT_EQ(indices[4], filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); + ASSERT_EQ(numerics.size(), 2); + EXPECT_EQ(numerics[0], 1); + EXPECT_EQ(numerics[1], 2); } TEST(LineDictionary, GetIndicesSamePatternMultipleTimes) { LineDictionary dictionary; - dictionary.addText("hp_copy_1 hp_copy_1"); - auto const indices = dictionary.getIndices("hp_copy_1 hp_copy_1"); - ASSERT_EQ(indices.size(), 3); - EXPECT_EQ(indices[0], 0); - EXPECT_EQ(indices[1], 1); - EXPECT_EQ(indices[2], 0); -} - -TEST(LineDictionary, GetIndicesWithExistingDictionary) { - LineDictionary dictionary; - dictionary.addText("unrelated_string"); - dictionary.addText("hp_copy_1"); - dictionary.addText("another_string"); - dictionary.addText(" "); - auto const indices = dictionary.getIndices("hp_copy_1 hp_copy_1"); - ASSERT_EQ(indices.size(), 3); - EXPECT_EQ(indices[0], 1); - EXPECT_EQ(indices[1], 3); - EXPECT_EQ(indices[2], 1); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1 hp_copy_1"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("hp_copy_1 hp_copy_1"); + ASSERT_EQ(indices.size(), 5); + EXPECT_EQ(indices[0], 0); // hp_copy + EXPECT_EQ(indices[1], filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); + EXPECT_EQ(indices[2], 1); // " " + EXPECT_EQ(indices[3], 0); // hp_copy + EXPECT_EQ(indices[4], filament::LineDictionaryUtils::DICTIONARY_NUMERIC_FLAG); + ASSERT_EQ(numerics.size(), 2); + EXPECT_EQ(numerics[0], 1); + EXPECT_EQ(numerics[1], 1); } TEST(LineDictionary, GetIndicesWithAdjacentPatterns) { LineDictionary dictionary; - dictionary.addText("hp_copy_1hp_copy_2"); - auto const indices = dictionary.getIndices("hp_copy_1hp_copy_2"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1hp_copy_2"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("hp_copy_1hp_copy_2"); ASSERT_EQ(indices.size(), 1); EXPECT_EQ(indices[0], 0); } TEST(LineDictionary, GetIndicesWithAdjacentPatternsNotInDictionary) { LineDictionary dictionary; - dictionary.addText("hp_copy_1"); - dictionary.addText("hp_copy_2"); - auto const indices = dictionary.getIndices("hp_copy_1hp_copy_2"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1"); + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_2"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("hp_copy_1hp_copy_2"); ASSERT_EQ(indices.size(), 0); } TEST(LineDictionary, GetIndicesWithMixedContent) { LineDictionary dictionary; - dictionary.addText("hp_copy_1"); - dictionary.addText(" "); - dictionary.addText("mp_copy_2"); - - // The query string contains patterns that are in the dictionary, - // but also content that is not. - auto const indices = dictionary.getIndices("prefix hp_copy_1 mp_copy_2 suffix"); - - // Since not all substrings of the query string are in the dictionary, - // getIndices should return an empty vector. + dictionary.addText(ShaderStage::FRAGMENT, "hp_copy_1"); + dictionary.addText(ShaderStage::FRAGMENT, " "); + dictionary.addText(ShaderStage::FRAGMENT, "mp_copy_2"); + dictionary.resolve(); + auto const [indices, numerics] = dictionary.tokenize("prefix hp_copy_1 mp_copy_2 suffix"); ASSERT_EQ(indices.size(), 0); } diff --git a/libs/matdbg/src/ShaderInfo.cpp b/libs/matdbg/src/ShaderInfo.cpp index 320e1b383a..ae76106b76 100644 --- a/libs/matdbg/src/ShaderInfo.cpp +++ b/libs/matdbg/src/ShaderInfo.cpp @@ -40,9 +40,19 @@ size_t getShaderCount(const ChunkContainer& container, ChunkType type) { return 0; } + bool const isTextChunk = ( + type == filamat::ChunkType::MaterialGlsl || + type == filamat::ChunkType::MaterialEssl1 || + type == filamat::ChunkType::MaterialWgsl || + type == filamat::ChunkType::MaterialMetal); + auto [start, end] = container.getChunkRange(type); Unflattener unflattener(start, end); + if (isTextChunk) { + unflattener.setCursor(unflattener.getCursor() + sizeof(uint16_t) * 4); + } + uint64_t shaderCount = 0; if (!unflattener.read(&shaderCount) || shaderCount == 0) { return 0; @@ -60,9 +70,19 @@ bool getShaderInfo(const ChunkContainer& container, ShaderInfo* info, ChunkType return false; } + bool const isTextChunk = ( + chunkType == filamat::ChunkType::MaterialGlsl || + chunkType == filamat::ChunkType::MaterialEssl1 || + chunkType == filamat::ChunkType::MaterialWgsl || + chunkType == filamat::ChunkType::MaterialMetal); + auto [start, end] = container.getChunkRange(chunkType); Unflattener unflattener(start, end); + if (isTextChunk) { + unflattener.setCursor(unflattener.getCursor() + sizeof(uint16_t) * 4); + } + uint64_t shaderCount = 0; if (!unflattener.read(&shaderCount) || shaderCount == 0) { return false; diff --git a/libs/matdbg/src/ShaderReplacer.cpp b/libs/matdbg/src/ShaderReplacer.cpp index 43054273b9..7716426fdd 100644 --- a/libs/matdbg/src/ShaderReplacer.cpp +++ b/libs/matdbg/src/ShaderReplacer.cpp @@ -371,7 +371,7 @@ ShaderIndex::ShaderIndex(ChunkType dictTag, ChunkType matTag, const filaflat::Ch void ShaderIndex::writeChunks(ostream& stream) { filamat::LineDictionary lines; for (const auto& record : mShaderRecords) { - lines.addText(record.shader); + lines.addText(record.stage, record.shader); } sortRecords(mShaderRecords); diff --git a/tools/matedit/src/ExternalCompile.cpp b/tools/matedit/src/ExternalCompile.cpp index efe6723f37..32a8863118 100644 --- a/tools/matedit/src/ExternalCompile.cpp +++ b/tools/matedit/src/ExternalCompile.cpp @@ -386,10 +386,10 @@ int externalCompile(utils::Path input, utils::Path output, bool preserveTextShad // Here we ONLY add GLSL and ESSL 1 types, as we're removing MSL completely. filamat::LineDictionary textDictionary; for (const auto& s : glslEntries) { - textDictionary.addText(s.shader); + textDictionary.addText(s.stage, s.shader); } for (const auto& s : essl1Entries) { - textDictionary.addText(s.shader); + textDictionary.addText(s.stage, s.shader); } // Add the re-generated text dictionary chunk and text-based shaders. diff --git a/tools/matinfo/src/main.cpp b/tools/matinfo/src/main.cpp index 316e9e4fc2..a484fdd970 100644 --- a/tools/matinfo/src/main.cpp +++ b/tools/matinfo/src/main.cpp @@ -64,7 +64,7 @@ struct Config { }; static void printUsage(const char* name) { - std::string execName(utils::Path(name).getName()); + std::string const execName(utils::Path(name).getName()); std::string usage( "MATINFO prints information about material files compiled with matc\n" "\n" @@ -590,7 +590,7 @@ static bool parseChunks(Config config, void* data, size_t size) { return false; } - size_t shaderCount = getShaderCount(container, filamat::ChunkType::MaterialWgsl); + size_t const shaderCount = getShaderCount(container, filamat::ChunkType::MaterialWgsl); info.resize(shaderCount); if (!getShaderInfo(container, info.data(), filamat::ChunkType::MaterialWgsl)) { std::cerr << "Failed to parse WebGPU chunk." << std::endl; @@ -632,8 +632,65 @@ static bool parseChunks(Config config, void* data, size_t size) { return false; } + std::vector counts(dictionary.size(), 0); + bool hasCounts = false; + + filamat::ChunkType chunkType = filamat::ChunkType::Unknown; + if (config.printDictionaryGLSL) { + chunkType = filamat::ChunkType::MaterialGlsl; + } else if (config.printDictionaryESSL1) { + chunkType = filamat::ChunkType::MaterialEssl1; + } else if (config.printDictionaryMetal) { + chunkType = filamat::ChunkType::MaterialMetal; + } else if (config.printDictionaryWGSL) { + chunkType = filamat::ChunkType::MaterialWgsl; + } + + size_t indicesSize = 0; + if (chunkType != filamat::ChunkType::Unknown) { + filaflat::MaterialChunk materialChunk(container); + if (materialChunk.initialize(chunkType)) { + indicesSize = materialChunk.getDictionaryOccurrences(counts); + hasCounts = true; + } + } + + size_t dictSize = 0; + for (uint32_t i = 0; i < dictionary.size(); i++) { + // Null-terminated string sizes map physically 1:1 with uncompressed Dictionary byte bounds + dictSize += dictionary[i].size(); + } + + std::cout << "Dictionary size: " << dictSize << " bytes" << std::endl; + if (hasCounts) { + std::cout << "Indices size: " << indicesSize << " bytes" << std::endl; + } + std::cout << std::endl; + + uint32_t index = 0; for (auto const& i : dictionary) { - std::cout << (const char*)i.data() << std::endl; + if (hasCounts && counts[index] == 0) { + index++; + continue; + } + + std::string str((const char*)i.data()); + // Replace \n with literal \n + size_t pos = 0; + while ((pos = str.find('\n', pos)) != std::string::npos) { + str.replace(pos, 1, "\\n"); + pos += 2; + } + if (hasCounts) { + int const bytes = (index < 240) ? 1 : ((index < 4080) ? 2 : 3); + std::cout << std::setw(6) << counts[index] << " | " + << std::setw(4) << index << " | " + << bytes << " | " + << str << std::endl; + } else { + std::cout << str << std::endl; + } + index++; } return true; @@ -670,21 +727,21 @@ static bool parseBinary(Config config, std::istream& in, long fileSize) { int main(int argc, char* argv[]) { Config config; - int optionIndex = handleArguments(argc, argv, &config); + int const optionIndex = handleArguments(argc, argv, &config); - int numArgs = argc - optionIndex; + int const numArgs = argc - optionIndex; if (numArgs < 1) { printUsage(argv[0]); return 1; } - Path src(argv[optionIndex]); + Path const src(argv[optionIndex]); if (!src.exists()) { std::cerr << "The source material " << src << " does not exist." << std::endl; return 1; } - long fileSize = static_cast(getFileSize(src.c_str())); + long const fileSize = static_cast(getFileSize(src.c_str())); if (fileSize <= 0) { std::cerr << "The source material " << src << " is invalid." << std::endl; return 1;