Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

matc: improvements to LineDictionary compression (#9078)

Browse Source 
We can significantly improve the effectiveness of the line dictionary
by splitting lines strategically. spirv optimization tends to create
new variable names (e.g. _1234) that are not always stable between
variants, hereby preventing lines from being duplicated. By simply
splitting lines around these patterns, we improve the effectiveness
of the compression.

The decompression is almost unchanged. To make the code easier, we 
now keep the '\n' at the end of lines in the dictionary. That's the
only change to the decompression side.

On certain materials like aiDefault.mat the compression ratio is
improved from 2.8x to 15x.


This also fixes a small but where the line dictionary relied on the
last line of the input to be a newline.
This commit is contained in:
Mathias Agopian
2025-08-12 12:58:35 -07:00
committed by GitHub
parent c0e7f0c664
commit f5c0fabb10
6 changed files with 307 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
NEW_RELEASE_NOTES.md
View File

@@ -14,4 +14,5 @@ appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md).
depth-only passes. [**Requires recompiling materials**]
- material: fix specularFactor in `LOW_QUALITY` mode. [**Requires recompiling materials**] to take effect.
- material: Add CRC32 validation for material packages [⚠️ **New Material Version**]
- material: Improve LineDictionary compression [⚠️ **New Material Version**]
- Filament is now targeting c++20 (was previously c++17)

3
libs/filaflat/src/MaterialChunk.cpp
View File

@@ -144,10 +144,9 @@ bool MaterialChunk::getTextShader(Unflattener unflattener,
}
const auto& content = dictionary[lineIndex];
// Replace null with newline.
// remove the terminating null character.
memcpy(&shaderContent[cursor], content.data(), content.size() - 1);
cursor += content.size() - 1;
shaderContent[cursor++] = '\n';
}
// Write the terminating null character.

14
libs/filamat/src/eiff/DictionaryTextChunk.cpp
View File

@@ -15,19 +15,27 @@
*/
#include "DictionaryTextChunk.h"
#include "Chunk.h"
#include "Flattener.h"
#include "LineDictionary.h"
#include <filament/MaterialChunkType.h>
#include <cstddef>
#include <utility>
namespace filamat {
DictionaryTextChunk::DictionaryTextChunk(LineDictionary&& dictionary, ChunkType chunkType) :
DictionaryTextChunk::DictionaryTextChunk(LineDictionary&& dictionary, ChunkType const chunkType) :
Chunk(chunkType), mDictionary(std::move(dictionary)) {
}
void DictionaryTextChunk::flatten(Flattener& f) {
// NumStrings
f.writeUint32(mDictionary.getLineCount());
f.writeUint32(mDictionary.getDictionaryLineCount());
// Strings
for (size_t i = 0 ; i < mDictionary.getLineCount() ; i++) {
for (LineDictionary::index_t i = 0, c = mDictionary.getDictionaryLineCount() ; i < c ; i++) {
f.writeString(mDictionary.getString(i).data());
}
}

245
libs/filamat/src/eiff/LineDictionary.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2017 The Android Open Source Project
* Copyright (C) 2025 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,50 +16,229 @@
#include "LineDictionary.h"
#include <utils/debug.h>
#include <utils/Log.h>
#include <utils/ostream.h>
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace filamat {
std::string_view LineDictionary::getString(size_t index) const noexcept {
LineDictionary::LineDictionary() = default;
LineDictionary::~LineDictionary() noexcept {
//printStatistics(utils::slog.d);
}
std::string const& LineDictionary::getString(index_t const index) const noexcept {
return *mStrings[index];
}
size_t LineDictionary::getLineCount() const {
size_t LineDictionary::getDictionaryLineCount() const {
return mStrings.size();
}
size_t LineDictionary::getIndex(std::string_view s) const noexcept {
if (auto iter = mLineIndices.find(s); iter != mLineIndices.end()) {
return iter->second;
}
return SIZE_MAX;
}
void LineDictionary::addText(const std::string& line) noexcept {
const char* s = line.c_str();
size_t cur = 0;
size_t pos = 0;
size_t len = 0;
while (s[cur] != '\0') {
pos = cur;
len = 0;
while (s[cur] != '\n') {
cur++;
len++;
std::vector<LineDictionary::index_t> LineDictionary::getIndices(
std::string_view const& line) const noexcept {
std::vector<index_t> result;
std::vector<std::string_view> const sublines = splitString(line);
for (std::string_view const& subline : sublines) {
if (auto iter = mLineIndices.find(subline); iter != mLineIndices.end()) {
result.push_back(iter->second.index);
}
std::string newLine(s + pos, len);
addLine(std::move(newLine));
cur++;
}
return result;
}
void LineDictionary::addText(std::string_view const text) noexcept {
size_t cur = 0;
size_t const len = text.length();
const char* s = text.data();
while (cur < len) {
// Start of the current line
size_t const pos = cur;
// Find the end of the current line or end of text
while (cur < len && s[cur] != '\n') {
cur++;
}
// If we found a newline, advance past it for the next iteration, ensuring '\n' is included
if (cur < len) {
cur++;
}
addLine({ s + pos, cur - pos });
}
}
void LineDictionary::addLine(const std::string&& line) noexcept {
// Never add a line twice.
if (mLineIndices.find(line) != mLineIndices.end()) {
return;
void LineDictionary::addLine(std::string_view const line) noexcept {
auto const lines = splitString(line);
for (std::string_view const& subline : lines) {
// Never add a line twice.
auto pos = mLineIndices.find(subline);
if (pos != mLineIndices.end()) {
pos->second.count++;
continue;
}
mStrings.emplace_back(std::make_unique<std::string>(subline));
mLineIndices.emplace(*mStrings.back(),
LineInfo{
.index = index_t(mStrings.size() - 1),
.count = 1 });
}
mStrings.emplace_back(std::make_unique<std::string>(line));
mLineIndices.emplace(*mStrings.back(), mStrings.size() - 1);
}
std::string_view LineDictionary::ltrim(std::string_view s) {
s.remove_prefix(std::distance(s.begin(), std::find_if(s.begin(), s.end(),
[](unsigned char const c) { return !std::isspace(c); })));
return { s.data(), s.size() };
}
std::pair<size_t, size_t> LineDictionary::findPattern(
std::string_view const line, size_t const offset) {
// Patterns are ordered from longest to shortest to ensure correct prefix matching.
static constexpr std::string_view kPatterns[] = { "hp_copy_", "mp_copy_", "_" };
const size_t line_len = line.length();
for (size_t i = offset; i < line_len; ++i) {
// A pattern must be a whole word (or at the start of the string).
if (i > 0 && std::isalnum(line[i - 1])) {
continue;
}
for (const auto& prefix : kPatterns) {
if (line.size() - i >= prefix.size() && line.substr(i, prefix.size()) == prefix) {
// A known prefix has been matched. Now, check for a sequence of digits.
size_t const startOfDigits = i + prefix.size();
if (startOfDigits < line_len && std::isdigit(line[startOfDigits])) {
size_t j = startOfDigits;
while (j < line_len && (j < startOfDigits + 6) && std::isdigit(line[j])) {
j++;
}
// We have a full pattern match (prefix + digits).
return { i, j - i };
}
// If a prefix is matched but not followed by digits, it's not a valid pattern.
// We break to the outer loop to continue searching from the next character,
// because we've already checked the longest possible prefix at this position.
break;
}
}
}
return { std::string_view::npos, 0 }; // No pattern found
}
std::vector<std::string_view> LineDictionary::splitString(std::string_view const line) {
std::vector<std::string_view> result;
size_t current_pos = 0;
if (line.empty()) {
result.push_back({});
return result;
}
while (current_pos < line.length()) {
auto const [match_pos, match_len] = findPattern(line, current_pos);
if (match_pos == std::string_view::npos) {
// No more patterns found, add the rest of the string.
result.push_back(line.substr(current_pos));
break;
}
// Add the part before the match.
if (match_pos > current_pos) {
result.push_back(line.substr(current_pos, match_pos - current_pos));
}
// Add the match itself.
result.push_back(line.substr(match_pos, match_len));
// Move cursor past the match.
current_pos = match_pos + match_len;
}
return result;
}
void LineDictionary::printStatistics(utils::io::ostream& stream) const noexcept {
std::vector<std::pair<std::string_view, LineInfo>> info;
for (auto const& pair : mLineIndices) {
info.push_back(pair);
}
// Sort by count, then by index.
std::sort(info.begin(), info.end(),
[](auto const& lhs, auto const& rhs) {
if (lhs.second.count != rhs.second.count) {
return lhs.second.count > rhs.second.count;
}
return lhs.second.index < rhs.second.index;
});
size_t total_size = 0;
size_t compressed_size = 0;
size_t total_lines = 0;
size_t indices_size = 0;
size_t indices_size_if_varlen = 0;
size_t indices_size_if_varlen_sorted = 0;
size_t i = 0;
using namespace utils;
// Print the dictionary.
stream << "Line dictionary:" << io::endl;
for (auto const& pair : info) {
compressed_size += pair.first.length();
total_size += pair.first.length() * pair.second.count;
total_lines += pair.second.count;
indices_size += sizeof(uint16_t) * pair.second.count;
if (pair.second.index <= 127) {
indices_size_if_varlen += sizeof(uint8_t) * pair.second.count;
} else {
indices_size_if_varlen += sizeof(uint16_t) * pair.second.count;
}
if (i <= 128) {
indices_size_if_varlen_sorted += sizeof(uint8_t) * pair.second.count;
} else {
indices_size_if_varlen_sorted += sizeof(uint16_t) * pair.second.count;
}
i++;
stream << " " << pair.second.count << ": " << pair.first << io::endl;
}
stream << "Total size: " << total_size << ", compressed size: " << compressed_size << io::endl;
stream << "Saved size: " << total_size - compressed_size << io::endl;
stream << "Unique lines: " << mLineIndices.size() << io::endl;
stream << "Total lines: " << total_lines << io::endl;
stream << "Compression ratio: " << double(total_size) / compressed_size << io::endl;
stream << "Average line length (total): " << double(total_size) / total_lines << io::endl;
stream << "Average line length (compressed): " << double(compressed_size) / mLineIndices.size() << io::endl;
stream << "Indices size: " << indices_size << io::endl;
stream << "Indices size (if varlen): " << indices_size_if_varlen << io::endl;
stream << "Indices size (if varlen, sorted): " << indices_size_if_varlen_sorted << io::endl;
// some data we gathered
// Total size: 751161, compressed size: 59818
// Saved size: 691343
// Unique lines: 3659
// Total lines: 61686
// Compression ratio: 12.557440904075696
// Average line length (total): 12.177171481373406
// Average line length (compressed): 16.34818256354195
// Total size: 751161, compressed size: 263215
// Saved size: 487946
// Unique lines: 4672
// Total lines: 23258
// Compression ratio: 2.8537925270216364
// Average line length (total): 32.296887092613296
// Average line length (compressed): 56.338827054794521
}
} // namespace filamat

50
libs/filamat/src/eiff/LineDictionary.h
View File

@@ -17,38 +17,70 @@
#ifndef TNT_FILAMAT_LINEDICTIONARY_H
#define TNT_FILAMAT_LINEDICTIONARY_H
#include <cstddef>
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
namespace utils::io {
class ostream;
}
namespace filamat {
// Establish a line <-> id mapping. Use for shader compression when each shader is sliced in lines
// and each line encoded into a 16 bit id.
class LineDictionary {
public:
LineDictionary() = default;
using index_t = uint32_t;
LineDictionary();
~LineDictionary() noexcept;
// Due to the presence of unique_ptr, disallow copy construction but allow move construction.
LineDictionary(LineDictionary const&) = delete;
LineDictionary(LineDictionary&&) = default;
void addText(const std::string& text) noexcept;
size_t getLineCount() const;
// Adds text to the dictionary, parsing it into lines.
void addText(std::string_view text) noexcept;
// Returns the total number of unique lines stored in the dictionary.
size_t getDictionaryLineCount() const;
// Checks if the dictionary is empty.
bool isEmpty() const noexcept {
return mStrings.empty();
}
std::string_view getString(size_t index) const noexcept;
size_t getIndex(std::string_view s) const noexcept;
// Retrieves a string by its index.
std::string const& getString(index_t index) const noexcept;
// Retrieves the indices of lines that match the given string view.
std::vector<index_t> getIndices(std::string_view const& line) const noexcept;
// Prints statistics about the dictionary to the given output stream.
void printStatistics(utils::io::ostream& stream) const noexcept;
private:
void addLine(const std::string&& line) noexcept;
// Adds a single line to the dictionary.
void addLine(std::string_view line) noexcept;
std::unordered_map<std::string_view, size_t> mLineIndices;
// Trims leading whitespace from a string view.
static std::string_view ltrim(std::string_view s);
// Splits a string view into a vector of string views based on delimiters.
static std::vector<std::string_view> splitString(std::string_view line);
// Finds a pattern within a string view starting from an offset.
static std::pair<size_t, size_t> findPattern(std::string_view line, size_t offset);
struct LineInfo {
index_t index;
uint32_t count;
};
std::unordered_map<std::string_view, LineInfo> mLineIndices;
std::vector<std::unique_ptr<std::string>> mStrings;
};

62
libs/filamat/src/eiff/MaterialTextChunk.cpp
View File

@@ -15,47 +15,67 @@
*/
#include "MaterialTextChunk.h"
#include "Flattener.h"
#include "LineDictionary.h"
#include "ShaderEntry.h"
#include <exception>
#include <utils/Log.h>
#include <utils/ostream.h>
#include <cstdint>
#include <cassert>
#include <cstddef>
#include <string_view>
#include <unordered_map>
#include <string>
namespace filamat {
void MaterialTextChunk::writeEntryAttributes(size_t entryIndex, Flattener& f) const noexcept {
void MaterialTextChunk::writeEntryAttributes(size_t const entryIndex, Flattener& f) const noexcept {
const TextEntry& entry = mEntries[entryIndex];
f.writeUint8(uint8_t(entry.shaderModel));
f.writeUint8(entry.variant.key);
f.writeUint8(uint8_t(entry.stage));
}
void compressShader(std::string_view src, Flattener &f, const LineDictionary& dictionary) {
void compressShader(std::string_view const src, Flattener &f, const LineDictionary& dictionary) {
if (dictionary.getDictionaryLineCount() > 65536) {
slog.e << "Dictionary is too large!" << io::endl;
std::terminate();
}
f.writeUint32(static_cast<uint32_t>(src.size() + 1));
f.writeValuePlaceholder();
size_t numLines = 0;
size_t cur = 0;
size_t const len = src.length();
const char* s = src.data();
while (s[cur] != '\0') {
size_t pos = cur;
size_t len = 0;
while (s[cur] != '\n') {
while (cur < len) {
// Start of the current line
size_t const pos = cur;
// Find the end of the current line or end of text
while (cur < len && s[cur] != '\n') {
cur++;
len++;
}
// If we found a newline, advance past it for the next iteration, ensuring '\n' is included
if (cur < len) {
cur++;
}
std::string_view const newLine{ s + pos, cur - pos };
auto const indices = dictionary.getIndices(newLine);
if (indices.empty()) {
slog.e << "Line not found in dictionary!" << io::endl;
std::terminate();
}
std::string_view newLine(s + pos, len);
size_t index = dictionary.getIndex(newLine);
if (index > UINT16_MAX) {
slog.e << "Dictionary returned line index > UINT16_MAX" << io::endl;
assert(false);
continue;
numLines += indices.size();
for (auto const index : indices) {
f.writeUint16(static_cast<uint16_t>(index));
}
f.writeUint16(static_cast<uint16_t>(index));
numLines += 1;
cur++;
}
f.writeValue(numLines);
}