From cdffc9eaa0010d1289d1ce63141081c013bd1177 Mon Sep 17 00:00:00 2001 From: yein Date: Wed, 10 Dec 2025 10:31:51 -0800 Subject: [PATCH] Allow embedding material source in cmat (#9484) * Allow embedding material source in cmat * Compress material string --- filament/include/filament/Material.h | 7 ++- filament/src/Material.cpp | 4 ++ filament/src/MaterialDefinition.cpp | 2 + filament/src/MaterialDefinition.h | 1 + filament/src/MaterialParser.cpp | 36 +++++++++++++ filament/src/MaterialParser.h | 2 + filament/src/details/Material.h | 4 ++ filament/test/CMakeLists.txt | 2 +- filament/test/filament_test_material.cpp | 50 ++++++++++++++++- .../include/filament/MaterialChunkType.h | 2 + libs/filamat/CMakeLists.txt | 5 +- .../filamat/include/filamat/MaterialBuilder.h | 7 +++ libs/filamat/src/MaterialBuilder.cpp | 11 ++++ .../src/eiff/CompressedStringChunk.cpp | 53 +++++++++++++++++++ libs/filamat/src/eiff/CompressedStringChunk.h | 46 ++++++++++++++++ libs/filamat/tests/test_filamat.cpp | 14 +++++ .../include/filament-matp/Config.h | 2 + tools/matc/src/matc/CommandlineConfig.cpp | 6 +++ tools/matc/src/matc/MaterialCompiler.cpp | 4 ++ 19 files changed, 254 insertions(+), 4 deletions(-) create mode 100644 libs/filamat/src/eiff/CompressedStringChunk.cpp create mode 100644 libs/filamat/src/eiff/CompressedStringChunk.h diff --git a/filament/include/filament/Material.h b/filament/include/filament/Material.h index c179da048f..3625bf6e9d 100644 --- a/filament/include/filament/Material.h +++ b/filament/include/filament/Material.h @@ -371,7 +371,12 @@ public: //! Indicates whether an existing parameter is a sampler or not. bool isSampler(const char* UTILS_NONNULL name) const noexcept; - + /** + * Returns a view of the material source (.mat which is a JSON-ish file) string, + * if it has been set. Otherwise, it returns a view of an empty string. + * The lifetime of the string_view is tied to the lifetime of the Material. + */ + std::string_view getSource() const noexcept; /** * * Gets the name of the transform field associated for the given sampler parameter. diff --git a/filament/src/Material.cpp b/filament/src/Material.cpp index bc9c7c93bc..d7923b2481 100644 --- a/filament/src/Material.cpp +++ b/filament/src/Material.cpp @@ -146,6 +146,10 @@ bool Material::isSampler(const char* name) const noexcept { return downcast(this)->isSampler(name); } +std::string_view Material::getSource() const noexcept { + return downcast(this)->getSource(); +} + const char* Material::getParameterTransformName(const char* samplerName) const noexcept { return downcast(this)->getParameterTransformName(samplerName); } diff --git a/filament/src/MaterialDefinition.cpp b/filament/src/MaterialDefinition.cpp index 3fbfb36f24..a8b0291128 100644 --- a/filament/src/MaterialDefinition.cpp +++ b/filament/src/MaterialDefinition.cpp @@ -257,6 +257,8 @@ void MaterialDefinition::processMain() { bool const hasFog = !(variantFilterMask & UserVariantFilterMask(UserVariantFilterBit::FOG)); perViewLayoutIndex = ColorPassDescriptorSet::getIndex(isLit, isSSR, hasFog); + + mMaterialParser->getSourceShader(&source); } void MaterialDefinition::processBlendingMode() { diff --git a/filament/src/MaterialDefinition.h b/filament/src/MaterialDefinition.h index 4632462c6d..3bbe46bcd8 100644 --- a/filament/src/MaterialDefinition.h +++ b/filament/src/MaterialDefinition.h @@ -113,6 +113,7 @@ struct MaterialDefinition { utils::CString name; uint64_t cacheId = 0; + utils::CString source; private: friend class MaterialCache; diff --git a/filament/src/MaterialParser.cpp b/filament/src/MaterialParser.cpp index 84f36f4c01..ea7af9e293 100644 --- a/filament/src/MaterialParser.cpp +++ b/filament/src/MaterialParser.cpp @@ -35,6 +35,8 @@ #include #include +#include + #include #include #include @@ -237,6 +239,40 @@ bool MaterialParser::getName(CString* cstring) const noexcept { return unflattener.read(cstring); } +bool MaterialParser::getSourceShader(CString* cstring) const noexcept { + auto [start, end] = mImpl.mChunkContainer.getChunkRange(MaterialSource); + // Source material is optional, treat it as a success. + if (start == end) return true; + + Unflattener unflattener(start, end); + + // First get the compressed blob. + const char* compressed; + size_t compressedSize = 0; + if (!unflattener.read(&compressed, &compressedSize)) { + return false; + } + + // Get the bound and decompress it. + const size_t decompressBound = + ZSTD_getFrameContentSize(compressed, compressedSize); + if (ZSTD_isError(decompressBound)) { + return false; + } + + auto dst_buffer = std::make_unique(decompressBound); + const size_t decompressed = + ZSTD_decompress( + dst_buffer.get(), decompressBound, + compressed, compressedSize); + if (ZSTD_isError(decompressed)) { + return false; + } + *cstring = CString { dst_buffer.get(), decompressed }; + + return true; +} + bool MaterialParser::getCacheId(uint64_t* cacheId) const noexcept { auto [start, end] = mImpl.mChunkContainer.getChunkRange(MaterialCacheId); if (start == end) return false; diff --git a/filament/src/MaterialParser.h b/filament/src/MaterialParser.h index 1b087bc524..c913bd5ace 100644 --- a/filament/src/MaterialParser.h +++ b/filament/src/MaterialParser.h @@ -145,6 +145,8 @@ public: return getMaterialChunk().hasShader(model, variant, stage); } + bool getSourceShader(utils::CString* cstring) const noexcept; + filaflat::MaterialChunk const& getMaterialChunk() const noexcept { return mImpl.mMaterialChunk; } diff --git a/filament/src/details/Material.h b/filament/src/details/Material.h index 318b3596db..f70d7e39ca 100644 --- a/filament/src/details/Material.h +++ b/filament/src/details/Material.h @@ -252,6 +252,10 @@ public: return mUseUboBatching; } + std::string_view getSource() const noexcept { + return mDefinition.source.c_str_safe(); + } + #if FILAMENT_ENABLE_MATDBG void applyPendingEdits() noexcept; diff --git a/filament/test/CMakeLists.txt b/filament/test/CMakeLists.txt index 4ae17274d7..ab9ac2d492 100644 --- a/filament/test/CMakeLists.txt +++ b/filament/test/CMakeLists.txt @@ -100,7 +100,7 @@ set_target_properties(test_material_parser PROPERTIES FOLDER Tests) add_executable(test_material filament_test_material.cpp ${RESGEN_SOURCE}) -target_link_libraries(test_material PRIVATE filament gtest) +target_link_libraries(test_material PRIVATE filamat filament gtest) target_compile_options(test_material PRIVATE ${COMPILER_FLAGS}) target_include_directories(test_material PRIVATE ${RESOURCE_DIR}) set_target_properties(test_material PROPERTIES FOLDER Tests) \ No newline at end of file diff --git a/filament/test/filament_test_material.cpp b/filament/test/filament_test_material.cpp index ab4d675b4a..df24ed8cde 100644 --- a/filament/test/filament_test_material.cpp +++ b/filament/test/filament_test_material.cpp @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include +#include + #include #include @@ -84,6 +85,53 @@ TEST(MaterialTransformName, QueryMultipleSamplersWithoutTransforms) { Engine::destroy(engine); } +TEST(Material, MaterialWithSourceMaterialSuccessfullyRetrieveSource) { + // Need to set a specific backend to create a proper MaterialParser. + Engine* engine = Engine::create(Engine::Backend::OPENGL); + + std::string shaderCode(R"( + void material(inout MaterialInputs material) { + prepareMaterial(material); + material.baseColor = vec4(1.); + } + )"); + filamat::MaterialBuilder builder; + builder.init(); + builder.materialSource(shaderCode); + filamat::Package result = builder.build(engine->getJobSystem()); + ASSERT_TRUE(result.isValid()); + + Material* material = Material::Builder() + .package(result.getData(), result.getSize()) + .build(*engine); + ASSERT_NE(material, nullptr); + + EXPECT_EQ(material->getSource(), shaderCode); + + engine->destroy(material); + Engine::destroy(engine); +} + + +TEST(Material, MaterialWithoutSourceMaterialReturnsEmptySource) { + // Need to set a specific backend to create a proper MaterialParser. + Engine* engine = Engine::create(Engine::Backend::OPENGL); + filamat::MaterialBuilder builder; + builder.init(); + filamat::Package result = builder.build(engine->getJobSystem()); + ASSERT_TRUE(result.isValid()); + + Material* material = Material::Builder() + .package(result.getData(), result.getSize()) + .build(*engine); + ASSERT_NE(material, nullptr); + + EXPECT_EQ(material->getSource(), ""); + + engine->destroy(material); + Engine::destroy(engine); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/libs/filabridge/include/filament/MaterialChunkType.h b/libs/filabridge/include/filament/MaterialChunkType.h index 73eef6a599..e321b3861d 100644 --- a/libs/filabridge/include/filament/MaterialChunkType.h +++ b/libs/filabridge/include/filament/MaterialChunkType.h @@ -101,6 +101,8 @@ enum UTILS_PUBLIC ChunkType : uint64_t { DictionaryMetalLibrary = charTo64bitNum("DIC_MLIB"), MaterialCrc32 = charTo64bitNum("MAT_CRC "), + + MaterialSource = charTo64bitNum("MAT_SRC "), }; } // namespace filamat diff --git a/libs/filamat/CMakeLists.txt b/libs/filamat/CMakeLists.txt index 7edfd1a4d5..58f8f6ab67 100644 --- a/libs/filamat/CMakeLists.txt +++ b/libs/filamat/CMakeLists.txt @@ -16,6 +16,7 @@ set(HDRS set(COMMON_PRIVATE_HDRS src/eiff/Chunk.h src/eiff/ChunkContainer.h + src/eiff/CompressedStringChunk.h src/eiff/DictionaryTextChunk.h src/eiff/Flattener.h src/eiff/LineDictionary.h @@ -28,6 +29,7 @@ set(COMMON_PRIVATE_HDRS set(COMMON_SRCS src/eiff/Chunk.cpp src/eiff/ChunkContainer.cpp + src/eiff/CompressedStringChunk.cpp src/eiff/DictionaryTextChunk.cpp src/eiff/LineDictionary.cpp src/eiff/MaterialTextChunk.cpp @@ -81,7 +83,7 @@ include_directories(${CMAKE_BINARY_DIR}) add_library(${TARGET} STATIC ${HDRS} ${PRIVATE_HDRS} ${SRCS}) target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) set_target_properties(${TARGET} PROPERTIES FOLDER Libs) -target_link_libraries(${TARGET} backend_headers shaders filabridge utils smol-v) +target_link_libraries(${TARGET} backend_headers shaders filabridge utils smol-v zstd) if (FILAMENT_SUPPORTS_WEBGPU) target_link_libraries(${TARGET} libtint) @@ -133,6 +135,7 @@ set(FILAMAT_DEPS spirv-cross-core spirv-cross-glsl spirv-cross-msl + zstd ) set(FILAMAT_COMBINED_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/libfilamat_combined.a") diff --git a/libs/filamat/include/filamat/MaterialBuilder.h b/libs/filamat/include/filamat/MaterialBuilder.h index 82ec6fabfa..5ca507f0ea 100644 --- a/libs/filamat/include/filamat/MaterialBuilder.h +++ b/libs/filamat/include/filamat/MaterialBuilder.h @@ -660,6 +660,12 @@ public: */ MaterialBuilder& useDefaultDepthVariant() noexcept; + /** + * Sets the source ASCII material (aka .mat file). + * The provided `source` string_view must remain valid until MaterialBuilder::build() is called. + */ + MaterialBuilder& materialSource(std::string_view source) noexcept; + /** * Build the material. If you are using the Filament engine with this library, you should use * the job system provided by Engine. @@ -905,6 +911,7 @@ private: ShaderCode mMaterialFragmentCode; ShaderCode mMaterialVertexCode; + std::string_view mMaterialSource; PropertyList mProperties; ParameterList mParameters; diff --git a/libs/filamat/src/MaterialBuilder.cpp b/libs/filamat/src/MaterialBuilder.cpp index fad063902c..7a99175f8e 100644 --- a/libs/filamat/src/MaterialBuilder.cpp +++ b/libs/filamat/src/MaterialBuilder.cpp @@ -31,6 +31,7 @@ #include "eiff/BlobDictionary.h" #include "eiff/ChunkContainer.h" +#include "eiff/CompressedStringChunk.h" #include "eiff/DictionarySpirvChunk.h" #include "eiff/DictionaryTextChunk.h" #include "eiff/LineDictionary.h" @@ -1251,6 +1252,11 @@ MaterialBuilder& MaterialBuilder::useLegacyMorphing() noexcept { return *this; } +MaterialBuilder& MaterialBuilder::materialSource(std::string_view source) noexcept { + mMaterialSource = source; + return *this; +} + Package MaterialBuilder::build(JobSystem& jobSystem) { if (materialBuilderClients == 0) { slog.e << "Error: MaterialBuilder::init() must be called before build()." << io::endl; @@ -1701,6 +1707,11 @@ void MaterialBuilder::writeCommonChunks(ChunkContainer& container, MaterialInfo& hasher({ frag.data(), frag.size() }))); container.emplace(MaterialCacheId, materialId); + if (!mMaterialSource.empty()) { + container.push( + MaterialSource, mMaterialSource, + CompressedStringChunk::CompressionLevel::MAX); + } } void MaterialBuilder::writeSurfaceChunks(ChunkContainer& container) const noexcept { diff --git a/libs/filamat/src/eiff/CompressedStringChunk.cpp b/libs/filamat/src/eiff/CompressedStringChunk.cpp new file mode 100644 index 0000000000..822f86508b --- /dev/null +++ b/libs/filamat/src/eiff/CompressedStringChunk.cpp @@ -0,0 +1,53 @@ +/* +* Copyright (C) 2025 The Android Open Source Project +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "CompressedStringChunk.h" + +#include + +namespace filamat { + +namespace { +int toZstdCompressionLevel(CompressedStringChunk::CompressionLevel compressionLevel) { + switch (compressionLevel) { + case CompressedStringChunk::CompressionLevel::MIN: + return ZSTD_minCLevel(); + case CompressedStringChunk::CompressionLevel::MAX: + return ZSTD_maxCLevel(); + case CompressedStringChunk::CompressionLevel::DEFAULT: + return ZSTD_defaultCLevel(); + } +} +} // namespace + +void CompressedStringChunk::flatten(filamat::Flattener& f) { + const size_t bufferBound = ZSTD_compressBound(mString.size()); + std::vector compressed(bufferBound); + + const size_t compressedSize = ZSTD_compress( + compressed.data(), compressed.size(), + mString.data(), mString.size(), + toZstdCompressionLevel(mCompressionLevel)); + + if (ZSTD_isError(compressedSize)) { + utils::slog.e << "Error compressing the input string." << utils::io::endl; + return; + } + + f.writeBlob((const char*) compressed.data(), compressedSize); +} + +} // namespace filamat \ No newline at end of file diff --git a/libs/filamat/src/eiff/CompressedStringChunk.h b/libs/filamat/src/eiff/CompressedStringChunk.h new file mode 100644 index 0000000000..40636377f7 --- /dev/null +++ b/libs/filamat/src/eiff/CompressedStringChunk.h @@ -0,0 +1,46 @@ +/* +* Copyright (C) 2025 The Android Open Source Project +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef TNT_COMPRESSEDSTRINGCHUNK_H +#define TNT_COMPRESSEDSTRINGCHUNK_H + +#include "Chunk.h" +#include "Flattener.h" + +#include + +namespace filamat { + +class CompressedStringChunk final : public Chunk { +public: + enum class CompressionLevel { MIN, MAX, DEFAULT }; + CompressedStringChunk( + ChunkType type, std::string_view string, CompressionLevel compressionLevel) + : Chunk(type), + mString(utils::CString(string.data(), string.size())), + mCompressionLevel(compressionLevel) {} + ~CompressedStringChunk() override = default; + +private: + void flatten(Flattener& f) override; + utils::CString mString; + CompressionLevel mCompressionLevel; +}; + +} // namespace filamat + + +#endif // TNT_COMPRESSEDSTRINGCHUNK_H diff --git a/libs/filamat/tests/test_filamat.cpp b/libs/filamat/tests/test_filamat.cpp index 0010b5f0c7..4f20e28bf2 100644 --- a/libs/filamat/tests/test_filamat.cpp +++ b/libs/filamat/tests/test_filamat.cpp @@ -964,6 +964,20 @@ TEST_F(MaterialCompiler, FeatureLevel0Ess3CallFails) { EXPECT_FALSE(result.isValid()); } +TEST_F(MaterialCompiler, EmbedMaterialSourceSucceeds) { + std::string shaderCode(R"( + void material(inout MaterialInputs material) { + prepareMaterial(material); + material.baseColor = vec4(1.); + } + )"); + filamat::MaterialBuilder builder; + builder.materialSource(shaderCode); + + filamat::Package result = builder.build(*jobSystem); + EXPECT_TRUE(result.isValid()); +} + #if FILAMENT_SUPPORTS_WEBGPU TEST_F(MaterialCompiler, WgslConversionBakedColor) { std::string bakedColorCodeFrag(R"( diff --git a/libs/filament-matp/include/filament-matp/Config.h b/libs/filament-matp/include/filament-matp/Config.h index 9f61bb7158..b36433c4cb 100644 --- a/libs/filament-matp/include/filament-matp/Config.h +++ b/libs/filament-matp/include/filament-matp/Config.h @@ -157,6 +157,7 @@ public: bool getInsertLineDirectives() const noexcept { return mInsertLineDirectives; } bool getInsertLineDirectiveChecks() const noexcept { return mInsertLineDirectiveChecks; } + bool getIncludeSourceMaterial() const noexcept { return mIncludeSourceMaterial; } protected: bool mDebug = false; @@ -179,6 +180,7 @@ protected: bool mIncludeEssl1 = true; bool mInsertLineDirectives = true; bool mInsertLineDirectiveChecks = true; + bool mIncludeSourceMaterial = false; }; } // namespace matp diff --git a/tools/matc/src/matc/CommandlineConfig.cpp b/tools/matc/src/matc/CommandlineConfig.cpp index ff32c49df0..8dd5c6c10d 100644 --- a/tools/matc/src/matc/CommandlineConfig.cpp +++ b/tools/matc/src/matc/CommandlineConfig.cpp @@ -70,6 +70,7 @@ static const option OPTIONS[] = { { "workarounds", required_argument, nullptr, 'W' }, { "no-insert-line-directives", no_argument, nullptr, 'n' }, { "no-insert-line-directive-check", no_argument, nullptr, 'N' }, + { "include-source-mat", no_argument, nullptr, 'm' }, { nullptr, 0, nullptr, 0 } // termination of the option list }; @@ -163,6 +164,8 @@ static void usage(char* name) { " #if defined(GL_GOOGLE_cpp_style_line_directive)\n" " Some drivers may complain about the use of cpp style #line directives\n" " if they don't support it.\n\n" + " --include-source-mat\n" + " Embeds the source ASCII material definition if set.\n" "Internal use and debugging only:\n" " --optimize-none, -g, -O0\n" @@ -406,6 +409,9 @@ bool CommandlineConfig::parse() { case 'N': mInsertLineDirectiveChecks = false; break; + case 'm': + mIncludeSourceMaterial = true; + break; } } diff --git a/tools/matc/src/matc/MaterialCompiler.cpp b/tools/matc/src/matc/MaterialCompiler.cpp index 749f6f7698..6d577a7cd3 100644 --- a/tools/matc/src/matc/MaterialCompiler.cpp +++ b/tools/matc/src/matc/MaterialCompiler.cpp @@ -98,6 +98,10 @@ bool MaterialCompiler::run(const matp::Config& config) { return false; } + if (config.getIncludeSourceMaterial()) { + builder.materialSource(buffer.get()); + } + // If we're reflecting parameters, the MaterialParser will have handled it inside of parse(). // We should return here to avoid actually building a material. if (config.getReflectionTarget() != matp::Config::Metadata::NONE) {