Allow embedding material source in cmat (#9484)

* Allow embedding material source in cmat
* Compress material string

filament/include/filament/Material.h

@@ -371,7 +371,12 @@ public:
     //! Indicates whether an existing parameter is a sampler or not.
     bool isSampler(const char* UTILS_NONNULL name) const noexcept;
 
-
+    /**
+     * Returns a view of the material source (.mat which is a JSON-ish file) string,
+     * if it has been set. Otherwise, it returns a view of an empty string.
+     * The lifetime of the string_view is tied to the lifetime of the Material.
+     */
+    std::string_view getSource() const noexcept;
     /**
      *
      * Gets the name of the transform field associated for the given sampler parameter.

filament/src/Material.cpp

@@ -146,6 +146,10 @@ bool Material::isSampler(const char* name) const noexcept {
     return downcast(this)->isSampler(name);
 }
 
+std::string_view Material::getSource() const noexcept {
+    return downcast(this)->getSource();
+}
+
 const char* Material::getParameterTransformName(const char* samplerName) const noexcept {
     return downcast(this)->getParameterTransformName(samplerName);
 }

filament/src/MaterialDefinition.cpp

@@ -257,6 +257,8 @@ void MaterialDefinition::processMain() {
     bool const hasFog = !(variantFilterMask & UserVariantFilterMask(UserVariantFilterBit::FOG));
 
     perViewLayoutIndex = ColorPassDescriptorSet::getIndex(isLit, isSSR, hasFog);
+
+    mMaterialParser->getSourceShader(&source);
 }
 
 void MaterialDefinition::processBlendingMode() {

filament/src/MaterialDefinition.h

@@ -113,6 +113,7 @@ struct MaterialDefinition {
 
     utils::CString name;
     uint64_t cacheId = 0;
+    utils::CString source;
 
 private:
     friend class MaterialCache;

filament/src/MaterialParser.cpp

@@ -35,6 +35,8 @@
 #include <backend/DriverEnums.h>
 #include <backend/Program.h>
 
+#include <zstd.h>
+
 #include <utils/compiler.h>
 #include <utils/CString.h>
 #include <utils/FixedCapacityVector.h>
@@ -237,6 +239,40 @@ bool MaterialParser::getName(CString* cstring) const noexcept {
    return unflattener.read(cstring);
 }
 
+bool MaterialParser::getSourceShader(CString* cstring) const noexcept {
+    auto [start, end] = mImpl.mChunkContainer.getChunkRange(MaterialSource);
+    // Source material is optional, treat it as a success.
+    if (start == end) return true;
+
+    Unflattener unflattener(start, end);
+
+    // First get the compressed blob.
+    const char* compressed;
+    size_t compressedSize = 0;
+    if (!unflattener.read(&compressed, &compressedSize)) {
+        return false;
+    }
+
+    // Get the bound and decompress it.
+    const size_t decompressBound =
+            ZSTD_getFrameContentSize(compressed, compressedSize);
+    if (ZSTD_isError(decompressBound)) {
+        return false;
+    }
+
+    auto dst_buffer = std::make_unique<char[]>(decompressBound);
+    const size_t decompressed =
+            ZSTD_decompress(
+                    dst_buffer.get(), decompressBound,
+                    compressed, compressedSize);
+    if (ZSTD_isError(decompressed)) {
+        return false;
+    }
+    *cstring = CString { dst_buffer.get(), decompressed };
+
+    return true;
+}
+
 bool MaterialParser::getCacheId(uint64_t* cacheId) const noexcept {
    auto [start, end] = mImpl.mChunkContainer.getChunkRange(MaterialCacheId);
     if (start == end) return false;

filament/src/MaterialParser.h

@@ -145,6 +145,8 @@ public:
         return getMaterialChunk().hasShader(model, variant, stage);
     }
 
+    bool getSourceShader(utils::CString* cstring) const noexcept;
+
     filaflat::MaterialChunk const& getMaterialChunk() const noexcept {
         return mImpl.mMaterialChunk;
     }

filament/src/details/Material.h

@@ -252,6 +252,10 @@ public:
         return mUseUboBatching;
     }
 
+    std::string_view getSource() const noexcept {
+        return mDefinition.source.c_str_safe();
+    }
+
 #if FILAMENT_ENABLE_MATDBG
     void applyPendingEdits() noexcept;
 

filament/test/CMakeLists.txt

@@ -100,7 +100,7 @@ set_target_properties(test_material_parser PROPERTIES FOLDER Tests)
 add_executable(test_material
         filament_test_material.cpp
         ${RESGEN_SOURCE})
-target_link_libraries(test_material PRIVATE filament gtest)
+target_link_libraries(test_material PRIVATE filamat filament gtest)
 target_compile_options(test_material PRIVATE ${COMPILER_FLAGS})
 target_include_directories(test_material PRIVATE ${RESOURCE_DIR})
 set_target_properties(test_material PROPERTIES FOLDER Tests)

filament/test/filament_test_material.cpp

@@ -13,9 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 #include <gtest/gtest.h>
 
+#include <filamat/MaterialBuilder.h>
+
 #include <filament/Engine.h>
 #include <filament/Material.h>
 
@@ -84,6 +85,53 @@ TEST(MaterialTransformName, QueryMultipleSamplersWithoutTransforms) {
     Engine::destroy(engine);
 }
 
+TEST(Material, MaterialWithSourceMaterialSuccessfullyRetrieveSource) {
+    // Need to set a specific backend to create a proper MaterialParser.
+    Engine* engine = Engine::create(Engine::Backend::OPENGL);
+
+    std::string shaderCode(R"(
+        void material(inout MaterialInputs material) {
+            prepareMaterial(material);
+            material.baseColor = vec4(1.);
+        }
+    )");
+    filamat::MaterialBuilder builder;
+    builder.init();
+    builder.materialSource(shaderCode);
+    filamat::Package result = builder.build(engine->getJobSystem());
+    ASSERT_TRUE(result.isValid());
+
+    Material* material = Material::Builder()
+                                 .package(result.getData(), result.getSize())
+                                 .build(*engine);
+    ASSERT_NE(material, nullptr);
+
+    EXPECT_EQ(material->getSource(), shaderCode);
+
+    engine->destroy(material);
+    Engine::destroy(engine);
+}
+
+
+TEST(Material, MaterialWithoutSourceMaterialReturnsEmptySource) {
+    // Need to set a specific backend to create a proper MaterialParser.
+    Engine* engine = Engine::create(Engine::Backend::OPENGL);
+    filamat::MaterialBuilder builder;
+    builder.init();
+    filamat::Package result = builder.build(engine->getJobSystem());
+    ASSERT_TRUE(result.isValid());
+
+    Material* material = Material::Builder()
+                                 .package(result.getData(), result.getSize())
+                                 .build(*engine);
+    ASSERT_NE(material, nullptr);
+
+    EXPECT_EQ(material->getSource(), "");
+
+    engine->destroy(material);
+    Engine::destroy(engine);
+}
+
 int main(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();

libs/filabridge/include/filament/MaterialChunkType.h

@@ -101,6 +101,8 @@ enum UTILS_PUBLIC ChunkType : uint64_t {
     DictionaryMetalLibrary = charTo64bitNum("DIC_MLIB"),
 
     MaterialCrc32 = charTo64bitNum("MAT_CRC "),
+
+    MaterialSource = charTo64bitNum("MAT_SRC "),
 };
 
 } // namespace filamat

libs/filamat/CMakeLists.txt

@@ -16,6 +16,7 @@ set(HDRS
 set(COMMON_PRIVATE_HDRS
         src/eiff/Chunk.h
         src/eiff/ChunkContainer.h
+        src/eiff/CompressedStringChunk.h
         src/eiff/DictionaryTextChunk.h
         src/eiff/Flattener.h
         src/eiff/LineDictionary.h
@@ -28,6 +29,7 @@ set(COMMON_PRIVATE_HDRS
 set(COMMON_SRCS
         src/eiff/Chunk.cpp
         src/eiff/ChunkContainer.cpp
+        src/eiff/CompressedStringChunk.cpp
         src/eiff/DictionaryTextChunk.cpp
         src/eiff/LineDictionary.cpp
         src/eiff/MaterialTextChunk.cpp
@@ -81,7 +83,7 @@ include_directories(${CMAKE_BINARY_DIR})
 add_library(${TARGET} STATIC ${HDRS} ${PRIVATE_HDRS} ${SRCS})
 target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR})
 set_target_properties(${TARGET} PROPERTIES FOLDER Libs)
-target_link_libraries(${TARGET} backend_headers shaders filabridge utils smol-v)
+target_link_libraries(${TARGET} backend_headers shaders filabridge utils smol-v zstd)
 
 if (FILAMENT_SUPPORTS_WEBGPU)
     target_link_libraries(${TARGET} libtint)
@@ -133,6 +135,7 @@ set(FILAMAT_DEPS
         spirv-cross-core
         spirv-cross-glsl
         spirv-cross-msl
+        zstd
         )
 
 set(FILAMAT_COMBINED_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/libfilamat_combined.a")

libs/filamat/include/filamat/MaterialBuilder.h

@@ -660,6 +660,12 @@ public:
      */
     MaterialBuilder& useDefaultDepthVariant() noexcept;
 
+    /**
+     * Sets the source ASCII material (aka .mat file).
+     * The provided `source` string_view must remain valid until MaterialBuilder::build() is called.
+     */
+    MaterialBuilder& materialSource(std::string_view source) noexcept;
+
     /**
      * Build the material. If you are using the Filament engine with this library, you should use
      * the job system provided by Engine.
@@ -905,6 +911,7 @@ private:
 
     ShaderCode mMaterialFragmentCode;
     ShaderCode mMaterialVertexCode;
+    std::string_view mMaterialSource;
 
     PropertyList mProperties;
     ParameterList mParameters;

libs/filamat/src/MaterialBuilder.cpp

@@ -31,6 +31,7 @@
 
 #include "eiff/BlobDictionary.h"
 #include "eiff/ChunkContainer.h"
+#include "eiff/CompressedStringChunk.h"
 #include "eiff/DictionarySpirvChunk.h"
 #include "eiff/DictionaryTextChunk.h"
 #include "eiff/LineDictionary.h"
@@ -1251,6 +1252,11 @@ MaterialBuilder& MaterialBuilder::useLegacyMorphing() noexcept {
     return *this;
 }
 
+MaterialBuilder& MaterialBuilder::materialSource(std::string_view source) noexcept {
+    mMaterialSource = source;
+    return *this;
+}
+
 Package MaterialBuilder::build(JobSystem& jobSystem) {
     if (materialBuilderClients == 0) {
         slog.e << "Error: MaterialBuilder::init() must be called before build()." << io::endl;
@@ -1701,6 +1707,11 @@ void MaterialBuilder::writeCommonChunks(ChunkContainer& container, MaterialInfo&
                     hasher({ frag.data(), frag.size() })));
 
     container.emplace<uint64_t>(MaterialCacheId, materialId);
+    if (!mMaterialSource.empty()) {
+        container.push<CompressedStringChunk>(
+                MaterialSource, mMaterialSource,
+                CompressedStringChunk::CompressionLevel::MAX);
+    }
 }
 
 void MaterialBuilder::writeSurfaceChunks(ChunkContainer& container) const noexcept {

libs/filamat/src/eiff/CompressedStringChunk.cpp

@@ -0,0 +1,53 @@
+/*
+* Copyright (C) 2025 The Android Open Source Project
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "CompressedStringChunk.h"
+
+#include <zstd.h>
+
+namespace filamat {
+
+namespace {
+int toZstdCompressionLevel(CompressedStringChunk::CompressionLevel compressionLevel) {
+    switch (compressionLevel) {
+        case CompressedStringChunk::CompressionLevel::MIN:
+            return ZSTD_minCLevel();
+        case CompressedStringChunk::CompressionLevel::MAX:
+            return ZSTD_maxCLevel();
+        case CompressedStringChunk::CompressionLevel::DEFAULT:
+            return ZSTD_defaultCLevel();
+    }
+}
+} // namespace
+
+void CompressedStringChunk::flatten(filamat::Flattener& f) {
+    const size_t bufferBound = ZSTD_compressBound(mString.size());
+    std::vector<std::uint8_t> compressed(bufferBound);
+
+    const size_t compressedSize = ZSTD_compress(
+            compressed.data(), compressed.size(),
+            mString.data(), mString.size(),
+            toZstdCompressionLevel(mCompressionLevel));
+
+    if (ZSTD_isError(compressedSize)) {
+        utils::slog.e << "Error compressing the input string." << utils::io::endl;
+        return;
+    }
+
+    f.writeBlob((const char*) compressed.data(), compressedSize);
+}
+
+} // namespace filamat

libs/filamat/src/eiff/CompressedStringChunk.h

@@ -0,0 +1,46 @@
+/*
+* Copyright (C) 2025 The Android Open Source Project
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef TNT_COMPRESSEDSTRINGCHUNK_H
+#define TNT_COMPRESSEDSTRINGCHUNK_H
+
+#include "Chunk.h"
+#include "Flattener.h"
+
+#include <utils/CString.h>
+
+namespace filamat {
+
+class CompressedStringChunk final : public Chunk {
+public:
+    enum class CompressionLevel { MIN, MAX, DEFAULT };
+    CompressedStringChunk(
+            ChunkType type, std::string_view string, CompressionLevel compressionLevel)
+            : Chunk(type),
+              mString(utils::CString(string.data(), string.size())),
+              mCompressionLevel(compressionLevel) {}
+    ~CompressedStringChunk() override = default;
+
+private:
+    void flatten(Flattener& f) override;
+    utils::CString mString;
+    CompressionLevel mCompressionLevel;
+};
+
+} // namespace filamat
+
+
+#endif // TNT_COMPRESSEDSTRINGCHUNK_H

libs/filamat/tests/test_filamat.cpp

@@ -964,6 +964,20 @@ TEST_F(MaterialCompiler, FeatureLevel0Ess3CallFails) {
   EXPECT_FALSE(result.isValid());
 }
 
+TEST_F(MaterialCompiler, EmbedMaterialSourceSucceeds) {
+    std::string shaderCode(R"(
+        void material(inout MaterialInputs material) {
+            prepareMaterial(material);
+            material.baseColor = vec4(1.);
+        }
+    )");
+    filamat::MaterialBuilder builder;
+    builder.materialSource(shaderCode);
+
+    filamat::Package result = builder.build(*jobSystem);
+    EXPECT_TRUE(result.isValid());
+}
+
 #if FILAMENT_SUPPORTS_WEBGPU
 TEST_F(MaterialCompiler, WgslConversionBakedColor) {
     std::string bakedColorCodeFrag(R"(

libs/filament-matp/include/filament-matp/Config.h

@@ -157,6 +157,7 @@ public:
 
     bool getInsertLineDirectives() const noexcept { return mInsertLineDirectives; }
     bool getInsertLineDirectiveChecks() const noexcept { return mInsertLineDirectiveChecks; }
+    bool getIncludeSourceMaterial() const noexcept { return mIncludeSourceMaterial; }
 
 protected:
     bool mDebug = false;
@@ -179,6 +180,7 @@ protected:
     bool mIncludeEssl1 = true;
     bool mInsertLineDirectives = true;
     bool mInsertLineDirectiveChecks = true;
+    bool mIncludeSourceMaterial = false;
 };
 
 } // namespace matp

tools/matc/src/matc/CommandlineConfig.cpp

@@ -70,6 +70,7 @@ static const option OPTIONS[] = {
         { "workarounds",       required_argument, nullptr, 'W' },
         { "no-insert-line-directives",       no_argument, nullptr, 'n' },
         { "no-insert-line-directive-check",       no_argument, nullptr, 'N' },
+        { "include-source-mat",       no_argument, nullptr, 'm' },
         { nullptr, 0, nullptr, 0 }  // termination of the option list
 };
 
@@ -163,6 +164,8 @@ static void usage(char* name) {
             "       #if defined(GL_GOOGLE_cpp_style_line_directive)\n"
             "       Some drivers may complain about the use of cpp style #line directives\n"
             "       if they don't support it.\n\n"
+            "   --include-source-mat\n"
+            "       Embeds the source ASCII material definition if set.\n"
 
             "Internal use and debugging only:\n"
             "   --optimize-none, -g, -O0\n"
@@ -406,6 +409,9 @@ bool CommandlineConfig::parse() {
             case 'N':
                 mInsertLineDirectiveChecks = false;
                 break;
+            case 'm':
+                mIncludeSourceMaterial = true;
+                break;
         }
     }
 

tools/matc/src/matc/MaterialCompiler.cpp

@@ -98,6 +98,10 @@ bool MaterialCompiler::run(const matp::Config& config) {
         return false;
     }
 
+    if (config.getIncludeSourceMaterial()) {
+        builder.materialSource(buffer.get());
+    }
+
     // If we're reflecting parameters, the MaterialParser will have handled it inside of parse().
     // We should return here to avoid actually building a material.
     if (config.getReflectionTarget() != matp::Config::Metadata::NONE) {