Introduce a new precompiled Metal material chunk type (#7749)

This change introduces a new chunk type to material files for precompiled Metal libraries. Previously, SPIR-V was the only binary type, so there's also a couple of refactor commits present here. Nothing is changed in Filament or matc yet.

BUGS=[333547148]
This commit is contained in:
Ben Doherty
2024-04-12 00:42:10 -04:00
committed by GitHub
parent 05ce5ead8d
commit cddd22da28
17 changed files with 206 additions and 56 deletions

View File

@@ -158,14 +158,16 @@ static constexpr const char* backendToString(Backend backend) {
}
/**
* Defines the shader language. Similar to the above backend enum, but the OpenGL backend can select
* between two shader languages: ESSL 1.0 and ESSL 3.0.
* Defines the shader language. Similar to the above backend enum, with some differences:
* - The OpenGL backend can select between two shader languages: ESSL 1.0 and ESSL 3.0.
* - The Metal backend can prefer precompiled Metal libraries, while falling back to MSL.
*/
enum class ShaderLanguage {
ESSL1 = 0,
ESSL3 = 1,
SPIRV = 2,
MSL = 3,
METAL_LIBRARY = 4,
};
static constexpr const char* shaderLanguageToString(ShaderLanguage shaderLanguage) {
@@ -178,6 +180,8 @@ static constexpr const char* shaderLanguageToString(ShaderLanguage shaderLanguag
return "SPIR-V";
case ShaderLanguage::MSL:
return "MSL";
case ShaderLanguage::METAL_LIBRARY:
return "Metal precompiled library";
}
}

View File

@@ -65,6 +65,10 @@ MaterialParser::MaterialParserDetails::MaterialParserDetails(ShaderLanguage lang
mMaterialTag = ChunkType::MaterialSpirv;
mDictionaryTag = ChunkType::DictionarySpirv;
break;
case ShaderLanguage::METAL_LIBRARY:
mMaterialTag = ChunkType::MaterialMetalLibrary;
mDictionaryTag = ChunkType::DictionaryMetalLibrary;
break;
}
}

View File

@@ -45,6 +45,7 @@ enum UTILS_PUBLIC ChunkType : uint64_t {
MaterialEssl1 = charTo64bitNum("MAT_ESS1"),
MaterialSpirv = charTo64bitNum("MAT_SPIR"),
MaterialMetal = charTo64bitNum("MAT_METL"),
MaterialMetalLibrary = charTo64bitNum("MAT_MLIB"),
MaterialShaderModels = charTo64bitNum("MAT_SMDL"),
MaterialSamplerBindings = charTo64bitNum("MAT_SAMP"),
MaterialUniformBindings = charTo64bitNum("MAT_UNIF"),
@@ -93,6 +94,7 @@ enum UTILS_PUBLIC ChunkType : uint64_t {
DictionaryText = charTo64bitNum("DIC_TEXT"),
DictionarySpirv = charTo64bitNum("DIC_SPIR"),
DictionaryMetalLibrary = charTo64bitNum("DIC_MLIB"),
};
} // namespace filamat

View File

@@ -74,7 +74,7 @@ private:
BlobDictionary const& dictionary, ShaderContent& shaderContent,
ShaderModel shaderModel, filament::Variant variant, ShaderStage shaderStage);
bool getSpirvShader(
bool getBinaryShader(
BlobDictionary const& dictionary, ShaderContent& shaderContent,
ShaderModel shaderModel, filament::Variant variant, ShaderStage shaderStage);
};

View File

@@ -78,6 +78,25 @@ bool DictionaryReader::unflatten(ChunkContainer const& container,
}
return true;
} else if (dictionaryTag == ChunkType::DictionaryMetalLibrary) {
uint32_t blobCount;
if (!unflattener.read(&blobCount)) {
return false;
}
dictionary.reserve(blobCount);
for (uint32_t i = 0; i < blobCount; i++) {
unflattener.skipAlignmentPadding();
const char* data;
size_t dataSize;
if (!unflattener.read(&data, &dataSize)) {
return false;
}
dictionary.emplace_back(dataSize);
memcpy(dictionary.back().data(), data, dictionary.back().size());
}
return true;
} else if (dictionaryTag == ChunkType::DictionaryText) {
uint32_t stringCount = 0;
if (!unflattener.read(&stringCount)) {

View File

@@ -153,7 +153,7 @@ bool MaterialChunk::getTextShader(Unflattener unflattener,
return true;
}
bool MaterialChunk::getSpirvShader(BlobDictionary const& dictionary,
bool MaterialChunk::getBinaryShader(BlobDictionary const& dictionary,
ShaderContent& shaderContent, ShaderModel shaderModel, filament::Variant variant, ShaderStage shaderStage) {
if (mBase == nullptr) {
@@ -186,7 +186,8 @@ bool MaterialChunk::getShader(ShaderContent& shaderContent, BlobDictionary const
case filamat::ChunkType::MaterialMetal:
return getTextShader(mUnflattener, dictionary, shaderContent, shaderModel, variant, stage);
case filamat::ChunkType::MaterialSpirv:
return getSpirvShader(dictionary, shaderContent, shaderModel, variant, stage);
case filamat::ChunkType::MaterialMetalLibrary:
return getBinaryShader(dictionary, shaderContent, shaderModel, variant, stage);
default:
return false;
}

View File

@@ -50,7 +50,8 @@ set(PRIVATE_HDRS
${COMMON_PRIVATE_HDRS}
src/eiff/BlobDictionary.h
src/eiff/DictionarySpirvChunk.h
src/eiff/MaterialSpirvChunk.h
src/eiff/DictionaryMetalLibraryChunk.h
src/eiff/MaterialBinaryChunk.h
src/GLSLPostProcessor.h
src/MetalArgumentBuffer.h
src/ShaderMinifier.h
@@ -63,7 +64,8 @@ set(SRCS
${COMMON_SRCS}
src/eiff/BlobDictionary.cpp
src/eiff/DictionarySpirvChunk.cpp
src/eiff/MaterialSpirvChunk.cpp
src/eiff/DictionaryMetalLibraryChunk.cpp
src/eiff/MaterialBinaryChunk.cpp
src/MetalArgumentBuffer.cpp
src/sca/ASTHelpers.cpp
src/sca/GLSLTools.cpp

View File

@@ -33,11 +33,12 @@
#include "eiff/LineDictionary.h"
#include "eiff/MaterialInterfaceBlockChunk.h"
#include "eiff/MaterialTextChunk.h"
#include "eiff/MaterialSpirvChunk.h"
#include "eiff/MaterialBinaryChunk.h"
#include "eiff/ChunkContainer.h"
#include "eiff/SimpleFieldChunk.h"
#include "eiff/DictionaryTextChunk.h"
#include "eiff/DictionarySpirvChunk.h"
#include "eiff/DictionaryMetalLibraryChunk.h"
#include <private/filament/BufferInterfaceBlock.h>
#include <private/filament/SamplerInterfaceBlock.h>
@@ -821,7 +822,7 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
Mutex entriesLock;
std::vector<TextEntry> glslEntries;
std::vector<TextEntry> essl1Entries;
std::vector<SpirvEntry> spirvEntries;
std::vector<BinaryEntry> spirvEntries;
std::vector<TextEntry> metalEntries;
LineDictionary textDictionary;
BlobDictionary spirvDictionary;
@@ -872,7 +873,7 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
std::string* pMsl = targetApiNeedsMsl ? &msl : nullptr;
TextEntry glslEntry{};
SpirvEntry spirvEntry{};
BinaryEntry spirvEntry{};
TextEntry metalEntry{};
glslEntry.shaderModel = params.shaderModel;
@@ -966,7 +967,7 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
case TargetApi::VULKAN:
assert(!spirv.empty());
spirvEntry.stage = v.stage;
spirvEntry.spirv = std::move(spirv);
spirvEntry.data = std::move(spirv);
spirvEntries.push_back(spirvEntry);
break;
case TargetApi::METAL:
@@ -1018,7 +1019,7 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
textDictionary.addText(s.shader);
}
for (auto& s : spirvEntries) {
std::vector<uint32_t> spirv = std::move(s.spirv);
std::vector<uint32_t> spirv = std::move(s.data);
s.dictionaryIndex = spirvDictionary.addBlob(spirv);
}
for (const auto& s : metalEntries) {
@@ -1041,11 +1042,11 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
dictionaryChunk.getDictionary(), ChunkType::MaterialEssl1);
}
// Emit SPIRV chunks (SpirvDictionaryReader and MaterialSpirvChunk).
// Emit SPIRV chunks (SpirvDictionaryReader and MaterialBinaryChunk).
if (!spirvEntries.empty()) {
const bool stripInfo = !mGenerateDebugInfo;
container.push<filamat::DictionarySpirvChunk>(std::move(spirvDictionary), stripInfo);
container.push<MaterialSpirvChunk>(std::move(spirvEntries));
container.push<MaterialBinaryChunk>(std::move(spirvEntries), ChunkType::MaterialSpirv);
}
// Emit Metal chunk (MaterialTextChunk).

View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "DictionaryMetalLibraryChunk.h"
namespace filamat {
DictionaryMetalLibraryChunk::DictionaryMetalLibraryChunk(BlobDictionary&& dictionary)
: Chunk(ChunkType::DictionaryMetalLibrary), mDictionary(std::move(dictionary)) {}
void DictionaryMetalLibraryChunk::flatten(Flattener& f) {
f.writeUint32(mDictionary.getBlobCount());
for (size_t i = 0 ; i < mDictionary.getBlobCount() ; i++) {
std::string_view blob = mDictionary.getBlob(i);
f.writeAlignmentPadding();
f.writeBlob((const char*) blob.data(), blob.size());
}
}
} // namespace filamat

View File

@@ -0,0 +1,43 @@
/*
* Copyright (C) 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TNT_FILAMAT_DIC_METAL_LIBRARY_CHUNK_H
#define TNT_FILAMAT_DIC_METAL_LIBRARY_CHUNK_H
#include <stdint.h>
#include <vector>
#include "Chunk.h"
#include "Flattener.h"
#include "BlobDictionary.h"
namespace filamat {
class DictionaryMetalLibraryChunk final : public Chunk {
public:
explicit DictionaryMetalLibraryChunk(BlobDictionary&& dictionary);
~DictionaryMetalLibraryChunk() = default;
private:
void flatten(Flattener& f) override;
BlobDictionary mDictionary;
bool mStripDebugInfo;
};
} // namespace filamat
#endif // TNT_FILAMAT_DIC_METAL_LIBRARY_CHUNK_H

View File

@@ -1,6 +1,6 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*DictionaryGlsl
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
@@ -14,16 +14,17 @@
* limitations under the License.
*/
#include "MaterialSpirvChunk.h"
#include "MaterialBinaryChunk.h"
namespace filamat {
MaterialSpirvChunk::MaterialSpirvChunk(const std::vector<SpirvEntry>&& entries) :
Chunk(ChunkType::MaterialSpirv), mEntries(entries) {}
MaterialBinaryChunk::MaterialBinaryChunk(
const std::vector<BinaryEntry>&& entries, ChunkType chunkType)
: Chunk(chunkType), mEntries(entries) {}
void MaterialSpirvChunk::flatten(Flattener &f) {
void MaterialBinaryChunk::flatten(Flattener &f) {
f.writeUint64(mEntries.size());
for (const SpirvEntry& entry : mEntries) {
for (const BinaryEntry& entry : mEntries) {
f.writeUint8(uint8_t(entry.shaderModel));
f.writeUint8(entry.variant.key);
f.writeUint8(uint8_t(entry.stage));

View File

@@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef TNT_FILAMAT_MATERIAL_SPIRV_CHUNK_H
#define TNT_FILAMAT_MATERIAL_SPIRV_CHUNK_H
#ifndef TNT_FILAMAT_MATERIAL_BINARY_CHUNK_H
#define TNT_FILAMAT_MATERIAL_BINARY_CHUNK_H
#include "Chunk.h"
#include "ShaderEntry.h"
@@ -24,17 +24,17 @@
namespace filamat {
class MaterialSpirvChunk final : public Chunk {
class MaterialBinaryChunk final : public Chunk {
public:
explicit MaterialSpirvChunk(const std::vector<SpirvEntry>&& entries);
~MaterialSpirvChunk() = default;
explicit MaterialBinaryChunk(const std::vector<BinaryEntry>&& entries, ChunkType type);
~MaterialBinaryChunk() = default;
private:
void flatten(Flattener& f) override;
const std::vector<SpirvEntry> mEntries;
const std::vector<BinaryEntry> mEntries;
};
} // namespace filamat
#endif // TNT_FILAMAT_MATERIAL_SPIRV_CHUNK_H
#endif // TNT_FILAMAT_MATERIAL_BINARY_CHUNK_H

View File

@@ -34,14 +34,14 @@ struct TextEntry {
std::string shader;
};
struct SpirvEntry {
struct BinaryEntry {
filament::backend::ShaderModel shaderModel;
filament::Variant variant;
filament::backend::ShaderStage stage;
size_t dictionaryIndex;
size_t dictionaryIndex; // maps to an index in the blob dictionary
// temporarily holds this entry's spirv until added to the dictionary
std::vector<uint32_t> spirv;
// temporarily holds this entry's binary data until added to the dictionary
std::vector<uint32_t> data;
};
} // namespace filamat

View File

@@ -53,6 +53,10 @@ ShaderExtractor::ShaderExtractor(backend::ShaderLanguage target, const void* dat
mMaterialTag = ChunkType::MaterialMetal;
mDictionaryTag = ChunkType::DictionaryText;
break;
case backend::ShaderLanguage::METAL_LIBRARY:
mMaterialTag = ChunkType::MaterialMetalLibrary;
mDictionaryTag = ChunkType::DictionaryMetalLibrary;
break;
case backend::ShaderLanguage::SPIRV:
mMaterialTag = ChunkType::MaterialSpirv;
mDictionaryTag = ChunkType::DictionarySpirv;

View File

@@ -35,7 +35,7 @@
#include "eiff/ChunkContainer.h"
#include "eiff/DictionarySpirvChunk.h"
#include "eiff/DictionaryTextChunk.h"
#include "eiff/MaterialSpirvChunk.h"
#include "eiff/MaterialBinaryChunk.h"
#include "eiff/MaterialTextChunk.h"
#include "eiff/LineDictionary.h"
@@ -73,7 +73,7 @@ private:
vector<TextEntry> mShaderRecords;
};
// Tiny database of data blobs that can import / export MaterialSpirvChunk and DictionarySpirvChunk.
// Tiny database of data blobs that can import / export MaterialBinaryChunk and DictionarySpirvChunk.
// The blobs are stored *after* they have been compressed by SMOL-V.
class BlobIndex {
public:
@@ -90,7 +90,7 @@ public:
private:
const ChunkType mDictTag;
const ChunkType mMatTag;
vector<SpirvEntry> mShaderRecords;
vector<BinaryEntry> mShaderRecords;
filaflat::BlobDictionary mDataBlobs;
};
@@ -364,7 +364,7 @@ BlobIndex::BlobIndex(ChunkType dictTag, ChunkType matTag, const filaflat::ChunkC
const auto& offsets = matChunk.getOffsets();
mShaderRecords.reserve(offsets.size());
for (auto [key, offset] : offsets) {
SpirvEntry info;
BinaryEntry info;
filaflat::MaterialChunk::decodeKey(key, &info.shaderModel, &info.variant, &info.stage);
info.dictionaryIndex = offset;
mShaderRecords.emplace_back(info);
@@ -391,7 +391,7 @@ void BlobIndex::writeChunks(ostream& stream) {
// Apply SMOL-V compression and write out the results.
filamat::ChunkContainer cc;
cc.push<MaterialSpirvChunk>(std::move(mShaderRecords));
cc.push<MaterialBinaryChunk>(std::move(mShaderRecords), ChunkType::MaterialSpirv);
cc.push<DictionarySpirvChunk>(std::move(blobs), false);
Flattener prepass = Flattener::getDryRunner();

View File

@@ -421,6 +421,9 @@ static bool printShaderInfo(ostream& text, const ChunkContainer& container, Chun
case ChunkType::MaterialMetal:
text << "Metal shaders:" << endl;
break;
case ChunkType::MaterialMetalLibrary:
text << "Metal precompiled shader libraries:" << endl;
break;
default:
assert(false && "Invalid shader ChunkType");
break;
@@ -455,6 +458,9 @@ bool TextWriter::writeMaterialInfo(const filaflat::ChunkContainer& container) {
if (!printShaderInfo(text, container, ChunkType::MaterialMetal)) {
return false;
}
if (!printShaderInfo(text, container, ChunkType::MaterialMetalLibrary)) {
return false;
}
printChunks(text, container);

View File

@@ -96,8 +96,10 @@ static void printUsage(const char* name) {
" Print the Vulkan dictionary\n\n"
" --web-server=[port], -w\n"
" Serve a web page at the given port (e.g. 8080)\n\n"
" --dump-binary=[index], -b\n"
" --dump-spirv-binary=[index], -b\n"
" Dump binary SPIRV for the nth Vulkan shader to 'out.spv'\n\n"
" --dump-metal-library=[index]\n"
" Dump binary Metal library for the nth Metal precompiled library to 'out.metallib'\n\n"
" --license\n"
" Print copyright and license information\n\n"
" --analyze-spirv=[index], -a\n"
@@ -124,21 +126,24 @@ static void license() {
static int handleArguments(int argc, char* argv[], Config* config) {
static constexpr const char* OPTSTR = "hla:g:G:s:v:b:m:b:w:Xxyz";
constexpr int DUMP_METAL_LIBRARY_OPTION = 1000;
static const struct option OPTIONS[] = {
{ "help", no_argument, nullptr, 'h' },
{ "license", no_argument, nullptr, 'l' },
{ "analyze-spirv", required_argument, nullptr, 'a' },
{ "print-glsl", required_argument, nullptr, 'g' },
{ "print-essl1", required_argument, nullptr, 'G' },
{ "print-spirv", required_argument, nullptr, 's' },
{ "print-vkglsl", required_argument, nullptr, 'v' },
{ "print-metal", required_argument, nullptr, 'm' },
{ "print-dic-glsl", no_argument, nullptr, 'x' },
{ "print-dic-essl1", no_argument, nullptr, 'X' },
{ "print-dic-metal", no_argument, nullptr, 'y' },
{ "print-dic-vk", no_argument, nullptr, 'z' },
{ "dump-binary", required_argument, nullptr, 'b' },
{ "web-server", required_argument, nullptr, 'w' },
{ "help", no_argument, nullptr, 'h' },
{ "license", no_argument, nullptr, 'l' },
{ "analyze-spirv", required_argument, nullptr, 'a' },
{ "print-glsl", required_argument, nullptr, 'g' },
{ "print-essl1", required_argument, nullptr, 'G' },
{ "print-spirv", required_argument, nullptr, 's' },
{ "print-vkglsl", required_argument, nullptr, 'v' },
{ "print-metal", required_argument, nullptr, 'm' },
{ "print-dic-glsl", no_argument, nullptr, 'x' },
{ "print-dic-essl1", no_argument, nullptr, 'X' },
{ "print-dic-metal", no_argument, nullptr, 'y' },
{ "print-dic-vk", no_argument, nullptr, 'z' },
{ "dump-binary", required_argument, nullptr, 'b' }, // backwards compatibility
{ "dump-spirv-binary", required_argument, nullptr, 'b' },
{ "dump-metal-library", required_argument, nullptr, DUMP_METAL_LIBRARY_OPTION },
{ "web-server", required_argument, nullptr, 'w' },
{ nullptr, 0, nullptr, 0 } // termination of the option list
};
@@ -185,6 +190,7 @@ static int handleArguments(int argc, char* argv[], Config* config) {
case 'm':
config->printMetal = true;
config->shaderIndex = static_cast<uint64_t>(std::stoi(arg));
config->binary = false;
break;
case 'w':
config->serverPort = std::stoi(arg);
@@ -201,6 +207,11 @@ static int handleArguments(int argc, char* argv[], Config* config) {
case 'z':
config->printDictionarySPIRV = true;
break;
case DUMP_METAL_LIBRARY_OPTION:
config->printMetal = true;
config->shaderIndex = static_cast<uint64_t>(std::stoi(arg));
config->binary = true;
break;
}
}
@@ -397,6 +408,11 @@ static void dumpSpirvBinary(const std::vector<uint32_t>& spirv, const std::strin
std::cout << "Binary SPIR-V dumped to " << filename << std::endl;
}
static void dumpBinary(const uint8_t* data, size_t size, const std::string& filename) {
std::ofstream out(filename, std::ofstream::binary);
out.write(reinterpret_cast<const char*>(data), size);
}
static bool parseChunks(Config config, void* data, size_t size) {
using namespace filament::matdbg;
ChunkContainer container(data, size);
@@ -517,13 +533,20 @@ static bool parseChunks(Config config, void* data, size_t size) {
}
if (config.printMetal) {
ShaderExtractor parser(filament::backend::ShaderLanguage::MSL, data, size);
const filament::backend::ShaderLanguage language = config.binary
? filament::backend::ShaderLanguage::METAL_LIBRARY
: filament::backend::ShaderLanguage::MSL;
ShaderExtractor parser(language, data, size);
if (!parser.parse()) {
return false;
}
info.resize(getShaderCount(container, filamat::ChunkType::MaterialMetal));
if (!getShaderInfo(container, info.data(), filamat::ChunkType::MaterialMetal)) {
const filamat::ChunkType chunkType = config.binary
? filamat::ChunkType::MaterialMetalLibrary
: filamat::ChunkType::MaterialMetal;
size_t shaderCount = getShaderCount(container, chunkType);
info.resize(shaderCount);
if (!getShaderInfo(container, info.data(), chunkType)) {
std::cerr << "Failed to parse Metal chunk." << std::endl;
return false;
}
@@ -534,8 +557,15 @@ static bool parseChunks(Config config, void* data, size_t size) {
}
const auto& item = info[config.shaderIndex];
parser.getShader(item.shaderModel, item.variant, item.pipelineStage, content);
std::cout << (const char*) content.data();
if (!parser.getShader(item.shaderModel, item.variant, item.pipelineStage, content)) {
return false;
}
if (config.binary) {
dumpBinary(content.data(), content.size(), "out.metallib");
} else {
std::cout << (const char*) content.data();
}
return true;
}