From 23ee5e276cfe54ba64642ad40844ae4edc1a603c Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Thu, 5 Sep 2019 15:29:43 -0700 Subject: [PATCH] matdbg server: add query for active programs. This uses the program cache to determine the set of variants that are actually being used. --- filament/src/Engine.cpp | 34 ++++---- filament/src/Material.cpp | 26 ++++++- filament/src/details/Material.h | 16 +++- libs/matdbg/README.md | 14 +++- libs/matdbg/include/matdbg/DebugServer.h | 18 +++-- libs/matdbg/include/matdbg/JsonWriter.h | 11 +++ libs/matdbg/src/DebugServer.cpp | 67 ++++++++++++---- libs/matdbg/src/JsonWriter.cpp | 98 ++++++++++++++++++++++-- tools/matinfo/src/main.cpp | 2 +- 9 files changed, 230 insertions(+), 56 deletions(-) diff --git a/filament/src/Engine.cpp b/filament/src/Engine.cpp index 1a7b960080..8653ee06d8 100644 --- a/filament/src/Engine.cpp +++ b/filament/src/Engine.cpp @@ -160,22 +160,6 @@ void FEngine::init() { mCommandStream = CommandStream(*mDriver, mCommandBufferQueue.getCircularBuffer()); DriverApi& driverApi = getDriverApi(); -#if FILAMENT_ENABLE_MATDBG - // Disable the web server for regression tests that occur in hermetic environments. - if (mBackend != backend::Backend::NOOP) { - debug.server = new matdbg::DebugServer(matdbg::ENGINE, 8080); - - // Sometimes the server can fail to spin up (e.g. if the above port is already in use). - // When this occurs, carry onward, developers can look at civetweb.txt for details. - if (!debug.server->isReady()) { - delete debug.server; - debug.server = nullptr; - } else { - debug.server->setEditCallback(FMaterial::onEditCallback); - } - } -#endif - mResourceAllocator = new fg::ResourceAllocator(driverApi); // Parse all post process shaders now, but create them lazily @@ -415,6 +399,24 @@ int FEngine::loop() { } slog.d << io::endl; } + +#if FILAMENT_ENABLE_MATDBG + // Disable the web server for regression tests that occur in hermetic environments. + if (mBackend != backend::Backend::NOOP) { + debug.server = new matdbg::DebugServer(mBackend, 8080); + + // Sometimes the server can fail to spin up (e.g. if the above port is already in use). + // When this occurs, carry onward, developers can look at civetweb.txt for details. + if (!debug.server->isReady()) { + delete debug.server; + debug.server = nullptr; + } else { + debug.server->setEditCallback(FMaterial::onEditCallback); + debug.server->setQueryCallback(FMaterial::onQueryCallback); + } + } +#endif + mDriver = platform->createDriver(mSharedGLContext); mDriverBarrier.latch(); if (UTILS_UNLIKELY(!mDriver)) { diff --git a/filament/src/Material.cpp b/filament/src/Material.cpp index 6b956698f7..7e47e20b9a 100644 --- a/filament/src/Material.cpp +++ b/filament/src/Material.cpp @@ -440,8 +440,13 @@ void FMaterial::applyPendingEdits() noexcept { mPendingEdits = nullptr; } -// Callback handler for the debug server, potentially called from any thread. This method is never -// called during normal operation and exists for debugging purposes only. +/** + * Callback handlers for the debug server, potentially called from any thread. These methods are never + * called during normal operation and exists for debugging purposes only. + * + * @{ + */ + void FMaterial::onEditCallback(void* userdata, const utils::CString& name, const void* packageData, size_t packageSize) { FMaterial* material = upcast((Material*) userdata); @@ -453,8 +458,21 @@ void FMaterial::onEditCallback(void* userdata, const utils::CString& name, const packageSize); } -MaterialParser* FMaterial::createParser(backend::Backend backend, const void* data, - size_t size) { +void FMaterial::onQueryCallback(void* userdata, uint16_t* pvariants) { + FMaterial* material = upcast((Material*) userdata); + uint16_t variants = 0; + auto& cachedPrograms = material->mCachedPrograms; + for (size_t i = 0, n = cachedPrograms.size(); i < n; ++i) { + if (cachedPrograms[i]) { + variants |= (1 << i); + } + } + *pvariants = variants; +} + + /** @}*/ + +MaterialParser* FMaterial::createParser(backend::Backend backend, const void* data, size_t size) { MaterialParser* materialParser = new MaterialParser(backend, data, size); bool materialOK = materialParser->parse() && materialParser->isShadingMaterial(); diff --git a/filament/src/details/Material.h b/filament/src/details/Material.h index 0a3ffa6ef5..ef7ae259fc 100644 --- a/filament/src/details/Material.h +++ b/filament/src/details/Material.h @@ -132,12 +132,22 @@ public: void destroyPrograms(FEngine& engine); - // Callback handler for the debug server, potentially called from any thread. The userdata - // argument has the same value that was passed to DebugServer::addMaterial(), which should - // be an instance of the public-facing Material. + /** + * Callback handlers for the debug server, potentially called from any thread. The userdata + * argument has the same value that was passed to DebugServer::addMaterial(), which should + * be an instance of the public-facing Material. + * @{ + */ + + /** Replaces the material package. */ static void onEditCallback(void* userdata, const utils::CString& name, const void* packageData, size_t packageSize); + /** Queries the program cache to check which variants are resident. */ + static void onQueryCallback(void* userdata, uint16_t* variants); + + /** @}*/ + static MaterialParser* createParser(backend::Backend backend, const void* data, size_t size); private: diff --git a/libs/matdbg/README.md b/libs/matdbg/README.md index 9e248fb35d..ba42af46cb 100644 --- a/libs/matdbg/README.md +++ b/libs/matdbg/README.md @@ -122,6 +122,16 @@ is equivalent to one of the items in the top-level array in `/api/materials`. --- +`/api/active` + +Returns an object that maps from material ids to their active shader variants. Example: + +```json +{"b38d4ad0": [1, 5 ] , "44ae2b62": [1, 2, 3 ] } +``` + +--- + `/api/shader?matid={id}&type=[glsl|spirv]&[glindex|vkindex|metalindex]={index}` Returns the entire shader code for the given variant. This is the only HTTP request that returns @@ -136,8 +146,8 @@ used to create the SPIR-V is not available. ## WebSocket messages Unlike HTTP requests, WebSocket messages can be pushed at any time and can travel in either -direction. In our homegrown protocol, every WebSocket message starts with a 4-character command -followed by a space character. Command arguments are delimited with spaces. +direction. In our homegrown protocol, every WebSocket message starts with a command that matches +\[A-Z\_]+ followed by a space character. Command arguments are delimited with spaces. Currently we support only one command. It travels from client to server. diff --git a/libs/matdbg/include/matdbg/DebugServer.h b/libs/matdbg/include/matdbg/DebugServer.h index 2f9ef02c1e..a9de5e0c6a 100644 --- a/libs/matdbg/include/matdbg/DebugServer.h +++ b/libs/matdbg/include/matdbg/DebugServer.h @@ -17,7 +17,6 @@ #ifndef MATDBG_DEBUGSERVER_H #define MATDBG_DEBUGSERVER_H -#include #include #include @@ -29,8 +28,6 @@ class CivetServer; namespace filament { namespace matdbg { -enum ServerMode { ENGINE, STANDALONE }; - /** * Server-side material debugger. * @@ -40,7 +37,7 @@ enum ServerMode { ENGINE, STANDALONE }; */ class DebugServer { public: - DebugServer(ServerMode mode, int port = 8080); + DebugServer(backend::Backend backend, int port); ~DebugServer(); /** @@ -50,6 +47,7 @@ public: void* userdata = nullptr); using EditCallback = void(*)(void* userdata, const utils::CString& name, const void*, size_t); + using QueryCallback = void(*)(void* userdata, uint16_t* variants); /** * Sets up a callback that allows the Filament engine to listen for shader edits. The callback @@ -57,6 +55,12 @@ public: */ void setEditCallback(EditCallback callback) { mEditCallback = callback; } + /** + * Sets up a callback that can ask the Filament engine which shader variants are active. The + * callback might be triggered from a secondary thread. + */ + void setQueryCallback(QueryCallback callback) { mQueryCallback = callback; } + bool isReady() const { return mServer; } private: @@ -68,10 +72,13 @@ private: size_t packageSize; utils::CString name; MaterialKey key; + uint16_t activeVariants; }; const MaterialRecord* getRecord(const MaterialKey& key) const; + void updateActiveVariants(); + /** * Replaces the entire content of a particular shader variant. The given shader index uses the * same ordering that the variants have within the package. @@ -79,7 +86,7 @@ private: bool handleEditCommand(const MaterialKey& mat, backend::Backend api, int shaderIndex, const char* newShaderContent, size_t newShaderLength); - UTILS_UNUSED const ServerMode mServerMode; + const backend::Backend mBackend; CivetServer* mServer; tsl::robin_map mMaterialRecords; @@ -87,6 +94,7 @@ private: utils::CString mJavascript; utils::CString mCss; EditCallback mEditCallback = nullptr; + QueryCallback mQueryCallback = nullptr; class FileRequestHandler* mFileHandler = nullptr; class RestRequestHandler* mRestHandler = nullptr; diff --git a/libs/matdbg/include/matdbg/JsonWriter.h b/libs/matdbg/include/matdbg/JsonWriter.h index 532fe3e8db..1ece76a48e 100644 --- a/libs/matdbg/include/matdbg/JsonWriter.h +++ b/libs/matdbg/include/matdbg/JsonWriter.h @@ -19,6 +19,8 @@ #include +#include + #include namespace filament { @@ -26,7 +28,16 @@ namespace matdbg { class JsonWriter { public: + + // Returns a JSON object describing the given material. bool writeMaterialInfo(const filaflat::ChunkContainer& package); + + // Returns a JSON array of the form [ backend, shaderIndex0, shaderIndex1, ... ] where each + // shader index is an active variant. Each bit in the activeVariants bitmask + // represents one of the possible variant combinations. + bool writeActiveInfo(const filaflat::ChunkContainer& package, backend::Backend backend, + uint16_t activeVariants); + const char* getJsonString() const; size_t getJsonSize() const; private: diff --git a/libs/matdbg/src/DebugServer.cpp b/libs/matdbg/src/DebugServer.cpp index da08082c7d..d3a0e90063 100644 --- a/libs/matdbg/src/DebugServer.cpp +++ b/libs/matdbg/src/DebugServer.cpp @@ -103,6 +103,7 @@ private: // GET /api/materials // GET /api/material?matid={id} // GET /api/shader?matid={id}&type=[glsl|spirv]&[glindex|vkindex|metalindex]={index} +// GET /api/active // class RestRequestHandler : public CivetHandler { public: @@ -117,6 +118,29 @@ public: return false; }; + if (uri == "/api/active") { + mServer->updateActiveVariants(); + mg_printf(conn, kSuccessHeader.c_str(), "application/json"); + mg_printf(conn, "{"); + int index = 0; + for (const auto& pair : mServer->mMaterialRecords) { + const auto& record = pair.second; + ChunkContainer package(record.package, record.packageSize); + if (!package.parse()) { + return error(__LINE__); + } + JsonWriter writer; + if (!writer.writeActiveInfo(package, mServer->mBackend, record.activeVariants)) { + return error(__LINE__); + } + const bool last = (++index) == mServer->mMaterialRecords.size(); + mg_printf(conn, "\"%8.8x\": %s %s", pair.first, writer.getJsonString(), + last ? "" : ","); + } + mg_printf(conn, "}"); + return true; + } + if (uri == "/api/matids") { mg_printf(conn, kSuccessHeader.c_str(), "application/json"); mg_printf(conn, "["); @@ -323,7 +347,7 @@ public: return true; } - // Every WebSocket message is prefixed with a 4-character command followed by a space. + // Every WebSocket message is prefixed with a command name followed by a space. // // For now we simply use istringstream for parsing, so command arguments are delimited // with space characters. @@ -341,19 +365,20 @@ public: const static StaticString kEditCmd = "EDIT "; const static size_t kEditCmdLength = kEditCmd.size(); - if (strncmp(data, kEditCmd.c_str(), kEditCmdLength)) { - slog.e << "Bad WebSocket message." << io::endl; - return false; + if (0 == strncmp(data, kEditCmd.c_str(), kEditCmdLength)) { + std::istringstream str(data + kEditCmdLength); + uint32_t matid; + int api; + int shaderIndex; + str >> std::hex >> matid >> std::dec >> api >> shaderIndex; + const char* source = data + kEditCmdLength + str.tellg(); + const size_t remaining = size - kEditCmdLength - str.tellg(); + mServer->handleEditCommand(matid, backend::Backend(api), shaderIndex, source, remaining); + return true; } - std::istringstream str(data + kEditCmdLength); - uint32_t matid; - int api; - int shaderIndex; - str >> std::hex >> matid >> std::dec >> api >> shaderIndex; - const char* source = data + kEditCmdLength + str.tellg(); - const size_t remaining = size - kEditCmdLength - str.tellg(); - mServer->handleEditCommand(matid, backend::Backend(api), shaderIndex, source, remaining); - return true; + + slog.e << "Bad WebSocket message." << io::endl; + return false; } void handleClose(CivetServer *server, const struct mg_connection *conn) override { @@ -374,7 +399,7 @@ private: struct mg_connection* mConnection = nullptr; }; -DebugServer::DebugServer(ServerMode mode, int port) : mServerMode(mode) { +DebugServer::DebugServer(Backend backend, int port) : mBackend(backend) { #if !SERVE_FROM_SOURCE_TREE mHtml = CString((const char*) MATDBG_RESOURCES_INDEX_DATA, MATDBG_RESOURCES_INDEX_SIZE - 1); mJavascript = CString((const char*) MATDBG_RESOURCES_SCRIPT_DATA, MATDBG_RESOURCES_SCRIPT_SIZE - 1); @@ -446,6 +471,15 @@ const DebugServer::MaterialRecord* DebugServer::getRecord(const MaterialKey& key return iter == mMaterialRecords.end() ? nullptr : &iter->second; } +void DebugServer::updateActiveVariants() { + if (mQueryCallback) { + for (auto& pair : mMaterialRecords) { + uint16_t& result = mMaterialRecords[pair.first].activeVariants; + mQueryCallback(pair.second.userdata, &result); + } + } +} + bool DebugServer::handleEditCommand(const MaterialKey& key, backend::Backend api, int shaderIndex, const char* source, size_t size) { const auto error = [](int line) { @@ -462,10 +496,9 @@ bool DebugServer::handleEditCommand(const MaterialKey& key, backend::Backend api return error(__LINE__); } - const backend::Backend apiType = (backend::Backend) api; std::vector infos; size_t shaderCount; - switch (apiType) { + switch (api) { case backend::Backend::OPENGL: { shaderCount = getShaderCount(package, ChunkType::MaterialGlsl); infos.resize(shaderCount); @@ -499,7 +532,7 @@ bool DebugServer::handleEditCommand(const MaterialKey& key, backend::Backend api } const ShaderInfo info = infos[shaderIndex]; - ShaderReplacer editor(apiType, package.getData(), package.getSize()); + ShaderReplacer editor(api, package.getData(), package.getSize()); if (!editor.replaceShaderSource(info.shaderModel, info.variant, info.pipelineStage, source, size)) { return error(__LINE__); } diff --git a/libs/matdbg/src/JsonWriter.cpp b/libs/matdbg/src/JsonWriter.cpp index 135fcae4ef..c8ebe0490a 100644 --- a/libs/matdbg/src/JsonWriter.cpp +++ b/libs/matdbg/src/JsonWriter.cpp @@ -40,9 +40,9 @@ using namespace utils; namespace filament { namespace matdbg { -static std::string arraySizeToString(uint64_t size) { +static string arraySizeToString(uint64_t size) { if (size > 1) { - std::string s = "["; + string s = "["; s += size; s += "]"; return s; @@ -115,7 +115,7 @@ static bool printParametersInfo(ostream& json, const ChunkContainer& container) return true; } -static void printShaderInfo(ostream& json, const std::vector& info) { +static void printShaderInfo(ostream& json, const vector& info) { for (uint64_t i = 0; i < info.size(); ++i) { const auto& item = info[i]; string variantString = ""; @@ -134,17 +134,17 @@ static void printShaderInfo(ostream& json, const std::vector& info) string ps = (item.pipelineStage == backend::ShaderType::VERTEX) ? "vertex " : "fragment"; json << " {" - << "\"index\": \"" << std::setw(2) << i << "\", " + << "\"index\": \"" << setw(2) << i << "\", " << "\"shaderModel\": \"" << toString(item.shaderModel) << "\", " << "\"pipelineStage\": \"" << ps << "\", " << "\"variantString\": \"" << variantString << "\", " - << "\"variant\": \"" << std::hex << int(item.variant) << std::dec << "\" }" + << "\"variant\": \"" << hex << int(item.variant) << dec << "\" }" << ((i == info.size() - 1) ? "\n" : ",\n"); } } static bool printGlslInfo(ostream& json, const ChunkContainer& container) { - std::vector info; + vector info; info.resize(getShaderCount(container, ChunkType::MaterialGlsl)); if (!getGlShaderInfo(container, info.data())) { return false; @@ -156,7 +156,7 @@ static bool printGlslInfo(ostream& json, const ChunkContainer& container) { } static bool printVkInfo(ostream& json, const ChunkContainer& container) { - std::vector info; + vector info; info.resize(getShaderCount(container, ChunkType::MaterialSpirv)); if (!getVkShaderInfo(container, info.data())) { return false; @@ -168,7 +168,7 @@ static bool printVkInfo(ostream& json, const ChunkContainer& container) { } static bool printMetalInfo(ostream& json, const ChunkContainer& container) { - std::vector info; + vector info; info.resize(getShaderCount(container, ChunkType::MaterialMetal)); if (!getMetalShaderInfo(container, info.data())) { return false; @@ -218,6 +218,47 @@ bool JsonWriter::writeMaterialInfo(const filaflat::ChunkContainer& container) { return true; } +bool JsonWriter::writeActiveInfo(const filaflat::ChunkContainer& package, + Backend backend, uint16_t activeVariants) { + vector shaders; + ostringstream json; + json << "[\""; + switch (backend) { + case Backend::OPENGL: + shaders.resize(getShaderCount(package, ChunkType::MaterialGlsl)); + getGlShaderInfo(package, shaders.data()); + json << "opengl"; + break; + case Backend::VULKAN: + shaders.resize(getShaderCount(package, ChunkType::MaterialSpirv)); + getVkShaderInfo(package, shaders.data()); + json << "vulkan"; + break; + case Backend::METAL: + shaders.resize(getShaderCount(package, ChunkType::MaterialMetal)); + getMetalShaderInfo(package, shaders.data()); + json << "metal"; + break; + default: + return false; + } + json << "\""; + for (uint8_t variant = 0; variant < VARIANT_COUNT; variant++) { + if (activeVariants & (1 << variant)) { + int shaderIndex = 0; + for (const auto& info : shaders) { + if (info.variant == variant) { + json << ", " << shaderIndex; + } + shaderIndex++; + } + } + } + json << "]"; + mJsonString = CString(json.str().c_str()); + return true; +} + const char* JsonWriter::getJsonString() const { return mJsonString.c_str(); } @@ -226,5 +267,46 @@ size_t JsonWriter::getJsonSize() const { return mJsonString.size(); } +bool JsonWriter::writeActiveInfo(const filaflat::ChunkContainer& package, + Backend backend, uint16_t activeVariants) { + vector shaders; + ostringstream json; + json << "[\""; + switch (backend) { + case Backend::OPENGL: + shaders.resize(getShaderCount(package, ChunkType::MaterialGlsl)); + getGlShaderInfo(package, shaders.data()); + json << "opengl"; + break; + case Backend::VULKAN: + shaders.resize(getShaderCount(package, ChunkType::MaterialSpirv)); + getVkShaderInfo(package, shaders.data()); + json << "vulkan"; + break; + case Backend::METAL: + shaders.resize(getShaderCount(package, ChunkType::MaterialMetal)); + getMetalShaderInfo(package, shaders.data()); + json << "metal"; + break; + default: + return false; + } + json << "\""; + for (uint8_t variant = 0; variant < VARIANT_COUNT; variant++) { + if (activeVariants & (1 << variant)) { + int shaderIndex = 0; + for (const auto& info : shaders) { + if (info.variant == variant) { + json << ", " << shaderIndex; + } + shaderIndex++; + } + } + } + json << "]"; + mJsonString = CString(json.str().c_str()); + return true; +} + } // namespace matdbg } // namespace filament diff --git a/tools/matinfo/src/main.cpp b/tools/matinfo/src/main.cpp index f364858a80..ec50e8e9e3 100644 --- a/tools/matinfo/src/main.cpp +++ b/tools/matinfo/src/main.cpp @@ -355,7 +355,7 @@ static bool parseChunks(Config config, void* data, size_t size) { if (config.serverPort) { // Spin up a web server on a secondary thread. - DebugServer server(STANDALONE, config.serverPort); + DebugServer server(Backend::DEFAULT, config.serverPort); // Notify the server that we have a filamat file. utils::CString name;