Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
135a3621761f206659038cf06441a7fb58d19b16
filament/libs/matdbg/src/DebugServer.cpp
LaiJF 135a362176 Fix typo (#5455)
2022-04-22 10:46:04 -04:00

659 lines
23 KiB
C++
Raw Blame History

/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <matdbg/DebugServer.h>
#include <CivetServer.h>
#include <utils/FixedCapacityVector.h>
#include <utils/Hash.h>
#include <utils/Log.h>
#include <spirv_glsl.hpp>
#include <spirv-tools/libspirv.h>
#include <matdbg/JsonWriter.h>
#include <matdbg/ShaderExtractor.h>
#include <matdbg/ShaderInfo.h>
#include <matdbg/ShaderReplacer.h>
#include <filaflat/ChunkContainer.h>
#include <tsl/robin_set.h>
#include <backend/DriverEnums.h>
#include "sca/GLSLTools.h"
#include <sstream>
#include <string>
using utils::FixedCapacityVector;
// If set to 0, this serves HTML from a resgen resource. Use 1 only during local development, which
// serves files directly from the source code tree.
#define SERVE_FROM_SOURCE_TREE 0
#if !SERVE_FROM_SOURCE_TREE
#include "matdbg_resources.h"
#endif
namespace filament {
namespace matdbg {
using namespace utils;
using namespace filament::backend;
using filaflat::ChunkContainer;
using filamat::ChunkType;
static const StaticString kSuccessHeader =
"HTTP/1.1 200 OK\r\nContent-Type: %s\r\n"
"Connection: close\r\n\r\n";
static const StaticString kErrorHeader =
"HTTP/1.1 404 Not Found\r\nContent-Type: %s\r\n"
"Connection: close\r\n\r\n";
static void spirvToAsm(struct mg_connection *conn, const uint32_t* spirv, size_t size) {
auto context = spvContextCreate(SPV_ENV_UNIVERSAL_1_0);
spv_text text = nullptr;
const uint32_t options = SPV_BINARY_TO_TEXT_OPTION_INDENT |
SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES;
spvBinaryToText(context, spirv, size / 4, options, &text, nullptr);
mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
mg_write(conn, text->str, text->length);
spvTextDestroy(text);
spvContextDestroy(context);
}
static void spirvToGlsl(struct mg_connection *conn, const uint32_t* spirv, size_t size) {
auto glsl = ShaderExtractor::spirvToGLSL(spirv, size / 4);
mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
mg_printf(conn, glsl.c_str(), glsl.size());
}
class FileRequestHandler : public CivetHandler {
public:
FileRequestHandler(DebugServer* server) : mServer(server) {}
bool handleGet(CivetServer *server, struct mg_connection *conn) {
const struct mg_request_info* request = mg_get_request_info(conn);
std::string uri(request->request_uri);
if (uri == "/" || uri == "/index.html") {
#if SERVE_FROM_SOURCE_TREE
mg_send_file(conn, "libs/matdbg/web/index.html");
#else
mg_printf(conn, kSuccessHeader.c_str(), "text/html");
mg_write(conn, mServer->mHtml.c_str(), mServer->mHtml.size());
#endif
return true;
}
if (uri == "/style.css") {
#if SERVE_FROM_SOURCE_TREE
mg_send_file(conn, "libs/matdbg/web/style.css");
#else
mg_printf(conn, kSuccessHeader.c_str(), "text/css");
mg_write(conn, mServer->mCss.c_str(), mServer->mCss.size());
#endif
return true;
}
if (uri == "/script.js") {
#if SERVE_FROM_SOURCE_TREE
mg_send_file(conn, "libs/matdbg/web/script.js");
#else
mg_printf(conn, kSuccessHeader.c_str(), "text/javascript");
mg_write(conn, mServer->mJavascript.c_str(), mServer->mJavascript.size());
#endif
return true;
}
slog.e << "DebugServer: bad request at line " << __LINE__ << ": " << uri << io::endl;
return false;
}
private:
DebugServer* mServer;
};
// Handles the following REST requests, where {id} is an 8-digit hex string.
//
// GET /api/matids
// GET /api/materials
// GET /api/material?matid={id}
// GET /api/shader?matid={id}&type=[glsl|spirv]&[glindex|vkindex|metalindex]={index}
// GET /api/active
//
class RestRequestHandler : public CivetHandler {
public:
RestRequestHandler(DebugServer* server) : mServer(server) {}
bool handleGet(CivetServer *server, struct mg_connection *conn) {
const struct mg_request_info* request = mg_get_request_info(conn);
std::string uri(request->local_uri);
const auto error = [request](int line) {
slog.e << "DebugServer: 404 at line " << line << ": " << request->local_uri
<< io::endl;
return false;
};
const auto softError = [request, conn](const char* msg) {
slog.e << "DebugServer: " << msg << ": " << request->query_string << io::endl;
mg_printf(conn, kErrorHeader.c_str(), "application/txt");
mg_write(conn, msg, strlen(msg));
return true;
};
if (uri == "/api/active") {
mServer->updateActiveVariants();
mg_printf(conn, kSuccessHeader.c_str(), "application/json");
mg_printf(conn, "{");
// If the backend has not been resolved to Vulkan, Metal, etc., then return an empty
// list. This can occur if the server is matinfo rather than an actual Filament session.
if (mServer->mBackend == backend::Backend::DEFAULT) {
mg_printf(conn, "}");
return true;
}
int index = 0;
for (const auto& pair : mServer->mMaterialRecords) {
const auto& record = pair.second;
ChunkContainer package(record.package, record.packageSize);
if (!package.parse()) {
return error(__LINE__);
}
JsonWriter writer;
if (!writer.writeActiveInfo(package, mServer->mBackend, record.activeVariants)) {
return error(__LINE__);
}
const bool last = (++index) == mServer->mMaterialRecords.size();
mg_printf(conn, "\"%8.8x\": %s %s", pair.first, writer.getJsonString(),
last ? "" : ",");
}
mg_printf(conn, "}");
return true;
}
if (uri == "/api/matids") {
mg_printf(conn, kSuccessHeader.c_str(), "application/json");
mg_printf(conn, "[");
int index = 0;
for (const auto& record : mServer->mMaterialRecords) {
const bool last = (++index) == mServer->mMaterialRecords.size();
mg_printf(conn, "\"%8.8x\" %s", record.first, last ? "" : ",");
}
mg_printf(conn, "]");
return true;
}
if (uri == "/api/materials") {
mg_printf(conn, kSuccessHeader.c_str(), "application/json");
mg_printf(conn, "[");
int index = 0;
for (const auto& record : mServer->mMaterialRecords) {
const bool last = (++index) == mServer->mMaterialRecords.size();
ChunkContainer package(record.second.package, record.second.packageSize);
if (!package.parse()) {
return error(__LINE__);
}
JsonWriter writer;
if (!writer.writeMaterialInfo(package)) {
return error(__LINE__);
}
mg_printf(conn, "{ \"matid\": \"%8.8x\", %s } %s", record.first,
writer.getJsonString(), last ? "" : ",");
}
mg_printf(conn, "]");
return true;
}
if (!request->query_string) {
return error(__LINE__);
}
const size_t qlength = strlen(request->query_string);
char matid[9] = {};
if (mg_get_var(request->query_string, qlength, "matid", matid, sizeof(matid)) < 0) {
return error(__LINE__);
}
const uint32_t id = strtol(matid, nullptr, 16);
const DebugServer::MaterialRecord* result = mServer->getRecord(id);
if (result == nullptr) {
return error(__LINE__);
}
ChunkContainer package(result->package, result->packageSize);
if (!package.parse()) {
return error(__LINE__);
}
if (uri == "/api/material") {
JsonWriter writer;
if (!writer.writeMaterialInfo(package)) {
return error(__LINE__);
}
mg_printf(conn, kSuccessHeader.c_str(), "application/json");
mg_printf(conn, "{ %s }", writer.getJsonString());
return true;
}
const StaticString glsl("glsl");
const StaticString msl("msl");
const StaticString spirv("spirv");
char type[6] = {};
if (mg_get_var(request->query_string, qlength, "type", type, sizeof(type)) < 0) {
return error(__LINE__);
}
CString language(type, strlen(type));
char glindex[4] = {};
char vkindex[4] = {};
char metalindex[4] = {};
mg_get_var(request->query_string, qlength, "glindex", glindex, sizeof(glindex));
mg_get_var(request->query_string, qlength, "vkindex", vkindex, sizeof(vkindex));
mg_get_var(request->query_string, qlength, "metalindex", metalindex, sizeof(metalindex));
if (!glindex[0] && !vkindex[0] && !metalindex[0]) {
return error(__LINE__);
}
if (uri != "/api/shader") {
return error(__LINE__);
}
if (glindex[0]) {
if (language != glsl) {
return softError("Only GLSL is supported.");
}
FixedCapacityVector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialGlsl));
if (!getGlShaderInfo(package, info.data())) {
return error(__LINE__);
}
const int shaderIndex = std::stoi(glindex);
if (shaderIndex >= info.size()) {
return error(__LINE__);
}
ShaderExtractor extractor(Backend::OPENGL, result->package, result->packageSize);
if (!extractor.parse()) {
return error(__LINE__);
}
const auto& item = info[shaderIndex];
filaflat::ShaderBuilder builder;
extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
mg_write(conn, builder.data(), builder.size() - 1);
return true;
}
if (vkindex[0]) {
ShaderExtractor extractor(Backend::VULKAN, result->package, result->packageSize);
if (!extractor.parse()) {
return error(__LINE__);
}
filaflat::ShaderBuilder builder;
FixedCapacityVector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialSpirv));
if (!getVkShaderInfo(package, info.data())) {
return error(__LINE__);
}
const int shaderIndex = std::stoi(vkindex);
if (shaderIndex >= info.size()) {
return error(__LINE__);
}
const auto& item = info[shaderIndex];
extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
if (language == spirv) {
spirvToAsm(conn, (const uint32_t*) builder.data(), builder.size());
return true;
}
if (language == glsl) {
spirvToGlsl(conn, (const uint32_t*) builder.data(), builder.size());
return true;
}
return softError("Only SPIRV is supported.");
}
if (metalindex[0]) {
ShaderExtractor extractor(Backend::METAL, result->package, result->packageSize);
if (!extractor.parse()) {
return error(__LINE__);
}
filaflat::ShaderBuilder builder;
FixedCapacityVector<ShaderInfo> info(getShaderCount(package, ChunkType::MaterialMetal));
if (!getMetalShaderInfo(package, info.data())) {
return error(__LINE__);
}
const int shaderIndex = std::stoi(metalindex);
if (shaderIndex >= info.size()) {
return error(__LINE__);
}
const auto& item = info[shaderIndex];
extractor.getShader(item.shaderModel, item.variant, item.pipelineStage, builder);
if (language == msl) {
mg_printf(conn, kSuccessHeader.c_str(), "application/txt");
mg_write(conn, builder.data(), builder.size() - 1);
return true;
}
return softError("Only MSL is supported.");
}
return error(__LINE__);
}
private:
DebugServer* mServer;
};
class WebSocketHandler : public CivetWebSocketHandler {
public:
WebSocketHandler(DebugServer* server) : mServer(server) {}
bool handleConnection(CivetServer *server, const struct mg_connection *conn) override {
return true;
}
void handleReadyState(CivetServer *server, struct mg_connection *conn) override {
mConnections.insert(conn);
}
bool handleData(CivetServer *server, struct mg_connection *conn, int bits, char *data,
size_t size) override {
// First check if this chunk is a continuation of a partial existing message.
if (mServer->mChunkedMessageRemaining > 0) {
const CString chunk(data, size);
const size_t pos = mServer->mChunkedMessage.size();
// Append the partial existing message.
mServer->mChunkedMessage = mServer->mChunkedMessage.insert(pos, chunk);
// Determine number of outstanding bytes.
if (size > mServer->mChunkedMessageRemaining) {
mServer->mChunkedMessageRemaining = 0;
} else {
mServer->mChunkedMessageRemaining -= size;
}
// Return early and wait for more chunks if some bytes are still outstanding.
if (mServer->mChunkedMessageRemaining > 0) {
return true;
}
data = mServer->mChunkedMessage.data();
size = mServer->mChunkedMessage.size();
// Ignore the handshake message that occurs after startup.
} else if (size < 8) {
return true;
}
mServer->mChunkedMessageRemaining = 0;
// Every WebSocket message is prefixed with a command name followed by a space.
//
// For now we simply use istringstream for parsing, so command arguments are delimited
// with space characters.
//
// The "API index" matches the values of filament::backend::Backend (zero is invalid).
//
// The "shader index" is a zero-based index into the list of variants using the order that
// they appear in the package, where each API (GL / VK / Metal) has its own list.
//
// Commands:
//
// EDIT [material id] [api index] [shader index] [shader length] [shader source....]
//
const static StaticString kEditCmd = "EDIT ";
const static size_t kEditCmdLength = kEditCmd.size();
if (0 == strncmp(data, kEditCmd.c_str(), kEditCmdLength)) {
std::istringstream str(data + kEditCmdLength);
uint32_t matid;
int api;
int shaderIndex;
int shaderLength;
str >> std::hex >> matid >> std::dec >> api >> shaderIndex >> shaderLength;
const char* source = data + kEditCmdLength + str.tellg() + 1;
const size_t remaining = size - kEditCmdLength - str.tellg();
// Return early and wait for more chunks if some bytes are still outstanding.
if (remaining < shaderLength + 1) {
mServer->mChunkedMessage = CString(data, size);
mServer->mChunkedMessageRemaining = shaderLength + 1 - remaining;
return true;
}
mServer->handleEditCommand(matid, backend::Backend(api), shaderIndex, source,
shaderLength);
return true;
}
const std::string firstFewChars(data, std::min(size, size_t(8)));
slog.e << "Bad WebSocket message. First few characters: "
<< "[" << firstFewChars << "]" << io::endl;
return false;
}
void handleClose(CivetServer *server, const struct mg_connection *conn) override {
struct mg_connection *key = const_cast<struct mg_connection *>(conn);
mConnections.erase(key);
}
// Notify all JavaScript clients that a new material package has been loaded.
void addMaterial(const DebugServer::MaterialRecord& material) {
for (auto connection : mConnections) {
char matid[9] = {};
snprintf(matid, sizeof(matid), "%8.8x", material.key);
mg_websocket_write(connection, MG_WEBSOCKET_OPCODE_TEXT, matid, 8);
}
}
private:
DebugServer* mServer;
tsl::robin_set<struct mg_connection*> mConnections;
};
DebugServer::DebugServer(Backend backend, int port) : mBackend(backend) {
#if !SERVE_FROM_SOURCE_TREE
mHtml = CString((const char*) MATDBG_RESOURCES_INDEX_DATA, MATDBG_RESOURCES_INDEX_SIZE - 1);
mJavascript = CString((const char*) MATDBG_RESOURCES_SCRIPT_DATA, MATDBG_RESOURCES_SCRIPT_SIZE - 1);
mCss = CString((const char*) MATDBG_RESOURCES_STYLE_DATA, MATDBG_RESOURCES_STYLE_SIZE - 1);
#endif
// By default the server spawns 50 threads so we override this to 10. According to the civetweb
// documentation, "it is recommended to use num_threads of at least 5, since browsers often
/// establish multiple connections to load a single web page, including all linked documents
// (CSS, JavaScript, images, ...)." If this count is too small, the web app basically hangs.
const char* kServerOptions[] = {
"listening_ports", "8080",
"num_threads", "10",
"error_log_file", "civetweb.txt",
nullptr
};
std::string portString = std::to_string(port);
kServerOptions[1] = portString.c_str();
mServer = new CivetServer(kServerOptions);
if (!mServer->getContext()) {
delete mServer;
mServer = nullptr;
slog.e << "Unable to start DebugServer, see civetweb.txt for details." << io::endl;
return;
}
mFileHandler = new FileRequestHandler(this);
mRestHandler = new RestRequestHandler(this);
mWebSocketHandler = new WebSocketHandler(this);
mServer->addHandler("/api", mRestHandler);
mServer->addHandler("", mFileHandler);
mServer->addWebSocketHandler("", mWebSocketHandler);
slog.i << "DebugServer listening at http://localhost:" << port << io::endl;
filamat::GLSLTools::init();
}
DebugServer::~DebugServer() {
filamat::GLSLTools::shutdown();
for (auto& pair : mMaterialRecords) {
delete [] pair.second.package;
}
delete mFileHandler;
delete mRestHandler;
delete mServer;
}
MaterialKey
DebugServer::addMaterial(const CString& name, const void* data, size_t size, void* userdata) {
filaflat::ChunkContainer* container = new filaflat::ChunkContainer(data, size);
if (!container->parse()) {
slog.e << "DebugServer: unable to parse material package: " << name.c_str() << io::endl;
return {};
}
const uint32_t seed = 42;
auto words = (const uint32_t*) data;
MaterialKey key = utils::hash::murmur3(words, size / 4, seed);
// Retain a copy of the package to permit queries after the client application has
// freed up the original material package.
uint8_t* package = new uint8_t[size];
memcpy(package, data, size);
MaterialRecord info = {userdata, package, size, name, key};
mMaterialRecords.insert({key, info});
mWebSocketHandler->addMaterial(info);
return key;
}
void DebugServer::removeMaterial(MaterialKey key) {
mMaterialRecords.erase(key);
}
const DebugServer::MaterialRecord* DebugServer::getRecord(const MaterialKey& key) const {
const auto& iter = mMaterialRecords.find(key);
return iter == mMaterialRecords.end() ? nullptr : &iter->second;
}
void DebugServer::updateActiveVariants() {
if (mQueryCallback) {
auto curr = mMaterialRecords.begin();
auto end = mMaterialRecords.end();
while (curr != end) {
auto& value = curr.value();
VariantList& result = value.activeVariants;
mQueryCallback(value.userdata, &result);
++curr;
}
}
}
bool DebugServer::handleEditCommand(const MaterialKey& key, backend::Backend api, int shaderIndex,
const char* source, size_t size) {
const auto error = [](int line) {
slog.e << "DebugServer: Unable to apply shader edit at line " << line << io::endl;
return false;
};
if (mMaterialRecords.find(key) == mMaterialRecords.end()) {
return error(__LINE__);
}
MaterialRecord& material = mMaterialRecords[key];
filaflat::ChunkContainer package(material.package, material.packageSize);
if (!package.parse()) {
return error(__LINE__);
}
FixedCapacityVector<ShaderInfo> infos;
size_t shaderCount;
switch (api) {
case backend::Backend::OPENGL: {
shaderCount = getShaderCount(package, ChunkType::MaterialGlsl);
infos.reserve(shaderCount);
infos.resize(shaderCount);
if (!getGlShaderInfo(package, infos.data())) {
return error(__LINE__);
}
break;
}
case backend::Backend::VULKAN: {
shaderCount = getShaderCount(package, ChunkType::MaterialSpirv);
infos.reserve(shaderCount);
infos.resize(shaderCount);
if (!getVkShaderInfo(package, infos.data())) {
return error(__LINE__);
}
break;
}
case backend::Backend::METAL: {
shaderCount = getShaderCount(package, ChunkType::MaterialMetal);
infos.reserve(shaderCount);
infos.resize(shaderCount);
if (!getMetalShaderInfo(package, infos.data())) {
return error(__LINE__);
}
break;
}
default:
error(__LINE__);
}
if (shaderIndex < 0 || shaderIndex >= shaderCount) {
return error(__LINE__);
}
const ShaderInfo info = infos[shaderIndex];
ShaderReplacer editor(api, package.getData(), package.getSize());
if (!editor.replaceShaderSource(info.shaderModel, info.variant, info.pipelineStage, source, size)) {
return error(__LINE__);
}
delete [] material.package;
material.package = editor.getEditedPackage();
material.packageSize = editor.getEditedSize();
if (mEditCallback) {
mEditCallback(material.userdata, material.name, material.package, material.packageSize);
}
return true;
}
} // namespace matdbg
} // namespace filament
Reference in New Issue View Git Blame Copy Permalink