fgviewer: add ability to look at rendertargets

2026-01-20 14:05:27 -08:00
17 changed files with 605 additions and 27 deletions
--- a/filament/backend/src/DataReshaper.h
+++ b/filament/backend/src/DataReshaper.h
@@ -19,13 +19,17 @@

 #include <backend/PixelBufferDescriptor.h>

+#include <cstdint>
 #include <cstring>
 #include <stddef.h>
 #include <stdint.h>

 #include <math/scalar.h>
+#include <math/half.h>

 #include <utils/debug.h>
+#include <utils/Log.h>
+#include <utils/ostream.h>

 namespace filament {
 namespace backend {
@@ -76,7 +80,7 @@ public:
        }
    }

-    // Converts a n-channel image of UBYTE, INT, UINT, or FLOAT to a different type.
+    // Converts a n-channel image of UBYTE, INT, UINT, HALF, or FLOAT to a different type.
    template<typename dstComponentType, typename srcComponentType>
    static void reshapeImage(uint8_t* UTILS_RESTRICT dest, const uint8_t* UTILS_RESTRICT src,
            size_t srcBytesPerRow,
@@ -91,6 +95,8 @@ public:
        UTILS_ASSUME(minChannelCount <= 4);
        dest += (dstRowOffset * dstBytesPerRow);
        const int inds[4] = { swizzle ? 2 : 0, 1, swizzle ? 0 : 2, 3 };
+
+        utils::slog.e <<"dstMaxValue=" << +dstMaxValue << utils::io::endl;
        for (size_t row = 0; row < height; ++row) {
            const srcComponentType* in = (const srcComponentType*) src;
            dstComponentType* out = (dstComponentType*)dest + (dstColumnOffset * dstChannelCount);
@@ -119,6 +125,7 @@ public:
    static bool reshapeImage(PixelBufferDescriptor* UTILS_RESTRICT dst, PixelDataType srcType,
            uint32_t srcChannelCount,  const uint8_t* UTILS_RESTRICT srcBytes, int srcBytesPerRow,
            int width, int height, bool swizzle) {
+        using namespace utils;
        size_t dstChannelCount;
        switch (dst->format) {
            case PixelDataFormat::R_INTEGER: dstChannelCount = 1; break;
@@ -129,7 +136,9 @@ public:
            case PixelDataFormat::RG: dstChannelCount = 2; break;
            case PixelDataFormat::RGB: dstChannelCount = 3; break;
            case PixelDataFormat::RGBA: dstChannelCount = 4; break;
-            default: return false;
+            default:
+                slog.e << "DataReshaper: unsupported dst->format: " << (int)dst->format << io::endl;
+                return false;
        }
        void (*reshaper)(uint8_t* dest, const uint8_t* src, size_t srcBytesPerRow,
                size_t srcChannelCount,
@@ -155,7 +164,10 @@ public:
                    case FLOAT: reshaper = reshapeImage<uint8_t, float>; break;
                    case INT: reshaper = reshapeImage<uint8_t, int32_t>; break;
                    case UINT: reshaper = reshapeImage<uint8_t, uint32_t>; break;
-                    default: return false;
+                    case HALF: reshaper = reshapeImage<uint8_t, math::half>; break;
+                    default:
+                        slog.e << "DataReshaper: UBYTE dst, unsupported srcType: " << (int)srcType << io::endl;
+                        return false;
                }
                break;
            case FLOAT:
@@ -164,7 +176,9 @@ public:
                    case FLOAT: reshaper = reshapeImage<float, float>; break;
                    case INT: reshaper = reshapeImage<float, int32_t>; break;
                    case UINT: reshaper = reshapeImage<float, uint32_t>; break;
-                    default: return false;
+                    default:
+                        slog.e << "DataReshaper: FLOAT dst, unsupported srcType: " << (int)srcType << io::endl;
+                        return false;
                }
                break;
            case INT:
@@ -173,7 +187,9 @@ public:
                    case FLOAT: reshaper = reshapeImage<int32_t, float>; break;
                    case INT: reshaper = reshapeImage<int32_t, int32_t>; break;
                    case UINT: reshaper = reshapeImage<int32_t, uint32_t>; break;
-                    default: return false;
+                    default:
+                        slog.e << "DataReshaper: INT dst, unsupported srcType: " << (int)srcType << io::endl;
+                        return false;
                }
                break;
            case UINT:
@@ -182,16 +198,21 @@ public:
                    case FLOAT: reshaper = reshapeImage<uint32_t, float>; break;
                    case INT: reshaper = reshapeImage<uint32_t, int32_t>; break;
                    case UINT: reshaper = reshapeImage<uint32_t, uint32_t>; break;
-                    default: return false;
+                    default:
+                        slog.e << "DataReshaper: UINT dst, unsupported srcType: " << (int)srcType << io::endl;
+                        return false;
                }
                break;
            case HALF:
                switch (srcType) {
                    case HALF: reshaper = copyImage; break;
-                    default: return false;
+                    default:
+                        slog.e << "DataReshaper: HALF dst, unsupported srcType: " << (int)srcType << io::endl;
+                        return false;
                }
                break;
            default:
+                slog.e << "DataReshaper: unsupported dst->type: " << (int)dst->type << io::endl;
                return false;
        }
        uint8_t* dstBytes = (uint8_t*) dst->buffer;
@@ -200,6 +221,7 @@ public:
        reshaper(dstBytes, srcBytes, srcBytesPerRow, srcChannelCount,
                dst->top, dst->left, dstBytesPerRow,
                dstChannelCount, width, height, swizzle);
+
        return true;
    }
 };
@@ -209,6 +231,7 @@ template<> inline int32_t getMaxValue() { return 0x7fffffff; }
 template<> inline uint32_t getMaxValue() { return 0xffffffff; }
 template<> inline uint16_t getMaxValue() { return 0x3c00; } // 0x3c00 is 1.0 in half-float.
 template<> inline uint8_t getMaxValue() { return 0xff; }
+template<> inline math::half getMaxValue() { return math::half(1.0f); }

 } // namespace backend
 } // namespace filament
--- a/filament/src/details/Engine.cpp
+++ b/filament/src/details/Engine.cpp
@@ -858,7 +858,13 @@ int FEngine::loop() {
 #endif
    if (fgviewerPortString != nullptr) {
        const int fgviewerPort = atoi(fgviewerPortString);
-        debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort);
+        debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort,
+                fgviewer::DebugServer::ReadbackRequest([this](utils::CString name,
+                        std::function<void(fgviewer::DebugServer::PixelBuffer,
+                                uint32_t, uint32_t,
+                                fgviewer::DebugServer::PixelDataFormat)> callback) {
+                    requestTextureReadback(name, std::move(callback));
+                }));

        // Sometimes the server can fail to spin up (e.g. if the above port is already in use).
        // When this occurs, carry onward, developers can look at civetweb.txt for details.
@@ -1678,4 +1684,13 @@ Engine::Config Engine::BuilderDetails::validateConfig(Config config) noexcept {
    return config;
 }

+#if FILAMENT_ENABLE_FGVIEWER
+void FEngine::requestTextureReadback(const CString& name,
+        std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                fgviewer::DebugServer::PixelDataFormat)>&& callback) {
+    std::unique_lock<utils::Mutex> lock(mReadbackRequestsMutex);
+    mReadbackRequests.emplace_back(ReadbackRequest{ name, std::move(callback) });
+}
+#endif
+
 } // namespace filament
--- a/filament/src/details/Engine.h
+++ b/filament/src/details/Engine.h
@@ -135,6 +135,7 @@ class TextureCache;
 * for a given context.
 */
 class FEngine : public Engine {
+    friend class FRenderer;
 public:
    void* operator new(std::size_t const size) noexcept {
        return utils::aligned_alloc(size, alignof(FEngine));
@@ -572,6 +573,12 @@ public:

    backend::Driver& getDriver() const noexcept { return *mDriver; }

+#if FILAMENT_ENABLE_FGVIEWER
+    void requestTextureReadback(utils::CString const& name,
+            std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                    fgviewer::DebugServer::PixelDataFormat)>&& callback);
+#endif
+
 private:
    explicit FEngine(Builder const& builder);
    void init();
@@ -703,6 +710,17 @@ private:

    bool mInitialized = false;

+#if FILAMENT_ENABLE_FGVIEWER
+    struct ReadbackRequest {
+        using Callback = std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                fgviewer::DebugServer::PixelDataFormat)>;
+        utils::CString name;
+        Callback callback;
+    };
+    utils::Mutex mReadbackRequestsMutex;
+    std::vector<ReadbackRequest> mReadbackRequests;
+#endif
+
    // Creation parameters
    Config mConfig;

--- a/filament/src/details/Renderer.cpp
+++ b/filament/src/details/Renderer.cpp
@@ -37,6 +37,7 @@
 #include "fg/FrameGraphResources.h"
 #include "fg/FrameGraphTexture.h"

+#include <mutex>
 #include <private/filament/EngineEnums.h>
 #include <private/filament/Variant.h>

@@ -60,6 +61,7 @@
 #include <utils/Allocator.h>
 #include <utils/bitset.h>
 #include <utils/JobSystem.h>
+#include <utils/Log.h>
 #include <utils/Logger.h>
 #include <utils/Panic.h>
 #include <utils/compiler.h>
@@ -1490,12 +1492,18 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {

    fg.forwardResource(fgViewRenderTarget, input);

+#if FILAMENT_ENABLE_FGVIEWER
+    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
+    if (UTILS_LIKELY(fgviewerServer)) {
+        readPixels(fg, blackboard);
+    }
+#endif
+
    fg.present(fgViewRenderTarget);

    fg.compile();

 #if FILAMENT_ENABLE_FGVIEWER
-    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
    if (UTILS_LIKELY(fgviewerServer)) {
        fgviewerServer->update(view.getViewHandle(), fg.getFrameGraphInfo(view.getName()));
    }
@@ -1507,10 +1515,91 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {

    fg.execute(driver);

+#if FILAMENT_ENABLE_FGVIEWER
+    if (UTILS_LIKELY(fgviewerServer)) {
+        fgviewerServer->tick();
+    }
+#endif
+
    // save the current history entry and destroy the oldest entry
    view.commitFrameHistory(engine);

    recordHighWatermark(commandArena.getListener().getHighWatermark());
 }

+#if FILAMENT_ENABLE_FGVIEWER
+
+void FRenderer::readPixels(FrameGraph& fg, Blackboard& blackboard) {
+
+    FEngine& engine = mEngine;
+    std::vector<FEngine::ReadbackRequest> requests;
+
+    { // scope for lock
+        std::unique_lock<utils::Mutex> lock(engine.mReadbackRequestsMutex);
+        requests = std::move(engine.mReadbackRequests);
+        engine.mReadbackRequests.clear();
+    }
+
+    for (auto& request : requests) {
+        FrameGraphId<FrameGraphTexture> fgTexture = fg.getTextureByName(request.name.c_str());
+        if (!fgTexture.isInitialized()) {
+            slog.w << "[fgviewer] Requested texture \"" << request.name.c_str_safe()
+                   << "\" not found in FrameGraph." << io::endl;
+            continue;
+        }
+
+        struct ReadbackPassData {
+            FrameGraphId<FrameGraphTexture> texture;
+        };
+        fg.addPass<ReadbackPassData>(
+                "Readback Pass",
+                [&](FrameGraph::Builder& builder, ReadbackPassData& data) {
+                    data.texture = builder.read(fgTexture, FrameGraphTexture::Usage::SAMPLEABLE);
+                    builder.sideEffect(); // Ensure this pass is not culled
+                },
+                [request = std::move(request)](FrameGraphResources const& resources,
+                        ReadbackPassData const& data, DriverApi& d) {
+                    const FrameGraphTexture::Descriptor& desc =
+                            resources.getDescriptor(data.texture);
+
+                    backend::PixelBufferDescriptor::PixelDataFormat format =
+                            PixelBufferDescriptor::PixelDataFormat::RGBA;
+                    backend::PixelBufferDescriptor::PixelDataType type =
+                            PixelBufferDescriptor::PixelDataType::UBYTE;
+                    fgviewer::DebugServer::PixelDataFormat targetFormat =
+                            fgviewer::DebugServer::Format::RGBA;
+                    const size_t bufferSize = desc.width * desc.height * 4;
+
+                    fgviewer::DebugServer::PixelBuffer pixelBuffer(bufferSize);
+                    void* bufferData = pixelBuffer.data();
+
+                    struct UserData {
+                        size_t width;
+                        size_t height;
+                        fgviewer::DebugServer::PixelDataFormat targetFormat;
+                        FEngine::ReadbackRequest::Callback callback;
+                        fgviewer::DebugServer::PixelBuffer pixelBuffer;
+                    };
+
+                    PixelBufferDescriptor pbd(
+                            bufferData, bufferSize, format, type, 1, 0, 0, 0, 0,
+                            [](void* buffer, size_t size, void* user) {
+                                std::unique_ptr<UserData> d {static_cast<UserData*>(user)};
+                                auto& callback = d->callback;
+                                callback(std::move(d->pixelBuffer), d->width, d->height, d->targetFormat);
+                            },
+                            new UserData{ desc.width, desc.height, targetFormat,
+                                std::move(request.callback), std::move(pixelBuffer) });
+
+                    MRT color(resources.getTexture(data.texture), 0, 0);
+                    Handle<HwRenderTarget> rt = d.createRenderTarget(TargetBufferFlags::COLOR0,
+                            desc.width, desc.height, 1, 1, color, {}, {});
+
+                    d.readPixels(rt, 0, 0, desc.width, desc.height, std::move(pbd));
+                    d.destroyRenderTarget(rt);
+                });
+    }
+}
+#endif
+
 } // namespace filament
--- a/filament/src/details/Renderer.h
+++ b/filament/src/details/Renderer.h
@@ -55,6 +55,7 @@
 namespace filament {

 class TextureCache;
+class Blackboard;

 namespace backend {
 class Driver;
@@ -198,6 +199,9 @@ private:

    void renderInternal(FView const* view, bool flush);
    void renderJob(RootArenaScope& rootArenaScope, FView& view);
+#if FILAMENT_ENABLE_FGVIEWER
+    void readPixels(FrameGraph& fg, Blackboard& blackboard);
+#endif

    static std::pair<float, math::float2> prepareUpscaler(math::float2 scale,
            TemporalAntiAliasingOptions const& taaOptions,
--- a/filament/src/fg/FrameGraph.cpp
+++ b/filament/src/fg/FrameGraph.cpp
@@ -582,6 +582,21 @@ fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char *viewName) con
 #endif
 }

+#if FILAMENT_ENABLE_FGVIEWER
+FrameGraphId<FrameGraphTexture> FrameGraph::getTextureByName(const char* name) const {
+    for (size_t i = 0; i < mResourceSlots.size(); ++i) {
+        const auto& slot = mResourceSlots[i];
+        const VirtualResource* resource = mResources[slot.rid];
+        if (strcmp(resource->name.data(), name) == 0) {
+            FrameGraphHandle handle((FrameGraphHandle::Index)i);
+            handle.version = slot.version;
+            return FrameGraphId<FrameGraphTexture>(handle);
+        }
+    }
+    return {};
+}
+#endif
+

 // ------------------------------------------------------------------------------------------------

--- a/filament/src/fg/FrameGraph.h
+++ b/filament/src/fg/FrameGraph.h
@@ -27,6 +27,7 @@

 #include "fg/details/DependencyGraph.h"
 #include "fg/details/Resource.h"
+#include "fg/details/ResourceCreationContext.h"
 #include "fg/details/Utilities.h"

 #include "backend/DriverApiForward.h"
@@ -36,6 +37,8 @@

 #include <functional>

+#include <utils/Log.h>
+
 #if FILAMENT_ENABLE_FGVIEWER
 #include <fgviewer/FrameGraphInfo.h>
 #else
@@ -448,8 +451,21 @@ public:
     * Export a fgviewer::FrameGraphInfo for current graph.
     * Note that this function should be called after FrameGraph::compile().
     */
+#if FILAMENT_ENABLE_FGVIEWER
    fgviewer::FrameGraphInfo getFrameGraphInfo(const char *viewName) const;

+    /**
+     * Retrieve a texture handle by its name. This is used for debugging/visualization tools.
+     * @param name Name of the resource
+     * @return Handle to the texture, or an uninitialized handle if not found.
+     */
+    FrameGraphId<FrameGraphTexture> getTextureByName(const char* name) const;
+#else
+    fgviewer::FrameGraphInfo getFrameGraphInfo(const char*) const {
+        return fgviewer::FrameGraphInfo();
+    }
+#endif
+
 private:
    friend class FrameGraphResources;
    friend class PassNode;
@@ -673,4 +689,4 @@ extern template FrameGraphId<FrameGraphTexture> FrameGraph::forwardResource(

 } // namespace filament

-#endif //TNT_FILAMENT_FG_FRAMEGRAPH_H
+#endif // TNT_FILAMENT_FG_FRAMEGRAPH_H
--- a/libs/fgviewer/CMakeLists.txt
+++ b/libs/fgviewer/CMakeLists.txt
@@ -20,10 +20,10 @@ set(PUBLIC_HDRS

 set(SRCS
    src/ApiHandler.cpp
-    src/ApiHandler.h
    src/DebugServer.cpp
    src/FrameGraphInfo.cpp
    src/JsonWriter.cpp
+    src/WebSocketHandler.cpp
 )

 # ==================================================================================================
@@ -68,6 +68,7 @@ target_link_libraries(${TARGET} PUBLIC
        civetweb
        fgviewer_resources
        utils
+        stb
 )

 target_include_directories(${TARGET} PRIVATE ${filamat_SOURCE_DIR}/src)
--- a/libs/fgviewer/include/fgviewer/DebugServer.h
+++ b/libs/fgviewer/include/fgviewer/DebugServer.h
@@ -22,14 +22,17 @@
 #include <utils/CString.h>
 #include <utils/Mutex.h>

+#include <functional>
 #include <unordered_map>
 #include <vector>
+#include <set>

 class CivetServer;

 namespace filament::fgviewer {

 using ViewHandle = uint32_t;
+class WebSocketHandler;

 /**
 * Server-side frame graph debugger.
@@ -39,10 +42,19 @@ using ViewHandle = uint32_t;
 */
 class DebugServer {
 public:
+    using PixelBuffer = std::vector<uint8_t>;
+    using PixelDataFormat = uint32_t;
+    struct Format {
+        static constexpr PixelDataFormat RGB = 0;
+        static constexpr PixelDataFormat RGBA = 1;
+    };
+    using ReadbackRequest = std::function<void(utils::CString,
+            std::function<void(PixelBuffer, uint32_t, uint32_t, PixelDataFormat)>)>;
+
    static std::string_view const kSuccessHeader;
    static std::string_view const kErrorHeader;

-    explicit DebugServer(int port);
+    explicit DebugServer(int port, ReadbackRequest&& readbackRequester);
    ~DebugServer();

    /**
@@ -54,26 +66,37 @@ public:
     * Notifies the debugger that the given view has been deleted.
     */
    void destroyView(ViewHandle h);
-
-    /**
-     * Updates the information for a given view.
-     */
    void update(ViewHandle h, FrameGraphInfo info);

+    void broadcast(const char* data, size_t len);
+    void tick();
+
    bool isReady() const { return mServer; }

 private:
+    void startMonitoring(const utils::CString& resourceName);
+    void stopMonitoring(const utils::CString& resourceName);
+
    CivetServer* mServer;

+    ReadbackRequest mReadbackRequester;
+
    std::unordered_map<ViewHandle, FrameGraphInfo> mViews;
    uint32_t mViewCounter = 0;
    mutable utils::Mutex mViewsMutex;

+    std::set<utils::CString> mMonitoredResources;
+    mutable utils::Mutex mMonitoredResourcesMutex;
+    uint32_t mFrameCount = 0;
+    uint32_t mCurrentMonitoredResourceIndex = 0;
+
    class FileRequestHandler* mFileHandler = nullptr;
    class ApiHandler* mApiHandler = nullptr;
+    class WebSocketHandler* mWebSocketHandler = nullptr;

    friend class FileRequestHandler;
    friend class ApiHandler;
+    friend class WebSocketHandler;
 };

 } // namespace filament::fgviewer
--- a/libs/fgviewer/src/ApiHandler.cpp
+++ b/libs/fgviewer/src/ApiHandler.cpp
@@ -49,6 +49,9 @@ auto const error = [](int line, std::string const& uri) {

 } // anonymous

+ApiHandler::ApiHandler(DebugServer* server) : mServer(server) {
+}
+
 bool ApiHandler::handleGet(CivetServer* server, struct mg_connection* conn) {
    struct mg_request_info const* request = mg_get_request_info(conn);
    std::string const& uri = request->local_uri;
@@ -95,6 +98,35 @@ bool ApiHandler::handleGet(CivetServer* server, struct mg_connection* conn) {
    return error(__LINE__, uri);
 }

+bool ApiHandler::handlePost(CivetServer* server, struct mg_connection* conn) {
+    struct mg_request_info const* request = mg_get_request_info(conn);
+    std::string const& uri = request->local_uri;
+
+    utils::CString resourceName;
+    char resourceNameStr[1024] = {};
+    if (mg_get_var(request->query_string, strlen(request->query_string), "resource", resourceNameStr, sizeof(resourceNameStr)) > 0) {
+        resourceName = resourceNameStr;
+    } else {
+        return error(__LINE__, uri);
+    }
+
+    if (uri.find("/api/monitor/start") == 0) {
+        utils::slog.i << "[fgviewer] API: start monitoring " << resourceName.c_str() << utils::io::endl;
+        mServer->startMonitoring(resourceName);
+        mg_printf(conn, kSuccessHeader.data(), "text/plain");
+        return true;
+    }
+
+    if (uri.find("/api/monitor/stop") == 0) {
+        utils::slog.i << "[fgviewer] API: stop monitoring " << resourceName.c_str() << utils::io::endl;
+        mServer->stopMonitoring(resourceName);
+        mg_printf(conn, kSuccessHeader.data(), "text/plain");
+        return true;
+    }
+
+    return error(__LINE__, uri);
+}
+
 void ApiHandler::updateFrameGraph(ViewHandle view_handle) {
    std::unique_lock const lock(mStatusMutex);
    snprintf(statusFrameGraphId, sizeof(statusFrameGraphId), "%8.8x", view_handle);
--- a/libs/fgviewer/src/ApiHandler.h
+++ b/libs/fgviewer/src/ApiHandler.h
@@ -34,13 +34,11 @@ struct FrameGraphInfo;
 //
 class ApiHandler : public CivetHandler {
 public:
-    explicit ApiHandler(DebugServer* server)
-        : mServer(server) {}
-    ~ApiHandler() = default;
+    explicit ApiHandler(DebugServer* server);
+    bool handleGet(CivetServer* server, struct mg_connection* conn) override;
+    bool handlePost(CivetServer *server, struct mg_connection *conn) override;

-    bool handleGet(CivetServer* server, struct mg_connection* conn);
-
-    void updateFrameGraph(ViewHandle view_handle);
+    void updateFrameGraph(uint32_t viewId);

 private:
    const FrameGraphInfo* getFrameGraphInfo(struct mg_request_info const* request);
--- a/libs/fgviewer/src/DebugServer.cpp
+++ b/libs/fgviewer/src/DebugServer.cpp
@@ -18,13 +18,20 @@
 #include <fgviewer/FrameGraphInfo.h>

 #include "ApiHandler.h"
+#include "WebSocketHandler.h"

 #include <CivetServer.h>

+
 #include <utils/Log.h>
 #include <utils/Mutex.h>
 #include <utils/ostream.h>

+#include <vector>
+#include <fstream>
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#include <stb_image_write.h>
+
 #include <mutex>
 #include <string>
 #include <string_view>
@@ -32,7 +39,7 @@

 // If set to 0, this serves HTML from a resgen resource. Use 1 only during local development, which
 // serves files directly from the source code tree.
-#define SERVE_FROM_SOURCE_TREE 0
+#define SERVE_FROM_SOURCE_TREE 1

 #if SERVE_FROM_SOURCE_TREE

@@ -104,7 +111,8 @@ private:
    DebugServer* mServer;
 };

-DebugServer::DebugServer(int port) {
+DebugServer::DebugServer(int port, ReadbackRequest&& readbackRequester)
+        : mReadbackRequester(std::move(readbackRequester)) {
 #if !SERVE_FROM_SOURCE_TREE
    ASSET_MAP["/index.html"] = {
        .mime = "text/html",
@@ -143,8 +151,10 @@ DebugServer::DebugServer(int port) {

    mFileHandler = new FileRequestHandler(this);
    mApiHandler = new ApiHandler(this);
+    mWebSocketHandler = new WebSocketHandler(this);

    mServer->addHandler("/api", mApiHandler);
+    mServer->addWebSocketHandler("/ws", mWebSocketHandler);
    mServer->addHandler("", mFileHandler);

    slog.i << "[fgviewer] DebugServer listening at http://localhost:" << port << io::endl;
@@ -155,6 +165,7 @@ DebugServer::~DebugServer() {

    delete mFileHandler;
    delete mApiHandler;
+    delete mWebSocketHandler;
    delete mServer;
 }

@@ -189,4 +200,124 @@ void DebugServer::update(ViewHandle h, FrameGraphInfo info) {
    mApiHandler->updateFrameGraph(h);
 }

+void DebugServer::startMonitoring(const utils::CString& resourceName) {
+    slog.i << "[fgviewer] DebugServer: adding " << resourceName.c_str_safe() << " to monitored list." << io::endl;
+    std::unique_lock<utils::Mutex> lock(mMonitoredResourcesMutex);
+    mMonitoredResources.insert(resourceName);
+}
+
+void DebugServer::stopMonitoring(const utils::CString& resourceName) {
+    slog.i << "[fgviewer] DebugServer: removing " << resourceName.c_str_safe() << " from monitored list." << io::endl;
+    std::unique_lock<utils::Mutex> lock(mMonitoredResourcesMutex);
+    mMonitoredResources.erase(resourceName);
+}
+
+void DebugServer::broadcast(const char* data, size_t len) {
+    mWebSocketHandler->broadcast(data, len);
+}
+
+void DebugServer::tick() {
+    static constexpr uint32_t UPDATE_INTERVAL = 100; // Update every 30 frames
+
+    mFrameCount++;
+    if (mFrameCount % UPDATE_INTERVAL != 0) {
+        return;
+    }
+
+    std::unique_lock<utils::Mutex> lock(mMonitoredResourcesMutex);
+    if (mMonitoredResources.empty()) {
+        return;
+    }
+
+    // Cycle through monitored resources
+    auto it = mMonitoredResources.begin();
+    std::advance(it, mCurrentMonitoredResourceIndex % mMonitoredResources.size());
+    utils::CString resourceToUpdate = *it;
+
+    slog.i << "[fgviewer] DebugServer: tick triggering readback for " << resourceToUpdate.c_str_safe() << io::endl;
+
+    mCurrentMonitoredResourceIndex++;
+
+    // Request readback from the engine
+    mReadbackRequester(resourceToUpdate, 
+        [this, resourceName = std::move(resourceToUpdate)](PixelBuffer pixelBuffer, uint32_t width, uint32_t height, PixelDataFormat format) {
+            // Encode to PNG and broadcast
+            if (pixelBuffer.empty()) {
+                slog.w << "[fgviewer] Readback for " << resourceName.c_str_safe() << " failed or returned empty buffer." << io::endl;
+                return;
+            }
+
+            // For simplicity, convert all to RGBA UBYTE for PNG. More robust format handling
+            // would be needed for a production system.
+            PixelBuffer rgbaBuffer;
+            rgbaBuffer.resize(width * height * 4);
+
+            // Simple conversion, assuming incoming is RGBA UBYTE for now, needs real conversion
+            // based on `format` if different formats are supported.
+            // For float textures (e.g., R32F, RGBA16F), a more complex tonemapping or scaling
+            // would be required before saving to 8-bit PNG.
+            // For now, if it's not RGBA UBYTE, we'll just copy, which might result in weird images.
+            if (format == Format::RGBA &&
+                pixelBuffer.size() == width * height * 4) {
+                std::copy(pixelBuffer.begin(), pixelBuffer.end(), rgbaBuffer.begin());
+            } else if (format == Format::RGB &&
+                       pixelBuffer.size() == width * height * 3) {
+                // Convert RGB to RGBA
+                for (size_t i = 0; i < width * height; ++i) {
+                    rgbaBuffer[i * 4 + 0] = pixelBuffer[i * 3 + 0];
+                    rgbaBuffer[i * 4 + 1] = pixelBuffer[i * 3 + 1];
+                    rgbaBuffer[i * 4 + 2] = pixelBuffer[i * 3 + 2];
+                    rgbaBuffer[i * 4 + 3] = 255; // Alpha
+                }
+            } else {
+                // Fallback for unsupported formats or if dimensions mismatch, fill with red.
+                slog.w << "[fgviewer] Unsupported pixel format or size mismatch for PNG conversion: "
+                       << (int)format << ", size: " << pixelBuffer.size() << io::endl;
+                std::fill(rgbaBuffer.begin(), rgbaBuffer.end(), 255);
+            }
+
+            PixelBuffer pngData;
+            auto stb_write_to_vector = [](void* context, void* data, int size) {
+                auto* buffer = static_cast<PixelBuffer*>(context);
+                buffer->insert(buffer->end(), static_cast<uint8_t*>(data),
+                               static_cast<uint8_t*>(data) + size);
+            };
+
+            int success = stbi_write_png_to_func(stb_write_to_vector, &pngData, (int)width, (int)height, 4, rgbaBuffer.data(), (int)width * 4);
+            slog.i << "[fgviewer] stbi_write_png_to_func result: " << success << " (bytes: " << pngData.size() << ")" << io::endl;
+
+            utils::slog.e << "pixel=" << (void*) pixelBuffer.data() << utils::io::endl;
+//            auto x = pixelBuffer.data();
+//            for (size_t i = 0; i < 16; ++i) {
+//                utils::slog.e << "[" << i << "]=" << (int) x[i] << utils::io::endl;
+//            }
+
+
+            if (success && !pngData.empty()) {
+                char const* actualName = resourceName.c_str_safe();
+                size_t actualNameLen = strlen(actualName);
+                {
+                    std::ofstream debugFile("/tmp/filament_debug.png", std::ios::binary);
+                    if (debugFile.is_open()) {
+                        debugFile.write(reinterpret_cast<const char*>(pngData.data()), pngData.size());
+                        debugFile.close();
+                        slog.i << "[fgviewer] Wrote PNG to /tmp/filament_debug.png" << io::endl;
+                    }
+                }
+                // Prefix with resource name + null terminator to identify the texture on the client side
+                std::vector<uint8_t> message;
+                message.reserve(actualNameLen + 1 + pngData.size());
+                message.insert(message.end(), (uint8_t*)actualName, (uint8_t*)actualName + actualNameLen);
+                message.push_back('\0');
+                message.insert(message.end(), pngData.begin(), pngData.end());
+                utils::slog.e << "sending bytes=" << message.size() <<
+                        " resource-size="<< actualNameLen <<  utils::io::endl;
+                mWebSocketHandler->broadcast((const char*)message.data(), message.size());
+            } else {
+                slog.e << "[fgviewer] Failed to encode PNG for " << resourceName.c_str_safe() << io::endl;
+            }
+        }
+    );
+}
+
 } // namespace filament::fgviewer
--- a/libs/fgviewer/src/WebSocketHandler.cpp
+++ b/libs/fgviewer/src/WebSocketHandler.cpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "WebSocketHandler.h"
+
+#include <fgviewer/DebugServer.h>
+#include <utils/Log.h>
+
+namespace filament::fgviewer {
+
+using namespace utils;
+
+WebSocketHandler::WebSocketHandler(DebugServer* server) : mServer(server) {
+}
+
+bool WebSocketHandler::handleConnection(CivetServer* server, const struct mg_connection* conn) {
+    slog.i << "[fgviewer] WebSocket connected." << io::endl;
+    return true;
+}
+
+void WebSocketHandler::handleReadyState(CivetServer* server, struct mg_connection* conn) {
+    std::unique_lock<utils::Mutex> lock(mMutex);
+    mConnections.insert(conn);
+}
+
+bool WebSocketHandler::handleData(CivetServer* server, struct mg_connection* conn, int bits,
+        char* data, size_t data_len) {
+    // For now, we don't expect any data from the client.
+    return true;
+}
+
+void WebSocketHandler::handleClose(CivetServer* server, const struct mg_connection* conn) {
+    slog.i << "[fgviewer] WebSocket disconnected." << io::endl;
+    std::unique_lock<utils::Mutex> lock(mMutex);
+    mConnections.erase(const_cast<struct mg_connection *>(conn));
+}
+
+void WebSocketHandler::broadcast(const char* data, size_t len) {
+    std::unique_lock<utils::Mutex> lock(mMutex);
+    for (auto* conn : mConnections) {
+        mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data, len);
+    }
+}
+
+} // namespace filament::fgviewer
--- a/libs/fgviewer/src/WebSocketHandler.h
+++ b/libs/fgviewer/src/WebSocketHandler.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FGVIEWER_WEBSOCKET_HANDLER_H
+#define FGVIEWER_WEBSOCKET_HANDLER_H
+
+#include <CivetServer.h>
+#include <utils/Mutex.h>
+#include <set>
+
+namespace filament::fgviewer {
+
+class DebugServer;
+
+class WebSocketHandler : public CivetWebSocketHandler {
+public:
+    explicit WebSocketHandler(DebugServer* server);
+
+    bool handleConnection(CivetServer *server, const struct mg_connection *conn) override;
+    void handleReadyState(CivetServer *server, struct mg_connection *conn) override;
+    bool handleData(CivetServer *server, struct mg_connection *conn, int bits, char *data, size_t data_len) override;
+    void handleClose(CivetServer *server, const struct mg_connection *conn) override;
+
+    void broadcast(const char* data, size_t len);
+
+private:
+    DebugServer* mServer;
+    utils::Mutex mMutex;
+    std::set<struct mg_connection *> mConnections;
+};
+
+} // namespace filament::fgviewer
+
+#endif // FGVIEWER_WEBSOCKET_HANDLER_H
--- a/libs/fgviewer/web/api.js
+++ b/libs/fgviewer/web/api.js
@@ -41,6 +41,50 @@ async function fetchFrameGraph(fgid) {
    return fgInfo;
 }

+async function startMonitoring(resourceName) {
+    await fetch(`api/monitor/start?resource=${resourceName}`, { method: 'POST' });
+}
+
+async function stopMonitoring(resourceName) {
+    await fetch(`api/monitor/stop?resource=${resourceName}`, { method: 'POST' });
+}
+
+function connectWebSocket(onImageReceived) {
+    const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+    const socket = new WebSocket(`${protocol}//${window.location.host}/ws`);
+    socket.binaryType = 'arraybuffer';
+
+    socket.onopen = () => {
+        console.log('WebSocket connected to /ws');
+    };
+
+    socket.onmessage = async (event) => {
+        const bytes = new Uint8Array(event.data);
+        const nullTerminatorIndex = bytes.indexOf(0);
+        if (nullTerminatorIndex !== -1) {
+            const resourceName = new TextDecoder().decode(bytes.slice(0, nullTerminatorIndex));
+            console.log(`[fgviewer] ------- Total bytes received for "${resourceName}" (${bytes.length} bytes, null index=${nullTerminatorIndex})`);                       const pngData = bytes.slice(nullTerminatorIndex + 1);
+            console.log(`[fgviewer] Received image for "${resourceName}" -- (${pngData.length} bytes)`);
+            const blob = new Blob([pngData], { type: 'image/png' });
+            const url = URL.createObjectURL(blob);
+            onImageReceived(resourceName, url);
+        } else {
+            console.warn('[fgviewer] Received WebSocket message without null terminator.');
+        }
+    };
+
+    socket.onclose = () => {
+        console.log('WebSocket disconnected. Retrying in 5s...');
+        setTimeout(() => connectWebSocket(onImageReceived), 5000);
+    };
+
+    socket.onerror = (error) => {
+        console.error('WebSocket error:', error);
+    };
+
+    return socket;
+}
+
 const STATUS_LOOP_TIMEOUT = 3000;

 const STATUS_CONNECTED = 1;
--- a/libs/fgviewer/web/app.js
+++ b/libs/fgviewer/web/app.js
@@ -15,8 +15,6 @@
 */

 import {LitElement, html, css, unsafeCSS, nothing} from "https://unpkg.com/lit@2.8.0?module";
-import { graphviz } from "https://cdn.skypack.dev/d3-graphviz@5.1.0";
-import * as d3 from "https://cdn.skypack.dev/d3@7";

 const kUntitledPlaceholder = "Untitled View";

@@ -199,6 +197,7 @@ class FrameGraphSidePanel extends LitElement {
            selectedFrameGraph: {type: String, attribute: 'selected-framegraph'},
            selectedResourceId: {type: Number, attribute: 'selected-resource'},
            viewMode: {type: String, attribute: 'view-mode'},
+            monitoredImages: {type: Object, attribute: false},

            database: {type: Object, state: true},
            framegraphs: {type: Array, state: true},
@@ -244,6 +243,16 @@ class FrameGraphSidePanel extends LitElement {
        }));
    }

+    _handleMonitorChange(e, resourceName) {
+        const eventName = e.target.checked ? 'start-monitoring' : 'stop-monitoring';
+        console.log(`[fgviewer-ui] Checkbox changed for ${resourceName}. Dispatching ${eventName}`);
+        this.dispatchEvent(new CustomEvent(eventName, {
+            detail: resourceName,
+            bubbles: true,
+            composed: true
+        }));
+    }
+
    _findCurrentResource() {
        if (!this.selectedFrameGraph)
            return null;
@@ -297,6 +306,10 @@ class FrameGraphSidePanel extends LitElement {
            if (!currentResource)
                return nothing;

+            const resourceName = currentResource.name;
+            const imageUrl = this.monitoredImages ? this.monitoredImages[resourceName] : null;
+            const isMonitored = this.monitoredImages && this.monitoredImages.hasOwnProperty(resourceName);
+
            return html`
                <menu-section title="${title}">
                    <div class="resource-content">
@@ -315,6 +328,19 @@ class FrameGraphSidePanel extends LitElement {
                            </ul>
                        </div>
                    ` : ''}
+                    <div class="resource-content" style="margin-top: 10px; border-top: 1px solid #444; padding-top: 10px;">
+                        <label>
+                            <input type="checkbox"
+                                   .checked="${isMonitored}"
+                                   @change="${(e) => this._handleMonitorChange(e, resourceName)}">
+                            Monitor Texture
+                        </label>
+                        ${imageUrl ? html`
+                            <div style="margin-top: 10px;">
+                                <img src="${imageUrl}" style="max-width: 100%; border: 1px solid #ccc;">
+                            </div>
+                        ` : nothing}
+                    </div>
                </menu-section>
            `;
        };
@@ -692,6 +718,14 @@ class FrameGraphViewer extends LitElement {
                }
        );

+        connectWebSocket((resourceName, imageUrl) => {
+            // Revoke old URL to avoid memory leaks
+            if (this.monitoredImages[resourceName]) {
+                URL.revokeObjectURL(this.monitoredImages[resourceName]);
+            }
+            this.monitoredImages = { ...this.monitoredImages, [resourceName]: imageUrl };
+        });
+
        let framegraphs = await fetchFrameGraphs();
        this.database = framegraphs;
    }
@@ -709,6 +743,7 @@ class FrameGraphViewer extends LitElement {
        this.selectedFrameGraph = null;
        this.selectedResourceId = -1;
        this.viewMode = VIEW_MODE_TABLE;
+        this.monitoredImages = {};
        this.init();

        this.addEventListener('select-framegraph',
@@ -728,6 +763,25 @@ class FrameGraphViewer extends LitElement {
                this.viewMode = ev.detail;
            }
        );
+
+        this.addEventListener('start-monitoring', (ev) => {
+            const resourceName = ev.detail;
+            console.log(`[fgviewer-app] Received start-monitoring for ${resourceName}. Calling API.`);
+            startMonitoring(resourceName);
+            // Optimistically mark as monitored (with null image initially)
+            this.monitoredImages = { ...this.monitoredImages, [resourceName]: null };
+            console.log('[fgviewer-app] monitoredImages updated:', this.monitoredImages);
+        });
+
+        this.addEventListener('stop-monitoring', (ev) => {
+            const resourceName = ev.detail;
+            console.log(`[fgviewer-app] Received stop-monitoring for ${resourceName}. Calling API.`);
+            stopMonitoring(resourceName);
+            const newImages = { ...this.monitoredImages };
+            delete newImages[resourceName];
+            this.monitoredImages = newImages;
+            console.log('[fgviewer-app] monitoredImages updated:', this.monitoredImages);
+        });
    }

    static get properties() {
@@ -737,6 +791,7 @@ class FrameGraphViewer extends LitElement {
            selectedFrameGraph: {type: String, state: true},
            selectedResourceId: {type: Number, state: true},
            viewMode: {type: String, state: true},
+            monitoredImages: {type: Object, state: true},
        }
    }

@@ -756,7 +811,8 @@ class FrameGraphViewer extends LitElement {
                ?connected="${this.connected}"
                selected-framegraph="${this.selectedFrameGraph}"
                selected-resource="${this.selectedResourceId}"
-                view-mode="${this.viewMode}">
+                view-mode="${this.viewMode}"
+                .monitoredImages="${this.monitoredImages}">
            </framegraph-sidepanel>
            
            <framegraph-table id="table"
--- a/libs/fgviewer/web/index.html
+++ b/libs/fgviewer/web/index.html
@@ -14,6 +14,14 @@
    }
  </style>
  <script src="api.js"></script>
+  <!-- 
+    We load d3 and d3-graphviz via script tags instead of ESM imports in app.js.
+    This is because d3-graphviz relies on viz.js (WASM), and loading it purely via 
+    ESM from CDNs often fails to locate the WASM file or handle the worker correctly.
+  -->
+  <script src="//d3js.org/d3.v7.min.js"></script>
+  <script src="https://unpkg.com/@hpcc-js/wasm@2.20.0/dist/index.min.js"></script>
+  <script src="https://unpkg.com/d3-graphviz@5.6.0/build/d3-graphviz.js"></script>
 </head>
 <body>
 <script src="app.js" type="module"></script>