Add a Renderer API to force skipping frames (#9313)

* Add a Renderer API to force skipping frames

Renderer::skipNextFrames(size_t) can be used to force filament to
pretend the next N frames must be skipped. This is mostly useful for
debugging.

* Add DebugOptions to Settings

We still need to move the "Debug" features of gltf_viewer to this,
but this give us a framework to do it. 
Currently there is one debug option that allows to set a number of
frames to skip.

ViewerGui propose a button to skip 10 frames using this framework

* Update libs/viewer/src/Settings.cpp

Co-authored-by: Powei Feng <powei@google.com>

* Update libs/viewer/src/Settings.cpp

Co-authored-by: Powei Feng <powei@google.com>

---------

Co-authored-by: Powei Feng <powei@google.com>

BUILDING.md

@@ -391,7 +391,7 @@ Alternatively, if you have node installed you can use the
 [live-server](https://www.npmjs.com/package/live-server) package, which automatically refreshes the
 web page when it detects a change.
 
-Each sample app has its own handwritten html file. Additionally the server folder contains assets
+Each sample app has its own handwritten html file. Additionally, the server folder contains assets
 such as meshes, textures, and materials.
 
 ## Running the native samples

android/filament-android/src/main/cpp/Renderer.cpp

@@ -208,3 +208,19 @@ Java_com_google_android_filament_Renderer_nSetVsyncTime(JNIEnv *, jclass,
     Renderer *renderer = (Renderer *) nativeRenderer;
     renderer->setVsyncTime(steadyClockTimeNano);
 }
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_google_android_filament_Renderer_nSkipNextFrames(JNIEnv *, jclass ,
+    jlong nativeRenderer, jint frameCount) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    renderer->skipNextFrames(frameCount);
+}
+
+extern "C"
+JNIEXPORT jint JNICALL
+Java_com_google_android_filament_Renderer_nGetFrameToSkipCount(JNIEnv *, jclass ,
+    jlong nativeRenderer) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    return renderer->getFrameToSkipCount();
+}

android/filament-android/src/main/java/com/google/android/filament/Renderer.java

@@ -732,6 +732,22 @@ public class Renderer {
         nResetUserTime(getNativeObject());
     }
 
+    /**
+     * Requests the next frameCount frames to be skipped. For Debugging.
+     * @param frameCount number of frames to skip.
+     */
+    public void skipNextFrames(int frameCount) {
+        nSkipNextFrames(getNativeObject(), frameCount);
+    }
+
+    /**
+     * Remainder count of frame to be skipped
+     * @return remaining frames to be skipped
+     */
+    public int getFrameToSkipCount() {
+        return nGetFrameToSkipCount(getNativeObject());
+    }
+
     public long getNativeObject() {
         if (mNativeObject == 0) {
             throw new IllegalStateException("Calling method on destroyed Renderer");
@@ -773,4 +789,7 @@ public class Renderer {
             float interval, float headRoomRatio, float scaleRate, int history);
     private static native void nSetClearOptions(long nativeRenderer,
             float r, float g, float b, float a, boolean clear, boolean discard);
+
+    private static native void nSkipNextFrames(long nativeObject, int frameCount);
+    private static native int nGetFrameToSkipCount(long nativeObject);
 }

filament/include/filament/Renderer.h

@@ -648,6 +648,19 @@ public:
      */
     void resetUserTime();
 
+
+    /**
+     * Requests the next frameCount frames to be skipped. For Debugging.
+     * @param frameCount number of frames to skip.
+     */
+    void skipNextFrames(size_t frameCount) const noexcept;
+
+    /**
+     * Remainder count of frame to be skipped
+     * @return remaining frames to be skipped
+     */
+    size_t getFrameToSkipCount() const noexcept;
+
 protected:
     // prevent heap allocation
     ~Renderer() = default;

filament/src/FrameSkipper.cpp

@@ -22,6 +22,8 @@
 #include <utils/debug.h>
 
 #include <algorithm>
+#include <cstdint>
+#include <limits>
 #include <utility>
 
 #include <stddef.h>
@@ -46,10 +48,16 @@ void FrameSkipper::terminate(DriverApi& driver) noexcept {
 }
 
 bool FrameSkipper::shouldRenderFrame(DriverApi& driver) const noexcept {
+
+    if (UTILS_UNLIKELY(mFrameToSkip)) {
+        mFrameToSkip--;
+        return false;
+    }
+
     auto& fences = mDelayedFences;
     if (fences.front()) {
         // Do we have a latency old fence?
-        auto status = driver.getFenceStatus(fences.front());
+        FenceStatus const status = driver.getFenceStatus(fences.front());
         if (UTILS_UNLIKELY(status == FenceStatus::TIMEOUT_EXPIRED)) {
             // The fence hasn't signaled yet, skip this frame
             return false;
@@ -75,4 +83,14 @@ void FrameSkipper::submitFrame(DriverApi& driver) noexcept {
     fences[last] = driver.createFence();
 }
 
+void FrameSkipper::skipNextFrames(size_t frameCount) const noexcept {
+    frameCount = std::min(frameCount, size_t(std::numeric_limits<decltype(mFrameToSkip)>::max()));
+    mFrameToSkip = uint16_t(frameCount);
+}
+
+size_t FrameSkipper::getFrameToSkipCount() const noexcept {
+    return mFrameToSkip;
+}
+
+
 } // namespace filament

filament/src/FrameSkipper.h

@@ -71,10 +71,16 @@ public:
 
     void submitFrame(backend::DriverApi& driver) noexcept;
 
+    // set frameCount frame to report as "skip". For debugging.
+    void skipNextFrames(size_t frameCount) const noexcept;
+    // return remaining number of frame to be skipped
+    size_t getFrameToSkipCount() const noexcept;
+
 private:
     using Container = std::array<backend::Handle<backend::HwFence>, MAX_FRAME_LATENCY>;
     Container mDelayedFences{};
     uint8_t const mLatency;
+    mutable uint16_t mFrameToSkip{};
 };
 
 } // namespace filament

filament/src/Renderer.cpp

@@ -87,6 +87,14 @@ void Renderer::resetUserTime() {
     downcast(this)->resetUserTime();
 }
 
+void Renderer::skipNextFrames(size_t frameCount) const noexcept {
+    downcast(this)->skipNextFrames(frameCount);
+}
+
+size_t Renderer::getFrameToSkipCount() const noexcept {
+    return downcast(this)->getFrameToSkipCount();
+}
+
 void Renderer::setDisplayInfo(const DisplayInfo& info) noexcept {
     downcast(this)->setDisplayInfo(info);
 }

filament/src/details/Renderer.h

@@ -83,10 +83,17 @@ public:
 
     void resetUserTime();
 
+    void skipNextFrames(size_t frameCount) const noexcept {
+        mFrameSkipper.skipNextFrames(frameCount);
+    }
+
+    size_t getFrameToSkipCount() const noexcept {
+        return mFrameSkipper.getFrameToSkipCount();
+    }
+
     // renders a single standalone view. The view must have a a custom rendertarget.
     void renderStandaloneView(FView const* view);
 
-
     void setPresentationTime(int64_t monotonic_clock_ns) const;
 
     void setVsyncTime(uint64_t steadyClockTimeNano) noexcept;

libs/viewer/include/viewer/Settings.h

@@ -51,6 +51,7 @@ struct Settings;
 struct ViewSettings;
 struct LightSettings;
 struct ViewerOptions;
+struct DebugOptions;
 
 enum class ToneMapping : uint8_t {
     LINEAR        = 0,
@@ -88,6 +89,8 @@ void applySettings(Engine* engine, const LightSettings& settings, IndirectLight*
         const utils::Entity* sceneLights, size_t sceneLightCount, LightManager* lm, Scene* scene, View* view);
 void applySettings(Engine* engine, const ViewerOptions& settings, Camera* camera, Skybox* skybox,
         Renderer* renderer);
+void applySettings(Engine* engine, const DebugOptions& settings,
+        Renderer* renderer);
 
 // Creates a new ColorGrading object based on the given settings.
 UTILS_PUBLIC
@@ -246,11 +249,16 @@ struct ViewerOptions {
     bool autoInstancingEnabled = false;
 };
 
+struct DebugOptions {
+    uint16_t skipFrames = 0;
+};
+
 struct Settings {
     ViewSettings view;
     MaterialSettings material;
     LightSettings lighting;
     ViewerOptions viewer;
+    DebugOptions debug;
 };
 
 } // namespace viewer

libs/viewer/src/AutomationEngine.cpp

@@ -201,6 +201,7 @@ void AutomationEngine::applySettings(Engine* engine, const char* json, size_t js
     Camera* camera = &content.view->getCamera();
     Skybox* skybox = content.scene->getSkybox();
     viewer::applySettings(engine, mSettings->viewer, camera, skybox, content.renderer);
+    viewer::applySettings(engine, mSettings->debug, content.renderer);
 }
 
 ColorGrading* AutomationEngine::getColorGrading(Engine* engine) {

libs/viewer/src/Settings.cpp

@@ -37,7 +37,6 @@
 using namespace utils;
 
 namespace filament::viewer {
-
 std::string_view to_string(color::ColorSpace const& colorspace) noexcept {
     using namespace color;
     if (colorspace == Rec709-Linear-D65) {
@@ -544,6 +543,26 @@ static int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, ViewerOp
     return i;
 }
 
+static int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, DebugOptions* out) {
+    CHECK_TOKTYPE(tokens[i], JSMN_OBJECT);
+    int size = tokens[i++].size;
+    for (int j = 0; j < size; ++j) {
+        const jsmntok_t tok = tokens[i];
+        CHECK_KEY(tok);
+        if (compare(tok, jsonChunk, "skipFrames") == 0) {
+            i = parse(tokens, i + 1, jsonChunk, &out->skipFrames);
+        } else {
+            slog.w << "Invalid debug options key: '" << STR(tok, jsonChunk) << "'" << io::endl;
+            i = parse(tokens, i + 1);
+        }
+        if (i < 0) {
+            slog.e << "Invalid debug options value: '" << STR(tok, jsonChunk) << "'" << io::endl;
+            return i;
+        }
+    }
+    return i;
+}
+
 int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, Settings* out) {
     CHECK_TOKTYPE(tokens[i], JSMN_OBJECT);
     int size = tokens[i++].size;
@@ -558,6 +577,8 @@ int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, Settings* out)
             i = parse(tokens, i + 1, jsonChunk, &out->lighting);
         } else if (compare(tok, jsonChunk, "viewer") == 0) {
             i = parse(tokens, i + 1, jsonChunk, &out->viewer);
+        } else if (compare(tok, jsonChunk, "debug") == 0) {
+            i = parse(tokens, i + 1, jsonChunk, &out->debug);
         } else {
             slog.w << "Invalid group key: '" << STR(tok, jsonChunk) << "'" << io::endl;
             i = parse(tokens, i + 1);
@@ -681,6 +702,12 @@ void applySettings(Engine* engine, const ViewerOptions& settings, Camera* camera
     }
 }
 
+void applySettings(Engine* engine, const DebugOptions& settings, Renderer* renderer) {
+    if (!renderer->getFrameToSkipCount()) {
+        renderer->skipNextFrames(settings.skipFrames);
+    }
+}
+
 constexpr ToneMapper* createToneMapper(const ColorGradingSettings& settings) noexcept {
     switch (settings.toneMapping) {
         case ToneMapping::LINEAR: return new LinearToneMapper;
@@ -914,6 +941,12 @@ static std::ostream& operator<<(std::ostream& out, const ViewerOptions& in) {
         << "}";
 }
 
+static std::ostream& operator<<(std::ostream& out, const DebugOptions& in) {
+    return out << "{\n"
+        << "\"skipFrames\": " << (in.skipFrames) << "\n"
+        << "}";
+}
+
 static std::ostream& operator<<(std::ostream& out, const DynamicLightingSettings& in) {
     return out << "{\n"
         << "\"zLightNear\": " << (in.zLightNear) << ",\n"
@@ -950,7 +983,8 @@ static std::ostream& operator<<(std::ostream& out, const Settings& in) {
         << "\"view\": " << (in.view) << ",\n"
         << "\"material\": " << (in.material) << ",\n"
         << "\"lighting\": " << (in.lighting) << ",\n"
-        << "\"viewer\": " << (in.viewer)
+        << "\"viewer\": " << (in.viewer) << ",\n"
+        << "\"debug\": " << (in.debug)
         << "}";
 }
 

libs/viewer/src/ViewerGui.cpp

@@ -1149,6 +1149,13 @@ void ViewerGui::updateUserInterface() {
 #endif
     }
 
+    if (ImGui::CollapsingHeader("Debug Options")) {
+        mSettings.debug.skipFrames = 0;
+        if (ImGui::Button("Skip 10 frames")) {
+            mSettings.debug.skipFrames = 10;
+        }
+    }
+
     colorGradingUI(mSettings, mRangePlot, mCurvePlot, mToneMapPlot);
 
     // At this point, all View settings have been modified,

samples/gltf_viewer.cpp

@@ -1143,6 +1143,10 @@ int main(int argc, char** argv) {
         Skybox* skybox = scene->getSkybox();
         applySettings(engine, app.viewer->getSettings().viewer, &camera, skybox, renderer);
 
+        // FIMXE: This applySettings() is done here instead of in AutomationEngine.cpp because
+        // we need access to the Renderer, which AutomationEngine does not provide.
+        applySettings(engine, app.viewer->getSettings().debug, renderer);
+
         // technically we don't need to do this each frame
         auto& tcm = engine->getTransformManager();
         TransformManager::Instance const& root = tcm.getInstance(app.rootTransformEntity);

web/filament-js/jsbindings.cpp

@@ -623,6 +623,10 @@ class_<Renderer>("Renderer")
         }
         engine->execute();
     }), allow_raw_pointers())
+    .function("getUserTime", &Renderer::getUserTime)
+    .function("resetUserTime", &Renderer::resetUserTime)
+    .function("skipNextFrames", &Renderer::skipNextFrames)
+    .function("getFrameToSkipCount", &Renderer::getFrameToSkipCount)
     .function("_setClearOptions", &Renderer::setClearOptions, allow_raw_pointers())
     .function("getClearOptions", &Renderer::getClearOptions)
     .function("setPresentationTime", &Renderer::setPresentationTime)