better GPU timings on Android

On drivers that don't implement timer queries properly, we use
fences instead, however we can't use a fence to determine the start
time of the GPU work (only the end time). To workaround this we
use the last call to glFlush() as a proxy.
At least on Pixel it works very well.

We also remove all mid-frame calls to flush(), we did this to kick
cpu work early, but it's not as efficient as overlapping the work
with the previous frame, it increases latency a bit though.

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -2741,11 +2741,13 @@ void OpenGLDriver::flush(int) {
     if (!gl.bugs.disable_glFlush) {
         glFlush();
     }
+    mTimerQueryImpl->flush();
 }
 
 void OpenGLDriver::finish(int) {
     DEBUG_MARKER()
     glFinish();
+    mTimerQueryImpl->flush();
     executeGpuCommandsCompleteOps();
     executeFrameBeginsOps();
     // since we executed a glFinish(), all pending tasks should be done

filament/backend/src/opengl/OpenGLDriver.h

@@ -125,7 +125,7 @@ public:
 
     struct GLTimerQuery : public backend::HwTimerQuery {
         struct State {
-            uint64_t elapsed = 0;
+            std::atomic<uint64_t> elapsed{};
             std::atomic_bool available{};
         };
         struct {

filament/backend/src/opengl/TimerQuery.cpp

@@ -39,6 +39,9 @@ TimerQueryNative::TimerQueryNative(OpenGLContext& context)
 
 TimerQueryNative::~TimerQueryNative() = default;
 
+void TimerQueryNative::flush() {
+}
+
 void TimerQueryNative::beginTimeElapsedQuery(GLTimerQuery* query) {
     gl.beginQuery(GL_TIME_ELAPSED, query->gl.query);
     CHECK_GL_ERROR(utils::slog.e)
@@ -107,32 +110,44 @@ void TimerQueryFence::enqueue(TimerQueryFence::Job&& job) {
     mCondition.notify_one();
 }
 
+void TimerQueryFence::flush() {
+    // Use calls to flush() as a proxy for when the GPU work started.
+    GLTimerQuery* query = mActiveQuery;
+    if (query) {
+        auto now = clock::now().time_since_epoch().count();
+        query->gl.emulation->elapsed.store(now, std::memory_order_relaxed);
+        //SYSTRACE_CONTEXT();
+        //SYSTRACE_ASYNC_BEGIN("gpu", query->gl.query);
+    }
+}
+
 void TimerQueryFence::beginTimeElapsedQuery(GLTimerQuery* query) {
-    Platform::Fence* fence = mPlatform.createFence();
-    if (!query->gl.emulation) {
+    assert(!mActiveQuery);
+    // We can't use a fence to figure out when a GPU operation starts (only when it finishes)
+    // so instead, we use when glFlush() was issued as a proxy.
+    if (UTILS_UNLIKELY(!query->gl.emulation)) {
         query->gl.emulation = std::make_shared<GLTimerQuery::State>();
     }
     query->gl.emulation->available.store(false);
-    std::weak_ptr<GLTimerQuery::State> weak = query->gl.emulation;
-    push([&platform = mPlatform, fence, weak]() {
-        auto emulation = weak.lock();
-        if (emulation) {
-            platform.waitFence(fence, FENCE_WAIT_FOR_EVER);
-            emulation->elapsed = clock::now().time_since_epoch().count();
-        }
-        platform.destroyFence(fence);
-    });
+    mActiveQuery = query;
 }
 
 void TimerQueryFence::endTimeElapsedQuery(GLTimerQuery* query) {
+    assert(mActiveQuery);
     Platform::Fence* fence = mPlatform.createFence();
     std::weak_ptr<GLTimerQuery::State> weak = query->gl.emulation;
+    mActiveQuery = nullptr;
+    //uint32_t cookie = cookie = query->gl.query;
     push([&platform = mPlatform, fence, weak]() {
         auto emulation = weak.lock();
         if (emulation) {
             platform.waitFence(fence, FENCE_WAIT_FOR_EVER);
-            emulation->elapsed = clock::now().time_since_epoch().count() - emulation->elapsed;
+            auto now = clock::now().time_since_epoch().count();
+            auto then = emulation->elapsed.load(std::memory_order_relaxed);
+            emulation->elapsed.store(now - then, std::memory_order_relaxed);
             emulation->available.store(true);
+            //SYSTRACE_CONTEXT();
+            //SYSTRACE_ASYNC_END("gpu", cookie);
         }
         platform.destroyFence(fence);
     });
@@ -152,6 +167,9 @@ TimerQueryFallback::TimerQueryFallback() = default;
 
 TimerQueryFallback::~TimerQueryFallback() = default;
 
+void TimerQueryFallback::flush() {
+}
+
 void TimerQueryFallback::beginTimeElapsedQuery(TimerQueryInterface::GLTimerQuery* query) {
     if (!query->gl.emulation) {
         query->gl.emulation = std::make_shared<GLTimerQuery::State>();

filament/backend/src/opengl/TimerQuery.h

@@ -41,6 +41,7 @@ protected:
 
 public:
     virtual ~TimerQueryInterface();
+    virtual void flush() = 0;
     virtual void beginTimeElapsedQuery(GLTimerQuery* query) = 0;
     virtual void endTimeElapsedQuery(GLTimerQuery* query) = 0;
     virtual bool queryResultAvailable(GLTimerQuery* query) = 0;
@@ -52,6 +53,7 @@ public:
     explicit TimerQueryNative(OpenGLContext& context);
     ~TimerQueryNative() override;
 private:
+    void flush() override;
     void beginTimeElapsedQuery(GLTimerQuery* query) override;
     void endTimeElapsedQuery(GLTimerQuery* query) override;
     bool queryResultAvailable(GLTimerQuery* query) override;
@@ -65,6 +67,7 @@ public:
     ~TimerQueryFence() override;
 private:
     using Job = std::function<void()>;
+    void flush() override;
     void beginTimeElapsedQuery(GLTimerQuery* query) override;
     void endTimeElapsedQuery(GLTimerQuery* query) override;
     bool queryResultAvailable(GLTimerQuery* query) override;
@@ -82,6 +85,7 @@ private:
     mutable utils::Condition mCondition;
     std::vector<Job> mQueue;
     bool mExitRequested = false;
+    GLTimerQuery* mActiveQuery = nullptr;
 };
 
 class TimerQueryFallback : public TimerQueryInterface {
@@ -89,6 +93,7 @@ public:
     explicit TimerQueryFallback();
     ~TimerQueryFallback() override;
 private:
+    void flush() override;
     void beginTimeElapsedQuery(GLTimerQuery* query) override;
     void endTimeElapsedQuery(GLTimerQuery* query) override;
     bool queryResultAvailable(GLTimerQuery* query) override;

filament/src/Renderer.cpp

@@ -247,7 +247,6 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) {
         // TODO: use the framegraph for the shadow passes
         RenderPass shadowMapPass = pass;
         view.renderShadowMaps(engine, driver, shadowMapPass);
-        driver.flush(); // Kick the GPU since we're done with this render target
         engine.flush(); // Wake-up the driver thread
     }