Add a way to query and set Display information

Renderer can now be given a DisplayInfo that contains some
important information about the current display. This will be used
for frame-pacing and dynamic-resolution.

This is most relevant on Android, where we can accurately query these
parameters. Added support for that in our samples.


This also sets the minSDK to 19 (including gltf viewer's), since this is
the version we support. We were cheating before by under-reporting our
minSdk.

android/build.gradle

@@ -24,11 +24,9 @@ buildscript {
     }
     def hasVulkan = project.hasProperty("filament_supports_vulkan")
 
-    // Our minSdkVersion is actually 19, we lie and say 14 here so apps don't have
-    // to increase their minSdkVersion unnecessarily. It is however up to them to
-    // ensure they do not initialize Filament on API levels < 19.
+    // Our minSdkVersion is 19.
     ext.versions = [
-        'minSdk': 14,
+        'minSdk': 19,
         'targetSdk': 29,
         'compileSdk': 29,
         'kotlin': '1.3.61',

android/filament-android/src/main/cpp/Renderer.cpp

@@ -144,3 +144,12 @@ Java_com_google_android_filament_Renderer_nResetUserTime(JNIEnv*, jclass, jlong
     Renderer *renderer = (Renderer *) nativeRenderer;
     renderer->resetUserTime();
 }
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_Renderer_nSetDisplayInfo(JNIEnv*, jclass, jlong nativeRenderer,
+        jfloat refreshRate, jlong presentationDeadlineNanos, jlong vsyncOffsetNanos) {
+    Renderer *renderer = (Renderer *) nativeRenderer;
+    renderer->setDisplayInfo({ .refreshRate = refreshRate,
+                               .presentationDeadlineNanos = (uint64_t)presentationDeadlineNanos,
+                               .vsyncOffsetNanos = (uint64_t)vsyncOffsetNanos });
+}

android/filament-android/src/main/java/com/google/android/filament/Renderer.java

@@ -45,6 +45,29 @@ public class Renderer {
     private final Engine mEngine;
     private long mNativeObject;
 
+    /**
+     * Information about the display this renderer is associated to
+     */
+    public static class DisplayInfo {
+        /**
+         * Refresh rate of the display in Hz. Set to 0 for offscreen or turn off frame-pacing.
+         * On Android you can use {@link android.view.Display#getRefreshRate()}.
+         */
+        public float refreshRate = 60.0f;
+
+        /**
+         * How far in advance a buffer must be queued for presentation at a given time in ns
+         * On Android you can use {@link android.view.Display#getPresentationDeadlineNanos()}.
+         */
+        public long presentationDeadlineNanos = 0;
+
+        /**
+         * Offset by which vsyncSteadyClockTimeNano provided in beginFrame() is offset in ns
+         * On Android you can use {@link android.view.Display#getAppVsyncOffsetNanos()}.
+         */
+        public long vsyncOffsetNanos = 0;
+    };
+
     /**
      * Indicates that the <code>dstSwapChain</code> passed into {@link #copyFrame} should be
      * committed after the frame has been copied.
@@ -75,6 +98,14 @@ public class Renderer {
         mNativeObject = nativeRenderer;
     }
 
+    /**
+     * Information about the display this Renderer is associated to. This information is needed
+     * to accurately compute dynamic-resolution scaling and for frame-pacing.
+     */
+    public void setDisplayInfo(@NonNull DisplayInfo info) {
+        nSetDisplayInfo(getNativeObject(), info.refreshRate, info.presentationDeadlineNanos, info.vsyncOffsetNanos);
+    }
+
     /**
      * Gets the {@link Engine} that created this <code>Renderer</code>.
      *
@@ -487,4 +518,6 @@ public class Renderer {
             Object handler, Runnable callback);
     private static native double nGetUserTime(long nativeRenderer);
     private static native void nResetUserTime(long nativeRenderer);
+    private static native void nSetDisplayInfo(long nativeRenderer,
+            float refreshRate, long presentationDeadlineNanos, long vsyncOffsetNanos);
 }

android/filament-android/src/main/java/com/google/android/filament/android/DisplayHelper.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.filament.android;
+
+import android.os.Build;
+import android.view.Display;
+
+import com.google.android.filament.Renderer;
+
+import androidx.annotation.NonNull;
+
+public class DisplayHelper {
+    public static long getAppVsyncOffsetNanos(Display display) {
+        if (Build.VERSION.SDK_INT >= 29) {
+            return display.getAppVsyncOffsetNanos();
+        }
+        return 0;
+    }
+
+    public static long getPresentationDeadlineNanos(Display display) {
+        if (Build.VERSION.SDK_INT >= 29) {
+            return display.getPresentationDeadlineNanos();
+        }
+        return 0;
+    }
+
+    public static long getRefreshPeriodNanos(Display display) {
+        return (long) (1000000000.0 / display.getRefreshRate());
+    }
+
+    public static float getRefreshRate(Display display) {
+        return display.getRefreshRate();
+    }
+
+
+    @NonNull
+    public static Renderer.DisplayInfo getDisplayInfo(Display display, @NonNull Renderer.DisplayInfo info) {
+        info.refreshRate = DisplayHelper.getRefreshRate(display);
+        info.presentationDeadlineNanos = DisplayHelper.getPresentationDeadlineNanos(display);
+        info.vsyncOffsetNanos = DisplayHelper.getAppVsyncOffsetNanos(display);
+        return info;
+    }
+}

android/filament-utils-android/src/main/java/com/google/android/filament/utils/ModelViewer.kt

@@ -21,6 +21,7 @@ import android.view.Surface
 import android.view.SurfaceView
 import android.view.TextureView
 import com.google.android.filament.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 import com.google.android.filament.gltfio.*
 import java.nio.Buffer
@@ -72,6 +73,7 @@ class ModelViewer : android.view.View.OnTouchListener {
     private val cameraManipulator: Manipulator
     private val gestureDetector: GestureDetector
     private val renderer: Renderer
+    private lateinit var surfaceView: SurfaceView
     private var swapChain: SwapChain? = null
     private var assetLoader: AssetLoader
     private var resourceLoader: ResourceLoader
@@ -122,6 +124,7 @@ class ModelViewer : android.view.View.OnTouchListener {
                 .viewport(surfaceView.width, surfaceView.height)
                 .build(Manipulator.Mode.ORBIT)
 
+        this.surfaceView = surfaceView
         gestureDetector = GestureDetector(surfaceView, cameraManipulator)
         uiHelper.renderCallback = SurfaceCallback()
         uiHelper.attachTo(surfaceView)
@@ -273,6 +276,7 @@ class ModelViewer : android.view.View.OnTouchListener {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-gltf-viewer/build.gradle

@@ -26,7 +26,7 @@ android {
     compileSdkVersion versions.compileSdk
     defaultConfig {
         applicationId "com.google.android.filament.gltf"
-        minSdkVersion 16
+        minSdkVersion 19
         targetSdkVersion versions.targetSdk
         missingDimensionStrategy 'functionality', 'full'
     }

android/samples/sample-hello-camera/src/main/java/com/google/android/filament/hellocam/MainActivity.kt

@@ -29,6 +29,7 @@ import androidx.core.app.ActivityCompat
 import com.google.android.filament.*
 import com.google.android.filament.RenderableManager.*
 import com.google.android.filament.VertexBuffer.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 
 import java.nio.ByteBuffer
@@ -375,6 +376,7 @@ class MainActivity : Activity(), ActivityCompat.OnRequestPermissionsResultCallba
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-hello-triangle/src/main/java/com/google/android/filament/hellotriangle/MainActivity.kt

@@ -28,6 +28,7 @@ import com.google.android.filament.*
 import com.google.android.filament.RenderableManager.PrimitiveType
 import com.google.android.filament.VertexBuffer.AttributeType
 import com.google.android.filament.VertexBuffer.VertexAttribute
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
@@ -302,6 +303,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface, uiHelper.swapChainFlags)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-image-based-lighting/src/main/java/com/google/android/filament/ibl/MainActivity.kt

@@ -25,6 +25,7 @@ import android.view.SurfaceView
 import android.view.animation.LinearInterpolator
 
 import com.google.android.filament.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 
 import java.nio.ByteBuffer
@@ -280,6 +281,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-lit-cube/src/main/java/com/google/android/filament/litcube/MainActivity.kt

@@ -28,6 +28,7 @@ import android.view.animation.LinearInterpolator
 import com.google.android.filament.*
 import com.google.android.filament.RenderableManager.*
 import com.google.android.filament.VertexBuffer.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 
 import java.nio.ByteBuffer
@@ -390,6 +391,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-material-builder/src/main/java/com/google/android/filament/material_builder/MainActivity.kt

@@ -25,6 +25,7 @@ import android.view.SurfaceView
 import android.view.animation.LinearInterpolator
 
 import com.google.android.filament.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 import com.google.android.filament.filamat.MaterialBuilder
 
@@ -320,6 +321,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-stream-test/src/main/java/com/google/android/filament/streamtest/MainActivity.kt

@@ -34,6 +34,7 @@ import java.nio.ByteOrder
 import java.nio.channels.Channels
 import android.opengl.*
 import android.view.MotionEvent
+import com.google.android.filament.android.DisplayHelper
 
 
 class MainActivity : Activity(), ActivityCompat.OnRequestPermissionsResultCallback {
@@ -362,6 +363,7 @@ class MainActivity : Activity(), ActivityCompat.OnRequestPermissionsResultCallba
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-texture-view/src/main/java/com/google/android/filament/textureview/MainActivity.kt

@@ -28,6 +28,7 @@ import com.google.android.filament.*
 import com.google.android.filament.RenderableManager.PrimitiveType
 import com.google.android.filament.VertexBuffer.AttributeType
 import com.google.android.filament.VertexBuffer.VertexAttribute
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
@@ -302,6 +303,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface, uiHelper.swapChainFlags)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(textureView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-textured-object/src/main/java/com/google/android/filament/textured/MainActivity.kt

@@ -25,6 +25,7 @@ import android.view.SurfaceView
 import android.view.animation.LinearInterpolator
 
 import com.google.android.filament.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.utils.*
 import com.google.android.filament.android.UiHelper
 
@@ -302,6 +303,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

android/samples/sample-transparent-view/src/main/java/com/google/android/filament/transparentrendering/MainActivity.kt

@@ -32,6 +32,7 @@ import android.widget.TextView
 import com.google.android.filament.*
 import com.google.android.filament.RenderableManager.*
 import com.google.android.filament.VertexBuffer.*
+import com.google.android.filament.android.DisplayHelper
 import com.google.android.filament.android.UiHelper
 
 import java.nio.ByteBuffer
@@ -320,6 +321,7 @@ class MainActivity : Activity() {
         override fun onNativeWindowChanged(surface: Surface) {
             swapChain?.let { engine.destroySwapChain(it) }
             swapChain = engine.createSwapChain(surface, uiHelper.swapChainFlags)
+            renderer.setDisplayInfo(DisplayHelper.getDisplayInfo(surfaceView.display, Renderer.DisplayInfo()))
         }
 
         override fun onDetachedFromSurface() {

filament/include/filament/Renderer.h

@@ -68,11 +68,31 @@ class PixelBufferDescriptor;
  */
 class UTILS_PUBLIC Renderer : public FilamentAPI {
 public:
-     /**
-      * Get the Engine that created this Renderer.
-      *
-      * @return A pointer to the Engine instance this Renderer is associated to.
-      */
+
+    struct DisplayInfo {
+        // refresh-rate of the display in Hz. set to 0 for offscreen or turn off frame-pacing.
+        float refreshRate = 60.0f;
+
+        // how far in advance a buffer must be queued for presentation at a given time in ns
+        uint64_t presentationDeadlineNanos = 0;
+
+        // offset by which vsyncSteadyClockTimeNano provided in beginFrame() is offset in ns
+        uint64_t vsyncOffsetNanos = 0;
+    };
+
+    /**
+     * Information about the display this Renderer is associated to. This information is needed
+     * to accurately compute dynamic-resolution scaling and for frame-pacing.
+     *
+     * @param info
+     */
+    void setDisplayInfo(const DisplayInfo& info) noexcept;
+
+    /**
+     * Get the Engine that created this Renderer.
+     *
+     * @return A pointer to the Engine instance this Renderer is associated to.
+     */
     Engine* getEngine() noexcept;
 
     /**

filament/src/Renderer.cpp

@@ -688,9 +688,15 @@ void FRenderer::copyFrame(FSwapChain* dstSwapChain, filament::Viewport const& ds
 
 bool FRenderer::beginFrame(FSwapChain* swapChain, uint64_t vsyncSteadyClockTimeNano,
         backend::FrameFinishedCallback callback, void* user) {
+    assert(swapChain);
+
     SYSTRACE_CALL();
 
-    assert(swapChain);
+    // get the timestamp as soon as possible
+    using namespace std::chrono;
+    const steady_clock::time_point now{ steady_clock::now() };
+    const steady_clock::time_point vsync{steady_clock::duration(vsyncSteadyClockTimeNano) };
+    const time_point<steady_clock> vsyncTp(vsyncSteadyClockTimeNano ? vsync : now);
 
     mFrameId++;
 
@@ -709,9 +715,7 @@ bool FRenderer::beginFrame(FSwapChain* swapChain, uint64_t vsyncSteadyClockTimeN
     // NOTE: this makes synchronous calls to the driver
     driver.updateStreams(&driver);
 
-    int64_t timestamp = vsyncSteadyClockTimeNano ?
-            vsyncSteadyClockTimeNano : std::chrono::steady_clock::now().time_since_epoch().count();
-    driver.beginFrame(timestamp, mFrameId, callback, user);
+    driver.beginFrame(vsyncTp.time_since_epoch().count(), mFrameId, callback, user);
 
     if (!mFrameSkipper.beginFrame()) {
         driver.endFrame(mFrameId);
@@ -724,7 +728,7 @@ bool FRenderer::beginFrame(FSwapChain* swapChain, uint64_t vsyncSteadyClockTimeN
     mFrameInfoManager.beginFrame(mFrameId);
 
     // latch the frame time
-    std::chrono::duration<double> time{ getUserTime() };
+    std::chrono::duration<double> time(vsync - mUserEpoch);
     float h = float(time.count());
     float l = float(time.count() - h);
     mShaderUserTime = { h, l, 0, 0 };
@@ -875,4 +879,8 @@ void Renderer::resetUserTime() {
     upcast(this)->resetUserTime();
 }
 
+void Renderer::setDisplayInfo(const DisplayInfo& info) noexcept {
+    upcast(this)->setDisplayInfo(info);
+}
+
 } // namespace filament

filament/src/details/Renderer.h

@@ -91,6 +91,10 @@ public:
     // Clean-up everything, this is typically called when the client calls Engine::destroyRenderer()
     void terminate(FEngine& engine);
 
+    void setDisplayInfo(DisplayInfo const& info) noexcept {
+        mDisplayInfo = info;
+    }
+
 private:
     friend class Renderer;
     using Command = RenderPass::Command;
@@ -154,6 +158,7 @@ private:
     bool mIsRGB8Supported : 1;
     Epoch mUserEpoch;
     math::float4 mShaderUserTime{};
+    DisplayInfo mDisplayInfo;
 
     // per-frame arena for this Renderer
     LinearAllocatorArena& mPerRenderPassArena;