From 613ae56cc10720613eb9a0e228dc7d84ab0e242d Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Fri, 7 May 2021 14:02:51 -0700 Subject: [PATCH] Add HDRLoader for JVM clients. This adds support for decoding HDR files, dropping them into the remote web page, and sending them over the wire. However the decoded results are not yet passed into IBLPrefilterContext, that will be subsequent PR. Rename KtxLoader to KTXLoader. The remote UI has been tweaked a bit, so that it shows a more helpful message when disconnected. --- android/filament-utils-android/CMakeLists.txt | 14 ++- .../src/main/cpp/HDRLoader.cpp | 86 +++++++++++++++++++ .../src/main/cpp/Utils.cpp | 26 ++++-- .../android/filament/utils/HDRLoader.kt | 50 +++++++++++ .../utils/{KtxLoader.kt => KTXLoader.kt} | 6 +- .../android/filament/gltf/MainActivity.kt | 18 +++- .../sample-textured-object/build.gradle | 2 +- web/samples/remote.html | 47 ++++------ 8 files changed, 206 insertions(+), 43 deletions(-) create mode 100644 android/filament-utils-android/src/main/cpp/HDRLoader.cpp create mode 100644 android/filament-utils-android/src/main/java/com/google/android/filament/utils/HDRLoader.kt rename android/filament-utils-android/src/main/java/com/google/android/filament/utils/{KtxLoader.kt => KTXLoader.kt} (92%) diff --git a/android/filament-utils-android/CMakeLists.txt b/android/filament-utils-android/CMakeLists.txt index 4cd4cd5e4e..d0e014ed48 100644 --- a/android/filament-utils-android/CMakeLists.txt +++ b/android/filament-utils-android/CMakeLists.txt @@ -21,15 +21,16 @@ add_library(civetweb STATIC IMPORTED) set_target_properties(civetweb PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libcivetweb.a) -include_directories(${FILAMENT_DIR}/include - .. - ../../libs/utils/include) +add_library(iblprefilter STATIC IMPORTED) +set_target_properties(iblprefilter PROPERTIES IMPORTED_LOCATION + ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libfilament-iblprefilter.a) set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libfilament-utils-jni.map") add_library(filament-utils-jni SHARED src/main/cpp/AutomationEngine.cpp src/main/cpp/Bookmark.cpp + src/main/cpp/HDRLoader.cpp src/main/cpp/Utils.cpp src/main/cpp/Manipulator.cpp src/main/cpp/RemoteServer.cpp @@ -38,6 +39,12 @@ add_library(filament-utils-jni SHARED ../common/NioUtils.cpp ) +target_include_directories(filament-utils-jni PRIVATE + ${FILAMENT_DIR}/include + .. + ../../third_party/stb + ../../libs/utils/include) + set_target_properties(filament-utils-jni PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/libfilament-utils-jni.symbols) @@ -46,6 +53,7 @@ target_link_libraries(filament-utils-jni gltfio-jni civetweb camutils + iblprefilter image viewer ) diff --git a/android/filament-utils-android/src/main/cpp/HDRLoader.cpp b/android/filament-utils-android/src/main/cpp/HDRLoader.cpp new file mode 100644 index 0000000000..0cce8784de --- /dev/null +++ b/android/filament-utils-android/src/main/cpp/HDRLoader.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include + +#include "common/NioUtils.h" + +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_HDR + +#include + +using namespace filament; +using namespace utils; + +using PixelBufferDescriptor = Texture::PixelBufferDescriptor; + +jlong nCreateHDRTexture(JNIEnv* env, jclass, + jlong nativeEngine, jobject javaBuffer, jint remaining, jint internalFormat) { + + Engine* engine = (Engine*) nativeEngine; + AutoBuffer buffer(env, javaBuffer, remaining); + Texture::InternalFormat textureFormat = (Texture::InternalFormat) internalFormat; + + auto dataPtr = (const stbi_uc*) buffer.getData(); + const size_t byteCount = buffer.getSize(); + + int width, height, nchan; + + float* const floatsPtr = stbi_loadf_from_memory(dataPtr, byteCount, &width, &height, &nchan, 3); + if (floatsPtr == nullptr) { + slog.e << "Unable to decode HDR image: " << stbi_failure_reason() << io::endl; + return 0; + } + + Texture* texture = Texture::Builder() + .width(width) + .height(height) + .levels(0xff) + .sampler(Texture::Sampler::SAMPLER_2D) + .format(textureFormat) + .build(*engine); + + if (texture == nullptr) { + slog.e << "Unable to create Filament Texture from HDR image." << io::endl; + stbi_image_free(floatsPtr); + return 0; + } + + PixelBufferDescriptor::Callback freeCallback = [](void* buf, size_t, void*) { + stbi_image_free(buf); + }; + + PixelBufferDescriptor pbd( + (void const* ) floatsPtr, + width * height * 3 * sizeof(float), + PixelBufferDescriptor::PixelDataFormat::RGB, + PixelBufferDescriptor::PixelDataType::FLOAT, + freeCallback); + + // Note that the setImage call could fail (e.g. due to an invalid combination of internal format + // and PixelDataFormat) but there is no way of detecting such a failure. + texture->setImage(*engine, 0, std::move(pbd)); + + texture->generateMipmaps(*engine); + + return (jlong) texture; +} diff --git a/android/filament-utils-android/src/main/cpp/Utils.cpp b/android/filament-utils-android/src/main/cpp/Utils.cpp index 1e30099568..77ed1b6520 100644 --- a/android/filament-utils-android/src/main/cpp/Utils.cpp +++ b/android/filament-utils-android/src/main/cpp/Utils.cpp @@ -28,7 +28,10 @@ using namespace filament; using namespace filament::math; using namespace image; -static jlong nCreateTexture(JNIEnv* env, jclass, +jlong nCreateHDRTexture(JNIEnv* env, jclass, + jlong nativeEngine, jobject javaBuffer, jint remaining, jint internalFormat); + +static jlong nCreateKTXTexture(JNIEnv* env, jclass, jlong nativeEngine, jobject javaBuffer, jint remaining, jboolean srgb) { Engine* engine = (Engine*) nativeEngine; AutoBuffer buffer(env, javaBuffer, remaining); @@ -80,15 +83,26 @@ JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void*) { return -1; } - jclass c = env->FindClass("com/google/android/filament/utils/KtxLoader"); - if (c == nullptr) return JNI_ERR; + int rc; - static const JNINativeMethod methods[] = { - {"nCreateTexture", "(JLjava/nio/Buffer;IZ)J", reinterpret_cast(nCreateTexture)}, + // KTXLoader + jclass ktxloaderClass = env->FindClass("com/google/android/filament/utils/KTXLoader"); + if (ktxloaderClass == nullptr) return JNI_ERR; + static const JNINativeMethod ktxMethods[] = { + {"nCreateKTXTexture", "(JLjava/nio/Buffer;IZ)J", reinterpret_cast(nCreateKTXTexture)}, {"nCreateIndirectLight", "(JLjava/nio/Buffer;IZ)J", reinterpret_cast(nCreateIndirectLight)}, {"nCreateSkybox", "(JLjava/nio/Buffer;IZ)J", reinterpret_cast(nCreateSkybox)}, }; - int rc = env->RegisterNatives(c, methods, sizeof(methods) / sizeof(JNINativeMethod)); + rc = env->RegisterNatives(ktxloaderClass, ktxMethods, sizeof(ktxMethods) / sizeof(JNINativeMethod)); + if (rc != JNI_OK) return rc; + + // HDRLoader + jclass hdrloaderClass = env->FindClass("com/google/android/filament/utils/HDRLoader"); + if (hdrloaderClass == nullptr) return JNI_ERR; + static const JNINativeMethod hdrMethods[] = { + {"nCreateHDRTexture", "(JLjava/nio/Buffer;II)J", reinterpret_cast(nCreateHDRTexture)}, + }; + rc = env->RegisterNatives(hdrloaderClass, hdrMethods, sizeof(hdrMethods) / sizeof(JNINativeMethod)); if (rc != JNI_OK) return rc; return JNI_VERSION_1_6; diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/HDRLoader.kt b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/HDRLoader.kt new file mode 100644 index 0000000000..e784e7fffa --- /dev/null +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/HDRLoader.kt @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.filament.utils + +import com.google.android.filament.Engine +import com.google.android.filament.Texture + +import java.nio.Buffer + +/** + * Utility for decoding an HDR file and producing a Filament texture. + */ +object HDRLoader { + class Options { + var desiredFormat = Texture.InternalFormat.RGB16F + } + + /** + * Consumes the content of an HDR file and produces a [Texture] object. + * + * @param engine Gets passed to the builder. + * @param buffer The content of the HDR File. + * @param options Loader options. + * @return The resulting Filament texture, or null on failure. + */ + fun createTexture(engine: Engine, buffer: Buffer, options: Options = Options()): Texture? { + val nativeEngine = engine.nativeObject + val nativeTexture = nCreateHDRTexture(nativeEngine, buffer, buffer.remaining(), options.desiredFormat.ordinal) + if (nativeTexture == 0L) { + return null; + } + return Texture(nativeTexture) + } + + private external fun nCreateHDRTexture(nativeEngine: Long, buffer: Buffer, remaining: Int, format: Int): Long +} \ No newline at end of file diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/KtxLoader.kt b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/KTXLoader.kt similarity index 92% rename from android/filament-utils-android/src/main/java/com/google/android/filament/utils/KtxLoader.kt rename to android/filament-utils-android/src/main/java/com/google/android/filament/utils/KTXLoader.kt index 328a836139..e71e7b17d3 100644 --- a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/KtxLoader.kt +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/KTXLoader.kt @@ -29,7 +29,7 @@ import java.nio.Buffer * KTX is a simple container format that makes it easy to bundle miplevels and cubemap faces * into a single file. */ -object KtxLoader { +object KTXLoader { class Options { var srgb = false } @@ -44,7 +44,7 @@ object KtxLoader { */ fun createTexture(engine: Engine, buffer: Buffer, options: Options = Options()): Texture { val nativeEngine = engine.nativeObject - val nativeTexture = nCreateTexture(nativeEngine, buffer, buffer.remaining(), options.srgb) + val nativeTexture = nCreateKTXTexture(nativeEngine, buffer, buffer.remaining(), options.srgb) return Texture(nativeTexture) } @@ -76,7 +76,7 @@ object KtxLoader { return Skybox(nativeSkybox) } - private external fun nCreateTexture(nativeEngine: Long, buffer: Buffer, remaining: Int, srgb: Boolean): Long + private external fun nCreateKTXTexture(nativeEngine: Long, buffer: Buffer, remaining: Int, srgb: Boolean): Long private external fun nCreateIndirectLight(nativeEngine: Long, buffer: Buffer, remaining: Int, srgb: Boolean): Long private external fun nCreateSkybox(nativeEngine: Long, buffer: Buffer, remaining: Int, srgb: Boolean): Long } \ No newline at end of file diff --git a/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt b/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt index f61c8e0648..0b6f6e4dae 100644 --- a/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt +++ b/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt @@ -115,11 +115,11 @@ class MainActivity : Activity() { val scene = modelViewer.scene val ibl = "default_env" readCompressedAsset("envs/$ibl/${ibl}_ibl.ktx").let { - scene.indirectLight = KtxLoader.createIndirectLight(engine, it) + scene.indirectLight = KTXLoader.createIndirectLight(engine, it) scene.indirectLight!!.intensity = 30_000.0f } readCompressedAsset("envs/$ibl/${ibl}_skybox.ktx").let { - scene.skybox = KtxLoader.createSkybox(engine, it) + scene.skybox = KTXLoader.createSkybox(engine, it) } } @@ -156,6 +156,18 @@ class MainActivity : Activity() { } } + private suspend fun loadHdr(message: RemoteServer.ReceivedMessage) { + withContext(Dispatchers.Main) { + val texture = HDRLoader.createTexture(modelViewer.engine, message.buffer) + if (texture == null) { + setStatusText("Could not decode HDR file.") + } else { + // TODO: Add Java bindings for IBLPrefilterContext and use them here. + setStatusText("Successfully decoded HDR file.") + } + } + } + private suspend fun loadZip(message: RemoteServer.ReceivedMessage) { // To alleviate memory pressure, remove the old model before deflating the zip. withContext(Dispatchers.Main) { @@ -266,6 +278,8 @@ class MainActivity : Activity() { CoroutineScope(Dispatchers.IO).launch { if (message.label.endsWith(".zip")) { loadZip(message) + } else if (message.label.endsWith(".hdr")) { + loadHdr(message) } else { loadGlb(message) } diff --git a/android/samples/sample-textured-object/build.gradle b/android/samples/sample-textured-object/build.gradle index ec37d9fcce..664398adc2 100644 --- a/android/samples/sample-textured-object/build.gradle +++ b/android/samples/sample-textured-object/build.gradle @@ -39,5 +39,5 @@ dependencies { implementation deps.kotlin implementation project(':filament-android') implementation project(':gltfio-android') - implementation project(':filament-utils-android') // required for KtxLoader + implementation project(':filament-utils-android') // required for KTXLoader } diff --git a/web/samples/remote.html b/web/samples/remote.html index 13fce5157f..b1af00f1c4 100644 --- a/web/samples/remote.html +++ b/web/samples/remote.html @@ -26,20 +26,6 @@ body { height: 100%; } -.status-area { - display: flex; - justify-content: center; - align-items: center; - width: 100%; - height: 100%; - max-width: 640px; - max-height: 32px; - background: burlywood; - border: solid 2px black; - border-bottom: none; - font-size: 12px; -} - .connection-settings { display: flex; justify-content: center; @@ -102,8 +88,7 @@ canvas { } .dropbox-area p { text-align: center; } -.instructions-area p { margin: 4px; } -.instructions-area { margin-bottom: 12px; } +.instructions-area { margin-top: 12px; } a { text-decoration: none; } a:visited { color: rgb(26, 65, 78); } .bad { background: lightcoral; } @@ -118,15 +103,9 @@ a:visited { color: rgb(26, 65, 78); } -
Disconnected
-
-
-

Drop a glb or a zip file here.

-
-
+

Disconnected.

-

adb forward tcp:8082 tcp:8082

@@ -252,7 +231,6 @@ class App { this.connectionUrl = document.getElementById("connection-url"); this.dropbox = document.getElementById("dropbox"); - this.status = document.getElementById("status"); this.canvas = document.getElementsByTagName("canvas")[0]; const engine = this.engine = Filament.Engine.create(this.canvas); @@ -339,7 +317,8 @@ class App { const file = event.dataTransfer.items[0].getAsFile(); const is_glb = file.name.match(/\.(glb)$/i); const is_zip = file.name.match(/\.(zip)$/i); - if (!is_glb && !is_zip) return; + const is_hdr = file.name.match(/\.(hdr)$/i); + if (!is_glb && !is_zip && !is_hdr) return; const files = event.dataTransfer.files; ([...files]).forEach(upload); }, false); @@ -373,9 +352,21 @@ class App { } updateDom() { - const connected = this.connection.isConnected(); - this.status.innerHTML = connected ? "Connected" : "Disconnected"; - this.status.style.backgroundColor = connected ? "#45d48d" : "burlywood"; + const instructions = document.getElementById("instructions"); + if (this.connection.isConnected()) { + instructions.style.visibility = "hidden"; + this.dropbox.innerHTML = `
+

Connected.

+

Drop a glb, zip, or hdr file here.

+
`; + } else { + instructions.style.visibility = "visible"; + this.dropbox.innerHTML = `
+

Disconnected.

+

Ensure app is active and port forwarding is enabled.

+

adb forward tcp:8082 tcp:8082

+
`; + } } startConnection() {