From 9ceb3b016cde2c4759924ec9563541bb827a9e1b Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Tue, 2 Mar 2021 19:28:32 -0800 Subject: [PATCH] sample-gltf-viewer: add drag-and-drop feature. This adds a WebSockets server to filament-utils-android. Unlike my first attempt (#3599), this does not use a piping server and is therefore much faster to use over a LAN. It also has lower user friction because there is no need to touch a button in the app or scan a QR code. --- CMakeLists.txt | 2 +- android/filament-utils-android/CMakeLists.txt | 6 + .../src/main/cpp/RemoteServer.cpp | 70 ++++++++ .../android/filament/utils/RemoteServer.java | 61 +++++++ .../samples/sample-gltf-viewer/build.gradle | 1 + .../android/filament/gltf/MainActivity.kt | 100 ++++++++++++ libs/viewer/CMakeLists.txt | 4 +- libs/viewer/include/viewer/AutomationSpec.h | 2 +- libs/viewer/include/viewer/RemoteServer.h | 61 +++++++ libs/viewer/src/RemoteServer.cpp | 152 ++++++++++++++++++ third_party/civetweb/tnt/CMakeLists.txt | 2 + 11 files changed, 458 insertions(+), 3 deletions(-) create mode 100644 android/filament-utils-android/src/main/cpp/RemoteServer.cpp create mode 100644 android/filament-utils-android/src/main/java/com/google/android/filament/utils/RemoteServer.java create mode 100644 libs/viewer/include/viewer/RemoteServer.h create mode 100644 libs/viewer/src/RemoteServer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ab228aee29..09e8aaf1ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -584,6 +584,7 @@ add_subdirectory(${LIBRARIES}/utils) add_subdirectory(${LIBRARIES}/viewer) add_subdirectory(${FILAMENT}/filament) add_subdirectory(${FILAMENT}/shaders) +add_subdirectory(${EXTERNAL}/civetweb/tnt) add_subdirectory(${EXTERNAL}/hat-trie/tnt) add_subdirectory(${EXTERNAL}/imgui/tnt) add_subdirectory(${EXTERNAL}/robin-map/tnt) @@ -606,7 +607,6 @@ if (FILAMENT_BUILD_FILAMAT OR IS_HOST_PLATFORM) # the material debugger requires filamat if (FILAMENT_ENABLE_MATDBG OR IS_HOST_PLATFORM) - add_subdirectory(${EXTERNAL}/civetweb/tnt) add_subdirectory(${LIBRARIES}/matdbg) endif() endif() diff --git a/android/filament-utils-android/CMakeLists.txt b/android/filament-utils-android/CMakeLists.txt index 1e003e2d6f..9c9e45d5b7 100644 --- a/android/filament-utils-android/CMakeLists.txt +++ b/android/filament-utils-android/CMakeLists.txt @@ -16,6 +16,10 @@ add_library(viewer STATIC IMPORTED) set_target_properties(viewer PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libviewer.a) +add_library(civetweb STATIC IMPORTED) +set_target_properties(civetweb PROPERTIES IMPORTED_LOCATION + ${FILAMENT_DIR}/third_party/${ANDROID_ABI}/libcivetweb.a) + include_directories(${FILAMENT_DIR}/include .. ../../libs/utils/include) @@ -27,6 +31,7 @@ add_library(filament-utils-jni SHARED src/main/cpp/Bookmark.cpp src/main/cpp/Utils.cpp src/main/cpp/Manipulator.cpp + src/main/cpp/RemoteServer.cpp ../common/CallbackUtils.cpp ../common/NioUtils.cpp @@ -38,6 +43,7 @@ set_target_properties(filament-utils-jni PROPERTIES LINK_DEPENDS # The ordering in the following list is important because CMake does not have dependency information. target_link_libraries(filament-utils-jni gltfio-jni + civetweb camutils image viewer diff --git a/android/filament-utils-android/src/main/cpp/RemoteServer.cpp b/android/filament-utils-android/src/main/cpp/RemoteServer.cpp new file mode 100644 index 0000000000..e118c5563c --- /dev/null +++ b/android/filament-utils-android/src/main/cpp/RemoteServer.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +using namespace filament::viewer; + +extern "C" JNIEXPORT jlong JNICALL +Java_com_google_android_filament_utils_RemoteServer_nCreate(JNIEnv* env, jclass, jint port) { + RemoteServer* server = new RemoteServer(port); + if (!server->isValid()) { + delete server; + return 0; + } + return (jlong) server; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_utils_RemoteServer_nDestroy(JNIEnv*, jclass, jlong native) { + RemoteServer* server = (RemoteServer*) native; + delete server; +} + +extern "C" JNIEXPORT jstring JNICALL +Java_com_google_android_filament_utils_RemoteServer_nPeekIncomingLabel(JNIEnv* env, jclass, jlong native) { + RemoteServer* server = (RemoteServer*) native; + IncomingMessage const* msg = server->peekIncomingMessage(); + return msg ? env->NewStringUTF(msg->label) : nullptr; +} + +extern "C" JNIEXPORT jint JNICALL +Java_com_google_android_filament_utils_RemoteServer_nPeekIncomingBufferLength(JNIEnv* env, jclass, jlong native) { + RemoteServer* server = (RemoteServer*) native; + IncomingMessage const* msg = server->peekIncomingMessage(); + return msg ? msg->bufferByteCount : 0; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_utils_RemoteServer_nAcquireIncomingMessage(JNIEnv* env, jclass, jlong native, jobject buffer, jint length) { + RemoteServer* server = (RemoteServer*) native; + IncomingMessage const* msg = server->acquireIncomingMessage(); + if (msg == nullptr) { + return; + } + + void* address = env->GetDirectBufferAddress(buffer); + if (address == nullptr) { + // This should never happen because the Java layer does allocateDirect. + return; + } + + memcpy(address, msg->buffer, length); + server->releaseIncomingMessage(msg); +} + diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/RemoteServer.java b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/RemoteServer.java new file mode 100644 index 0000000000..32abee88c4 --- /dev/null +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/RemoteServer.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.filament.utils; + +import androidx.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class RemoteServer { + private long mNativeObject; + + public static class IncomingMessage { + public String label; + public ByteBuffer buffer; + } + + public RemoteServer(int port) { + mNativeObject = nCreate(port); + if (mNativeObject == 0) throw new IllegalStateException("Couldn't create RemoteServer"); + } + + public @Nullable IncomingMessage acquireIncomingMessage() { + int length = nPeekIncomingBufferLength(mNativeObject); + if (length == 0) { + return null; + } + IncomingMessage message = new IncomingMessage(); + message.label = nPeekIncomingLabel(mNativeObject); + message.buffer = ByteBuffer.allocateDirect(length); + message.buffer.order(ByteOrder.LITTLE_ENDIAN); + nAcquireIncomingMessage(mNativeObject, message.buffer, length); + return message; + } + + @Override + protected void finalize() throws Throwable { + nDestroy(mNativeObject); + super.finalize(); + } + + private static native long nCreate(int port); + private static native String nPeekIncomingLabel(long nativeObject); + private static native int nPeekIncomingBufferLength(long nativeObject); + private static native void nAcquireIncomingMessage(long nativeObject, ByteBuffer buffer, int length); + private static native void nDestroy(long nativeObject); +} diff --git a/android/samples/sample-gltf-viewer/build.gradle b/android/samples/sample-gltf-viewer/build.gradle index c33bd2bda4..a1349f5fba 100644 --- a/android/samples/sample-gltf-viewer/build.gradle +++ b/android/samples/sample-gltf-viewer/build.gradle @@ -36,4 +36,5 @@ dependencies { implementation project(':filament-android') implementation project(':gltfio-android') implementation project(':filament-utils-android') + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.3' } diff --git a/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt b/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt index 3f5cc7fced..4c5ae4168c 100644 --- a/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt +++ b/android/samples/sample-gltf-viewer/src/main/java/com/google/android/filament/gltf/MainActivity.kt @@ -19,18 +19,29 @@ package com.google.android.filament.gltf import android.annotation.SuppressLint import android.app.Activity import android.os.Bundle +import android.util.Log import android.view.* import android.widget.Toast import com.google.android.filament.utils.KtxLoader import com.google.android.filament.utils.ModelViewer +import com.google.android.filament.utils.RemoteServer import com.google.android.filament.utils.Utils +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.io.ByteArrayInputStream +import java.io.IOException +import java.nio.Buffer import java.nio.ByteBuffer +import java.util.zip.ZipInputStream class MainActivity : Activity() { companion object { // Load the library for the utility layer, which in turn loads gltfio and the Filament core. init { Utils.init() } + private const val TAG = "gltf-viewer" } private lateinit var surfaceView: SurfaceView @@ -39,6 +50,7 @@ class MainActivity : Activity() { private lateinit var modelViewer: ModelViewer private val doubleTapListener = DoubleTapListener() private lateinit var doubleTapDetector: GestureDetector + private var remoteServer: RemoteServer? = null @SuppressLint("ClickableViewAccessibility") override fun onCreate(savedInstanceState: Bundle?) { @@ -75,6 +87,8 @@ class MainActivity : Activity() { val bloomOptions = modelViewer.view.bloomOptions bloomOptions.enabled = true modelViewer.view.bloomOptions = bloomOptions + + remoteServer = RemoteServer(8082) } private fun createRenderables() { @@ -108,6 +122,73 @@ class MainActivity : Activity() { return ByteBuffer.wrap(bytes) } + private fun setStatusText(text: String) { + runOnUiThread { + Toast.makeText(applicationContext, text, Toast.LENGTH_SHORT).show() + } + } + + private suspend fun loadGlb(message: RemoteServer.IncomingMessage) { + withContext(Dispatchers.Main) { + modelViewer.destroyModel() + modelViewer.loadModelGlb(message.buffer) + modelViewer.transformToUnitCube() + } + } + + private suspend fun loadZip(message: RemoteServer.IncomingMessage) { + val zipfileBytes = ByteArray(message.buffer.remaining()) + message.buffer.get(zipfileBytes) + + var gltfPath: String? = null + val pathToBufferMapping = withContext(Dispatchers.IO) { + val deflater = ZipInputStream(ByteArrayInputStream(zipfileBytes)) + val mapping = HashMap() + while (true) { + val entry = deflater.nextEntry ?: break + if (entry.isDirectory) continue + if (entry.name.startsWith("__MACOSX")) continue + val uri = entry.name + val byteArray = deflater.readBytes() + Log.i(TAG, "Deflated ${byteArray.size} bytes from $uri") + val buffer = ByteBuffer.wrap(byteArray) + mapping[uri] = buffer + if (uri.endsWith(".gltf")) { + gltfPath = uri + } + } + mapping + } + + if (gltfPath == null) { + setStatusText( "Could not find .gltf in the zip.") + return + } + + setStatusText( "Received all model data.") + + val gltfBuffer = pathToBufferMapping[gltfPath]!! + + // The gltf is often not at the root level (e.g. if a folder is zipped) so + // we need to extract its path in order to resolve the embedded uri strings. + var gltfPrefix = gltfPath!!.substringBeforeLast('/', "") + if (gltfPrefix.isNotEmpty()) { + gltfPrefix += "/" + } + + withContext(Dispatchers.Main) { + modelViewer.destroyModel() + modelViewer.loadModelGltf(gltfBuffer) { uri -> + val path = gltfPrefix + uri + if (!pathToBufferMapping.contains(path)) { + Log.e(TAG, "Could not find $path in the zip.") + } + pathToBufferMapping[path]!! + } + modelViewer.transformToUnitCube() + } + } + override fun onResume() { super.onResume() choreographer.postFrameCallback(frameScheduler) @@ -137,6 +218,25 @@ class MainActivity : Activity() { } modelViewer.render(frameTimeNanos) + + val message = remoteServer?.acquireIncomingMessage() + if (message != null) { + Log.i("Filament", "Downloaded ${message.label} ${message.buffer.capacity()}") + + CoroutineScope(Dispatchers.IO).launch { + try { + if (message.label.endsWith(".zip")) { + loadZip(message) + } else { + loadGlb(message) + } + } catch (exc: IOException) { + setStatusText( "URL fetch failed.") + Log.e(TAG, "URL fetch failed", exc) + } + } + + } } } diff --git a/libs/viewer/CMakeLists.txt b/libs/viewer/CMakeLists.txt index 9a2ababd03..0035672d56 100644 --- a/libs/viewer/CMakeLists.txt +++ b/libs/viewer/CMakeLists.txt @@ -10,6 +10,7 @@ set(PUBLIC_HDR_DIR include) set(PUBLIC_HDRS include/viewer/AutomationEngine.h include/viewer/AutomationSpec.h + include/viewer/RemoteServer.h include/viewer/Settings.h include/viewer/SimpleViewer.h ) @@ -18,6 +19,7 @@ set(SRCS src/jsonParseUtils.h src/AutomationEngine.cpp src/AutomationSpec.cpp + src/RemoteServer.cpp src/Settings.cpp src/SimpleViewer.cpp ) @@ -26,7 +28,7 @@ set(SRCS # Include and target definitions # ================================================================================================== add_library(${TARGET} STATIC ${PUBLIC_HDRS} ${SRCS}) -target_link_libraries(${TARGET} PUBLIC imgui filament gltfio_core filagui jsmn) +target_link_libraries(${TARGET} PUBLIC imgui filament gltfio_core filagui jsmn civetweb) target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) # ================================================================================================== diff --git a/libs/viewer/include/viewer/AutomationSpec.h b/libs/viewer/include/viewer/AutomationSpec.h index 26d584feca..8f282ce04d 100644 --- a/libs/viewer/include/viewer/AutomationSpec.h +++ b/libs/viewer/include/viewer/AutomationSpec.h @@ -81,4 +81,4 @@ private: } // namespace viewer } // namespace filament -#endif // VIEWER_AUTOMATION_H +#endif // VIEWER_AUTOMATION_SPEC_H diff --git a/libs/viewer/include/viewer/RemoteServer.h b/libs/viewer/include/viewer/RemoteServer.h new file mode 100644 index 0000000000..e3259cf2dc --- /dev/null +++ b/libs/viewer/include/viewer/RemoteServer.h @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIEWER_REMOTE_SERVER_H +#define VIEWER_REMOTE_SERVER_H + +#include +#include + +class CivetServer; + +namespace filament { +namespace viewer { + +class WsHandler; + +struct IncomingMessage { + char* label; + char* buffer; + size_t bufferByteCount; + size_t messageUid; +}; + +class RemoteServer { +public: + RemoteServer(int port = 8082); + ~RemoteServer(); + bool isValid() const { return mCivetServer; } + IncomingMessage const* peekIncomingMessage() const; + IncomingMessage const* acquireIncomingMessage(); + void releaseIncomingMessage(IncomingMessage const* message); + +private: + void enqueueIncomingMessage(IncomingMessage* message); + CivetServer* mCivetServer = nullptr; + WsHandler* mWsHandler = nullptr; + size_t mNextMessageUid = 0; + size_t mOldestMessageUid = 0; + static const size_t kMessageCapacity = 4; + IncomingMessage* mIncomingMessages[kMessageCapacity] = {}; + mutable std::mutex mIncomingMessagesMutex; + friend class WsHandler; +}; + +} // namespace viewer +} // namespace filament + +#endif // VIEWER_REMOTE_SERVER_H diff --git a/libs/viewer/src/RemoteServer.cpp b/libs/viewer/src/RemoteServer.cpp new file mode 100644 index 0000000000..2c1ee69b6a --- /dev/null +++ b/libs/viewer/src/RemoteServer.cpp @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include + +#include + +using namespace utils; + +namespace filament { +namespace viewer { + +class WsHandler : public CivetWebSocketHandler { + public: + WsHandler(RemoteServer* server) : mServer(server) {} + ~WsHandler() { delete mIncomingMessage; } + bool handleData(CivetServer* server, struct mg_connection*, int, char* , size_t) override; + private: + RemoteServer* mServer; + std::vector mChunk; + IncomingMessage* mIncomingMessage = nullptr; +}; + +RemoteServer::RemoteServer(int port) { + const char* kServerOptions[] = { + "listening_ports", "8082", + "num_threads", "2", + "error_log_file", "civetweb.txt", + nullptr, + }; + std::string portString = std::to_string(port); + kServerOptions[1] = portString.c_str(); + mCivetServer = new CivetServer(kServerOptions); + if (!mCivetServer->getContext()) { + slog.e << "Unable to start RemoteServer, see civetweb.txt for details." << io::endl; + delete mCivetServer; + mCivetServer = nullptr; + mWsHandler = nullptr; + return; + } + mWsHandler = new WsHandler(this); + mCivetServer->addWebSocketHandler("", mWsHandler); + slog.i << "RemoteServer listening at ws://localhost:" << port << io::endl; +} + +RemoteServer::~RemoteServer() { + delete mCivetServer; + delete mWsHandler; + for (auto msg : mIncomingMessages) { + releaseIncomingMessage(msg); + } +} + +IncomingMessage const * RemoteServer::peekIncomingMessage() const { + std::lock_guard lock(mIncomingMessagesMutex); + const size_t oldest = mOldestMessageUid; + for (auto msg : mIncomingMessages) { if (msg && msg->messageUid == oldest) return msg; } + return nullptr; +} + +IncomingMessage const * RemoteServer::acquireIncomingMessage() { + std::lock_guard lock(mIncomingMessagesMutex); + const size_t oldest = mOldestMessageUid; + for (auto& msg : mIncomingMessages) { + if (msg && msg->messageUid == oldest) { + auto result = msg; + msg = nullptr; + ++mOldestMessageUid; + return result; + } + } + return nullptr; +} + +void RemoteServer::enqueueIncomingMessage(IncomingMessage* message) { + std::lock_guard lock(mIncomingMessagesMutex); + for (auto& msg : mIncomingMessages) { + if (!msg) { + message->messageUid = mNextMessageUid++; + msg = message; + return; + } + } + slog.e << "Discarding message, message queue overflow." << io::endl; +} + +void RemoteServer::releaseIncomingMessage(IncomingMessage const* message) { + if (message) { + delete[] message->label; + delete[] message->buffer; + delete message; + } +} + +// NOTE: This is invoked off the main thread. +bool WsHandler::handleData(CivetServer* server, struct mg_connection* conn, int bits, + char* data, size_t size) { + const bool final = bits & 0x80; + const int opcode = bits & 0xf; + if (opcode == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE) { + return true; + } + + // Append this frame to the aggregated chunk. + mChunk.insert(mChunk.end(), data, data + size); + + // If this message part still has outstanding frames, return early. + if (!final) { + return true; + } + + // Part 1 of the message is the label. + if (mIncomingMessage == nullptr) { + mIncomingMessage = new IncomingMessage({}); + mIncomingMessage->label = new char[mChunk.size() + 1]{}; + memcpy(mIncomingMessage->label, mChunk.data(), mChunk.size()); + mChunk.clear(); + return true; + } + + // Part 2 of the message is the buffer. + auto message = mIncomingMessage; + message->bufferByteCount = mChunk.size(); + message->buffer = new char[message->bufferByteCount]; + memcpy(message->buffer, mChunk.data(), message->bufferByteCount); + mChunk.clear(); + + // We have all parts, so go ahead and enqueue the incoming message. + mServer->enqueueIncomingMessage(mIncomingMessage); + mIncomingMessage = nullptr; + return true; +} + +} // namespace viewer +} // namespace filament diff --git a/third_party/civetweb/tnt/CMakeLists.txt b/third_party/civetweb/tnt/CMakeLists.txt index 5ea92dc359..e372b77b13 100644 --- a/third_party/civetweb/tnt/CMakeLists.txt +++ b/third_party/civetweb/tnt/CMakeLists.txt @@ -32,3 +32,5 @@ target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) target_link_libraries(${TARGET} ${CMAKE_DL_LIBS}) target_compile_options(${TARGET} PRIVATE $<$:-fPIC>) + +install(TARGETS ${TARGET} ARCHIVE DESTINATION third_party/${DIST_DIR})