From d85f8164c0d028377913e3d5a06aad54ebfaca7b Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Wed, 6 Nov 2019 11:11:15 -0800 Subject: [PATCH] Stream API: add support for acquire / release semantics. Previously, Streams had two modes (native and texid), this adds a third mode called "acquired", which allows for copy-free synchronized external textures in OpenGL and paves the way for Vulkan. The native mode is now deprecated but texid mode needs to stay around until all clients can be upgraded. In an early prototype, this functionality was added directly into Texture but required quite a bit of additional state tracking, so the Stream API seemed like a better fit. This API is probably not necessary for Metal due to Metal's shared ownership semantics. This has been tested with a new Android sample that will be added in a subsequent PR. --- android/common/CallbackUtils.cpp | 64 +++--- android/common/CallbackUtils.h | 31 +-- .../filament-android/src/main/cpp/Stream.cpp | 57 +++++- .../com/google/android/filament/Stream.java | 53 ++++- .../backend/include/backend/DriverEnums.h | 17 ++ .../include/private/backend/DriverAPI.inc | 4 +- .../include/private/backend/OpenGLPlatform.h | 5 + filament/backend/src/Driver.cpp | 13 ++ filament/backend/src/DriverBase.h | 6 +- filament/backend/src/metal/MetalDriver.mm | 10 +- filament/backend/src/opengl/OpenGLDriver.cpp | 182 ++++++++++++++---- filament/backend/src/opengl/OpenGLDriver.h | 11 +- filament/backend/src/opengl/PlatformEGL.cpp | 33 ++++ filament/backend/src/opengl/PlatformEGL.h | 2 + filament/backend/src/vulkan/VulkanDriver.cpp | 10 +- filament/include/filament/Stream.h | 29 ++- filament/src/Stream.cpp | 22 ++- filament/src/details/Stream.h | 7 +- 18 files changed, 435 insertions(+), 121 deletions(-) diff --git a/android/common/CallbackUtils.cpp b/android/common/CallbackUtils.cpp index cc0c6d259b..905f66e2f7 100644 --- a/android/common/CallbackUtils.cpp +++ b/android/common/CallbackUtils.cpp @@ -27,39 +27,6 @@ struct { jmethodID execute; } gCallbackUtils; -JniCallback* JniCallback::make(filament::Engine* engine, - JNIEnv* env, jobject handler, jobject callback) { - void* that = engine->streamAlloc(sizeof(JniCallback), alignof(JniCallback)); - return new (that) JniCallback(env, handler, callback); -} - -JniCallback::JniCallback(JNIEnv* env, jobject handler, jobject callback) - : mEnv(env) - , mHandler(env->NewGlobalRef(handler)) - , mCallback(env->NewGlobalRef(callback)) { -} - -JniCallback::~JniCallback() { - if (mHandler && mCallback) { -#ifdef ANDROID - if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) { - mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback); - } -#endif - if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) { - mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback); - } - } - mEnv->DeleteGlobalRef(mHandler); - mEnv->DeleteGlobalRef(mCallback); -} - -void JniCallback::invoke(void*, size_t, void* user) { - JniCallback* data = reinterpret_cast(user); - // don't call delete here, because we don't own the storage - data->~JniCallback(); -} - JniBufferCallback* JniBufferCallback::make(filament::Engine* engine, JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer) { void* that = engine->streamAlloc(sizeof(JniBufferCallback), alignof(JniBufferCallback)); @@ -95,6 +62,37 @@ void JniBufferCallback::invoke(void*, size_t, void* user) { data->~JniBufferCallback(); } +JniImageCallback* JniImageCallback::make(filament::Engine* engine, + JNIEnv* env, jobject handler, jobject callback, long image) { + void* that = engine->streamAlloc(sizeof(JniImageCallback), alignof(JniImageCallback)); + return new (that) JniImageCallback(env, handler, callback, image); +} + +JniImageCallback::JniImageCallback(JNIEnv* env, jobject handler, jobject callback, long image) + : mEnv(env) + , mHandler(env->NewGlobalRef(handler)) + , mCallback(env->NewGlobalRef(callback)) + , mImage(image) { } + +JniImageCallback::~JniImageCallback() { + if (mHandler && mCallback) { +#ifdef ANDROID + if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) { + mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback); + } +#endif + if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) { + mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback); + } + } + mEnv->DeleteGlobalRef(mHandler); + mEnv->DeleteGlobalRef(mCallback); +} + +void JniImageCallback::invoke(void* image, void* user) { + reinterpret_cast(user)->~JniImageCallback(); +} + void registerCallbackUtils(JNIEnv *env) { #ifdef ANDROID gCallbackUtils.handlerClass = env->FindClass("android/os/Handler"); diff --git a/android/common/CallbackUtils.h b/android/common/CallbackUtils.h index 4501db3fea..f9cd1ec625 100644 --- a/android/common/CallbackUtils.h +++ b/android/common/CallbackUtils.h @@ -23,21 +23,6 @@ #include -struct JniCallback { - static JniCallback* make(filament::Engine* engine, - JNIEnv* env, jobject handler, jobject callback); - - static void invoke(void* buffer, size_t n, void* user); - -private: - JniCallback(JNIEnv* env, jobject handler, jobject callback); - ~JniCallback(); - - JNIEnv* mEnv; - jobject mHandler; - jobject mCallback; -}; - struct JniBufferCallback { static JniBufferCallback* make(filament::Engine* engine, JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer); @@ -53,3 +38,19 @@ private: jobject mCallback; AutoBuffer mBuffer; }; + +struct JniImageCallback { + static JniImageCallback* make(filament::Engine* engine, JNIEnv* env, jobject handler, + jobject runnable, long image); + + static void invoke(void* image, void* user); + +private: + JniImageCallback(JNIEnv* env, jobject handler, jobject runnable, long image); + ~JniImageCallback(); + + JNIEnv* mEnv; + jobject mHandler; + jobject mCallback; + long mImage; +}; diff --git a/android/filament-android/src/main/cpp/Stream.cpp b/android/filament-android/src/main/cpp/Stream.cpp index 53e8097a8d..271a926103 100644 --- a/android/filament-android/src/main/cpp/Stream.cpp +++ b/android/filament-android/src/main/cpp/Stream.cpp @@ -24,6 +24,26 @@ #include "common/NioUtils.h" #include "common/CallbackUtils.h" +#ifdef ANDROID + +#if __has_include() +#include +#else +struct AHardwareBuffer; +typedef struct AHardwareBuffer AHardwareBuffer; +#endif + +#include + +#include + +template +static void loadSymbol(T*& pfn, const char *symbol) noexcept { + pfn = (T*)dlsym(RTLD_DEFAULT, symbol); +} + +#endif + using namespace filament; using namespace backend; @@ -108,10 +128,10 @@ Java_com_google_android_filament_Stream_nBuilderBuild(JNIEnv*, jclass, return (jlong) builder->builder()->build(*engine); } -extern "C" JNIEXPORT jboolean JNICALL -Java_com_google_android_filament_Stream_nIsNative(JNIEnv*, jclass, jlong nativeStream) { +extern "C" JNIEXPORT jint JNICALL +Java_com_google_android_filament_Stream_nGetStreamType(JNIEnv*, jclass, jlong nativeStream) { Stream* stream = (Stream*) nativeStream; - return (jboolean) stream->isNativeStream(); + return (jint) stream->getStreamType(); } extern "C" JNIEXPORT void JNICALL @@ -160,3 +180,34 @@ Java_com_google_android_filament_Stream_nGetTimestamp(JNIEnv*, jclass, jlong nat Stream *stream = (Stream *) nativeStream; return stream->getTimestamp(); } + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_Stream_nSetAcquiredImage(JNIEnv* env, jclass, jlong nativeStream, + jlong nativeEngine, jobject hwbuffer, jobject handler, jobject runnable) { + Engine* engine = (Engine*) nativeEngine; + Stream* stream = (Stream*) nativeStream; + +#ifdef ANDROID + + AHardwareBuffer* (*AHardwareBuffer_fromHardwareBuffer)(JNIEnv*, jobject); + loadSymbol(AHardwareBuffer_fromHardwareBuffer, "AHardwareBuffer_fromHardwareBuffer"); + + AHardwareBuffer* nativeBuffer = AHardwareBuffer_fromHardwareBuffer(env, hwbuffer); + if (!nativeBuffer) { + __android_log_print(ANDROID_LOG_INFO, "Filament", "Unable to obtain native HardwareBuffer."); + return; + } + + auto* callback = JniImageCallback::make(engine, env, handler, runnable, (long) nativeBuffer); + +#else + + // TODO: for non-Android platforms, it is unclear how to go from "jobject" to "void*" + // For now this code is reserved for future use. + auto* callback = JniImageCallback::make(engine, env, handler, runnable, 0); + void* nativeBuffer = nullptr; + +#endif + + stream->setAcquiredImage((void*) nativeBuffer, &JniImageCallback::invoke, callback); +} diff --git a/android/filament-android/src/main/java/com/google/android/filament/Stream.java b/android/filament-android/src/main/java/com/google/android/filament/Stream.java index 97f055bdc0..e35dfcb315 100644 --- a/android/filament-android/src/main/java/com/google/android/filament/Stream.java +++ b/android/filament-android/src/main/java/com/google/android/filament/Stream.java @@ -33,6 +33,20 @@ public class Stream { private long mNativeObject; private long mNativeEngine; + /** + * Represents the immutable stream type. + */ + public enum StreamType { + /** Not synchronized but copy-free. Good for video. */ + NATIVE, + + /** Synchronized, but GL-only and incurs copies. Good for AR on older devices. */ + TEXTURE_ID, + + /** Synchronized, copy-free, and take a release callback. Good for AR on newer devices. */ + ACQUIRED, + }; + Stream(long nativeStream, Engine engine) { mNativeObject = nativeStream; mNativeEngine = engine.getNativeObject(); @@ -40,6 +54,12 @@ public class Stream { /** * Use Builder to construct an Stream object instance. + * + * By default, Stream objects are {@link StreamType#ACQUIRED ACQUIRED} and must have external images pushed to them via + * {@line #setAcquiredImage}. + * + * To create a {@link StreamType#NATIVE NATIVE} or {@link StreamType#TEXTURE_ID TEXTURE_ID} stream, call one of the 
stream
methods + * on the builder. */ public static class Builder { @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) // Keep to finalize native resources @@ -55,8 +75,8 @@ public class Stream { } /** - * Creates a native stream. Native streams can sample data directly from an - * opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture} + * Creates a {@link StreamType#NATIVE NATIVE} stream. Native streams can sample data + * directly from an opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture} * on Android. * * @param streamSource an opaque native stream handle, e.g.: on Android this must be a @@ -74,7 +94,7 @@ public class Stream { } /** - * Creates a copy stream. A copy stream will sample data from the supplied + * Creates a {@link StreamType#TEXTURE_ID TEXTURE_ID} stream. A copy stream will sample data from the supplied * external texture and copy it into an internal private texture. * *

Currently only OpenGL external texture ids are supported.

@@ -150,12 +170,24 @@ public class Stream { } /** - * Indicates whether this Stream is a native stream or a copy stream. - * - * @return true if this is a native Stream, false otherwise. + * Indicates whether this Stream is NATIVE, TEXTURE_ID, or ACQUIRED. */ - public boolean isNative() { - return nIsNative(getNativeObject()); + public StreamType getStreamType() { + return StreamType.values()[nGetStreamType(getNativeObject())]; + } + + /** + * Updates an 
ACQUIRED
stream with an image that is guaranteed to be used in the next frame. + * + * This method should be called on the same thread that calls {#link Renderer#beginFrame}, which is + * also where the callback is invoked. + * + * @param hwbuffer {@link android.hardware.HardwareBuffer HardwareBuffer} (requires API level 26) + * @param handler {@link java.util.concurrent.Executor Executor} or {@link android.os.Handler Handler}. + * @param callback a callback invoked by handler when the hwbuffer can be released. + */ + public void setAcquiredImage(Object hwbuffer, Object handler, Runnable callback) { + nSetAcquiredImage(getNativeObject(), mNativeEngine, hwbuffer, handler, callback); } /** @@ -286,6 +318,7 @@ public class Stream { private static native void nBuilderHeight(long nativeStreamBuilder, int height); private static native long nBuilderBuild(long nativeStreamBuilder, long nativeEngine); + private static native int nGetStreamType(long nativeStream); private static native void nSetDimensions(long nativeStream, int width, int height); private static native int nReadPixels(long nativeStream, long nativeEngine, int xoffset, int yoffset, int width, int height, @@ -293,6 +326,6 @@ public class Stream { int left, int top, int type, int alignment, int stride, int format, Object handler, Runnable callback); private static native long nGetTimestamp(long nativeStream); - - private static native boolean nIsNative(long nativeStream); + private static native void nSetAcquiredImage(long nativeStream, long nativeEngine, + Object hwbuffer, Object handler, Runnable callback); } diff --git a/filament/backend/include/backend/DriverEnums.h b/filament/backend/include/backend/DriverEnums.h index f5c790dcc1..93ffab70b3 100644 --- a/filament/backend/include/backend/DriverEnums.h +++ b/filament/backend/include/backend/DriverEnums.h @@ -624,6 +624,23 @@ enum class BlendFunction : uint8_t { SRC_ALPHA_SATURATE //!< f(src, dst) = (1,1,1) * min(src.a, 1 - dst.a), 1 }; +//! Stream for external textures +enum class StreamType { + NATIVE, + TEXTURE_ID, + ACQUIRED, +}; + +//! Releases an ACQUIRED external texture, guaranteed to be called on the application thread. +using StreamCallback = void(*)(void* image, void* user); + +//! Bundles the state of an ACQUIRED stream. +struct AcquiredImage { + void* image = nullptr; + backend::StreamCallback callback = nullptr; + void* userData = nullptr; +}; + //! Vertex attribute descriptor struct Attribute { //! attribute is normalized (remapped between 0 and 1) diff --git a/filament/backend/include/private/backend/DriverAPI.inc b/filament/backend/include/private/backend/DriverAPI.inc index 869538b0c6..c3334be0ab 100644 --- a/filament/backend/include/private/backend/DriverAPI.inc +++ b/filament/backend/include/private/backend/DriverAPI.inc @@ -215,7 +215,9 @@ DECL_DRIVER_API_N(destroyStream, backend::StreamHandle, sh) */ DECL_DRIVER_API_SYNCHRONOUS_0(void, terminate) -DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStream, void*, stream) +DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamNative, void*, stream) +DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamAcquired) +DECL_DRIVER_API_SYNCHRONOUS_N(void, setAcquiredImage, backend::StreamHandle, stream, void*, image, backend::StreamCallback, cb, void*, userData) DECL_DRIVER_API_SYNCHRONOUS_N(void, setStreamDimensions, backend::StreamHandle, stream, uint32_t, width, uint32_t, height) DECL_DRIVER_API_SYNCHRONOUS_N(int64_t, getStreamTimestamp, backend::StreamHandle, stream) DECL_DRIVER_API_SYNCHRONOUS_N(void, updateStreams, backend::DriverApi*, driver) diff --git a/filament/backend/include/private/backend/OpenGLPlatform.h b/filament/backend/include/private/backend/OpenGLPlatform.h index 7df84cd238..24b014b10b 100644 --- a/filament/backend/include/private/backend/OpenGLPlatform.h +++ b/filament/backend/include/private/backend/OpenGLPlatform.h @@ -107,6 +107,11 @@ public: // called once before a SAMPLER_EXTERNAL texture is destroyed. virtual void destroyExternalImage(void* texture) noexcept {} + // allows platforms to convert the given image object into a new type (e.g. HardwareBuffer => EGLImage) + // by default, performs no conversion. + virtual AcquiredImage createAcquiredImage(void* hwbuffer, backend::StreamCallback cb, void* userData) noexcept { + return {hwbuffer, cb, userData}; + } }; } // namespace backend diff --git a/filament/backend/src/Driver.cpp b/filament/backend/src/Driver.cpp index 333637c2c3..a2582de013 100644 --- a/filament/backend/src/Driver.cpp +++ b/filament/backend/src/Driver.cpp @@ -43,9 +43,15 @@ DriverBase::~DriverBase() noexcept { void DriverBase::purge() noexcept { std::vector buffersToPurge; + std::vector imagesToPurge; std::unique_lock lock(mPurgeLock); std::swap(buffersToPurge, mBufferToPurge); + std::swap(imagesToPurge, mImagesToPurge); lock.unlock(); // don't remove this, it ensures mBufferToPurge is destroyed without lock held + for (auto& image : imagesToPurge) { + image.callback(image.image, image.userData); + } + // When the BufferDescriptors go out of scope, their destructors invoke their callbacks. } void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept { @@ -53,6 +59,13 @@ void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept { mBufferToPurge.push_back(std::move(buffer)); } +// This is called from an async driver method so it's in the GL thread, but purge is called +// on the user thread. This is typically called 0 or 1 times per frame. +void DriverBase::scheduleRelease(AcquiredImage&& image) noexcept { + std::lock_guard lock(mPurgeLock); + mImagesToPurge.push_back(std::move(image)); +} + // ------------------------------------------------------------------------------------------------ Driver::~Driver() noexcept = default; diff --git a/filament/backend/src/DriverBase.h b/filament/backend/src/DriverBase.h index 6de6ae2f7d..dcbfaecbe7 100644 --- a/filament/backend/src/DriverBase.h +++ b/filament/backend/src/DriverBase.h @@ -133,8 +133,9 @@ struct HwSwapChain : public HwBase { struct HwStream : public HwBase { HwStream() = default; - explicit HwStream(Platform::Stream* stream) : stream(stream) { } + explicit HwStream(Platform::Stream* stream) : stream(stream), streamType(StreamType::NATIVE) { } Platform::Stream* stream = nullptr; + StreamType streamType = StreamType::ACQUIRED; uint32_t width = 0; uint32_t height = 0; }; @@ -168,9 +169,12 @@ protected: void scheduleDestroySlow(BufferDescriptor&& buffer) noexcept; + void scheduleRelease(AcquiredImage&& image) noexcept; + private: std::mutex mPurgeLock; std::vector mBufferToPurge; + std::vector mImagesToPurge; }; diff --git a/filament/backend/src/metal/MetalDriver.mm b/filament/backend/src/metal/MetalDriver.mm index b1d2f28cc8..c0b4fe384f 100644 --- a/filament/backend/src/metal/MetalDriver.mm +++ b/filament/backend/src/metal/MetalDriver.mm @@ -392,10 +392,18 @@ ShaderModel MetalDriver::getShaderModel() const noexcept { #endif } -Handle MetalDriver::createStream(void* stream) { +Handle MetalDriver::createStreamNative(void* stream) { return {}; } +Handle MetalDriver::createStreamAcquired() { + return {}; +} + +void MetalDriver::setAcquiredImage(Handle sh, void* image, backend::StreamCallback cb, + void* userData) { +} + void MetalDriver::setStreamDimensions(Handle stream, uint32_t width, uint32_t height) { diff --git a/filament/backend/src/opengl/OpenGLDriver.cpp b/filament/backend/src/opengl/OpenGLDriver.cpp index ffda0bbc7a..e72d4a9788 100644 --- a/filament/backend/src/opengl/OpenGLDriver.cpp +++ b/filament/backend/src/opengl/OpenGLDriver.cpp @@ -1015,6 +1015,7 @@ void OpenGLDriver::createStreamFromTextureIdR(Handle sh, s->width = width; s->height = height; s->gl.externalTextureId = static_cast(externalTextureId); + s->streamType = StreamType::TEXTURE_ID; glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read); glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write); for (auto& info : s->user_thread.infos) { @@ -1161,9 +1162,9 @@ void OpenGLDriver::destroyStream(Handle sh) { if (pos != externalStreams.end()) { detachStream(*pos); } - if (s->isNativeStream()) { + if (s->streamType == StreamType::NATIVE) { mPlatform.destroyStream(s->stream); - } else { + } else if (s->streamType == StreamType::TEXTURE_ID) { glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read); glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write); if (s->gl.fbo) { @@ -1182,13 +1183,35 @@ void OpenGLDriver::destroyStream(Handle sh) { // These are called on the application's thread // ------------------------------------------------------------------------------------------------ -Handle OpenGLDriver::createStream(void* nativeStream) { +Handle OpenGLDriver::createStreamNative(void* nativeStream) { Handle sh( allocateHandle(sizeof(GLStream)) ); Platform::Stream* stream = mPlatform.createStream(nativeStream); construct(sh, stream); return sh; } +Handle OpenGLDriver::createStreamAcquired() { + Handle sh(allocateHandle(sizeof(GLStream))); + construct(sh); + return sh; +} + +// Stashes an acquired external image and a release callback. The image is not bound to OpenGL until +// the subsequent call to beginFrame (see updateStreamAcquired). +// +// setAcquiredImage should be called by the user outside of beginFrame / endFrame, and should be +// called only once per frame. If the user pushes images to the same stream multiple times in a +// single frame, we emit a warning and honor only the final image, but still invoke all callbacks. +void OpenGLDriver::setAcquiredImage(Handle sh, void* hwbuffer, + backend::StreamCallback cb, void* userData) { + GLStream* glstream = handle_cast(sh); + if (glstream->user_thread.pending.image) { + scheduleRelease(std::move(glstream->user_thread.pending)); + slog.w << "Acquired image is set more than once per frame." << io::endl; + } + glstream->user_thread.pending = mPlatform.createAcquiredImage(hwbuffer, cb, userData); +} + void OpenGLDriver::updateStreams(DriverApi* driver) { if (UTILS_UNLIKELY(!mExternalStreams.empty())) { OpenGLBlitter::State state; @@ -1198,13 +1221,17 @@ void OpenGLDriver::updateStreams(DriverApi* driver) { GLStream* s = static_cast(t->hwStream); if (UTILS_UNLIKELY(s == nullptr)) { // this can happen because we're called synchronously and the setExternalStream() - // call may bot have been processed yet. + // call may not have been processed yet. continue; } - if (!s->isNativeStream()) { + if (s->streamType == StreamType::TEXTURE_ID) { state.setup(); - updateStream(t, driver); + updateStreamTexId(t, driver); + } + + if (s->streamType == StreamType::ACQUIRED) { + updateStreamAcquired(t, driver); } } } @@ -1718,10 +1745,12 @@ void OpenGLDriver::cancelExternalImage(void* image) { } void OpenGLDriver::setExternalImage(Handle th, void* image) { - GLTexture* t = handle_cast(th); - auto& gl = mContext; + mPlatform.setExternalImage(image, handle_cast(th)); + setExternalTexture(handle_cast(th), image); +} - mPlatform.setExternalImage(image, t); +void OpenGLDriver::setExternalTexture(GLTexture* t, void* image) { + auto& gl = mContext; // TODO: move this logic to PlatformEGL. if (gl.ext.OES_EGL_image_external_essl3) { @@ -1748,7 +1777,7 @@ void OpenGLDriver::setExternalStream(Handle th, Handle sh) if (UTILS_LIKELY(sh)) { GLStream* s = handle_cast(sh); if (UTILS_LIKELY(!t->hwStream)) { - // we're not attached alread + // we're not attached already attachStream(t, s); } else { if (s->stream != t->hwStream->stream) { @@ -1768,20 +1797,26 @@ void OpenGLDriver::attachStream(GLTexture* t, GLStream* hwStream) noexcept { auto& gl = mContext; mExternalStreams.push_back(t); - if (hwStream->isNativeStream()) { - mPlatform.attach(hwStream->stream, t->gl.id); - } else { - assert(t->target == SamplerType::SAMPLER_EXTERNAL); - // The texture doesn't need a texture name anymore, get rid of it - gl.unbindTexture(t->gl.target, t->gl.id); - glDeleteTextures(1, &t->gl.id); - t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + switch (hwStream->streamType) { + case StreamType::NATIVE: + mPlatform.attach(hwStream->stream, t->gl.id); + break; + case StreamType::TEXTURE_ID: + assert(t->target == SamplerType::SAMPLER_EXTERNAL); + // The texture doesn't need a texture name anymore, get rid of it + gl.unbindTexture(t->gl.target, t->gl.id); + glDeleteTextures(1, &t->gl.id); + t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + break; + case StreamType::ACQUIRED: + break; } t->hwStream = hwStream; } UTILS_NOINLINE void OpenGLDriver::detachStream(GLTexture* t) noexcept { + auto& gl = mContext; auto& streams = mExternalStreams; auto pos = std::find(streams.begin(), streams.end(), t); if (pos != streams.end()) { @@ -1789,30 +1824,57 @@ void OpenGLDriver::detachStream(GLTexture* t) noexcept { } GLStream* s = static_cast(t->hwStream); - if (s->isNativeStream()) { - mPlatform.detach(t->hwStream->stream); - // this deletes the texture id + switch (s->streamType) { + case StreamType::NATIVE: + mPlatform.detach(t->hwStream->stream); + // ^ this deletes the texture id + break; + case StreamType::TEXTURE_ID: + break; + case StreamType::ACQUIRED: + gl.unbindTexture(t->gl.target, t->gl.id); + glDeleteTextures(1, &t->gl.id); + break; } + glGenTextures(1, &t->gl.id); + t->hwStream = nullptr; } UTILS_NOINLINE -void OpenGLDriver::replaceStream(GLTexture* t, GLStream* hwStream) noexcept { - GLStream* s = static_cast(t->hwStream); - if (s->isNativeStream()) { - mPlatform.detach(t->hwStream->stream); - // this deletes the texture id +void OpenGLDriver::replaceStream(GLTexture* texture, GLStream* newStream) noexcept { + assert(newStream && "Do not use replaceStream to detach a stream."); + + // This could be implemented via detachStream + attachStream but inlining allows + // a few small optimizations, like not touching the mExternalStreams list. + + GLStream* oldStream = static_cast(texture->hwStream); + switch (oldStream->streamType) { + case StreamType::NATIVE: + mPlatform.detach(texture->hwStream->stream); + // ^ this deletes the texture id + break; + case StreamType::TEXTURE_ID: + case StreamType::ACQUIRED: + break; } - if (hwStream->isNativeStream()) { - glGenTextures(1, &t->gl.id); - mPlatform.attach(hwStream->stream, t->gl.id); - } else { - assert(t->target == SamplerType::SAMPLER_EXTERNAL); - t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + switch (newStream->streamType) { + case StreamType::NATIVE: + glGenTextures(1, &texture->gl.id); + mPlatform.attach(newStream->stream, texture->gl.id); + break; + case StreamType::TEXTURE_ID: + assert(texture->target == SamplerType::SAMPLER_EXTERNAL); + texture->gl.id = newStream->user_thread.read[newStream->user_thread.cur]; + break; + case StreamType::ACQUIRED: + // Just re-use the old texture id. + break; } - t->hwStream = hwStream; + + texture->hwStream = newStream; } void OpenGLDriver::beginRenderPass(Handle rth, @@ -2114,19 +2176,51 @@ void OpenGLDriver::setViewportScissor(Viewport const& viewportScissor) noexcept gl.enable(GL_SCISSOR_TEST); } -/* - * This is called in the user thread - */ +// Binds the external image stashed in the associated stream. +// +// updateStreamAcquired() and setAcquiredImage() are both called from on the application's thread +// and therefore do not require synchronization. The former is always called immediately before +// beginFrame, the latter is called by the user from anywhere outside beginFrame / endFrame. +void OpenGLDriver::updateStreamAcquired(GLTexture* gltexture, DriverApi* driver) noexcept { + SYSTRACE_CALL(); + + GLStream* glstream = static_cast(gltexture->hwStream); + assert(glstream); + assert(glstream->streamType == StreamType::ACQUIRED); + + // If there's no pending image, do nothing. Note that GL_OES_EGL_image does not let you pass + // NULL to glEGLImageTargetTexture2DOES, and there is no concept of "detaching" an EGLimage from + // a texture. + if (glstream->user_thread.pending.image == nullptr) { + return; + } + + AcquiredImage previousImage = glstream->user_thread.acquired; + glstream->user_thread.acquired = glstream->user_thread.pending; + glstream->user_thread.pending = {0}; + + // Bind the stashed EGLImage to its corresponding GL texture as soon as we start making the GL + // calls for the upcoming frame. + void* image = glstream->user_thread.acquired.image; + driver->queueCommand([this, gltexture, image, previousImage]() { + setExternalTexture(gltexture, image); + if (previousImage.image) { + whenGpuCommandsComplete([this, previousImage]() { + scheduleRelease(AcquiredImage(previousImage)); + }); + } + }); +} #define DEBUG_NO_EXTERNAL_STREAM_COPY false -void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept { +void OpenGLDriver::updateStreamTexId(GLTexture* t, DriverApi* driver) noexcept { SYSTRACE_CALL(); auto& gl = mContext; GLStream* s = static_cast(t->hwStream); assert(s); - assert(!s->isNativeStream()); + assert(s->streamType == StreamType::TEXTURE_ID); // round-robin to the next texture name if (UTILS_UNLIKELY(DEBUG_NO_EXTERNAL_STREAM_COPY || @@ -2136,7 +2230,7 @@ void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept { // also make sure that this texture is still associated with the same stream auto& streams = mExternalStreams; if (UTILS_LIKELY(std::find(streams.begin(), streams.end(), t) != streams.end()) && - (t->hwStream == s)) { + (t->hwStream == s)) { t->gl.id = s->gl.externalTextureId; } }); @@ -2221,7 +2315,13 @@ void OpenGLDriver::readStreamPixels(Handle sh, auto& gl = mContext; GLStream* s = handle_cast(sh); - if (UTILS_LIKELY(!s->isNativeStream())) { + + if (UTILS_UNLIKELY(s->streamType == StreamType::ACQUIRED)) { + PANIC_LOG("readStreamPixels with ACQUIRED streams is not yet implemented."); + return; + } + + if (UTILS_LIKELY(s->streamType == StreamType::TEXTURE_ID)) { GLuint tid = s->gl.externalTexture2DId; if (tid == 0) { return; @@ -2510,7 +2610,7 @@ void OpenGLDriver::beginFrame(int64_t monotonic_clock_ns, uint32_t frameId, OpenGLPlatform& platform = mPlatform; for (GLTexture const* t : mExternalStreams) { assert(t && t->hwStream); - if (static_cast(t->hwStream)->isNativeStream()) { + if (t->hwStream->streamType == StreamType::NATIVE) { assert(t->hwStream->stream); platform.updateTexImage(t->hwStream->stream, &static_cast(t->hwStream)->user_thread.timestamp); diff --git a/filament/backend/src/opengl/OpenGLDriver.h b/filament/backend/src/opengl/OpenGLDriver.h index 7b96ef6187..5586f66d40 100644 --- a/filament/backend/src/opengl/OpenGLDriver.h +++ b/filament/backend/src/opengl/OpenGLDriver.h @@ -124,7 +124,6 @@ public: struct GLStream : public backend::HwStream { static constexpr size_t ROUND_ROBIN_TEXTURE_COUNT = 3; // 3 maximum using HwStream::HwStream; - bool isNativeStream() const { return gl.externalTextureId == 0; } struct Info { // storage for the read/write textures below backend::Platform::ExternalTexture* ets = nullptr; @@ -143,8 +142,9 @@ public: GLuint fbo = 0; } gl; // 20 bytes + /* - * The fields below are access from the main application thread + * The fields below are accessed from the main application thread * (not the GL thread) */ struct { @@ -155,6 +155,8 @@ public: Info infos[ROUND_ROBIN_TEXTURE_COUNT]; // 48 bytes int64_t timestamp = 0; uint8_t cur = 0; + backend::AcquiredImage acquired; + backend::AcquiredImage pending; } user_thread; }; @@ -373,10 +375,13 @@ private: backend::OpenGLPlatform& mPlatform; OpenGLBlitter* mOpenGLBlitter = nullptr; - void updateStream(GLTexture* t, backend::DriverApi* driver) noexcept; + void updateStreamTexId(GLTexture* t, backend::DriverApi* driver) noexcept; + void updateStreamAcquired(GLTexture* t, backend::DriverApi* driver) noexcept; void updateBuffer(GLenum target, GLBuffer* buffer, backend::BufferDescriptor const& p, uint32_t alignment = 16) noexcept; void updateTextureLodRange(GLTexture* texture, int8_t targetLevel) noexcept; + void setExternalTexture(GLTexture* t, void* image); + void whenGpuCommandsComplete(std::function fn) noexcept; void executeGpuCommandsCompleteOps() noexcept; std::vector>> mGpuCommandCompleteOps; diff --git a/filament/backend/src/opengl/PlatformEGL.cpp b/filament/backend/src/opengl/PlatformEGL.cpp index 377170ef26..6a0127cc26 100644 --- a/filament/backend/src/opengl/PlatformEGL.cpp +++ b/filament/backend/src/opengl/PlatformEGL.cpp @@ -542,6 +542,39 @@ void PlatformEGL::destroyExternalImage(void* texture) noexcept { glDeleteTextures(1, &t->gl.id); } +backend::AcquiredImage PlatformEGL::createAcquiredImage(void* hwbuffer, backend::StreamCallback userCallback, void* userData) noexcept { + + // Convert the AHardwareBuffer to EGLImage. + EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID((const AHardwareBuffer*) hwbuffer); + if (!clientBuffer) { + slog.e << "Unable to get EGLClientBuffer from AHardwareBuffer." << io::endl; + return {}; + } + // Note that this cannot be used to stream protected video (for now) because we do not set EGL_PROTECTED_CONTENT_EXT. + EGLint attrs[] = { EGL_NONE, EGL_NONE }; + EGLImageKHR eglImage = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, clientBuffer, attrs); + if (eglImage == EGL_NO_IMAGE_KHR) { + slog.e << "eglCreateImageKHR returned no image." << io::endl; + return {}; + } + + // Destroy the EGLImage before invoking the user's callback. + AcquiredImage* closure = new AcquiredImage(); + closure->callback = userCallback; + closure->image = hwbuffer; + closure->userData = userData; + auto patchedCallback = [](void* image, void* userdata) { + if (eglDestroyImageKHR(eglGetCurrentDisplay(), (EGLImageKHR) image) == EGL_FALSE) { + slog.e << "eglDestroyImageKHR failed." << io::endl; + } + backend::AcquiredImage* closure = (backend::AcquiredImage*) userdata; + closure->callback(closure->image, closure->userData); + delete closure; + }; + + return {eglImage, patchedCallback, closure}; +} + void PlatformEGL::initializeGlExtensions() noexcept { unordered_string_set glExtensions; GLint n; diff --git a/filament/backend/src/opengl/PlatformEGL.h b/filament/backend/src/opengl/PlatformEGL.h index f68705006e..19db1f10ad 100644 --- a/filament/backend/src/opengl/PlatformEGL.h +++ b/filament/backend/src/opengl/PlatformEGL.h @@ -69,6 +69,8 @@ public: void createExternalImageTexture(void* texture) noexcept final; void destroyExternalImage(void* texture) noexcept final; + backend::AcquiredImage createAcquiredImage(void* hwbuffer, backend::StreamCallback cb, void* userData) noexcept final; + int getOSVersion() const noexcept final; private: diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 16908c9a94..24a5c15f44 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -545,10 +545,18 @@ void VulkanDriver::destroySwapChain(Handle sch) { void VulkanDriver::destroyStream(Handle sh) { } -Handle VulkanDriver::createStream(void* nativeStream) { +Handle VulkanDriver::createStreamNative(void* nativeStream) { return {}; } +Handle VulkanDriver::createStreamAcquired() { + return {}; +} + +void VulkanDriver::setAcquiredImage(Handle sh, void* image, backend::StreamCallback cb, + void* userData) { +} + void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { } diff --git a/filament/include/filament/Stream.h b/filament/include/filament/Stream.h index a9c7d3af02..9a1b664796 100644 --- a/filament/include/filament/Stream.h +++ b/filament/include/filament/Stream.h @@ -37,7 +37,18 @@ class UTILS_PUBLIC Stream : public FilamentAPI { struct BuilderDetails; public: - //! Use Builder to construct an Stream object instance + using Callback = backend::StreamCallback; + using StreamType = backend::StreamType; + + /** + * Constructs a Stream object instance. + * + * By default, Stream objects are ACQUIRED and must have external images pushed to them via + * 
Stream::setAcquiredImage
. + * + * To create a NATIVE or TEXTURE_ID stream, call one of the 
stream
methods + * on the builder. + */ class Builder : public BuilderBase { friend struct BuilderDetails; public: @@ -49,7 +60,7 @@ public: Builder& operator=(Builder&& rhs) noexcept; /** - * Creates a native stream. Native streams can sample data directly from an + * Creates a NATIVE stream. Native streams can sample data directly from an * opaque platform object such as a SurfaceTexture on Android. * * @param stream An opaque native stream handle. e.g.: on Android this is an @@ -60,7 +71,7 @@ public: Builder& stream(void* stream) noexcept; /** - * Creates a copy stream. A copy stream will sample data from the supplied + * Creates a TEXTURE_ID stream. This will sample data from the supplied * external texture and copy it into an internal private texture. * * @param externalTextureId An opaque texture id (typically a GLuint created with glGenTextures) @@ -107,9 +118,17 @@ public: }; /** - * Indicates whether this stream is a native stream or a copy stream. + * Indicates whether this stream is a NATIVE stream, TEXTURE_ID stream, or ACQUIRED stream. */ - bool isNativeStream() const noexcept; + StreamType getStreamType() const noexcept; + + /** + * Updates an ACQUIRED stream with an image that is guaranteed to be used in the next frame. + * + * This method should be called on the same thread that calls Renderer::beginFrame, which is + * also where the callback is invoked. + */ + void setAcquiredImage(void* image, Callback callback, void* userdata) noexcept; /** * Updates the size of the incoming stream. Whether this value is used is diff --git a/filament/src/Stream.cpp b/filament/src/Stream.cpp index a4fbdb18c2..923ee72d78 100644 --- a/filament/src/Stream.cpp +++ b/filament/src/Stream.cpp @@ -84,6 +84,10 @@ namespace details { FStream::FStream(FEngine& engine, const Builder& builder) noexcept : mEngine(engine), + mStreamType( + builder->mExternalTextureId ? StreamType::TEXTURE_ID : + (builder->mStream ? StreamType::NATIVE : StreamType::ACQUIRED) + ), mNativeStream(builder->mStream), mExternalTextureId(builder->mExternalTextureId), mWidth(builder->mWidth), @@ -91,10 +95,12 @@ FStream::FStream(FEngine& engine, const Builder& builder) noexcept if (mNativeStream) { // Note: this is a synchronous call. On Android, this calls back into Java. - mStreamHandle = engine.getDriverApi().createStream(mNativeStream); + mStreamHandle = engine.getDriverApi().createStreamNative(mNativeStream); } else if (mExternalTextureId) { mStreamHandle = engine.getDriverApi().createStreamFromTextureId( mExternalTextureId, mWidth, mHeight); + } else { + mStreamHandle = engine.getDriverApi().createStreamAcquired(); } } @@ -102,6 +108,10 @@ void FStream::terminate(FEngine& engine) noexcept { engine.getDriverApi().destroyStream(mStreamHandle); } +void FStream::setAcquiredImage(void* image, Callback callback, void* userdata) noexcept { + mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, callback, userdata); +} + void FStream::setDimensions(uint32_t width, uint32_t height) noexcept { mWidth = width; mHeight = height; @@ -116,7 +126,7 @@ void FStream::setDimensions(uint32_t width, uint32_t height) noexcept { void FStream::readPixels(uint32_t xoffset, uint32_t yoffset, uint32_t width, uint32_t height, backend::PixelBufferDescriptor&& buffer) noexcept { - if (isExternalTextureId()) { + if (getStreamType() == StreamType::TEXTURE_ID) { // this works only on external texture id streams const size_t sizeNeeded = PixelBufferDescriptor::computeDataSize( @@ -150,8 +160,12 @@ int64_t FStream::getTimestamp() const noexcept { using namespace details; -bool Stream::isNativeStream() const noexcept { - return upcast(this)->isNativeStream(); +StreamType Stream::getStreamType() const noexcept { + return upcast(this)->getStreamType(); +} + +void Stream::setAcquiredImage(void* image, Callback callback, void* userdata) noexcept { + upcast(this)->setAcquiredImage(image, callback, userdata); } void Stream::setDimensions(uint32_t width, uint32_t height) noexcept { diff --git a/filament/src/details/Stream.h b/filament/src/details/Stream.h index bed808cb9f..96fb5dbfcb 100644 --- a/filament/src/details/Stream.h +++ b/filament/src/details/Stream.h @@ -37,14 +37,14 @@ public: backend::Handle getHandle() const noexcept { return mStreamHandle; } + void setAcquiredImage(void* image, Callback callback, void* userdata) noexcept; + void setDimensions(uint32_t width, uint32_t height) noexcept; void readPixels(uint32_t xoffset, uint32_t yoffset, uint32_t width, uint32_t height, backend::PixelBufferDescriptor&& buffer) noexcept; - bool isNativeStream() const noexcept { return mNativeStream != nullptr; } - - bool isExternalTextureId() const noexcept { return !isNativeStream(); } + StreamType getStreamType() const noexcept { return mStreamType; } uint32_t getWidth() const noexcept { return mWidth; } @@ -54,6 +54,7 @@ public: private: FEngine& mEngine; + const StreamType mStreamType; backend::Handle mStreamHandle; void* mNativeStream = nullptr; intptr_t mExternalTextureId;