diff --git a/android/common/CallbackUtils.cpp b/android/common/CallbackUtils.cpp index cc0c6d259b..905f66e2f7 100644 --- a/android/common/CallbackUtils.cpp +++ b/android/common/CallbackUtils.cpp @@ -27,39 +27,6 @@ struct { jmethodID execute; } gCallbackUtils; -JniCallback* JniCallback::make(filament::Engine* engine, - JNIEnv* env, jobject handler, jobject callback) { - void* that = engine->streamAlloc(sizeof(JniCallback), alignof(JniCallback)); - return new (that) JniCallback(env, handler, callback); -} - -JniCallback::JniCallback(JNIEnv* env, jobject handler, jobject callback) - : mEnv(env) - , mHandler(env->NewGlobalRef(handler)) - , mCallback(env->NewGlobalRef(callback)) { -} - -JniCallback::~JniCallback() { - if (mHandler && mCallback) { -#ifdef ANDROID - if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) { - mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback); - } -#endif - if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) { - mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback); - } - } - mEnv->DeleteGlobalRef(mHandler); - mEnv->DeleteGlobalRef(mCallback); -} - -void JniCallback::invoke(void*, size_t, void* user) { - JniCallback* data = reinterpret_cast(user); - // don't call delete here, because we don't own the storage - data->~JniCallback(); -} - JniBufferCallback* JniBufferCallback::make(filament::Engine* engine, JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer) { void* that = engine->streamAlloc(sizeof(JniBufferCallback), alignof(JniBufferCallback)); @@ -95,6 +62,37 @@ void JniBufferCallback::invoke(void*, size_t, void* user) { data->~JniBufferCallback(); } +JniImageCallback* JniImageCallback::make(filament::Engine* engine, + JNIEnv* env, jobject handler, jobject callback, long image) { + void* that = engine->streamAlloc(sizeof(JniImageCallback), alignof(JniImageCallback)); + return new (that) JniImageCallback(env, handler, callback, image); +} + +JniImageCallback::JniImageCallback(JNIEnv* env, jobject handler, jobject callback, long image) + : mEnv(env) + , mHandler(env->NewGlobalRef(handler)) + , mCallback(env->NewGlobalRef(callback)) + , mImage(image) { } + +JniImageCallback::~JniImageCallback() { + if (mHandler && mCallback) { +#ifdef ANDROID + if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.handlerClass)) { + mEnv->CallBooleanMethod(mHandler, gCallbackUtils.post, mCallback); + } +#endif + if (mEnv->IsInstanceOf(mHandler, gCallbackUtils.executorClass)) { + mEnv->CallVoidMethod(mHandler, gCallbackUtils.execute, mCallback); + } + } + mEnv->DeleteGlobalRef(mHandler); + mEnv->DeleteGlobalRef(mCallback); +} + +void JniImageCallback::invoke(void* image, void* user) { + reinterpret_cast(user)->~JniImageCallback(); +} + void registerCallbackUtils(JNIEnv *env) { #ifdef ANDROID gCallbackUtils.handlerClass = env->FindClass("android/os/Handler"); diff --git a/android/common/CallbackUtils.h b/android/common/CallbackUtils.h index 4501db3fea..f9cd1ec625 100644 --- a/android/common/CallbackUtils.h +++ b/android/common/CallbackUtils.h @@ -23,21 +23,6 @@ #include -struct JniCallback { - static JniCallback* make(filament::Engine* engine, - JNIEnv* env, jobject handler, jobject callback); - - static void invoke(void* buffer, size_t n, void* user); - -private: - JniCallback(JNIEnv* env, jobject handler, jobject callback); - ~JniCallback(); - - JNIEnv* mEnv; - jobject mHandler; - jobject mCallback; -}; - struct JniBufferCallback { static JniBufferCallback* make(filament::Engine* engine, JNIEnv* env, jobject handler, jobject callback, AutoBuffer&& buffer); @@ -53,3 +38,19 @@ private: jobject mCallback; AutoBuffer mBuffer; }; + +struct JniImageCallback { + static JniImageCallback* make(filament::Engine* engine, JNIEnv* env, jobject handler, + jobject runnable, long image); + + static void invoke(void* image, void* user); + +private: + JniImageCallback(JNIEnv* env, jobject handler, jobject runnable, long image); + ~JniImageCallback(); + + JNIEnv* mEnv; + jobject mHandler; + jobject mCallback; + long mImage; +}; diff --git a/android/filament-android/src/main/cpp/Stream.cpp b/android/filament-android/src/main/cpp/Stream.cpp index 53e8097a8d..271a926103 100644 --- a/android/filament-android/src/main/cpp/Stream.cpp +++ b/android/filament-android/src/main/cpp/Stream.cpp @@ -24,6 +24,26 @@ #include "common/NioUtils.h" #include "common/CallbackUtils.h" +#ifdef ANDROID + +#if __has_include() +#include +#else +struct AHardwareBuffer; +typedef struct AHardwareBuffer AHardwareBuffer; +#endif + +#include + +#include + +template +static void loadSymbol(T*& pfn, const char *symbol) noexcept { + pfn = (T*)dlsym(RTLD_DEFAULT, symbol); +} + +#endif + using namespace filament; using namespace backend; @@ -108,10 +128,10 @@ Java_com_google_android_filament_Stream_nBuilderBuild(JNIEnv*, jclass, return (jlong) builder->builder()->build(*engine); } -extern "C" JNIEXPORT jboolean JNICALL -Java_com_google_android_filament_Stream_nIsNative(JNIEnv*, jclass, jlong nativeStream) { +extern "C" JNIEXPORT jint JNICALL +Java_com_google_android_filament_Stream_nGetStreamType(JNIEnv*, jclass, jlong nativeStream) { Stream* stream = (Stream*) nativeStream; - return (jboolean) stream->isNativeStream(); + return (jint) stream->getStreamType(); } extern "C" JNIEXPORT void JNICALL @@ -160,3 +180,34 @@ Java_com_google_android_filament_Stream_nGetTimestamp(JNIEnv*, jclass, jlong nat Stream *stream = (Stream *) nativeStream; return stream->getTimestamp(); } + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_Stream_nSetAcquiredImage(JNIEnv* env, jclass, jlong nativeStream, + jlong nativeEngine, jobject hwbuffer, jobject handler, jobject runnable) { + Engine* engine = (Engine*) nativeEngine; + Stream* stream = (Stream*) nativeStream; + +#ifdef ANDROID + + AHardwareBuffer* (*AHardwareBuffer_fromHardwareBuffer)(JNIEnv*, jobject); + loadSymbol(AHardwareBuffer_fromHardwareBuffer, "AHardwareBuffer_fromHardwareBuffer"); + + AHardwareBuffer* nativeBuffer = AHardwareBuffer_fromHardwareBuffer(env, hwbuffer); + if (!nativeBuffer) { + __android_log_print(ANDROID_LOG_INFO, "Filament", "Unable to obtain native HardwareBuffer."); + return; + } + + auto* callback = JniImageCallback::make(engine, env, handler, runnable, (long) nativeBuffer); + +#else + + // TODO: for non-Android platforms, it is unclear how to go from "jobject" to "void*" + // For now this code is reserved for future use. + auto* callback = JniImageCallback::make(engine, env, handler, runnable, 0); + void* nativeBuffer = nullptr; + +#endif + + stream->setAcquiredImage((void*) nativeBuffer, &JniImageCallback::invoke, callback); +} diff --git a/android/filament-android/src/main/java/com/google/android/filament/Stream.java b/android/filament-android/src/main/java/com/google/android/filament/Stream.java index 97f055bdc0..e35dfcb315 100644 --- a/android/filament-android/src/main/java/com/google/android/filament/Stream.java +++ b/android/filament-android/src/main/java/com/google/android/filament/Stream.java @@ -33,6 +33,20 @@ public class Stream { private long mNativeObject; private long mNativeEngine; + /** + * Represents the immutable stream type. + */ + public enum StreamType { + /** Not synchronized but copy-free. Good for video. */ + NATIVE, + + /** Synchronized, but GL-only and incurs copies. Good for AR on older devices. */ + TEXTURE_ID, + + /** Synchronized, copy-free, and take a release callback. Good for AR on newer devices. */ + ACQUIRED, + }; + Stream(long nativeStream, Engine engine) { mNativeObject = nativeStream; mNativeEngine = engine.getNativeObject(); @@ -40,6 +54,12 @@ public class Stream { /** * Use Builder to construct an Stream object instance. + * + * By default, Stream objects are {@link StreamType#ACQUIRED ACQUIRED} and must have external images pushed to them via + * {@line #setAcquiredImage}. + * + * To create a {@link StreamType#NATIVE NATIVE} or {@link StreamType#TEXTURE_ID TEXTURE_ID} stream, call one of the
stream
methods + * on the builder. */ public static class Builder { @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) // Keep to finalize native resources @@ -55,8 +75,8 @@ public class Stream { } /** - * Creates a native stream. Native streams can sample data directly from an - * opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture} + * Creates a {@link StreamType#NATIVE NATIVE} stream. Native streams can sample data + * directly from an opaque platform object such as a {@link android.graphics.SurfaceTexture SurfaceTexture} * on Android. * * @param streamSource an opaque native stream handle, e.g.: on Android this must be a @@ -74,7 +94,7 @@ public class Stream { } /** - * Creates a copy stream. A copy stream will sample data from the supplied + * Creates a {@link StreamType#TEXTURE_ID TEXTURE_ID} stream. A copy stream will sample data from the supplied * external texture and copy it into an internal private texture. * *

Currently only OpenGL external texture ids are supported.

@@ -150,12 +170,24 @@ public class Stream { } /** - * Indicates whether this Stream is a native stream or a copy stream. - * - * @return true if this is a native Stream, false otherwise. + * Indicates whether this Stream is NATIVE, TEXTURE_ID, or ACQUIRED. */ - public boolean isNative() { - return nIsNative(getNativeObject()); + public StreamType getStreamType() { + return StreamType.values()[nGetStreamType(getNativeObject())]; + } + + /** + * Updates an
ACQUIRED
stream with an image that is guaranteed to be used in the next frame. + * + * This method should be called on the same thread that calls {#link Renderer#beginFrame}, which is + * also where the callback is invoked. + * + * @param hwbuffer {@link android.hardware.HardwareBuffer HardwareBuffer} (requires API level 26) + * @param handler {@link java.util.concurrent.Executor Executor} or {@link android.os.Handler Handler}. + * @param callback a callback invoked by handler when the hwbuffer can be released. + */ + public void setAcquiredImage(Object hwbuffer, Object handler, Runnable callback) { + nSetAcquiredImage(getNativeObject(), mNativeEngine, hwbuffer, handler, callback); } /** @@ -286,6 +318,7 @@ public class Stream { private static native void nBuilderHeight(long nativeStreamBuilder, int height); private static native long nBuilderBuild(long nativeStreamBuilder, long nativeEngine); + private static native int nGetStreamType(long nativeStream); private static native void nSetDimensions(long nativeStream, int width, int height); private static native int nReadPixels(long nativeStream, long nativeEngine, int xoffset, int yoffset, int width, int height, @@ -293,6 +326,6 @@ public class Stream { int left, int top, int type, int alignment, int stride, int format, Object handler, Runnable callback); private static native long nGetTimestamp(long nativeStream); - - private static native boolean nIsNative(long nativeStream); + private static native void nSetAcquiredImage(long nativeStream, long nativeEngine, + Object hwbuffer, Object handler, Runnable callback); } diff --git a/filament/backend/include/backend/DriverEnums.h b/filament/backend/include/backend/DriverEnums.h index f5c790dcc1..93ffab70b3 100644 --- a/filament/backend/include/backend/DriverEnums.h +++ b/filament/backend/include/backend/DriverEnums.h @@ -624,6 +624,23 @@ enum class BlendFunction : uint8_t { SRC_ALPHA_SATURATE //!< f(src, dst) = (1,1,1) * min(src.a, 1 - dst.a), 1 }; +//! Stream for external textures +enum class StreamType { + NATIVE, + TEXTURE_ID, + ACQUIRED, +}; + +//! Releases an ACQUIRED external texture, guaranteed to be called on the application thread. +using StreamCallback = void(*)(void* image, void* user); + +//! Bundles the state of an ACQUIRED stream. +struct AcquiredImage { + void* image = nullptr; + backend::StreamCallback callback = nullptr; + void* userData = nullptr; +}; + //! Vertex attribute descriptor struct Attribute { //! attribute is normalized (remapped between 0 and 1) diff --git a/filament/backend/include/private/backend/DriverAPI.inc b/filament/backend/include/private/backend/DriverAPI.inc index 869538b0c6..c3334be0ab 100644 --- a/filament/backend/include/private/backend/DriverAPI.inc +++ b/filament/backend/include/private/backend/DriverAPI.inc @@ -215,7 +215,9 @@ DECL_DRIVER_API_N(destroyStream, backend::StreamHandle, sh) */ DECL_DRIVER_API_SYNCHRONOUS_0(void, terminate) -DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStream, void*, stream) +DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamNative, void*, stream) +DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamAcquired) +DECL_DRIVER_API_SYNCHRONOUS_N(void, setAcquiredImage, backend::StreamHandle, stream, void*, image, backend::StreamCallback, cb, void*, userData) DECL_DRIVER_API_SYNCHRONOUS_N(void, setStreamDimensions, backend::StreamHandle, stream, uint32_t, width, uint32_t, height) DECL_DRIVER_API_SYNCHRONOUS_N(int64_t, getStreamTimestamp, backend::StreamHandle, stream) DECL_DRIVER_API_SYNCHRONOUS_N(void, updateStreams, backend::DriverApi*, driver) diff --git a/filament/backend/include/private/backend/OpenGLPlatform.h b/filament/backend/include/private/backend/OpenGLPlatform.h index 7df84cd238..24b014b10b 100644 --- a/filament/backend/include/private/backend/OpenGLPlatform.h +++ b/filament/backend/include/private/backend/OpenGLPlatform.h @@ -107,6 +107,11 @@ public: // called once before a SAMPLER_EXTERNAL texture is destroyed. virtual void destroyExternalImage(void* texture) noexcept {} + // allows platforms to convert the given image object into a new type (e.g. HardwareBuffer => EGLImage) + // by default, performs no conversion. + virtual AcquiredImage createAcquiredImage(void* hwbuffer, backend::StreamCallback cb, void* userData) noexcept { + return {hwbuffer, cb, userData}; + } }; } // namespace backend diff --git a/filament/backend/src/Driver.cpp b/filament/backend/src/Driver.cpp index 333637c2c3..a2582de013 100644 --- a/filament/backend/src/Driver.cpp +++ b/filament/backend/src/Driver.cpp @@ -43,9 +43,15 @@ DriverBase::~DriverBase() noexcept { void DriverBase::purge() noexcept { std::vector buffersToPurge; + std::vector imagesToPurge; std::unique_lock lock(mPurgeLock); std::swap(buffersToPurge, mBufferToPurge); + std::swap(imagesToPurge, mImagesToPurge); lock.unlock(); // don't remove this, it ensures mBufferToPurge is destroyed without lock held + for (auto& image : imagesToPurge) { + image.callback(image.image, image.userData); + } + // When the BufferDescriptors go out of scope, their destructors invoke their callbacks. } void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept { @@ -53,6 +59,13 @@ void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept { mBufferToPurge.push_back(std::move(buffer)); } +// This is called from an async driver method so it's in the GL thread, but purge is called +// on the user thread. This is typically called 0 or 1 times per frame. +void DriverBase::scheduleRelease(AcquiredImage&& image) noexcept { + std::lock_guard lock(mPurgeLock); + mImagesToPurge.push_back(std::move(image)); +} + // ------------------------------------------------------------------------------------------------ Driver::~Driver() noexcept = default; diff --git a/filament/backend/src/DriverBase.h b/filament/backend/src/DriverBase.h index 6de6ae2f7d..dcbfaecbe7 100644 --- a/filament/backend/src/DriverBase.h +++ b/filament/backend/src/DriverBase.h @@ -133,8 +133,9 @@ struct HwSwapChain : public HwBase { struct HwStream : public HwBase { HwStream() = default; - explicit HwStream(Platform::Stream* stream) : stream(stream) { } + explicit HwStream(Platform::Stream* stream) : stream(stream), streamType(StreamType::NATIVE) { } Platform::Stream* stream = nullptr; + StreamType streamType = StreamType::ACQUIRED; uint32_t width = 0; uint32_t height = 0; }; @@ -168,9 +169,12 @@ protected: void scheduleDestroySlow(BufferDescriptor&& buffer) noexcept; + void scheduleRelease(AcquiredImage&& image) noexcept; + private: std::mutex mPurgeLock; std::vector mBufferToPurge; + std::vector mImagesToPurge; }; diff --git a/filament/backend/src/metal/MetalDriver.mm b/filament/backend/src/metal/MetalDriver.mm index b1d2f28cc8..c0b4fe384f 100644 --- a/filament/backend/src/metal/MetalDriver.mm +++ b/filament/backend/src/metal/MetalDriver.mm @@ -392,10 +392,18 @@ ShaderModel MetalDriver::getShaderModel() const noexcept { #endif } -Handle MetalDriver::createStream(void* stream) { +Handle MetalDriver::createStreamNative(void* stream) { return {}; } +Handle MetalDriver::createStreamAcquired() { + return {}; +} + +void MetalDriver::setAcquiredImage(Handle sh, void* image, backend::StreamCallback cb, + void* userData) { +} + void MetalDriver::setStreamDimensions(Handle stream, uint32_t width, uint32_t height) { diff --git a/filament/backend/src/opengl/OpenGLDriver.cpp b/filament/backend/src/opengl/OpenGLDriver.cpp index ffda0bbc7a..e72d4a9788 100644 --- a/filament/backend/src/opengl/OpenGLDriver.cpp +++ b/filament/backend/src/opengl/OpenGLDriver.cpp @@ -1015,6 +1015,7 @@ void OpenGLDriver::createStreamFromTextureIdR(Handle sh, s->width = width; s->height = height; s->gl.externalTextureId = static_cast(externalTextureId); + s->streamType = StreamType::TEXTURE_ID; glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read); glGenTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write); for (auto& info : s->user_thread.infos) { @@ -1161,9 +1162,9 @@ void OpenGLDriver::destroyStream(Handle sh) { if (pos != externalStreams.end()) { detachStream(*pos); } - if (s->isNativeStream()) { + if (s->streamType == StreamType::NATIVE) { mPlatform.destroyStream(s->stream); - } else { + } else if (s->streamType == StreamType::TEXTURE_ID) { glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.read); glDeleteTextures(GLStream::ROUND_ROBIN_TEXTURE_COUNT, s->user_thread.write); if (s->gl.fbo) { @@ -1182,13 +1183,35 @@ void OpenGLDriver::destroyStream(Handle sh) { // These are called on the application's thread // ------------------------------------------------------------------------------------------------ -Handle OpenGLDriver::createStream(void* nativeStream) { +Handle OpenGLDriver::createStreamNative(void* nativeStream) { Handle sh( allocateHandle(sizeof(GLStream)) ); Platform::Stream* stream = mPlatform.createStream(nativeStream); construct(sh, stream); return sh; } +Handle OpenGLDriver::createStreamAcquired() { + Handle sh(allocateHandle(sizeof(GLStream))); + construct(sh); + return sh; +} + +// Stashes an acquired external image and a release callback. The image is not bound to OpenGL until +// the subsequent call to beginFrame (see updateStreamAcquired). +// +// setAcquiredImage should be called by the user outside of beginFrame / endFrame, and should be +// called only once per frame. If the user pushes images to the same stream multiple times in a +// single frame, we emit a warning and honor only the final image, but still invoke all callbacks. +void OpenGLDriver::setAcquiredImage(Handle sh, void* hwbuffer, + backend::StreamCallback cb, void* userData) { + GLStream* glstream = handle_cast(sh); + if (glstream->user_thread.pending.image) { + scheduleRelease(std::move(glstream->user_thread.pending)); + slog.w << "Acquired image is set more than once per frame." << io::endl; + } + glstream->user_thread.pending = mPlatform.createAcquiredImage(hwbuffer, cb, userData); +} + void OpenGLDriver::updateStreams(DriverApi* driver) { if (UTILS_UNLIKELY(!mExternalStreams.empty())) { OpenGLBlitter::State state; @@ -1198,13 +1221,17 @@ void OpenGLDriver::updateStreams(DriverApi* driver) { GLStream* s = static_cast(t->hwStream); if (UTILS_UNLIKELY(s == nullptr)) { // this can happen because we're called synchronously and the setExternalStream() - // call may bot have been processed yet. + // call may not have been processed yet. continue; } - if (!s->isNativeStream()) { + if (s->streamType == StreamType::TEXTURE_ID) { state.setup(); - updateStream(t, driver); + updateStreamTexId(t, driver); + } + + if (s->streamType == StreamType::ACQUIRED) { + updateStreamAcquired(t, driver); } } } @@ -1718,10 +1745,12 @@ void OpenGLDriver::cancelExternalImage(void* image) { } void OpenGLDriver::setExternalImage(Handle th, void* image) { - GLTexture* t = handle_cast(th); - auto& gl = mContext; + mPlatform.setExternalImage(image, handle_cast(th)); + setExternalTexture(handle_cast(th), image); +} - mPlatform.setExternalImage(image, t); +void OpenGLDriver::setExternalTexture(GLTexture* t, void* image) { + auto& gl = mContext; // TODO: move this logic to PlatformEGL. if (gl.ext.OES_EGL_image_external_essl3) { @@ -1748,7 +1777,7 @@ void OpenGLDriver::setExternalStream(Handle th, Handle sh) if (UTILS_LIKELY(sh)) { GLStream* s = handle_cast(sh); if (UTILS_LIKELY(!t->hwStream)) { - // we're not attached alread + // we're not attached already attachStream(t, s); } else { if (s->stream != t->hwStream->stream) { @@ -1768,20 +1797,26 @@ void OpenGLDriver::attachStream(GLTexture* t, GLStream* hwStream) noexcept { auto& gl = mContext; mExternalStreams.push_back(t); - if (hwStream->isNativeStream()) { - mPlatform.attach(hwStream->stream, t->gl.id); - } else { - assert(t->target == SamplerType::SAMPLER_EXTERNAL); - // The texture doesn't need a texture name anymore, get rid of it - gl.unbindTexture(t->gl.target, t->gl.id); - glDeleteTextures(1, &t->gl.id); - t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + switch (hwStream->streamType) { + case StreamType::NATIVE: + mPlatform.attach(hwStream->stream, t->gl.id); + break; + case StreamType::TEXTURE_ID: + assert(t->target == SamplerType::SAMPLER_EXTERNAL); + // The texture doesn't need a texture name anymore, get rid of it + gl.unbindTexture(t->gl.target, t->gl.id); + glDeleteTextures(1, &t->gl.id); + t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + break; + case StreamType::ACQUIRED: + break; } t->hwStream = hwStream; } UTILS_NOINLINE void OpenGLDriver::detachStream(GLTexture* t) noexcept { + auto& gl = mContext; auto& streams = mExternalStreams; auto pos = std::find(streams.begin(), streams.end(), t); if (pos != streams.end()) { @@ -1789,30 +1824,57 @@ void OpenGLDriver::detachStream(GLTexture* t) noexcept { } GLStream* s = static_cast(t->hwStream); - if (s->isNativeStream()) { - mPlatform.detach(t->hwStream->stream); - // this deletes the texture id + switch (s->streamType) { + case StreamType::NATIVE: + mPlatform.detach(t->hwStream->stream); + // ^ this deletes the texture id + break; + case StreamType::TEXTURE_ID: + break; + case StreamType::ACQUIRED: + gl.unbindTexture(t->gl.target, t->gl.id); + glDeleteTextures(1, &t->gl.id); + break; } + glGenTextures(1, &t->gl.id); + t->hwStream = nullptr; } UTILS_NOINLINE -void OpenGLDriver::replaceStream(GLTexture* t, GLStream* hwStream) noexcept { - GLStream* s = static_cast(t->hwStream); - if (s->isNativeStream()) { - mPlatform.detach(t->hwStream->stream); - // this deletes the texture id +void OpenGLDriver::replaceStream(GLTexture* texture, GLStream* newStream) noexcept { + assert(newStream && "Do not use replaceStream to detach a stream."); + + // This could be implemented via detachStream + attachStream but inlining allows + // a few small optimizations, like not touching the mExternalStreams list. + + GLStream* oldStream = static_cast(texture->hwStream); + switch (oldStream->streamType) { + case StreamType::NATIVE: + mPlatform.detach(texture->hwStream->stream); + // ^ this deletes the texture id + break; + case StreamType::TEXTURE_ID: + case StreamType::ACQUIRED: + break; } - if (hwStream->isNativeStream()) { - glGenTextures(1, &t->gl.id); - mPlatform.attach(hwStream->stream, t->gl.id); - } else { - assert(t->target == SamplerType::SAMPLER_EXTERNAL); - t->gl.id = hwStream->user_thread.read[hwStream->user_thread.cur]; + switch (newStream->streamType) { + case StreamType::NATIVE: + glGenTextures(1, &texture->gl.id); + mPlatform.attach(newStream->stream, texture->gl.id); + break; + case StreamType::TEXTURE_ID: + assert(texture->target == SamplerType::SAMPLER_EXTERNAL); + texture->gl.id = newStream->user_thread.read[newStream->user_thread.cur]; + break; + case StreamType::ACQUIRED: + // Just re-use the old texture id. + break; } - t->hwStream = hwStream; + + texture->hwStream = newStream; } void OpenGLDriver::beginRenderPass(Handle rth, @@ -2114,19 +2176,51 @@ void OpenGLDriver::setViewportScissor(Viewport const& viewportScissor) noexcept gl.enable(GL_SCISSOR_TEST); } -/* - * This is called in the user thread - */ +// Binds the external image stashed in the associated stream. +// +// updateStreamAcquired() and setAcquiredImage() are both called from on the application's thread +// and therefore do not require synchronization. The former is always called immediately before +// beginFrame, the latter is called by the user from anywhere outside beginFrame / endFrame. +void OpenGLDriver::updateStreamAcquired(GLTexture* gltexture, DriverApi* driver) noexcept { + SYSTRACE_CALL(); + + GLStream* glstream = static_cast(gltexture->hwStream); + assert(glstream); + assert(glstream->streamType == StreamType::ACQUIRED); + + // If there's no pending image, do nothing. Note that GL_OES_EGL_image does not let you pass + // NULL to glEGLImageTargetTexture2DOES, and there is no concept of "detaching" an EGLimage from + // a texture. + if (glstream->user_thread.pending.image == nullptr) { + return; + } + + AcquiredImage previousImage = glstream->user_thread.acquired; + glstream->user_thread.acquired = glstream->user_thread.pending; + glstream->user_thread.pending = {0}; + + // Bind the stashed EGLImage to its corresponding GL texture as soon as we start making the GL + // calls for the upcoming frame. + void* image = glstream->user_thread.acquired.image; + driver->queueCommand([this, gltexture, image, previousImage]() { + setExternalTexture(gltexture, image); + if (previousImage.image) { + whenGpuCommandsComplete([this, previousImage]() { + scheduleRelease(AcquiredImage(previousImage)); + }); + } + }); +} #define DEBUG_NO_EXTERNAL_STREAM_COPY false -void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept { +void OpenGLDriver::updateStreamTexId(GLTexture* t, DriverApi* driver) noexcept { SYSTRACE_CALL(); auto& gl = mContext; GLStream* s = static_cast(t->hwStream); assert(s); - assert(!s->isNativeStream()); + assert(s->streamType == StreamType::TEXTURE_ID); // round-robin to the next texture name if (UTILS_UNLIKELY(DEBUG_NO_EXTERNAL_STREAM_COPY || @@ -2136,7 +2230,7 @@ void OpenGLDriver::updateStream(GLTexture* t, DriverApi* driver) noexcept { // also make sure that this texture is still associated with the same stream auto& streams = mExternalStreams; if (UTILS_LIKELY(std::find(streams.begin(), streams.end(), t) != streams.end()) && - (t->hwStream == s)) { + (t->hwStream == s)) { t->gl.id = s->gl.externalTextureId; } }); @@ -2221,7 +2315,13 @@ void OpenGLDriver::readStreamPixels(Handle sh, auto& gl = mContext; GLStream* s = handle_cast(sh); - if (UTILS_LIKELY(!s->isNativeStream())) { + + if (UTILS_UNLIKELY(s->streamType == StreamType::ACQUIRED)) { + PANIC_LOG("readStreamPixels with ACQUIRED streams is not yet implemented."); + return; + } + + if (UTILS_LIKELY(s->streamType == StreamType::TEXTURE_ID)) { GLuint tid = s->gl.externalTexture2DId; if (tid == 0) { return; @@ -2510,7 +2610,7 @@ void OpenGLDriver::beginFrame(int64_t monotonic_clock_ns, uint32_t frameId, OpenGLPlatform& platform = mPlatform; for (GLTexture const* t : mExternalStreams) { assert(t && t->hwStream); - if (static_cast(t->hwStream)->isNativeStream()) { + if (t->hwStream->streamType == StreamType::NATIVE) { assert(t->hwStream->stream); platform.updateTexImage(t->hwStream->stream, &static_cast(t->hwStream)->user_thread.timestamp); diff --git a/filament/backend/src/opengl/OpenGLDriver.h b/filament/backend/src/opengl/OpenGLDriver.h index 7b96ef6187..5586f66d40 100644 --- a/filament/backend/src/opengl/OpenGLDriver.h +++ b/filament/backend/src/opengl/OpenGLDriver.h @@ -124,7 +124,6 @@ public: struct GLStream : public backend::HwStream { static constexpr size_t ROUND_ROBIN_TEXTURE_COUNT = 3; // 3 maximum using HwStream::HwStream; - bool isNativeStream() const { return gl.externalTextureId == 0; } struct Info { // storage for the read/write textures below backend::Platform::ExternalTexture* ets = nullptr; @@ -143,8 +142,9 @@ public: GLuint fbo = 0; } gl; // 20 bytes + /* - * The fields below are access from the main application thread + * The fields below are accessed from the main application thread * (not the GL thread) */ struct { @@ -155,6 +155,8 @@ public: Info infos[ROUND_ROBIN_TEXTURE_COUNT]; // 48 bytes int64_t timestamp = 0; uint8_t cur = 0; + backend::AcquiredImage acquired; + backend::AcquiredImage pending; } user_thread; }; @@ -373,10 +375,13 @@ private: backend::OpenGLPlatform& mPlatform; OpenGLBlitter* mOpenGLBlitter = nullptr; - void updateStream(GLTexture* t, backend::DriverApi* driver) noexcept; + void updateStreamTexId(GLTexture* t, backend::DriverApi* driver) noexcept; + void updateStreamAcquired(GLTexture* t, backend::DriverApi* driver) noexcept; void updateBuffer(GLenum target, GLBuffer* buffer, backend::BufferDescriptor const& p, uint32_t alignment = 16) noexcept; void updateTextureLodRange(GLTexture* texture, int8_t targetLevel) noexcept; + void setExternalTexture(GLTexture* t, void* image); + void whenGpuCommandsComplete(std::function fn) noexcept; void executeGpuCommandsCompleteOps() noexcept; std::vector>> mGpuCommandCompleteOps; diff --git a/filament/backend/src/opengl/PlatformEGL.cpp b/filament/backend/src/opengl/PlatformEGL.cpp index 377170ef26..6a0127cc26 100644 --- a/filament/backend/src/opengl/PlatformEGL.cpp +++ b/filament/backend/src/opengl/PlatformEGL.cpp @@ -542,6 +542,39 @@ void PlatformEGL::destroyExternalImage(void* texture) noexcept { glDeleteTextures(1, &t->gl.id); } +backend::AcquiredImage PlatformEGL::createAcquiredImage(void* hwbuffer, backend::StreamCallback userCallback, void* userData) noexcept { + + // Convert the AHardwareBuffer to EGLImage. + EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID((const AHardwareBuffer*) hwbuffer); + if (!clientBuffer) { + slog.e << "Unable to get EGLClientBuffer from AHardwareBuffer." << io::endl; + return {}; + } + // Note that this cannot be used to stream protected video (for now) because we do not set EGL_PROTECTED_CONTENT_EXT. + EGLint attrs[] = { EGL_NONE, EGL_NONE }; + EGLImageKHR eglImage = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, clientBuffer, attrs); + if (eglImage == EGL_NO_IMAGE_KHR) { + slog.e << "eglCreateImageKHR returned no image." << io::endl; + return {}; + } + + // Destroy the EGLImage before invoking the user's callback. + AcquiredImage* closure = new AcquiredImage(); + closure->callback = userCallback; + closure->image = hwbuffer; + closure->userData = userData; + auto patchedCallback = [](void* image, void* userdata) { + if (eglDestroyImageKHR(eglGetCurrentDisplay(), (EGLImageKHR) image) == EGL_FALSE) { + slog.e << "eglDestroyImageKHR failed." << io::endl; + } + backend::AcquiredImage* closure = (backend::AcquiredImage*) userdata; + closure->callback(closure->image, closure->userData); + delete closure; + }; + + return {eglImage, patchedCallback, closure}; +} + void PlatformEGL::initializeGlExtensions() noexcept { unordered_string_set glExtensions; GLint n; diff --git a/filament/backend/src/opengl/PlatformEGL.h b/filament/backend/src/opengl/PlatformEGL.h index f68705006e..19db1f10ad 100644 --- a/filament/backend/src/opengl/PlatformEGL.h +++ b/filament/backend/src/opengl/PlatformEGL.h @@ -69,6 +69,8 @@ public: void createExternalImageTexture(void* texture) noexcept final; void destroyExternalImage(void* texture) noexcept final; + backend::AcquiredImage createAcquiredImage(void* hwbuffer, backend::StreamCallback cb, void* userData) noexcept final; + int getOSVersion() const noexcept final; private: diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 16908c9a94..24a5c15f44 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -545,10 +545,18 @@ void VulkanDriver::destroySwapChain(Handle sch) { void VulkanDriver::destroyStream(Handle sh) { } -Handle VulkanDriver::createStream(void* nativeStream) { +Handle VulkanDriver::createStreamNative(void* nativeStream) { return {}; } +Handle VulkanDriver::createStreamAcquired() { + return {}; +} + +void VulkanDriver::setAcquiredImage(Handle sh, void* image, backend::StreamCallback cb, + void* userData) { +} + void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { } diff --git a/filament/include/filament/Stream.h b/filament/include/filament/Stream.h index a9c7d3af02..9a1b664796 100644 --- a/filament/include/filament/Stream.h +++ b/filament/include/filament/Stream.h @@ -37,7 +37,18 @@ class UTILS_PUBLIC Stream : public FilamentAPI { struct BuilderDetails; public: - //! Use Builder to construct an Stream object instance + using Callback = backend::StreamCallback; + using StreamType = backend::StreamType; + + /** + * Constructs a Stream object instance. + * + * By default, Stream objects are ACQUIRED and must have external images pushed to them via + *
Stream::setAcquiredImage
. + * + * To create a NATIVE or TEXTURE_ID stream, call one of the
stream
methods + * on the builder. + */ class Builder : public BuilderBase { friend struct BuilderDetails; public: @@ -49,7 +60,7 @@ public: Builder& operator=(Builder&& rhs) noexcept; /** - * Creates a native stream. Native streams can sample data directly from an + * Creates a NATIVE stream. Native streams can sample data directly from an * opaque platform object such as a SurfaceTexture on Android. * * @param stream An opaque native stream handle. e.g.: on Android this is an @@ -60,7 +71,7 @@ public: Builder& stream(void* stream) noexcept; /** - * Creates a copy stream. A copy stream will sample data from the supplied + * Creates a TEXTURE_ID stream. This will sample data from the supplied * external texture and copy it into an internal private texture. * * @param externalTextureId An opaque texture id (typically a GLuint created with glGenTextures) @@ -107,9 +118,17 @@ public: }; /** - * Indicates whether this stream is a native stream or a copy stream. + * Indicates whether this stream is a NATIVE stream, TEXTURE_ID stream, or ACQUIRED stream. */ - bool isNativeStream() const noexcept; + StreamType getStreamType() const noexcept; + + /** + * Updates an ACQUIRED stream with an image that is guaranteed to be used in the next frame. + * + * This method should be called on the same thread that calls Renderer::beginFrame, which is + * also where the callback is invoked. + */ + void setAcquiredImage(void* image, Callback callback, void* userdata) noexcept; /** * Updates the size of the incoming stream. Whether this value is used is diff --git a/filament/src/Stream.cpp b/filament/src/Stream.cpp index a4fbdb18c2..923ee72d78 100644 --- a/filament/src/Stream.cpp +++ b/filament/src/Stream.cpp @@ -84,6 +84,10 @@ namespace details { FStream::FStream(FEngine& engine, const Builder& builder) noexcept : mEngine(engine), + mStreamType( + builder->mExternalTextureId ? StreamType::TEXTURE_ID : + (builder->mStream ? StreamType::NATIVE : StreamType::ACQUIRED) + ), mNativeStream(builder->mStream), mExternalTextureId(builder->mExternalTextureId), mWidth(builder->mWidth), @@ -91,10 +95,12 @@ FStream::FStream(FEngine& engine, const Builder& builder) noexcept if (mNativeStream) { // Note: this is a synchronous call. On Android, this calls back into Java. - mStreamHandle = engine.getDriverApi().createStream(mNativeStream); + mStreamHandle = engine.getDriverApi().createStreamNative(mNativeStream); } else if (mExternalTextureId) { mStreamHandle = engine.getDriverApi().createStreamFromTextureId( mExternalTextureId, mWidth, mHeight); + } else { + mStreamHandle = engine.getDriverApi().createStreamAcquired(); } } @@ -102,6 +108,10 @@ void FStream::terminate(FEngine& engine) noexcept { engine.getDriverApi().destroyStream(mStreamHandle); } +void FStream::setAcquiredImage(void* image, Callback callback, void* userdata) noexcept { + mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, callback, userdata); +} + void FStream::setDimensions(uint32_t width, uint32_t height) noexcept { mWidth = width; mHeight = height; @@ -116,7 +126,7 @@ void FStream::setDimensions(uint32_t width, uint32_t height) noexcept { void FStream::readPixels(uint32_t xoffset, uint32_t yoffset, uint32_t width, uint32_t height, backend::PixelBufferDescriptor&& buffer) noexcept { - if (isExternalTextureId()) { + if (getStreamType() == StreamType::TEXTURE_ID) { // this works only on external texture id streams const size_t sizeNeeded = PixelBufferDescriptor::computeDataSize( @@ -150,8 +160,12 @@ int64_t FStream::getTimestamp() const noexcept { using namespace details; -bool Stream::isNativeStream() const noexcept { - return upcast(this)->isNativeStream(); +StreamType Stream::getStreamType() const noexcept { + return upcast(this)->getStreamType(); +} + +void Stream::setAcquiredImage(void* image, Callback callback, void* userdata) noexcept { + upcast(this)->setAcquiredImage(image, callback, userdata); } void Stream::setDimensions(uint32_t width, uint32_t height) noexcept { diff --git a/filament/src/details/Stream.h b/filament/src/details/Stream.h index bed808cb9f..96fb5dbfcb 100644 --- a/filament/src/details/Stream.h +++ b/filament/src/details/Stream.h @@ -37,14 +37,14 @@ public: backend::Handle getHandle() const noexcept { return mStreamHandle; } + void setAcquiredImage(void* image, Callback callback, void* userdata) noexcept; + void setDimensions(uint32_t width, uint32_t height) noexcept; void readPixels(uint32_t xoffset, uint32_t yoffset, uint32_t width, uint32_t height, backend::PixelBufferDescriptor&& buffer) noexcept; - bool isNativeStream() const noexcept { return mNativeStream != nullptr; } - - bool isExternalTextureId() const noexcept { return !isNativeStream(); } + StreamType getStreamType() const noexcept { return mStreamType; } uint32_t getWidth() const noexcept { return mWidth; } @@ -54,6 +54,7 @@ public: private: FEngine& mEngine; + const StreamType mStreamType; backend::Handle mStreamHandle; void* mNativeStream = nullptr; intptr_t mExternalTextureId;