From a2beaf0582c4e429ae1393c42feffed26ff52ae3 Mon Sep 17 00:00:00 2001 From: "daemyung jang(danny.jang)" Date: Tue, 11 Apr 2023 01:38:26 +0900 Subject: [PATCH] Support the external image on macOS (#6689) * Support the external image on macOS Implement CocoaExternalImage. * Fix to take an onwership of the external image * Correct incorrect comments * Rename a function explicitly Make a function name to know copying RECTANGLE to TEXTURE2D. * Do lazy initialization Create CocoaExternalImage::SharedGl when it's needed. * Fix a crash when engine is terminated Destroy the external image shared gl before gl context is destroyed. * Remove an useless variable --- NEW_RELEASE_NOTES.md | 1 + filament/backend/CMakeLists.txt | 1 + .../backend/platforms/PlatformCocoaGL.h | 4 + .../src/opengl/platforms/CocoaExternalImage.h | 90 +++++++ .../opengl/platforms/CocoaExternalImage.mm | 235 ++++++++++++++++++ .../src/opengl/platforms/PlatformCocoaGL.mm | 53 ++++ 6 files changed, 384 insertions(+) create mode 100644 filament/backend/src/opengl/platforms/CocoaExternalImage.h create mode 100644 filament/backend/src/opengl/platforms/CocoaExternalImage.mm diff --git a/NEW_RELEASE_NOTES.md b/NEW_RELEASE_NOTES.md index b746940d78..4d56a0d11d 100644 --- a/NEW_RELEASE_NOTES.md +++ b/NEW_RELEASE_NOTES.md @@ -10,3 +10,4 @@ appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md). - materials: improved size reduction of OpenGL/Metal shaders by ~65% when compiling materials with size optimizations (`matc -S`) [⚠️ **Recompile Materials**] +opengl: support the external image on macOS diff --git a/filament/backend/CMakeLists.txt b/filament/backend/CMakeLists.txt index 7c7457befb..c42b95d9b7 100644 --- a/filament/backend/CMakeLists.txt +++ b/filament/backend/CMakeLists.txt @@ -96,6 +96,7 @@ if (FILAMENT_SUPPORTS_OPENGL AND NOT FILAMENT_USE_EXTERNAL_GLES3 AND NOT FILAMEN list(APPEND SRCS src/opengl/platforms/CocoaTouchExternalImage.mm) elseif (APPLE) list(APPEND SRCS src/opengl/platforms/PlatformCocoaGL.mm) + list(APPEND SRCS src/opengl/platforms/CocoaExternalImage.mm) elseif (WEBGL) list(APPEND SRCS src/opengl/platforms/PlatformWebGL.cpp) elseif (LINUX) diff --git a/filament/backend/include/backend/platforms/PlatformCocoaGL.h b/filament/backend/include/backend/platforms/PlatformCocoaGL.h index f32d0a25bb..97188852a3 100644 --- a/filament/backend/include/backend/platforms/PlatformCocoaGL.h +++ b/filament/backend/include/backend/platforms/PlatformCocoaGL.h @@ -57,6 +57,10 @@ protected: void destroySwapChain(SwapChain* swapChain) noexcept override; void makeCurrent(SwapChain* drawSwapChain, SwapChain* readSwapChain) noexcept override; void commit(SwapChain* swapChain) noexcept override; + OpenGLPlatform::ExternalTexture* createExternalImageTexture() noexcept override; + void destroyExternalImage(ExternalTexture* texture) noexcept override; + void retainExternalImage(void* externalImage) noexcept override; + bool setExternalImage(void* externalImage, ExternalTexture* texture) noexcept override; private: PlatformCocoaGLImpl* pImpl = nullptr; diff --git a/filament/backend/src/opengl/platforms/CocoaExternalImage.h b/filament/backend/src/opengl/platforms/CocoaExternalImage.h new file mode 100644 index 0000000000..38c3a9d1ff --- /dev/null +++ b/filament/backend/src/opengl/platforms/CocoaExternalImage.h @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FILAMENT_DRIVER_OPENGL_COCOA_EXTERNAL_IMAGE +#define FILAMENT_DRIVER_OPENGL_COCOA_EXTERNAL_IMAGE + +#include +#include + +#include "../gl_headers.h" + +namespace filament::backend { + +class CocoaExternalImage final : public OpenGLPlatform::ExternalTexture { +public: + /** + * GL objects that can be shared across multiple instances of CocoaExternalImage. + */ + class SharedGl { + public: + SharedGl() noexcept; + ~SharedGl() noexcept; + + SharedGl(const SharedGl&) = delete; + SharedGl& operator=(const SharedGl&) = delete; + + GLuint program = 0; + GLuint sampler = 0; + GLuint fragmentShader = 0; + GLuint vertexShader = 0; + }; + + CocoaExternalImage(const CVOpenGLTextureCacheRef textureCache, + const SharedGl& sharedGl) noexcept; + ~CocoaExternalImage() noexcept; + + /** + * Set this external image to the passed-in CVPixelBuffer. + * Afterwards, calling glGetTexture returns the GL texture name backed by the CVPixelBuffer. + */ + bool set(CVPixelBufferRef p) noexcept; + + GLuint getGlTexture() const noexcept; + GLuint getInternalFormat() const noexcept; + GLuint getTarget() const noexcept; + +private: + void release() noexcept; + CVOpenGLTextureRef createTextureFromImage(CVPixelBufferRef image) noexcept; + GLuint encodeCopyRectangleToTexture2D(GLuint rectangle, size_t width, size_t height) noexcept; + + class State { + public: + void save() noexcept; + void restore() noexcept; + + private: + GLint activeTexture = 0; + GLint textureBinding = { 0 }; + GLint samplerBinding = { 0 }; + GLint framebuffer = 0; + GLint viewport[4] = { 0 }; + GLint vertexAttrib = 0; + } mState; + + GLuint mFBO = 0; + const SharedGl& mSharedGl; + GLuint mRgbaTexture = 0; + + const CVOpenGLTextureCacheRef mTextureCache; + CVPixelBufferRef mImage = nullptr; + CVOpenGLTextureRef mTexture = nullptr; +}; + +} // namespace filament::backend + +#endif diff --git a/filament/backend/src/opengl/platforms/CocoaExternalImage.mm b/filament/backend/src/opengl/platforms/CocoaExternalImage.mm new file mode 100644 index 0000000000..4d8e08b9dc --- /dev/null +++ b/filament/backend/src/opengl/platforms/CocoaExternalImage.mm @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define COREVIDEO_SILENCE_GL_DEPRECATION + +#include "CocoaExternalImage.h" +#include +#include "../GLUtils.h" + +namespace filament::backend { + +static const char *s_vertex = R"SHADER(#version 410 core +void main() { + float x = -1.0 + float(((gl_VertexID & 1) <<2)); + float y = -1.0 + float(((gl_VertexID & 2) <<1)); + gl_Position=vec4(x, y, 0.0, 1.0); +} +)SHADER"; + +static const char *s_fragment = R"SHADER(#version 410 core +precision mediump float; + +uniform sampler2DRect rectangle; +layout(location = 0) out vec4 fragColor; + +void main() { + fragColor = texture(rectangle, gl_FragCoord.xy); +} +)SHADER"; + +CocoaExternalImage::SharedGl::SharedGl() noexcept { + glGenSamplers(1, &sampler); + glSamplerParameteri(sampler, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glSamplerParameteri(sampler, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glSamplerParameteri(sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glSamplerParameteri(sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glSamplerParameteri(sampler, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE); + + GLint status; + + vertexShader = glCreateShader(GL_VERTEX_SHADER); + glShaderSource(vertexShader, 1, &s_vertex, nullptr); + glCompileShader(vertexShader); + glGetShaderiv(vertexShader, GL_COMPILE_STATUS, &status); + assert_invariant(status == GL_TRUE); + + fragmentShader = glCreateShader(GL_FRAGMENT_SHADER); + glShaderSource(fragmentShader, 1, &s_fragment, nullptr); + glCompileShader(fragmentShader); + glGetShaderiv(fragmentShader, GL_COMPILE_STATUS, &status); + assert_invariant(status == GL_TRUE); + + program = glCreateProgram(); + glAttachShader(program, vertexShader); + glAttachShader(program, fragmentShader); + glLinkProgram(program); + glGetProgramiv(program, GL_LINK_STATUS, &status); + assert_invariant(status == GL_TRUE); + + // Save current program state. + GLint currentProgram; + glGetIntegerv(GL_CURRENT_PROGRAM, ¤tProgram); + + glUseProgram(program); + GLint samplerLoc = glGetUniformLocation(program, "rectangle"); + glUniform1i(samplerLoc, 0); + + // Restore state. + glUseProgram(currentProgram); +} + +CocoaExternalImage::SharedGl::~SharedGl() noexcept { + glDeleteSamplers(1, &sampler); + glDetachShader(program, vertexShader); + glDetachShader(program, fragmentShader); + glDeleteShader(vertexShader); + glDeleteShader(fragmentShader); + glDeleteProgram(program); +} + +CocoaExternalImage::CocoaExternalImage(const CVOpenGLTextureCacheRef textureCache, + const SharedGl &sharedGl) noexcept : mSharedGl(sharedGl), mTextureCache(textureCache) { + glGenFramebuffers(1, &mFBO); + CHECK_GL_ERROR(utils::slog.e) +} + +CocoaExternalImage::~CocoaExternalImage() noexcept { + glDeleteFramebuffers(1, &mFBO); + release(); +} + +bool CocoaExternalImage::set(CVPixelBufferRef image) noexcept { + // Release references to a previous external image, if we're holding any. + release(); + + if (!image) { + return false; + } + + OSType formatType = CVPixelBufferGetPixelFormatType(image); + ASSERT_POSTCONDITION(formatType == kCVPixelFormatType_32BGRA, + "macOS external images must be 32BGRA format."); + + // The pixel buffer must be locked whenever we do rendering with it. We'll unlock it before + // releasing. + UTILS_UNUSED_IN_RELEASE CVReturn lockStatus = CVPixelBufferLockBaseAddress(image, 0); + assert_invariant(lockStatus == kCVReturnSuccess); + + mImage = image; + mTexture = createTextureFromImage(image); + mRgbaTexture = encodeCopyRectangleToTexture2D(CVOpenGLTextureGetName(mTexture), + CVPixelBufferGetWidth(image), CVPixelBufferGetHeight(image)); + CHECK_GL_ERROR(utils::slog.e) + + return true; +} + +GLuint CocoaExternalImage::getGlTexture() const noexcept { + return mRgbaTexture; +} + +GLuint CocoaExternalImage::getInternalFormat() const noexcept { + if (mRgbaTexture) { + return GL_RGBA8; + } + return 0; +} + +GLuint CocoaExternalImage::getTarget() const noexcept { + if (mRgbaTexture) { + return GL_TEXTURE_2D; + } + return 0; +} + +void CocoaExternalImage::release() noexcept { + if (mImage) { + CVPixelBufferUnlockBaseAddress(mImage, 0); + CVPixelBufferRelease(mImage); + } + if (mTexture) { + CFRelease(mTexture); + } + if (mRgbaTexture) { + glDeleteTextures(1, &mRgbaTexture); + mRgbaTexture = 0; + } +} + +CVOpenGLTextureRef CocoaExternalImage::createTextureFromImage(CVPixelBufferRef image) noexcept { + CVOpenGLTextureRef texture = nullptr; + UTILS_UNUSED_IN_RELEASE CVReturn success = + CVOpenGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + mTextureCache, image, nil, &texture); + assert_invariant(success == kCVReturnSuccess); + + return texture; +} + +GLuint CocoaExternalImage::encodeCopyRectangleToTexture2D(GLuint rectangle, + size_t width, size_t height) noexcept { + GLuint texture; + glGenTextures(1, &texture); + + mState.save(); + + // Create a texture to hold the result of the blit image. + glBindTexture(GL_TEXTURE_2D, texture); + glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, width, height); + CHECK_GL_ERROR(utils::slog.e) + + // source textures + glBindSampler(0, mSharedGl.sampler); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_RECTANGLE, rectangle); + CHECK_GL_ERROR(utils::slog.e) + + // destination texture + glBindFramebuffer(GL_FRAMEBUFFER, mFBO); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0); + CHECK_GL_ERROR(utils::slog.e) + + CHECK_GL_FRAMEBUFFER_STATUS(utils::slog.e, GL_FRAMEBUFFER) + CHECK_GL_ERROR(utils::slog.e) + + // draw + glViewport(0, 0, width, height); + CHECK_GL_ERROR(utils::slog.e) + glUseProgram(mSharedGl.program); + CHECK_GL_ERROR(utils::slog.e) + glDisableVertexAttribArray(0); + glDrawArrays(GL_TRIANGLES, 0, 3); + CHECK_GL_ERROR(utils::slog.e) + + mState.restore(); + CHECK_GL_ERROR(utils::slog.e) + + return texture; +} + +void CocoaExternalImage::State::save() noexcept { + glGetIntegerv(GL_ACTIVE_TEXTURE, &activeTexture); + glGetIntegerv(GL_TEXTURE_BINDING_2D, &textureBinding); + glGetIntegerv(GL_SAMPLER_BINDING, &samplerBinding); + glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, &framebuffer); + glGetIntegerv(GL_VIEWPORT, viewport); + glGetVertexAttribiv(0, GL_VERTEX_ATTRIB_ARRAY_ENABLED, &vertexAttrib); +} + +void CocoaExternalImage::State::restore() noexcept { + glActiveTexture(activeTexture); + glBindTexture(GL_TEXTURE_2D, textureBinding); + glBindSampler(0, samplerBinding); + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer); + glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); + + if (vertexAttrib) { + glEnableVertexAttribArray(0); + } +} + +} // namespace filament::backend diff --git a/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm b/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm index 55d5cb7925..34a76078ae 100644 --- a/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm +++ b/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm @@ -28,6 +28,7 @@ #include #include +#include "CocoaExternalImage.h" namespace filament::backend { @@ -49,6 +50,8 @@ struct PlatformCocoaGLImpl { NSOpenGLContext* mGLContext = nullptr; CocoaGLSwapChain* mCurrentSwapChain = nullptr; std::vector mHeadlessSwapChains; + CVOpenGLTextureCacheRef mTextureCache = nullptr; + std::unique_ptr mExternalImageSharedGl; void updateOpenGLContext(NSView *nsView, bool resetView, bool clearView); }; @@ -159,6 +162,12 @@ Driver* PlatformCocoaGL::createDriver(void* sharedContext, const Platform::Drive int result = bluegl::bind(); ASSERT_POSTCONDITION(!result, "Unable to load OpenGL entry points."); + + UTILS_UNUSED_IN_RELEASE CVReturn success = CVOpenGLTextureCacheCreate(kCFAllocatorDefault, nullptr, + [pImpl->mGLContext CGLContextObj], [pImpl->mGLContext.pixelFormat CGLPixelFormatObj], nullptr, + &pImpl->mTextureCache); + assert_invariant(success == kCVReturnSuccess); + return OpenGLPlatform::createDefaultDriver(this, sharedContext, driverConfig); } @@ -167,6 +176,8 @@ int PlatformCocoaGL::getOSVersion() const noexcept { } void PlatformCocoaGL::terminate() noexcept { + CFRelease(pImpl->mTextureCache); + pImpl->mExternalImageSharedGl.reset(); pImpl->mGLContext = nil; bluegl::unbind(); } @@ -256,6 +267,9 @@ void PlatformCocoaGL::makeCurrent(Platform::SwapChain* drawSwapChain, void PlatformCocoaGL::commit(Platform::SwapChain* swapChain) noexcept { [pImpl->mGLContext flushBuffer]; + + // This needs to be done periodically. + CVOpenGLTextureCacheFlush(pImpl->mTextureCache, 0); } bool PlatformCocoaGL::pumpEvents() noexcept { @@ -266,6 +280,45 @@ bool PlatformCocoaGL::pumpEvents() noexcept { return true; } +OpenGLPlatform::ExternalTexture* PlatformCocoaGL::createExternalImageTexture() noexcept { + if (!pImpl->mExternalImageSharedGl) { + pImpl->mExternalImageSharedGl = std::make_unique(); + } + + ExternalTexture* outTexture = new CocoaExternalImage(pImpl->mTextureCache, + *pImpl->mExternalImageSharedGl); + + // the actual id/target will be set in setExternalImage. + outTexture->id = 0; + outTexture->target = GL_TEXTURE_2D; + return outTexture; +} + +void PlatformCocoaGL::destroyExternalImage(ExternalTexture* texture) noexcept { + auto* p = static_cast(texture); + delete p; +} + +void PlatformCocoaGL::retainExternalImage(void* externalImage) noexcept { + // Take ownership of the passed in buffer. It will be released the next time + // setExternalImage is called, or when the texture is destroyed. + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef) externalImage; + CVPixelBufferRetain(pixelBuffer); +} + +bool PlatformCocoaGL::setExternalImage(void* externalImage, ExternalTexture* texture) noexcept { + CVPixelBufferRef cvPixelBuffer = (CVPixelBufferRef) externalImage; + CocoaExternalImage* cocoaExternalImage = static_cast(texture); + if (!cocoaExternalImage->set(cvPixelBuffer)) { + return false; + } + texture->target = cocoaExternalImage->getTarget(); + texture->id = cocoaExternalImage->getGlTexture(); + // we used to set the internalFormat, but it's not used anywhere on the gl backend side + // cocoaExternalImage->getInternalFormat(); + return true; +} + void PlatformCocoaGLImpl::updateOpenGLContext(NSView *nsView, bool resetView, bool clearView) { NSOpenGLContext* glContext = mGLContext;