From 8c8e0ce8a382aff93b73f62aff992eaaf12663cf Mon Sep 17 00:00:00 2001 From: Ben Doherty Date: Wed, 18 Aug 2021 10:57:50 -0700 Subject: [PATCH] Metal: implement texture swizzling for external textures (#4514) --- filament/backend/CMakeLists.txt | 1 + .../backend/src/metal/MetalExternalImage.h | 14 ++++++- .../backend/src/metal/MetalExternalImage.mm | 36 ++++++++++++++--- filament/backend/src/metal/MetalHandles.mm | 21 +++------- filament/backend/src/metal/MetalUtils.h | 34 ++++++++++++++++ filament/backend/src/metal/MetalUtils.mm | 40 +++++++++++++++++++ filament/include/filament/Texture.h | 2 +- 7 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 filament/backend/src/metal/MetalUtils.h create mode 100644 filament/backend/src/metal/MetalUtils.mm diff --git a/filament/backend/CMakeLists.txt b/filament/backend/CMakeLists.txt index c7846fd618..76ff5c5ab2 100644 --- a/filament/backend/CMakeLists.txt +++ b/filament/backend/CMakeLists.txt @@ -117,6 +117,7 @@ if (FILAMENT_SUPPORTS_METAL) src/metal/MetalResourceTracker.cpp src/metal/MetalState.mm src/metal/MetalTimerQuery.mm + src/metal/MetalUtils.mm ) list(APPEND SRCS ${METAL_SRCS}) diff --git a/filament/backend/src/metal/MetalExternalImage.h b/filament/backend/src/metal/MetalExternalImage.h index 45f7c0d75a..f5ad259999 100644 --- a/filament/backend/src/metal/MetalExternalImage.h +++ b/filament/backend/src/metal/MetalExternalImage.h @@ -17,6 +17,8 @@ #ifndef TNT_METALEXTERNALIMAGE_H #define TNT_METALEXTERNALIMAGE_H +#import + #include #include @@ -34,7 +36,11 @@ class MetalExternalImage { public: - MetalExternalImage(MetalContext& context) noexcept; + MetalExternalImage(MetalContext& context, + TextureSwizzle r = TextureSwizzle::CHANNEL_0, + TextureSwizzle g = TextureSwizzle::CHANNEL_1, + TextureSwizzle b = TextureSwizzle::CHANNEL_2, + TextureSwizzle a = TextureSwizzle::CHANNEL_3) noexcept; /** * @return true, if this MetalExternalImage is holding a live external image. Returns false @@ -94,6 +100,7 @@ private: CVMetalTextureRef createTextureFromImage(CVPixelBufferRef image, MTLPixelFormat format, size_t plane); id createRgbTexture(size_t width, size_t height); + id createSwizzledTextureView(CVMetalTextureRef texture) const; void ensureComputePipelineState(); id encodeColorConversionPass(id inYPlane, id inCbCrTexture, id outTexture); @@ -105,8 +112,10 @@ private: // If the external image has a single plane, mImage and mTexture hold references to the image // and created Metal texture, respectively. + // mTextureView is a view of mTexture with any swizzling applied. CVPixelBufferRef mImage = nullptr; CVMetalTextureRef mTexture = nullptr; + id mTextureView = nullptr; size_t mWidth = 0; size_t mHeight = 0; @@ -114,6 +123,9 @@ private: // texture. id mRgbTexture = nil; + struct { + TextureSwizzle r, g, b, a; + } swizzle; }; } // namespace metal diff --git a/filament/backend/src/metal/MetalExternalImage.mm b/filament/backend/src/metal/MetalExternalImage.mm index e321852b10..8f2ac737c1 100644 --- a/filament/backend/src/metal/MetalExternalImage.mm +++ b/filament/backend/src/metal/MetalExternalImage.mm @@ -17,6 +17,8 @@ #include "MetalExternalImage.h" #include "MetalContext.h" +#include "MetalEnums.h" +#include "MetalUtils.h" #include #include @@ -69,7 +71,8 @@ ycbcrToRgb(texture2d inYTexture [[texture(0)]], } )"; -MetalExternalImage::MetalExternalImage(MetalContext& context) noexcept : mContext(context) { } +MetalExternalImage::MetalExternalImage(MetalContext& context, TextureSwizzle r, TextureSwizzle g, + TextureSwizzle b, TextureSwizzle a) noexcept : mContext(context), swizzle{r, g, b ,a} { } bool MetalExternalImage::isValid() const noexcept { return mRgbTexture != nil || mImage != nullptr; @@ -95,6 +98,7 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept { if (planeCount == 0) { mImage = image; mTexture = createTextureFromImage(image, MTLPixelFormatBGRA8Unorm, 0); + mTextureView = createSwizzledTextureView(mTexture); mWidth = CVPixelBufferGetWidth(image); mHeight = CVPixelBufferGetHeight(image); } @@ -108,12 +112,14 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept { mWidth = CVPixelBufferGetWidthOfPlane(image, Y_PLANE); mHeight = CVPixelBufferGetHeightOfPlane(image, Y_PLANE); - mRgbTexture = createRgbTexture(mWidth, mHeight); - + id rgbTexture = createRgbTexture(mWidth, mHeight); id commandBuffer = encodeColorConversionPass( CVMetalTextureGetTexture(yPlane), CVMetalTextureGetTexture(cbcrPlane), - mRgbTexture); + rgbTexture); + + mRgbTexture = createTextureViewWithSwizzle(rgbTexture, + getSwizzleChannels(swizzle.r, swizzle.g, swizzle.b, swizzle.a)); [commandBuffer addCompletedHandler:^(id o) { CVBufferRelease(yPlane); @@ -154,6 +160,7 @@ void MetalExternalImage::set(CVPixelBufferRef image, size_t plane) noexcept { const MTLPixelFormat format = getPlaneFormat(plane); assert_invariant(format != MTLPixelFormatInvalid); mTexture = createTextureFromImage(image, format, plane); + mTextureView = createSwizzledTextureView(mTexture); } id MetalExternalImage::getMetalTextureForDraw() const noexcept { @@ -173,7 +180,8 @@ id MetalExternalImage::getMetalTextureForDraw() const noexcept { CVBufferRetain(mTexture); } - return CVMetalTextureGetTexture(mTexture); + assert_invariant(mTextureView); + return mTextureView; } CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVPixelBufferRef image, @@ -206,6 +214,7 @@ void MetalExternalImage::unset() { mImage = nullptr; mTexture = nullptr; + mTextureView = nil; mRgbTexture = nil; mWidth = 0; mHeight = 0; @@ -221,6 +230,23 @@ id MetalExternalImage::createRgbTexture(size_t width, size_t height) return [mContext.device newTextureWithDescriptor:descriptor]; } +id MetalExternalImage::createSwizzledTextureView(CVMetalTextureRef ref) const { + id texture = CVMetalTextureGetTexture(ref); + const bool isDefaultSwizzle = + swizzle.r == TextureSwizzle::CHANNEL_0 && + swizzle.g == TextureSwizzle::CHANNEL_1 && + swizzle.b == TextureSwizzle::CHANNEL_2 && + swizzle.a == TextureSwizzle::CHANNEL_3; + if (!isDefaultSwizzle && mContext.supportsTextureSwizzling) { + // Even though we've already checked supportsTextureSwizzling, we still need to guard these + // calls with @availability, otherwise the API usage will generate compiler warnings. + if (@available(iOS 13, *)) { + texture = createTextureViewWithSwizzle(texture, + getSwizzleChannels(swizzle.r, swizzle.g, swizzle.b, swizzle.a)); + } + } + return texture; +} void MetalExternalImage::ensureComputePipelineState() { if (mContext.externalImageComputePipelineState != nil) { diff --git a/filament/backend/src/metal/MetalHandles.mm b/filament/backend/src/metal/MetalHandles.mm index a4176bf3c7..f9c965a089 100644 --- a/filament/backend/src/metal/MetalHandles.mm +++ b/filament/backend/src/metal/MetalHandles.mm @@ -18,6 +18,7 @@ #include "MetalBlitter.h" #include "MetalEnums.h" +#include "MetalUtils.h" #include @@ -380,7 +381,7 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le TextureUsage usage, TextureSwizzle r, TextureSwizzle g, TextureSwizzle b, TextureSwizzle a) noexcept : HwTexture(target, levels, samples, width, height, depth, format, usage), context(context), - externalImage(context) { + externalImage(context, r, g, b, a) { devicePixelFormat = decidePixelFormat(&context, format); ASSERT_POSTCONDITION(devicePixelFormat != MTLPixelFormatInvalid, "Texture format not supported."); @@ -475,25 +476,15 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le g == TextureSwizzle::CHANNEL_1 && b == TextureSwizzle::CHANNEL_2 && a == TextureSwizzle::CHANNEL_3; - // If texture is nil, then it must be a SAMPLER_EXTERNAL texture. We'll ignore this case for now. - // TODO: implement swizzling for external textures. + // If texture is nil, then it must be a SAMPLER_EXTERNAL texture. + // Swizzling for external textures is handled inside MetalExternalImage. if (!isDefaultSwizzle && texture && context.supportsTextureSwizzling) { // Even though we've already checked context.supportsTextureSwizzling, we still need to // guard these calls with @availability, otherwise the API usage will generate compiler // warnings. if (@available(iOS 13, *)) { - NSUInteger slices = texture.arrayLength; - if (texture.textureType == MTLTextureTypeCube || - texture.textureType == MTLTextureTypeCubeArray) { - slices *= 6; - } - NSUInteger mips = texture.mipmapLevelCount; - MTLTextureSwizzleChannels swizzle = getSwizzleChannels(r, g, b, a); - swizzledTextureView = [texture newTextureViewWithPixelFormat:texture.pixelFormat - textureType:texture.textureType - levels:NSMakeRange(0, mips) - slices:NSMakeRange(0, slices) - swizzle:swizzle]; + swizzledTextureView = + createTextureViewWithSwizzle(texture, getSwizzleChannels(r, g, b, a)); } } } diff --git a/filament/backend/src/metal/MetalUtils.h b/filament/backend/src/metal/MetalUtils.h new file mode 100644 index 0000000000..f97e03fd1a --- /dev/null +++ b/filament/backend/src/metal/MetalUtils.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_DRIVER_METALUTILS_H +#define TNT_FILAMENT_DRIVER_METALUTILS_H + +#include + +namespace filament::backend::metal { + +/** + * Creates a texture view of the passed-in texture with the given swizzle. If swizzle is the default + * swizzle, simply returns texture. + */ +API_AVAILABLE(ios(13.0)) +id createTextureViewWithSwizzle(id texture, + MTLTextureSwizzleChannels swizzle); + +} // namespace filament::backend::metal + +#endif //TNT_FILAMENT_DRIVER_METALUTILS_H diff --git a/filament/backend/src/metal/MetalUtils.mm b/filament/backend/src/metal/MetalUtils.mm new file mode 100644 index 0000000000..528ac0b8ed --- /dev/null +++ b/filament/backend/src/metal/MetalUtils.mm @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "MetalUtils.h" + +namespace filament::backend::metal { + +id createTextureViewWithSwizzle(id texture, + MTLTextureSwizzleChannels swizzle) { + if (swizzle.red == MTLTextureSwizzleRed && swizzle.green == MTLTextureSwizzleGreen && + swizzle.blue == MTLTextureSwizzleBlue && swizzle.alpha == MTLTextureSwizzleAlpha) { + return texture; + } + NSUInteger slices = texture.arrayLength; + if (texture.textureType == MTLTextureTypeCube || + texture.textureType == MTLTextureTypeCubeArray) { + slices *= 6; + } + NSUInteger mips = texture.mipmapLevelCount; + return [texture newTextureViewWithPixelFormat:texture.pixelFormat + textureType:texture.textureType + levels:NSMakeRange(0, mips) + slices:NSMakeRange(0, slices) + swizzle:swizzle]; +} + +} diff --git a/filament/include/filament/Texture.h b/filament/include/filament/Texture.h index 71550ba648..2656023e16 100644 --- a/filament/include/filament/Texture.h +++ b/filament/include/filament/Texture.h @@ -181,7 +181,7 @@ public: /** * Specifies how a texture's channels map to color components * - * Texture Swizzle is only supported is isTextureSwizzleSupported() returns true. + * Texture Swizzle is only supported if isTextureSwizzleSupported() returns true. * * @param r texture channel for red component * @param g texture channel for green component