Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Metal: implement texture swizzling for external textures (#4514)

Browse Source
This commit is contained in:
Ben Doherty
2021-08-18 10:57:50 -07:00
committed by GitHub
parent 1b9dea5b75
commit 8c8e0ce8a3
7 changed files with 126 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
filament/backend/CMakeLists.txt
+1
View File
@@ -117,6 +117,7 @@ if (FILAMENT_SUPPORTS_METAL)
src/metal/MetalResourceTracker.cpp
src/metal/MetalState.mm
src/metal/MetalTimerQuery.mm
src/metal/MetalUtils.mm
)
list(APPEND SRCS ${METAL_SRCS})
filament/backend/src/metal/MetalExternalImage.h
+13 -1
View File
@@ -17,6 +17,8 @@
#ifndef TNT_METALEXTERNALIMAGE_H
#define TNT_METALEXTERNALIMAGE_H
#import <backend/DriverEnums.h>
#include <CoreVideo/CoreVideo.h>
#include <Metal/Metal.h>
@@ -34,7 +36,11 @@ class MetalExternalImage {
public:
MetalExternalImage(MetalContext& context) noexcept;
MetalExternalImage(MetalContext& context,
TextureSwizzle r = TextureSwizzle::CHANNEL_0,
TextureSwizzle g = TextureSwizzle::CHANNEL_1,
TextureSwizzle b = TextureSwizzle::CHANNEL_2,
TextureSwizzle a = TextureSwizzle::CHANNEL_3) noexcept;
/**
* @return true, if this MetalExternalImage is holding a live external image. Returns false
@@ -94,6 +100,7 @@ private:
CVMetalTextureRef createTextureFromImage(CVPixelBufferRef image, MTLPixelFormat format,
size_t plane);
id<MTLTexture> createRgbTexture(size_t width, size_t height);
id<MTLTexture> createSwizzledTextureView(CVMetalTextureRef texture) const;
void ensureComputePipelineState();
id<MTLCommandBuffer> encodeColorConversionPass(id<MTLTexture> inYPlane, id<MTLTexture>
inCbCrTexture, id<MTLTexture> outTexture);
@@ -105,8 +112,10 @@ private:
// If the external image has a single plane, mImage and mTexture hold references to the image
// and created Metal texture, respectively.
// mTextureView is a view of mTexture with any swizzling applied.
CVPixelBufferRef mImage = nullptr;
CVMetalTextureRef mTexture = nullptr;
id<MTLTexture> mTextureView = nullptr;
size_t mWidth = 0;
size_t mHeight = 0;
@@ -114,6 +123,9 @@ private:
// texture.
id<MTLTexture> mRgbTexture = nil;
struct {
TextureSwizzle r, g, b, a;
} swizzle;
};
} // namespace metal
filament/backend/src/metal/MetalExternalImage.mm
+31 -5
View File
@@ -17,6 +17,8 @@
#include "MetalExternalImage.h"
#include "MetalContext.h"
#include "MetalEnums.h"
#include "MetalUtils.h"
#include <utils/Panic.h>
#include <utils/trap.h>
@@ -69,7 +71,8 @@ ycbcrToRgb(texture2d<half, access::read> inYTexture [[texture(0)]],
}
)";
MetalExternalImage::MetalExternalImage(MetalContext& context) noexcept : mContext(context) { }
MetalExternalImage::MetalExternalImage(MetalContext& context, TextureSwizzle r, TextureSwizzle g,
TextureSwizzle b, TextureSwizzle a) noexcept : mContext(context), swizzle{r, g, b ,a} { }
bool MetalExternalImage::isValid() const noexcept {
return mRgbTexture != nil || mImage != nullptr;
@@ -95,6 +98,7 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
if (planeCount == 0) {
mImage = image;
mTexture = createTextureFromImage(image, MTLPixelFormatBGRA8Unorm, 0);
mTextureView = createSwizzledTextureView(mTexture);
mWidth = CVPixelBufferGetWidth(image);
mHeight = CVPixelBufferGetHeight(image);
}
@@ -108,12 +112,14 @@ void MetalExternalImage::set(CVPixelBufferRef image) noexcept {
mWidth = CVPixelBufferGetWidthOfPlane(image, Y_PLANE);
mHeight = CVPixelBufferGetHeightOfPlane(image, Y_PLANE);
mRgbTexture = createRgbTexture(mWidth, mHeight);
id<MTLTexture> rgbTexture = createRgbTexture(mWidth, mHeight);
id <MTLCommandBuffer> commandBuffer = encodeColorConversionPass(
CVMetalTextureGetTexture(yPlane),
CVMetalTextureGetTexture(cbcrPlane),
mRgbTexture);
rgbTexture);
mRgbTexture = createTextureViewWithSwizzle(rgbTexture,
getSwizzleChannels(swizzle.r, swizzle.g, swizzle.b, swizzle.a));
[commandBuffer addCompletedHandler:^(id <MTLCommandBuffer> o) {
CVBufferRelease(yPlane);
@@ -154,6 +160,7 @@ void MetalExternalImage::set(CVPixelBufferRef image, size_t plane) noexcept {
const MTLPixelFormat format = getPlaneFormat(plane);
assert_invariant(format != MTLPixelFormatInvalid);
mTexture = createTextureFromImage(image, format, plane);
mTextureView = createSwizzledTextureView(mTexture);
}
id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
@@ -173,7 +180,8 @@ id<MTLTexture> MetalExternalImage::getMetalTextureForDraw() const noexcept {
CVBufferRetain(mTexture);
}
return CVMetalTextureGetTexture(mTexture);
assert_invariant(mTextureView);
return mTextureView;
}
CVMetalTextureRef MetalExternalImage::createTextureFromImage(CVPixelBufferRef image,
@@ -206,6 +214,7 @@ void MetalExternalImage::unset() {
mImage = nullptr;
mTexture = nullptr;
mTextureView = nil;
mRgbTexture = nil;
mWidth = 0;
mHeight = 0;
@@ -221,6 +230,23 @@ id<MTLTexture> MetalExternalImage::createRgbTexture(size_t width, size_t height)
return [mContext.device newTextureWithDescriptor:descriptor];
}
id<MTLTexture> MetalExternalImage::createSwizzledTextureView(CVMetalTextureRef ref) const {
id<MTLTexture> texture = CVMetalTextureGetTexture(ref);
const bool isDefaultSwizzle =
swizzle.r == TextureSwizzle::CHANNEL_0 &&
swizzle.g == TextureSwizzle::CHANNEL_1 &&
swizzle.b == TextureSwizzle::CHANNEL_2 &&
swizzle.a == TextureSwizzle::CHANNEL_3;
if (!isDefaultSwizzle && mContext.supportsTextureSwizzling) {
// Even though we've already checked supportsTextureSwizzling, we still need to guard these
// calls with @availability, otherwise the API usage will generate compiler warnings.
if (@available(iOS 13, *)) {
texture = createTextureViewWithSwizzle(texture,
getSwizzleChannels(swizzle.r, swizzle.g, swizzle.b, swizzle.a));
}
}
return texture;
}
void MetalExternalImage::ensureComputePipelineState() {
if (mContext.externalImageComputePipelineState != nil) {
filament/backend/src/metal/MetalHandles.mm
+6 -15
View File
@@ -18,6 +18,7 @@
#include "MetalBlitter.h"
#include "MetalEnums.h"
#include "MetalUtils.h"
#include <filament/SwapChain.h>
@@ -380,7 +381,7 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
TextureUsage usage, TextureSwizzle r, TextureSwizzle g, TextureSwizzle b,
TextureSwizzle a) noexcept
: HwTexture(target, levels, samples, width, height, depth, format, usage), context(context),
externalImage(context) {
externalImage(context, r, g, b, a) {
devicePixelFormat = decidePixelFormat(&context, format);
ASSERT_POSTCONDITION(devicePixelFormat != MTLPixelFormatInvalid, "Texture format not supported.");
@@ -475,25 +476,15 @@ MetalTexture::MetalTexture(MetalContext& context, SamplerType target, uint8_t le
g == TextureSwizzle::CHANNEL_1 &&
b == TextureSwizzle::CHANNEL_2 &&
a == TextureSwizzle::CHANNEL_3;
// If texture is nil, then it must be a SAMPLER_EXTERNAL texture. We'll ignore this case for now.
// TODO: implement swizzling for external textures.
// If texture is nil, then it must be a SAMPLER_EXTERNAL texture.
// Swizzling for external textures is handled inside MetalExternalImage.
if (!isDefaultSwizzle && texture && context.supportsTextureSwizzling) {
// Even though we've already checked context.supportsTextureSwizzling, we still need to
// guard these calls with @availability, otherwise the API usage will generate compiler
// warnings.
if (@available(iOS 13, *)) {
NSUInteger slices = texture.arrayLength;
if (texture.textureType == MTLTextureTypeCube ||
texture.textureType == MTLTextureTypeCubeArray) {
slices *= 6;
}
NSUInteger mips = texture.mipmapLevelCount;
MTLTextureSwizzleChannels swizzle = getSwizzleChannels(r, g, b, a);
swizzledTextureView = [texture newTextureViewWithPixelFormat:texture.pixelFormat
textureType:texture.textureType
levels:NSMakeRange(0, mips)
slices:NSMakeRange(0, slices)
swizzle:swizzle];
swizzledTextureView =
createTextureViewWithSwizzle(texture, getSwizzleChannels(r, g, b, a));
}
}
}
filament/backend/src/metal/MetalUtils.h
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TNT_FILAMENT_DRIVER_METALUTILS_H
#define TNT_FILAMENT_DRIVER_METALUTILS_H
#include <Metal/Metal.h>
namespace filament::backend::metal {
/**
* Creates a texture view of the passed-in texture with the given swizzle. If swizzle is the default
* swizzle, simply returns texture.
*/
API_AVAILABLE(ios(13.0))
id<MTLTexture> createTextureViewWithSwizzle(id<MTLTexture> texture,
MTLTextureSwizzleChannels swizzle);
} // namespace filament::backend::metal
#endif //TNT_FILAMENT_DRIVER_METALUTILS_H
filament/backend/src/metal/MetalUtils.mm
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "MetalUtils.h"
namespace filament::backend::metal {
id<MTLTexture> createTextureViewWithSwizzle(id<MTLTexture> texture,
MTLTextureSwizzleChannels swizzle) {
if (swizzle.red == MTLTextureSwizzleRed && swizzle.green == MTLTextureSwizzleGreen &&
swizzle.blue == MTLTextureSwizzleBlue && swizzle.alpha == MTLTextureSwizzleAlpha) {
return texture;
}
NSUInteger slices = texture.arrayLength;
if (texture.textureType == MTLTextureTypeCube ||
texture.textureType == MTLTextureTypeCubeArray) {
slices *= 6;
}
NSUInteger mips = texture.mipmapLevelCount;
return [texture newTextureViewWithPixelFormat:texture.pixelFormat
textureType:texture.textureType
levels:NSMakeRange(0, mips)
slices:NSMakeRange(0, slices)
swizzle:swizzle];
}
}
filament/include/filament/Texture.h
+1 -1
View File
@@ -181,7 +181,7 @@ public:
/**
* Specifies how a texture's channels map to color components
*
* Texture Swizzle is only supported is isTextureSwizzleSupported() returns true.
* Texture Swizzle is only supported if isTextureSwizzleSupported() returns true.
*
* @param r texture channel for red component
* @param g texture channel for green component