diff --git a/filament/backend/src/DataReshaper.h b/filament/backend/src/DataReshaper.h index 20e53507bf..2217969f46 100644 --- a/filament/backend/src/DataReshaper.h +++ b/filament/backend/src/DataReshaper.h @@ -24,19 +24,24 @@ namespace filament { namespace backend { -// This little utility adds padding to multi-channel interleaved data by inserting dummy values, or -// discards trailing channels. This is useful for platforms that only accept 4-component data, since -// users often wish to submit (or receive) 3-component data. +// Provides an alpha value when expanding 3-channel images to 4-channel. +// Also used as a normalization scale when converting between numeric types. +template inline componentType getMaxValue(); + class DataReshaper { public: - template::max()> + + // Adds padding to multi-channel interleaved data by inserting dummy values, or discards + // trailing channels. This is useful for platforms that only accept 4-component data, since + // users often wish to submit (or receive) 3-component data. + template static void reshape(void* dest, const void* src, size_t numSrcBytes) { + const componentType maxValue = getMaxValue(); const componentType* in = (const componentType*) src; componentType* out = (componentType*) dest; - const size_t srcWordCount = (numSrcBytes / sizeof(componentType)) / srcChannelCount; + const size_t width = (numSrcBytes / sizeof(componentType)) / srcChannelCount; const int minChannelCount = filament::math::min(srcChannelCount, dstChannelCount); - for (size_t word = 0; word < srcWordCount; ++word) { + for (size_t column = 0; column < width; ++column) { for (size_t channel = 0; channel < minChannelCount; ++channel) { out[channel] = in[channel]; } @@ -48,37 +53,114 @@ public: } } - template::max()> - static void reshapeImage(uint8_t* dest, const uint8_t* src, size_t srcBytesPerRow, - size_t dstBytesPerRow, size_t height, bool swizzle03) { - const size_t srcWordCount = (srcBytesPerRow / sizeof(componentType)) / srcChannelCount; - const int minChannelCount = filament::math::min(srcChannelCount, dstChannelCount); + // Converts a 4-channel image of UBYTE, INT, UINT, or FLOAT to a different type. + template + static void reshapeImage(uint8_t* dest, const uint8_t* src, size_t srcBytesPerRow, + size_t dstBytesPerRow, size_t dstChannelCount, size_t height, bool swizzle, bool flip) { + const size_t srcChannelCount = 4; + const dstComponentType dstMaxValue = getMaxValue(); + const srcComponentType srcMaxValue = getMaxValue(); + const size_t width = (srcBytesPerRow / sizeof(srcComponentType)) / srcChannelCount; + const size_t minChannelCount = filament::math::min(srcChannelCount, dstChannelCount); assert(minChannelCount <= 4); - int inds[4] = {0, 1, 2, 3}; - if (swizzle03) { - inds[0] = 2; - inds[2] = 0; + const int inds[4] = {swizzle ? 2 : 0, 1, swizzle ? 0 : 2, 3}; + + int srcStride; + if (flip) { + src += srcBytesPerRow * (height - 1); + srcStride = -srcBytesPerRow; + } else { + srcStride = srcBytesPerRow; } + for (size_t row = 0; row < height; ++row) { - const componentType* in = (const componentType*) src; - componentType* out = (componentType*) dest; - for (size_t word = 0; word < srcWordCount; ++word) { + const srcComponentType* in = (const srcComponentType*) src; + dstComponentType* out = (dstComponentType*) dest; + for (size_t column = 0; column < width; ++column) { for (size_t channel = 0; channel < minChannelCount; ++channel) { - out[channel] = in[inds[channel]]; + out[channel] = in[inds[channel]] * dstMaxValue / srcMaxValue; } for (size_t channel = srcChannelCount; channel < dstChannelCount; ++channel) { - out[channel] = maxValue; + out[channel] = dstMaxValue; } in += srcChannelCount; out += dstChannelCount; } - src += srcBytesPerRow; + src += srcStride; dest += dstBytesPerRow; } } + + // Converts a 4-channel image of UBYTE, INT, UINT, or FLOAT to a different type. + static bool reshapeImage(PixelBufferDescriptor* dst, PixelDataType srcType, + const uint8_t* srcBytes, int srcBytesPerRow, int width, int height, bool swizzle, + bool flip) { + size_t dstChannelCount; + switch (dst->format) { + case PixelDataFormat::RGB: dstChannelCount = 3; break; + case PixelDataFormat::RGBA: dstChannelCount = 4; break; + default: return false; + } + void (*reshaper)(uint8_t*, const uint8_t*, size_t, size_t, size_t, size_t, bool, bool) + = nullptr; + constexpr auto UBYTE = PixelDataType::UBYTE, FLOAT = PixelDataType::FLOAT, + UINT = PixelDataType::UINT, INT = PixelDataType::INT; + switch (dst->type) { + case UBYTE: + switch (srcType) { + case UBYTE: reshaper = reshapeImage; break; + case FLOAT: reshaper = reshapeImage; break; + case INT: reshaper = reshapeImage; break; + case UINT: reshaper = reshapeImage; break; + default: return false; + } + break; + case FLOAT: + switch (srcType) { + case UBYTE: reshaper = reshapeImage; break; + case FLOAT: reshaper = reshapeImage; break; + case INT: reshaper = reshapeImage; break; + case UINT: reshaper = reshapeImage; break; + default: return false; + } + break; + case INT: + switch (srcType) { + case UBYTE: reshaper = reshapeImage; break; + case FLOAT: reshaper = reshapeImage; break; + case INT: reshaper = reshapeImage; break; + case UINT: reshaper = reshapeImage; break; + default: return false; + } + break; + case UINT: + switch (srcType) { + case UBYTE: reshaper = reshapeImage; break; + case FLOAT: reshaper = reshapeImage; break; + case INT: reshaper = reshapeImage; break; + case UINT: reshaper = reshapeImage; break; + default: return false; + } + break; + default: + return false; + } + uint8_t* dstBytes = (uint8_t*) dst->buffer; + const int dstBytesPerRow = PixelBufferDescriptor::computeDataSize(dst->format, dst->type, + dst->stride ? dst->stride : width, 1, dst->alignment); + reshaper(dstBytes, srcBytes, srcBytesPerRow, dstBytesPerRow, dstChannelCount, height, + swizzle, flip); + return true; + } + }; +template<> inline float getMaxValue() { return 1.0f; } +template<> inline int32_t getMaxValue() { return 0x7fffffff; } +template<> inline uint32_t getMaxValue() { return 0xffffffff; } +template<> inline uint16_t getMaxValue() { return 0x3c00; } // 0x3c00 is 1.0 in half-float. +template<> inline uint8_t getMaxValue() { return 0xff; } + } // namespace backend } // namespace filament diff --git a/filament/backend/src/DriverBase.h b/filament/backend/src/DriverBase.h index 2fd026f8a8..d481cb9191 100644 --- a/filament/backend/src/DriverBase.h +++ b/filament/backend/src/DriverBase.h @@ -167,7 +167,7 @@ public: explicit DriverBase(Dispatcher* dispatcher) noexcept; ~DriverBase() noexcept override; - void purge() noexcept final; + void purge() noexcept override; Dispatcher& getDispatcher() noexcept final { return *mDispatcher; } diff --git a/filament/backend/src/TextureReshaper.cpp b/filament/backend/src/TextureReshaper.cpp index 6549d5de89..08212708e8 100644 --- a/filament/backend/src/TextureReshaper.cpp +++ b/filament/backend/src/TextureReshaper.cpp @@ -38,8 +38,7 @@ TextureReshaper::TextureReshaper(TextureFormat requestedFormat) noexcept { const size_t reshapedSize = p.size / 6 * 8; // reshaping from 6 to 8 bytes per pixel void* reshapeBuffer = malloc(reshapedSize); ASSERT_POSTCONDITION(reshapeBuffer, "Could not allocate memory to reshape pixels."); - // 0x3c00 is 1.0 in 16 bit floating point. - DataReshaper::reshape(reshapeBuffer, p.buffer, p.size); + DataReshaper::reshape(reshapeBuffer, p.buffer, p.size); PixelBufferDescriptor reshaped(reshapeBuffer, reshapedSize, PixelBufferDescriptor::PixelDataFormat::RGBA, diff --git a/filament/backend/src/vulkan/VulkanContext.cpp b/filament/backend/src/vulkan/VulkanContext.cpp index c024ceb0d9..ca428e1950 100644 --- a/filament/backend/src/vulkan/VulkanContext.cpp +++ b/filament/backend/src/vulkan/VulkanContext.cpp @@ -367,7 +367,6 @@ void createSwapChain(VulkanContext& context, VulkanSurfaceContext& surfaceContex for (const VkSurfaceFormatKHR& format : surfaceContext.surfaceFormats) { if (format.format == VK_FORMAT_R8G8B8A8_UNORM) { surfaceContext.surfaceFormat = format; - break; } } const auto compositionCaps = caps.supportedCompositeAlpha; diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 8339307b24..1bd7c58bd6 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1293,25 +1293,22 @@ void VulkanDriver::stopCapture(int) { } -void VulkanDriver::readPixels(Handle src, - uint32_t x, uint32_t y, uint32_t width, uint32_t height, - PixelBufferDescriptor&& pbd) { - // TODO: add support for all types listed in the Renderer docstring for readPixels. - assert(pbd.type == PixelBufferDescriptor::PixelDataType::UBYTE); - +void VulkanDriver::readPixels(Handle src, uint32_t x, uint32_t y, + uint32_t width, uint32_t height, PixelBufferDescriptor&& pbd) { const VkDevice device = mContext.device; + const VulkanRenderTarget* srcTarget = handle_cast(mHandleMap, src); + const VulkanTexture* srcTexture = srcTarget->getColor(0).texture; + const VkFormat swapChainFormat = mContext.currentSurface->surfaceFormat.format; + const VkFormat srcFormat = srcTexture ? srcTexture->vkformat : swapChainFormat; + const bool swizzle = srcFormat == VK_FORMAT_B8G8R8A8_UNORM; // Create a host visible, linearly tiled image as a staging area. VkImageCreateInfo imageInfo { .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, - .format = VK_FORMAT_R8G8B8A8_UNORM, - .extent = { - .width = width, - .height = height, - .depth = 1, - }, + .format = srcFormat, + .extent = { width, height, 1 }, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, @@ -1336,10 +1333,8 @@ void VulkanDriver::readPixels(Handle src, vkAllocateMemory(device, &allocInfo, nullptr, &stagingMemory); vkBindImageMemory(device, stagingImage, stagingMemory, 0); - // TODO: Should we allow readPixels within beginFrame / endFrame? - - assert(mContext.currentCommands == nullptr); - acquireWorkCommandBuffer(mContext); + // TODO: replace waitForIdle with an image barrier coupled with acquireWorkCommandBuffer. + waitForIdle(mContext); // Transition the staging image layout. @@ -1347,9 +1342,12 @@ void VulkanDriver::readPixels(Handle src, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); + const uint8_t srcMipLevel = srcTarget->getColor(0).level; + VkImageCopy imageCopyRegion = { .srcSubresource = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = srcMipLevel, .layerCount = 1, }, .srcOffset = { @@ -1369,11 +1367,10 @@ void VulkanDriver::readPixels(Handle src, // Transition the source image layout (which might be the swap chain) - VulkanRenderTarget* srcTarget = handle_cast(mHandleMap, src); VkImage srcImage = srcTarget->getColor(0).image; VulkanTexture::transitionImageLayout(mContext.work.cmdbuffer, srcImage, - VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, srcMipLevel, 1, 1, + VK_IMAGE_ASPECT_COLOR_BIT); // Perform the blit. @@ -1383,16 +1380,15 @@ void VulkanDriver::readPixels(Handle src, // Restore the source image layout. - VulkanTexture* srcTexture = srcTarget->getColor(0).texture; if (srcTexture || mContext.currentSurface->presentQueue) { const VkImageLayout present = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; VulkanTexture::transitionImageLayout(mContext.work.cmdbuffer, srcImage, VK_IMAGE_LAYOUT_UNDEFINED, srcTexture ? getTextureLayout(srcTexture->usage) : present, - 0, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); + srcMipLevel, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); } else { VulkanTexture::transitionImageLayout(mContext.work.cmdbuffer, srcImage, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, - 0, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); + srcMipLevel, 1, 1, VK_IMAGE_ASPECT_COLOR_BIT); } // Transition the staging image layout to GENERAL. @@ -1440,28 +1436,18 @@ void VulkanDriver::readPixels(Handle src, vkMapMemory(device, stagingMemory, 0, VK_WHOLE_SIZE, 0, (void**) &srcPixels); srcPixels += subResourceLayout.offset; - uint8_t* dstPixels = (uint8_t*) closure->buffer; - const uint32_t dstStride = closure->stride ? closure->stride : width; - const int dstBytesPerRow = PixelBufferDescriptor::computeDataSize(closure->format, - closure->type, dstStride, 1, closure->alignment); - const int srcBytesPerRow = subResourceLayout.rowPitch; - const VkFormat swapChainFormat = mContext.currentSurface->surfaceFormat.format; - const bool swizzle = !srcTexture && swapChainFormat == VK_FORMAT_B8G8R8A8_UNORM; - - switch (closure->format) { - case PixelDataFormat::RGB: - case PixelDataFormat::RGB_INTEGER: - DataReshaper::reshapeImage(dstPixels, srcPixels, srcBytesPerRow, - dstBytesPerRow, height, swizzle); - break; - case PixelDataFormat::RGBA: - case PixelDataFormat::RGBA_INTEGER: - DataReshaper::reshapeImage(dstPixels, srcPixels, srcBytesPerRow, - dstBytesPerRow, height, swizzle); - break; - default: - utils::slog.e << "ReadPixels: invalid PixelDataFormat" << utils::io::endl; - break; + // TODO: investigate why this Y-flip conditional exists. At least two SwiftShader-based + // tests (viewer_basic_test.cc and gltf_viewer batch mode) seem to require "false". However + // test_ReadPixels.cpp with MoltenVK requires "true" to be consistent with OpenGL and Metal. + // One hypothesis is that this is due to the layout of the SwiftShader backbuffer. + #ifdef FILAMENT_USE_SWIFTSHADER + constexpr bool flipY = false; + #else + constexpr bool flipY = true; + #endif + if (!DataReshaper::reshapeImage(closure, getComponentType(srcFormat), srcPixels, + subResourceLayout.rowPitch, width, height, swizzle, flipY)) { + utils::slog.e << "Unsupported PixelDataFormat or PixelDataType" << utils::io::endl; } vkUnmapMemory(device, stagingMemory); diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index ecba991476..ad6acb80ce 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -77,6 +77,17 @@ private: VulkanDriver(VulkanDriver const&) = delete; VulkanDriver& operator = (VulkanDriver const&) = delete; + void purge() noexcept override { + // First we trigger garbage collection of Vulkan resources. This ensures that transient + // resources (e.g. the ReadPixels staging buffer) are removed if their refcount is 0, which + // allows related BufferDescriptors to move to the purge list. + mDisposer.gc(); + + // Next, allow the base class to clean up the purge list in order to trigger the + // BufferDescriptor destructors, which in turn triggers the user-provided callbacks. + DriverBase::purge(); + } + private: backend::VulkanPlatform& mContextManager; diff --git a/filament/backend/src/vulkan/VulkanUtility.cpp b/filament/backend/src/vulkan/VulkanUtility.cpp index 2a4d27dfc6..615de8b730 100644 --- a/filament/backend/src/vulkan/VulkanUtility.cpp +++ b/filament/backend/src/vulkan/VulkanUtility.cpp @@ -281,5 +281,101 @@ VkFrontFace getFrontFace(bool inverseFrontFaces) { VkFrontFace::VK_FRONT_FACE_CLOCKWISE : VkFrontFace::VK_FRONT_FACE_COUNTER_CLOCKWISE; } +PixelDataType getComponentType(VkFormat format) { + switch (format) { + case VK_FORMAT_R8_UNORM: + case VK_FORMAT_R8_SNORM: + case VK_FORMAT_R8_USCALED: + case VK_FORMAT_R8_SSCALED: + case VK_FORMAT_R8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_R8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_R8_SRGB: + case VK_FORMAT_R8G8_UNORM: + case VK_FORMAT_R8G8_SNORM: + case VK_FORMAT_R8G8_USCALED: + case VK_FORMAT_R8G8_SSCALED: + case VK_FORMAT_R8G8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_R8G8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_R8G8_SRGB: + case VK_FORMAT_R8G8B8_UNORM: + case VK_FORMAT_R8G8B8_SNORM: + case VK_FORMAT_R8G8B8_USCALED: + case VK_FORMAT_R8G8B8_SSCALED: + case VK_FORMAT_R8G8B8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_R8G8B8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_R8G8B8_SRGB: + case VK_FORMAT_B8G8R8_UNORM: return PixelDataType::UBYTE; + case VK_FORMAT_B8G8R8_SNORM: return PixelDataType::BYTE; + case VK_FORMAT_B8G8R8_USCALED: + case VK_FORMAT_B8G8R8_SSCALED: + case VK_FORMAT_B8G8R8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_B8G8R8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_B8G8R8_SRGB: + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_R8G8B8A8_SNORM: + case VK_FORMAT_R8G8B8A8_USCALED: + case VK_FORMAT_R8G8B8A8_SSCALED: + case VK_FORMAT_R8G8B8A8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_R8G8B8A8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_B8G8R8A8_SNORM: + case VK_FORMAT_B8G8R8A8_USCALED: + case VK_FORMAT_B8G8R8A8_SSCALED: + case VK_FORMAT_B8G8R8A8_UINT: return PixelDataType::UBYTE; + case VK_FORMAT_B8G8R8A8_SINT: return PixelDataType::BYTE; + case VK_FORMAT_B8G8R8A8_SRGB: + case VK_FORMAT_A8B8G8R8_UNORM_PACK32: + case VK_FORMAT_A8B8G8R8_SNORM_PACK32: + case VK_FORMAT_A8B8G8R8_USCALED_PACK32: + case VK_FORMAT_A8B8G8R8_SSCALED_PACK32: + case VK_FORMAT_A8B8G8R8_UINT_PACK32: return PixelDataType::UBYTE; + case VK_FORMAT_A8B8G8R8_SINT_PACK32: return PixelDataType::BYTE; + case VK_FORMAT_A8B8G8R8_SRGB_PACK32: return PixelDataType::UBYTE; + case VK_FORMAT_R16_UNORM: + case VK_FORMAT_R16_SNORM: + case VK_FORMAT_R16_USCALED: + case VK_FORMAT_R16_SSCALED: + case VK_FORMAT_R16_UINT: return PixelDataType::USHORT; + case VK_FORMAT_R16_SINT: return PixelDataType::SHORT; + case VK_FORMAT_R16_SFLOAT: return PixelDataType::HALF; + case VK_FORMAT_R16G16_UNORM: + case VK_FORMAT_R16G16_SNORM: + case VK_FORMAT_R16G16_USCALED: + case VK_FORMAT_R16G16_SSCALED: + case VK_FORMAT_R16G16_UINT: return PixelDataType::USHORT; + case VK_FORMAT_R16G16_SINT: return PixelDataType::SHORT; + case VK_FORMAT_R16G16_SFLOAT: return PixelDataType::HALF; + case VK_FORMAT_R16G16B16_UNORM: + case VK_FORMAT_R16G16B16_SNORM: + case VK_FORMAT_R16G16B16_USCALED: + case VK_FORMAT_R16G16B16_SSCALED: + case VK_FORMAT_R16G16B16_UINT: return PixelDataType::USHORT; + case VK_FORMAT_R16G16B16_SINT: return PixelDataType::SHORT; + case VK_FORMAT_R16G16B16_SFLOAT: return PixelDataType::HALF; + case VK_FORMAT_R16G16B16A16_UNORM: + case VK_FORMAT_R16G16B16A16_SNORM: + case VK_FORMAT_R16G16B16A16_USCALED: + case VK_FORMAT_R16G16B16A16_SSCALED: + case VK_FORMAT_R16G16B16A16_UINT: return PixelDataType::USHORT; + case VK_FORMAT_R16G16B16A16_SINT: return PixelDataType::SHORT; + case VK_FORMAT_R16G16B16A16_SFLOAT: return PixelDataType::HALF; + case VK_FORMAT_R32_UINT: return PixelDataType::UINT; + case VK_FORMAT_R32_SINT: return PixelDataType::INT; + case VK_FORMAT_R32_SFLOAT: return PixelDataType::FLOAT; + case VK_FORMAT_R32G32_UINT: return PixelDataType::UINT; + case VK_FORMAT_R32G32_SINT: return PixelDataType::INT; + case VK_FORMAT_R32G32_SFLOAT: return PixelDataType::FLOAT; + case VK_FORMAT_R32G32B32_UINT: return PixelDataType::UINT; + case VK_FORMAT_R32G32B32_SINT: return PixelDataType::INT; + case VK_FORMAT_R32G32B32_SFLOAT: return PixelDataType::FLOAT; + case VK_FORMAT_R32G32B32A32_UINT: return PixelDataType::UINT; + case VK_FORMAT_R32G32B32A32_SINT: return PixelDataType::INT; + case VK_FORMAT_R32G32B32A32_SFLOAT: return PixelDataType::FLOAT; + default: assert(false && "Unknown data type, conversion is not supported."); + } + return {}; +} + } // namespace filament } // namespace backend diff --git a/filament/backend/src/vulkan/VulkanUtility.h b/filament/backend/src/vulkan/VulkanUtility.h index 1cff95b5da..d7f3765d63 100644 --- a/filament/backend/src/vulkan/VulkanUtility.h +++ b/filament/backend/src/vulkan/VulkanUtility.h @@ -32,6 +32,7 @@ VkCompareOp getCompareOp(SamplerCompareFunc func); VkBlendFactor getBlendFactor(BlendFunction mode); VkCullModeFlags getCullMode(CullingMode mode); VkFrontFace getFrontFace(bool inverseFrontFaces); +PixelDataType getComponentType(VkFormat format); } // namespace filament } // namespace backend