diff --git a/CMakeLists.txt b/CMakeLists.txt index bbb84cfe05..9e290c70fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,7 @@ find_package(embree 3.0 QUIET PATHS /usr/lib64/cmake) if (embree_FOUND AND NOT ANDROID) message("Found embree in ${embree_DIR}") add_definitions(-DFILAMENT_HAS_EMBREE) + set(DENOISE_LIBRARY OpenImageDenoise) else() message("Embree not found, pipeline features are disabled.") endif() @@ -420,6 +421,7 @@ if (NOT ANDROID AND NOT WEBGL AND NOT IOS) add_subdirectory(${EXTERNAL}/libz/tnt) add_subdirectory(${EXTERNAL}/skylight/tnt) add_subdirectory(${EXTERNAL}/tinyexr/tnt) + add_subdirectory(${EXTERNAL}/OpenImageDenoise) add_subdirectory(${TOOLS}/cmgen) add_subdirectory(${TOOLS}/filamesh) diff --git a/libs/gltfio/src/AssetPipeline.cpp b/libs/gltfio/src/AssetPipeline.cpp index 9a4ccac0a4..e75f7dc1c1 100644 --- a/libs/gltfio/src/AssetPipeline.cpp +++ b/libs/gltfio/src/AssetPipeline.cpp @@ -744,6 +744,7 @@ void Pipeline::bakeAmbientOcclusion(const cgltf_data* sourceAsset, image::Linear .meshes(meshes, numMeshes) .outputPlane(filament::rays::AMBIENT_OCCLUSION, target) .uvCamera() + .denoise() .tileCallback(onTile, userData) .doneCallback(onDone, userData); @@ -762,6 +763,7 @@ void Pipeline::renderAmbientOcclusion(const cgltf_data* sourceAsset, image::Line .meshes(meshes, numMeshes) .outputPlane(filament::rays::AMBIENT_OCCLUSION, target) .filmCamera(camera) + .denoise() .tileCallback(onTile, userData) .doneCallback(onDone, userData); diff --git a/libs/image/include/image/ImageOps.h b/libs/image/include/image/ImageOps.h index af14a3dab4..e2af2a66f8 100644 --- a/libs/image/include/image/ImageOps.h +++ b/libs/image/include/image/ImageOps.h @@ -77,6 +77,9 @@ LinearImage edtFromCoordField(const LinearImage& coordField, bool sqrt); // Dereferences the given coordinate field. Useful for creating Voronoi diagrams or dilated images. LinearImage voronoiFromCoordField(const LinearImage& coordField, const LinearImage& src); +// Copies content of a source image into a target image. Requires width/height/channels to match. +void blitImage(LinearImage& target, const LinearImage& source); + } // namespace image diff --git a/libs/image/src/ImageOps.cpp b/libs/image/src/ImageOps.cpp index d873dd1bea..fb097d8c84 100644 --- a/libs/image/src/ImageOps.cpp +++ b/libs/image/src/ImageOps.cpp @@ -413,4 +413,13 @@ LinearImage voronoiFromCoordField(const LinearImage& coordField, const LinearIma return result; } +void blitImage(LinearImage& target, const LinearImage& source) { + ASSERT_PRECONDITION(source.getWidth() == target.getWidth(), "Images must have same width."); + ASSERT_PRECONDITION(source.getHeight() == target.getHeight(), "Images must have same height."); + ASSERT_PRECONDITION(source.getChannels() == target.getChannels(), + "Images must have same number of channels."); + memcpy(target.getPixelRef(), source.getPixelRef(), + sizeof(float) * source.getWidth() * source.getHeight() * source.getChannels()); +} + } // namespace image diff --git a/libs/rays/CMakeLists.txt b/libs/rays/CMakeLists.txt index 53cdfb3925..4c54e7d33d 100644 --- a/libs/rays/CMakeLists.txt +++ b/libs/rays/CMakeLists.txt @@ -23,7 +23,7 @@ set(SRCS include_directories(${PUBLIC_HDR_DIR} ${EMBREE_INCLUDE_DIRS}) add_library(${TARGET} STATIC ${PUBLIC_HDRS} ${SRCS}) -target_link_libraries(${TARGET} PUBLIC math utils image ${EMBREE_LIBRARY}) +target_link_libraries(${TARGET} PUBLIC math utils image ${EMBREE_LIBRARY} ${DENOISE_LIBRARY}) target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) # ================================================================================================== diff --git a/libs/rays/include/rays/PathTracer.h b/libs/rays/include/rays/PathTracer.h index b187344564..405d7cb9eb 100644 --- a/libs/rays/include/rays/PathTracer.h +++ b/libs/rays/include/rays/PathTracer.h @@ -77,6 +77,7 @@ public: size_t samplesPerPixel = 256; SimpleCamera filmCamera; bool uvCamera = false; + bool denoise = false; TileCallback tileCallback = nullptr; void* tileUserData = nullptr; DoneCallback doneCallback = nullptr; @@ -110,6 +111,11 @@ public: */ Builder& uvCamera(); + /** + * Executes OpenImageDenoise after rendering the complete image. + */ + Builder& denoise(); + /** * Signals that a region within each render target has become available, typically used for * progress notification. This can be called from any thread. diff --git a/libs/rays/src/PathTracer.cpp b/libs/rays/src/PathTracer.cpp index 4a04b070a3..7e814807ed 100644 --- a/libs/rays/src/PathTracer.cpp +++ b/libs/rays/src/PathTracer.cpp @@ -27,6 +27,8 @@ #ifdef FILAMENT_HAS_EMBREE #include #include + +#include #endif using namespace filament::math; @@ -99,6 +101,11 @@ PathTracer::Builder& PathTracer::Builder::uvCamera() { return *this; } +PathTracer::Builder& PathTracer::Builder::denoise() { + mConfig.denoise = true; + return *this; +} + PathTracer::Builder& PathTracer::Builder::tileCallback(TileCallback onTile, void* userData) { mConfig.tileCallback = onTile; mConfig.tileUserData = userData; @@ -292,7 +299,57 @@ static void dilateCharts(EmbreeContext* context) { }; auto coords = image::computeCoordField(meshNormals, presence, nullptr); auto dilated = image::voronoiFromCoordField(coords, ao); - memcpy(ao.getPixelRef(), dilated.getPixelRef(), sizeof(float) * ao.getWidth() * ao.getHeight()); + blitImage(ao, dilated); +} + +static void denoise(EmbreeContext* context) { + LinearImage ao = context->config.renderTargets[(int) AMBIENT_OCCLUSION]; + const LinearImage& meshNormals = context->config.renderTargets[(int) MESH_NORMALS]; + + // The denoiser requires color inputs, so convert 1-chan to 3-chan. + const size_t width = ao.getWidth(); + const size_t height = ao.getHeight(); + LinearImage denoiseSource = combineChannels({ ao, ao, ao }); + + // Construct a fake albedo image, which for our purposes can be white everywhere that a surface + // is present. This is optional but the denoise library doesn't produce good results without it. + LinearImage fakeAlbedo(width, height, 3); + for (int32_t row = 0; row < height; ++row) { + for (uint32_t col = 0; col < width; ++col) { + const float* normal = meshNormals.getPixelRef(col, row); + float* albedo = fakeAlbedo.getPixelRef(col, row); + albedo[0] = albedo[1] = albedo[2] = (normal[0] == EMPTY_SENTINEL ? 0.0f : 1.0f); + } + } + + // Invoke the denoiser. + LinearImage denoiseTarget(width, height, 3); + OIDNDevice device = oidnNewDevice(OIDN_DEVICE_TYPE_DEFAULT); + oidnCommitDevice(device); + OIDNFilter filter = oidnNewFilter(device, "RT"); + oidnSetSharedFilterImage(filter, "color", denoiseSource.getPixelRef(), + OIDN_FORMAT_FLOAT3, width, height, 0, 0, 0); + oidnSetSharedFilterImage(filter, "normal", (void*) meshNormals.getPixelRef(), + OIDN_FORMAT_FLOAT3, width, height, 0, 0, 0); + oidnSetSharedFilterImage(filter, "albedo", fakeAlbedo.getPixelRef(), + OIDN_FORMAT_FLOAT3, width, height, 0, 0, 0); + oidnSetSharedFilterImage(filter, "output", denoiseTarget.getPixelRef(), + OIDN_FORMAT_FLOAT3, width, height, 0, 0, 0); + oidnCommitFilter(filter); + oidnExecuteFilter(filter); + + // Check for errors. + const char* errorMessage; + if (oidnGetDeviceError(device, &errorMessage) != OIDN_ERROR_NONE) { + printf("OpenImageDenoise Error: %s\n", errorMessage); + oidnReleaseFilter(filter); + oidnReleaseDevice(device); + return; + } + oidnReleaseFilter(filter); + oidnReleaseDevice(device); + + blitImage(ao, extractChannel(denoiseTarget, 0)); } static void renderTileFromGbuffer(EmbreeContext* context, PixelRectangle rect) { @@ -482,6 +539,9 @@ bool PathTracer::render() { context->config.tileUserData); }, [](EmbreeContext* context) { dilateCharts(context); + if (context->config.denoise) { + denoise(context); + } context->config.doneCallback(context->config.doneUserData); rtcReleaseScene(context->scene); rtcReleaseDevice(context->device); @@ -499,6 +559,9 @@ bool PathTracer::render() { context->config.tileCallback(rect.topLeft, rect.bottomRight, context->config.tileUserData); }, [](EmbreeContext* context) { + if (context->config.denoise) { + denoise(context); + } context->config.doneCallback(context->config.doneUserData); rtcReleaseScene(context->scene); rtcReleaseDevice(context->device); diff --git a/samples/gltf_baker.cpp b/samples/gltf_baker.cpp index c74f9d8bbf..baf9f12b90 100644 --- a/samples/gltf_baker.cpp +++ b/samples/gltf_baker.cpp @@ -440,6 +440,7 @@ static void renderAsset(App& app) { auto onRenderDone = [](void* userData) { App* app = (App*) userData; app->requestStatePop = true; + app->requestOverlayUpdate = true; delete app->pipeline; }; app.pipeline->renderAmbientOcclusion(asset, app.ambientOcclusion, camera, onRenderTile, @@ -498,6 +499,7 @@ static void bakeAsset(App& app) { auto onRenderDone = [](void* userData) { App* app = (App*) userData; delete app->pipeline; + app->requestOverlayUpdate = true; app->state = BAKED; }; app.pipeline->bakeAmbientOcclusion(asset, app.ambientOcclusion, onRenderTile, onRenderDone,