From ee6f3fb1dc5fc255f30a01d04335f9ac2bbd7634 Mon Sep 17 00:00:00 2001 From: Mathias Agopian Date: Thu, 18 Jan 2024 15:34:38 -0800 Subject: [PATCH] Add sharpening option to TAA We use the RCAS algorithm from FSR1. This is useful for when TAA upscaling is enabled and similar to to what FSR2 is doing. --- .../com/google/android/filament/View.java | 4 + filament/include/filament/Options.h | 1 + filament/src/PostProcessManager.cpp | 125 +++++++++++------- filament/src/PostProcessManager.h | 7 + filament/src/details/Renderer.cpp | 3 +- filament/src/fg/details/Resource.h | 2 +- filament/src/materials/fsr/fsr_rcas.mat | 5 + libs/viewer/src/Settings_generated.cpp | 3 + libs/viewer/src/ViewerGui.cpp | 1 + web/filament-js/extensions_generated.js | 1 + web/filament-js/filament.d.ts | 4 + web/filament-js/jsbindings_generated.cpp | 1 + 12 files changed, 103 insertions(+), 54 deletions(-) diff --git a/android/filament-android/src/main/java/com/google/android/filament/View.java b/android/filament-android/src/main/java/com/google/android/filament/View.java index 22b37a6d12..28cb268c23 100644 --- a/android/filament-android/src/main/java/com/google/android/filament/View.java +++ b/android/filament-android/src/main/java/com/google/android/filament/View.java @@ -1940,6 +1940,10 @@ public class View { * texturing lod bias (typically -1 or -2) */ public float lodBias = -1.0f; + /** + * post-TAA sharpen, especially useful when upscaling is true. + */ + public float sharpness = 0.0f; /** * enables or disables temporal anti-aliasing */ diff --git a/filament/include/filament/Options.h b/filament/include/filament/Options.h index 93ae2c8c23..450bbc1062 100644 --- a/filament/include/filament/Options.h +++ b/filament/include/filament/Options.h @@ -438,6 +438,7 @@ struct TemporalAntiAliasingOptions { float filterWidth = 1.0f; //!< reconstruction filter width typically between 0.2 (sharper, aliased) and 1.5 (smoother) float feedback = 0.12f; //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA). float lodBias = -1.0f; //!< texturing lod bias (typically -1 or -2) + float sharpness = 0.0f; //!< post-TAA sharpen, especially useful when upscaling is true. bool enabled = false; //!< enables or disables temporal anti-aliasing bool upscaling = false; //!< 4x TAA upscaling. Disables Dynamic Resolution. [BETA] diff --git a/filament/src/PostProcessManager.cpp b/filament/src/PostProcessManager.cpp index 6d966dc8f5..9b29fe132d 100644 --- a/filament/src/PostProcessManager.cpp +++ b/filament/src/PostProcessManager.cpp @@ -115,7 +115,7 @@ PostProcessManager::PostProcessMaterial::PostProcessMaterial() noexcept { } PostProcessManager::PostProcessMaterial::PostProcessMaterial(MaterialInfo const& info) noexcept - : PostProcessMaterial() { + : PostProcessMaterial() { mData = info.data; // aliased to mMaterial mSize = info.size; mConstants = info.constants; @@ -167,7 +167,7 @@ void PostProcessManager::PostProcessMaterial::loadMaterial(FEngine& engine) cons mHasMaterial = true; auto builder = Material::Builder(); builder.package(mData, mSize); - for (auto const& constant : mConstants) { + for (auto const& constant: mConstants) { std::visit([&](auto&& arg) { builder.constant(constant.name.data(), constant.name.size(), arg); }, constant.value); @@ -189,14 +189,15 @@ PipelineState PostProcessManager::PostProcessMaterial::getPipelineState( FMaterial* const material = getMaterial(engine); material->prepareProgram(Variant{ variantKey }); return { - .program = material->getProgram(Variant{variantKey}), + .program = material->getProgram(Variant{ variantKey }), .rasterState = material->getRasterState(), .scissor = material->getDefaultInstance()->getScissor() }; } UTILS_NOINLINE -FMaterialInstance* PostProcessManager::PostProcessMaterial::getMaterialInstance(FEngine& engine) const noexcept { +FMaterialInstance* PostProcessManager::PostProcessMaterial::getMaterialInstance( + FEngine& engine) const noexcept { FMaterial* const material = getMaterial(engine); return material->getDefaultInstance(); } @@ -233,14 +234,15 @@ const PostProcessManager::JitterSequence<32> PostProcessManager::PostProcessManager(FEngine& engine) noexcept : mEngine(engine), - mWorkaroundSplitEasu(false), - mWorkaroundAllowReadOnlyAncillaryFeedbackLoop(false) { + mWorkaroundSplitEasu(false), + mWorkaroundAllowReadOnlyAncillaryFeedbackLoop(false) { } PostProcessManager::~PostProcessManager() noexcept = default; UTILS_NOINLINE -void PostProcessManager::registerPostProcessMaterial(std::string_view name, MaterialInfo const& info) { +void PostProcessManager::registerPostProcessMaterial(std::string_view name, + MaterialInfo const& info) { mMaterialRegistry.try_emplace(name, info); } @@ -323,13 +325,13 @@ void PostProcessManager::init() noexcept { driver.isWorkaroundNeeded(Workaround::ALLOW_READ_ONLY_ANCILLARY_FEEDBACK_LOOP); #pragma nounroll - for (auto const& info : sMaterialListFeatureLevel0) { + for (auto const& info: sMaterialListFeatureLevel0) { registerPostProcessMaterial(info.name, info); } if (mEngine.getActiveFeatureLevel() >= FeatureLevel::FEATURE_LEVEL_1) { #pragma nounroll - for (auto const& info : sMaterialList) { + for (auto const& info: sMaterialList) { registerPostProcessMaterial(info.name, info); } } @@ -2767,24 +2769,77 @@ FrameGraphId PostProcessManager::taa(FrameGraph& fg, }); input = colorGradingConfig.asSubpass ? taaPass->tonemappedOutput : taaPass->output; + auto history = input; + + // optional sharpen pass from FSR1 + if (taaOptions.sharpness > 0.0f) { + input = rcas(fg, taaOptions.sharpness, + input, fg.getDescriptor(input), colorGradingConfig.translucent); + } struct ExportColorHistoryData { FrameGraphId color; }; - auto& exportHistoryPass = fg.addPass("Export TAA history", + fg.addPass("Export TAA history", [&](FrameGraph::Builder& builder, auto& data) { // We need to use sideEffect here to ensure this pass won't be culled. // The "output" of this pass is going to be used during the next frame as // an "import". builder.sideEffect(); - data.color = builder.sample(input); // FIXME: an access must be declared for detach(), why? - }, [¤t](FrameGraphResources const& resources, auto const& data, - backend::DriverApi&) { - resources.detach(data.color, - ¤t.color, ¤t.desc); + data.color = builder.sample(history); // FIXME: an access must be declared for detach(), why? + }, [¤t](FrameGraphResources const& resources, auto const& data, auto&) { + resources.detach(data.color, ¤t.color, ¤t.desc); }); - return exportHistoryPass->color; + return input; +} + +FrameGraphId PostProcessManager::rcas( + FrameGraph& fg, + float sharpness, + FrameGraphId input, + FrameGraphTexture::Descriptor const& outDesc, + bool translucent) { + + struct QuadBlitData { + FrameGraphId input; + FrameGraphId output; + }; + + auto& ppFsrRcas = fg.addPass("FidelityFX FSR1 Rcas", + [&](FrameGraph::Builder& builder, auto& data) { + data.input = builder.sample(input); + data.input = builder.createTexture("FFX FSR1 Rcas output", outDesc); + data.input = builder.declareRenderPass(data.input); + }, + [=](FrameGraphResources const& resources, + auto const& data, DriverApi& driver) { + + auto input = resources.getTexture(data.input); + auto out = resources.getRenderPassInfo(); + auto const& outputDesc = resources.getDescriptor(data.input); + + auto& material = getPostProcessMaterial("fsr_rcas"); + FMaterialInstance* const mi = material.getMaterialInstance(mEngine); + + FSRUniforms uniforms; + FSR_SharpeningSetup(&uniforms, { .sharpness = 2.0f - 2.0f * sharpness }); + mi->setParameter("RcasCon", uniforms.RcasCon); + mi->setParameter("color", input, {}); // uses texelFetch + mi->setParameter("resolution", float4{ + outputDesc.width, outputDesc.height, + 1.0f / outputDesc.width, 1.0f / outputDesc.height }); + mi->commit(driver); + mi->use(driver); + + const uint8_t variant = uint8_t( + translucent ? PostProcessVariant::TRANSLUCENT : PostProcessVariant::OPAQUE); + + PipelineState const pipeline(material.getPipelineState(mEngine, variant)); + render(out, pipeline, driver); + }); + + return ppFsrRcas->output; } FrameGraphId PostProcessManager::upscale(FrameGraph& fg, bool translucent, @@ -2947,41 +3002,9 @@ FrameGraphId PostProcessManager::upscale(FrameGraph& fg, bool auto output = ppQuadBlit->output; // if we had to take the low quality fallback, we still do the "sharpen pass" - if (dsrOptions.sharpness > 0.0f && (dsrOptions.quality != QualityLevel::LOW || lowQualityFallback)) { - auto& ppFsrRcas = fg.addPass("FidelityFX FSR1 Rcas", - [&](FrameGraph::Builder& builder, auto& data) { - data.input = builder.sample(output); - data.output = builder.createTexture("FFX FSR1 Rcas output", outDesc); - data.output = builder.declareRenderPass(data.output); - }, - [=](FrameGraphResources const& resources, - auto const& data, DriverApi& driver) { - - auto color = resources.getTexture(data.input); - auto out = resources.getRenderPassInfo(); - auto const& outputDesc = resources.getDescriptor(data.output); - - auto& material = getPostProcessMaterial("fsr_rcas"); - FMaterialInstance* const mi = material.getMaterialInstance(mEngine); - - FSRUniforms uniforms; - FSR_SharpeningSetup(&uniforms, { .sharpness = 2.0f - 2.0f * dsrOptions.sharpness }); - mi->setParameter("RcasCon", uniforms.RcasCon); - mi->setParameter("color", color, { }); // uses texelFetch - mi->setParameter("resolution", float4{ - outputDesc.width, outputDesc.height, - 1.0f / outputDesc.width, 1.0f / outputDesc.height }); - mi->commit(driver); - mi->use(driver); - - const uint8_t variant = uint8_t(translucent ? - PostProcessVariant::TRANSLUCENT : PostProcessVariant::OPAQUE); - - PipelineState const pipeline(material.getPipelineState(mEngine, variant)); - render(out, pipeline, driver); - }); - - output = ppFsrRcas->output; + if (dsrOptions.sharpness > 0.0f && + (dsrOptions.quality != QualityLevel::LOW || lowQualityFallback)) { + output = rcas(fg, dsrOptions.sharpness, output, outDesc, translucent); } // we rely on automatic culling of unused render passes diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h index 84ba18bdc7..bcffc50a01 100644 --- a/filament/src/PostProcessManager.h +++ b/filament/src/PostProcessManager.h @@ -242,6 +242,13 @@ public: filament::Viewport const& vp, FrameGraphTexture::Descriptor const& outDesc, backend::SamplerMagFilter filter) noexcept; + FrameGraphId rcas( + FrameGraph& fg, + float sharpness, + FrameGraphId input, + FrameGraphTexture::Descriptor const& outDesc, + bool translucent); + // upscale/downscale blitter using shaders FrameGraphId blit(FrameGraph& fg, bool translucent, FrameGraphId input, diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp index b046b5b7c6..3c7567295b 100644 --- a/filament/src/details/Renderer.cpp +++ b/filament/src/details/Renderer.cpp @@ -985,8 +985,7 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) { auto& history = view.getFrameHistory(); auto& current = history.getCurrent(); current.ssr.projection = projection; - resources.detach(data.history, - ¤t.ssr.color, ¤t.ssr.desc); + resources.detach(data.history, ¤t.ssr.color, ¤t.ssr.desc); }); } diff --git a/filament/src/fg/details/Resource.h b/filament/src/fg/details/Resource.h index 966b4bc338..6c3bffaf58 100644 --- a/filament/src/fg/details/Resource.h +++ b/filament/src/fg/details/Resource.h @@ -132,7 +132,7 @@ public: Descriptor descriptor; SubResourceDescriptor subResourceDescriptor; - // weather the resource was detached + // whether the resource was detached bool detached = false; // An Edge with added data from this resource diff --git a/filament/src/materials/fsr/fsr_rcas.mat b/filament/src/materials/fsr/fsr_rcas.mat index 185e62d288..cfdf037204 100644 --- a/filament/src/materials/fsr/fsr_rcas.mat +++ b/filament/src/materials/fsr/fsr_rcas.mat @@ -43,6 +43,8 @@ fragment { #define FSR_RCAS_F 1 #if !POST_PROCESS_OPAQUE # define FSR_RCAS_PASSTHROUGH_ALPHA + // todo: make this a spec constant + # define FSR_RCAS_DENOISE #endif #include "ffx_fsr1.h" @@ -61,6 +63,9 @@ fragment { #endif p, materialParams.RcasCon); + // todo: make this an option + postProcess.color.rgb = max(vec3(0), postProcess.color.rgb); + #if POST_PROCESS_OPAQUE postProcess.color.a = 1.0; #endif diff --git a/libs/viewer/src/Settings_generated.cpp b/libs/viewer/src/Settings_generated.cpp index 763832de35..3b7c5f6236 100644 --- a/libs/viewer/src/Settings_generated.cpp +++ b/libs/viewer/src/Settings_generated.cpp @@ -723,6 +723,8 @@ int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, TemporalAntiAli i = parse(tokens, i + 1, jsonChunk, &out->feedback); } else if (compare(tok, jsonChunk, "lodBias") == 0) { i = parse(tokens, i + 1, jsonChunk, &out->lodBias); + } else if (compare(tok, jsonChunk, "sharpness") == 0) { + i = parse(tokens, i + 1, jsonChunk, &out->sharpness); } else if (compare(tok, jsonChunk, "enabled") == 0) { i = parse(tokens, i + 1, jsonChunk, &out->enabled); } else if (compare(tok, jsonChunk, "upscaling") == 0) { @@ -762,6 +764,7 @@ std::ostream& operator<<(std::ostream& out, const TemporalAntiAliasingOptions& i << "\"filterWidth\": " << (in.filterWidth) << ",\n" << "\"feedback\": " << (in.feedback) << ",\n" << "\"lodBias\": " << (in.lodBias) << ",\n" + << "\"sharpness\": " << (in.sharpness) << ",\n" << "\"enabled\": " << to_string(in.enabled) << ",\n" << "\"upscaling\": " << to_string(in.upscaling) << ",\n" << "\"filterHistory\": " << to_string(in.filterHistory) << ",\n" diff --git a/libs/viewer/src/ViewerGui.cpp b/libs/viewer/src/ViewerGui.cpp index c8c46224ec..9d352e3600 100644 --- a/libs/viewer/src/ViewerGui.cpp +++ b/libs/viewer/src/ViewerGui.cpp @@ -809,6 +809,7 @@ void ViewerGui::updateUserInterface() { ImGui::Combo("Box Clipping", &boxClipping, "Accurate\0Clamp\0None\0\0"); ImGui::Combo("Box Type", &boxType, "AABB\0Variance\0Both\0\0"); ImGui::SliderFloat("Variance Gamma", &mSettings.view.taa.varianceGamma, 0.75f, 1.25f); + ImGui::SliderFloat("RCAS", &mSettings.view.taa.sharpness, 0.0f, 1.0f); mSettings.view.taa.boxClipping = (TemporalAntiAliasingOptions::BoxClipping)boxClipping; mSettings.view.taa.boxType = (TemporalAntiAliasingOptions::BoxType)boxType; mSettings.view.taa.jitterPattern = (TemporalAntiAliasingOptions::JitterPattern)jitterSequence; diff --git a/web/filament-js/extensions_generated.js b/web/filament-js/extensions_generated.js index e6b526ecaf..a0d69778c4 100644 --- a/web/filament-js/extensions_generated.js +++ b/web/filament-js/extensions_generated.js @@ -141,6 +141,7 @@ Filament.loadGeneratedExtensions = function() { filterWidth: 1.0, feedback: 0.12, lodBias: -1.0, + sharpness: 0.0, enabled: false, upscaling: false, filterHistory: true, diff --git a/web/filament-js/filament.d.ts b/web/filament-js/filament.d.ts index 71e12dbd9a..17d874c352 100644 --- a/web/filament-js/filament.d.ts +++ b/web/filament-js/filament.d.ts @@ -1684,6 +1684,10 @@ export interface View$TemporalAntiAliasingOptions { * texturing lod bias (typically -1 or -2) */ lodBias?: number; + /** + * post-TAA sharpen, especially useful when upscaling is true. + */ + sharpness?: number; /** * enables or disables temporal anti-aliasing */ diff --git a/web/filament-js/jsbindings_generated.cpp b/web/filament-js/jsbindings_generated.cpp index a05065a2f4..07fd54e786 100644 --- a/web/filament-js/jsbindings_generated.cpp +++ b/web/filament-js/jsbindings_generated.cpp @@ -121,6 +121,7 @@ value_object("View$TemporalAntiAliasingOption .field("filterWidth", &View::TemporalAntiAliasingOptions::filterWidth) .field("feedback", &View::TemporalAntiAliasingOptions::feedback) .field("lodBias", &View::TemporalAntiAliasingOptions::lodBias) + .field("sharpness", &View::TemporalAntiAliasingOptions::sharpness) .field("enabled", &View::TemporalAntiAliasingOptions::enabled) .field("upscaling", &View::TemporalAntiAliasingOptions::upscaling) .field("filterHistory", &View::TemporalAntiAliasingOptions::filterHistory)