diff --git a/android/filament-android/src/main/java/com/google/android/filament/View.java b/android/filament-android/src/main/java/com/google/android/filament/View.java index 6de6fd557f..680c8d9fd3 100644 --- a/android/filament-android/src/main/java/com/google/android/filament/View.java +++ b/android/filament-android/src/main/java/com/google/android/filament/View.java @@ -2041,10 +2041,6 @@ public class View { * use an AABB neighborhood */ AABB, - /** - * use the variance of the neighborhood (not recommended) - */ - VARIANCE, /** * use both AABB and variance */ @@ -2075,7 +2071,7 @@ public class View { } /** - * reconstruction filter width typically between 1 (sharper) and 2 (smoother) + * @deprecated has no effect. */ public float filterWidth = 1.0f; /** @@ -2095,9 +2091,9 @@ public class View { */ public boolean enabled = false; /** - * 4x TAA upscaling. Disables Dynamic Resolution. [BETA] + * Upscaling factor. Disables Dynamic Resolution. [BETA] */ - public boolean upscaling = false; + public float upscaling = 1.0f; /** * whether to filter the history buffer */ @@ -2110,6 +2106,10 @@ public class View { * whether to use the YcoCg color-space for history rejection */ public boolean useYCoCg = false; + /** + * set to true for HDR content + */ + public boolean hdr = true; /** * type of color gamut box */ diff --git a/filament/include/filament/Options.h b/filament/include/filament/Options.h index 07110ee196..422a0222d1 100644 --- a/filament/include/filament/Options.h +++ b/filament/include/filament/Options.h @@ -489,16 +489,15 @@ struct MultiSampleAntiAliasingOptions { * @see setTemporalAntiAliasingOptions() */ struct TemporalAntiAliasingOptions { - float filterWidth = 1.0f; //!< reconstruction filter width typically between 1 (sharper) and 2 (smoother) + float filterWidth = 1.0f; //!< @deprecated has no effect. float feedback = 0.12f; //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA). float lodBias = -1.0f; //!< texturing lod bias (typically -1 or -2) float sharpness = 0.0f; //!< post-TAA sharpen, especially useful when upscaling is true. bool enabled = false; //!< enables or disables temporal anti-aliasing - bool upscaling = false; //!< 4x TAA upscaling. Disables Dynamic Resolution. [BETA] + float upscaling = 1.0f; //!< Upscaling factor. Disables Dynamic Resolution. [BETA] enum class BoxType : uint8_t { AABB, //!< use an AABB neighborhood - VARIANCE, //!< use the variance of the neighborhood (not recommended) AABB_VARIANCE //!< use both AABB and variance }; @@ -519,6 +518,7 @@ struct TemporalAntiAliasingOptions { bool filterHistory = true; //!< whether to filter the history buffer bool filterInput = true; //!< whether to apply the reconstruction filter to the input bool useYCoCg = false; //!< whether to use the YcoCg color-space for history rejection + bool hdr = true; //!< set to true for HDR content BoxType boxType = BoxType::AABB; //!< type of color gamut box BoxClipping boxClipping = BoxClipping::ACCURATE; //!< clipping algorithm JitterPattern jitterPattern = JitterPattern::HALTON_23_X16; //! Jitter Pattern diff --git a/filament/src/PostProcessManager.cpp b/filament/src/PostProcessManager.cpp index 26a5a4d9cc..975e429500 100644 --- a/filament/src/PostProcessManager.cpp +++ b/filament/src/PostProcessManager.cpp @@ -2911,11 +2911,12 @@ void PostProcessManager::configureTemporalAntiAliasingMaterial( FMaterial* const ma = getPostProcessMaterial("taa").getMaterial(mEngine); bool dirty = false; - setConstantParameter(ma, "upscaling", taaOptions.upscaling, dirty); + setConstantParameter(ma, "upscaling", taaOptions.upscaling > 1.0f, dirty); setConstantParameter(ma, "historyReprojection", taaOptions.historyReprojection, dirty); setConstantParameter(ma, "filterHistory", taaOptions.filterHistory, dirty); setConstantParameter(ma, "filterInput", taaOptions.filterInput, dirty); setConstantParameter(ma, "useYCoCg", taaOptions.useYCoCg, dirty); + setConstantParameter(ma, "hdr", taaOptions.hdr, dirty); setConstantParameter(ma, "preventFlickering", taaOptions.preventFlickering, dirty); setConstantParameter(ma, "boxType", int32_t(taaOptions.boxType), dirty); setConstantParameter(ma, "boxClipping", int32_t(taaOptions.boxClipping), dirty); @@ -2984,6 +2985,8 @@ FMaterialInstance* PostProcessManager::configureColorGradingMaterial( FrameGraphId PostProcessManager::taa(FrameGraph& fg, FrameGraphId input, FrameGraphId const depth, + filament::Viewport const& xvp, + filament::Viewport const& vp, FrameHistory& frameHistory, FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa, TemporalAntiAliasingOptions const& taaOptions, @@ -3013,9 +3016,9 @@ FrameGraphId PostProcessManager::taa(FrameGraph& fg, auto const& taaPass = fg.addPass("TAA", [&](FrameGraph::Builder& builder, auto& data) { auto desc = fg.getDescriptor(input); - if (taaOptions.upscaling) { - desc.width *= 2; - desc.height *= 2; + if (taaOptions.upscaling > 1.0f) { + desc.width *= taaOptions.upscaling; + desc.height *= taaOptions.upscaling; } data.color = builder.sample(input); data.depth = builder.sample(depth); @@ -3047,55 +3050,15 @@ FrameGraphId PostProcessManager::taa(FrameGraph& fg, 0, 0, 0, 1 }}; - constexpr float2 sampleOffsets[9] = { - { -1.0f, -1.0f }, { 0.0f, -1.0f }, { 1.0f, -1.0f }, { -1.0f, 0.0f }, - { 0.0f, 0.0f }, - { 1.0f, 0.0f }, { -1.0f, 1.0f }, { 0.0f, 1.0f }, { 1.0f, 1.0f }, - }; - - constexpr float2 subSampleOffsets[4] = { - { -0.25f, 0.25f }, - { 0.25f, 0.25f }, - { 0.25f, -0.25f }, - { -0.25f, -0.25f } - }; - - UTILS_UNUSED - auto const lanczos = [](float const x, float const a) -> float { - if (x <= std::numeric_limits::epsilon()) { - return 1.0f; - } - if (std::abs(x) <= a) { - return (a * std::sin(f::PI * x) * std::sin(f::PI * x / a)) - / ((f::PI * f::PI) * (x * x)); - } - return 0.0f; - }; - - float const filterWidth = std::clamp(taaOptions.filterWidth, 1.0f, 2.0f); - float4 sum = 0.0; - float4 weights[9]; - - // this doesn't get vectorized (probably because of exp()), so don't bother - // unrolling it. - UTILS_NOUNROLL - for (size_t i = 0; i < 9; i++) { - float2 const o = sampleOffsets[i]; - for (size_t j = 0; j < 4; j++) { - float2 const subPixelOffset = taaOptions.upscaling ? subSampleOffsets[j] : float2{ 0 }; - float2 const d = (o - (current.jitter - subPixelOffset)) / filterWidth; - weights[i][j] = lanczos(length(d), filterWidth); - } - sum += weights[i]; - } - for (auto& w : weights) { - w /= sum; - } - auto out = resources.getRenderPassInfo(); - auto color = resources.getTexture(data.color); - auto depth = resources.getTexture(data.depth); - auto history = resources.getTexture(data.history); + + auto const color = resources.getTexture(data.color); + auto const depth = resources.getTexture(data.depth); + auto const history = resources.getTexture(data.history); + + auto const& colorDesc = resources.getDescriptor(data.color); + auto const& historyDesc = resources.getDescriptor(data.history); + auto const& material = getPostProcessMaterial("taa"); PostProcessVariant const variant = colorGradingConfig.translucent ? @@ -3106,17 +3069,37 @@ FrameGraphId PostProcessManager::taa(FrameGraph& fg, FMaterialInstance* mi = getMaterialInstance(ma); mi->setParameter("color", color, SamplerParams{}); // nearest mi->setParameter("depth", depth, SamplerParams{}); // nearest - mi->setParameter("alpha", taaOptions.feedback); mi->setParameter("history", history, SamplerParams{ .filterMag = SamplerMagFilter::LINEAR, .filterMin = SamplerMinFilter::LINEAR }); - mi->setParameter("filterWeights", weights, 9); - mi->setParameter("jitter", current.jitter); + mi->setParameter("alpha", taaOptions.feedback); + + // jitter is negated here because the material wants the sample jitter offset + // not the projection matrix offset + mi->setParameter("jitter", -current.jitter); + mi->setParameter("scale", taaOptions.upscaling); + mi->setParameter("reprojection", mat4f{ historyProjection * inverse(current.projection) } * normalizedToClip); + mi->setParameter("colorViewport", + float4{ + float(xvp.left) / colorDesc.width, + float(xvp.bottom) / colorDesc.height, + float(xvp.width) / colorDesc.width, + float(xvp.height) / colorDesc.height, + }); + + mi->setParameter("colorResolution", + float4{ colorDesc.width, colorDesc.height, + 1.0f / colorDesc.width, 1.0f / colorDesc.height }); + + mi->setParameter("historyResolution", + float4{ historyDesc.width, historyDesc.height, + 1.0f / historyDesc.width, 1.0f / historyDesc.height }); + mi->commit(driver, getUboManager()); mi->use(driver); diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h index 48762aed53..5ec108e153 100644 --- a/filament/src/PostProcessManager.h +++ b/filament/src/PostProcessManager.h @@ -235,6 +235,8 @@ public: FrameGraphId taa(FrameGraph& fg, FrameGraphId input, FrameGraphId depth, + Viewport const& xvp, + Viewport const& vp, FrameHistory& frameHistory, FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa, TemporalAntiAliasingOptions const& taaOptions, diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp index a2e082c10c..1a2dc8a240 100644 --- a/filament/src/details/Renderer.cpp +++ b/filament/src/details/Renderer.cpp @@ -263,7 +263,9 @@ std::pair FRenderer::prepareUpscaler(float2 const scale, if (taaOptions.enabled) { bias += taaOptions.lodBias; if (taaOptions.upscaling) { - derivativesScale = 0.5f; + if (taaOptions.upscaling > 1.0f) { + derivativesScale = 1.0f / taaOptions.upscaling; + } } } return { bias, derivativesScale }; @@ -688,13 +690,13 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) { // This configures post-process materials by setting constant parameters if (taaOptions.enabled) { ppm.configureTemporalAntiAliasingMaterial(taaOptions); - if (taaOptions.upscaling) { + if (taaOptions.upscaling > 1.0f) { // for now TAA upscaling is incompatible with regular dsr dsrOptions.enabled = false; // also, upscaling doesn't work well with quater-resolution SSAO aoOptions.resolution = 1.0; // Currently we only support a fixed TAA upscaling ratio - scale = 0.5f; + scale = 1.0f / taaOptions.upscaling; } } } @@ -781,14 +783,14 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) { // Without post-processing, we usually draw directly into // the SwapChain, and we might want to keep it this way. - auto round = [](uint32_t const x) { + auto ceil16 = [](uint32_t const x) { constexpr uint32_t rounding = 16u; return (x + (rounding - 1u)) & ~(rounding - 1u); }; // compute the new rendering width and height, multiple of 16. - const float width = float(round(svp.width )) + 2.0f * guardBand; - const float height = float(round(svp.height)) + 2.0f * guardBand; + const float width = float(ceil16(svp.width )) + 2.0f * guardBand; + const float height = float(ceil16(svp.height)) + 2.0f * guardBand; // scale the field-of-view up, so it covers exactly the extra pixels const float3 clipSpaceScaling{ @@ -1309,16 +1311,26 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) { // TAA for color pass if (taaOptions.enabled) { - input = ppm.taa(fg, input, depth, view.getFrameHistory(), &FrameHistoryEntry::taa, + input = ppm.taa(fg, input, depth, xvp, vp, + view.getFrameHistory(), &FrameHistoryEntry::taa, taaOptions, colorGradingConfig); - if (taaOptions.upscaling) { + if (taaOptions.upscaling > 1.0f) { scale = 1.0f; scaled = false; + + // xvp = vp; + // xvp.left = xvp.bottom = 0; + // svp = xvp; + UTILS_UNUSED_IN_RELEASE auto const& inputDesc = fg.getDescriptor(input); svp.width = inputDesc.width; svp.height = inputDesc.height; - xvp.width *= 2; - xvp.height *= 2; + // FIXME: this rounds xvp incorrectly + + xvp.left *= taaOptions.upscaling; + xvp.bottom *= taaOptions.upscaling; + xvp.width *= taaOptions.upscaling; + xvp.height *= taaOptions.upscaling; } } diff --git a/filament/src/details/View.cpp b/filament/src/details/View.cpp index 6d77b982e4..9b8f056a11 100644 --- a/filament/src/details/View.cpp +++ b/filament/src/details/View.cpp @@ -320,9 +320,9 @@ float2 FView::updateScale(FEngine& engine, // relative scaling ("velocity" control) const float scale = mScale.x * mScale.y * command; - const float w = float(mViewport.width); - const float h = float(mViewport.height); if (scale < 1.0f && !options.homogeneousScaling) { + const float w = float(mViewport.width); + const float h = float(mViewport.height); // figure out the major and minor axis const float major = std::max(w, h); const float minor = std::min(w, h); @@ -351,7 +351,7 @@ float2 FView::updateScale(FEngine& engine, const auto s = mScale; mScale = clamp(s, options.minScale, options.maxScale); - // disable the integration term when we're outside the controllable range + // Disable the integration term when we're outside the controllable range // (i.e. we clamped). This help not to have to wait too long for the Integral term // to kick in after a clamping event. mPidController.setIntegralInhibitionEnabled(mScale != s); diff --git a/filament/src/materials/antiAliasing/taa/taa.mat b/filament/src/materials/antiAliasing/taa/taa.mat index a3d6e58d48..f174bb00fd 100644 --- a/filament/src/materials/antiAliasing/taa/taa.mat +++ b/filament/src/materials/antiAliasing/taa/taa.mat @@ -12,13 +12,18 @@ material { filterable: false }, { - type : float, - name : alpha + type : float4, + name : colorViewport, + precision: high }, { type : sampler2d, name : history }, + { + type : float, + name : alpha + }, { type : mat4, name : reprojection, @@ -30,8 +35,19 @@ material { precision: high }, { - type : float4[9], - name : filterWeights + type : float, + name : scale, + precision: high + }, + { + type : float4, + name : colorResolution, + precision: high + }, + { + type : float4, + name : historyResolution, + precision: high } ], constants : [ @@ -75,6 +91,11 @@ material { name : preventFlickering, default : false }, + { + type : bool, + name : hdr, + default : true + }, { type : float, name : varianceGamma, @@ -91,7 +112,13 @@ material { vertex { void postProcessVertex(inout PostProcessVertexInputs postProcess) { - postProcess.vertex.xy = uvToRenderTargetUV(postProcess.normalizedUV); + highp vec2 uv = postProcess.normalizedUV; + // input UVs + //postProcess.vertex.xy = uvToRenderTargetUV( + // materialParams.colorViewport.xy + uv * materialParams.colorViewport.zw); + postProcess.vertex.xy = uvToRenderTargetUV(uv); + // history UVs + postProcess.vertex.zw = uvToRenderTargetUV(uv); } } @@ -101,10 +128,8 @@ fragment { // min/max neighborhood #define BOX_TYPE_AABB 0 -// Variance based neighborhood -#define BOX_TYPE_VARIANCE 1 // uses both min/max and variance -#define BOX_TYPE_AABB_VARIANCE 2 +#define BOX_TYPE_AABB_VARIANCE 1 /* Clipping algorithm */ @@ -127,9 +152,8 @@ float lumaYCoCg(const vec3 c) { return c.x; } -float luma(const vec3 c) { - return (materialConstants_useYCoCg && materialConstants_boxClipping != BOX_CLIPPING_NONE) ? - lumaYCoCg(c) : lumaRGB(c); +float luma(const vec3 c, const bool isYcoCg) { + return isYcoCg ? lumaYCoCg(c) : lumaRGB(c); } vec3 tonemap(const vec3 c) { @@ -153,6 +177,28 @@ vec3 untonemap(const vec3 c) { return c * rcp(max(epsilon, 1.0 - max3(c))); } +// 1-D Lanczos-2 approximation, input is squared and must be [-4, 4] +float lanczos2_SquaredInputApproximate(highp float x2) { + highp float wA = x2 - 4.0; + highp float wB = x2 * wA - wA; + wA *= wA; + wB *= wA; + return wB * (1.0 / 64.0); +} + +// 1-D Lanczos-2 filter, input must be [-2, 2] +float lanczos2(float v) { + return lanczos2_SquaredInputApproximate(v * v); +} + +// 1-D Lanczos-2 filter, input must be [-2, 2] +float lanczos2(float v, float scale) { + if (materialConstants_upscaling) { + v = clamp(v * scale, -2.0, 2.0); + } + return lanczos2(v); +} + vec3 RGB_YCoCg(const vec3 c) { float Y = dot(c.rgb, vec3( 1, 2, 1) * 0.25); float Co = dot(c.rgb, vec3( 2, 0, -2) * 0.25); @@ -176,12 +222,15 @@ vec4 clipToBox(const int quality, const float epsilon = 0.0001; if (quality == BOX_CLIPPING_ACCURATE) { - vec4 r = c - h; - vec3 ir = 1.0 / (epsilon + r.rgb); - vec3 rmax = (boxmax - h.rgb) * ir; - vec3 rmin = (boxmin - h.rgb) * ir; - vec3 imin = min(rmax, rmin); - return h + r * saturate(max3(imin)); + // don't clip if we're already in the box + if (any(lessThan(h.rgb, boxmin)) || any(greaterThan(h.rgb, boxmax))) { + vec4 r = c - h; + vec3 ir = 1.0 / (epsilon + r.rgb); + vec3 rmax = (boxmax - h.rgb) * ir; + vec3 rmin = (boxmin - h.rgb) * ir; + vec3 imin = min(rmax, rmin); + return h + r * saturate(max3(imin)); + } } else if (quality == BOX_CLIPPING_CLAMP) { return vec4(clamp(h.rgb, boxmin, boxmax), h.a); } @@ -190,17 +239,23 @@ vec4 clipToBox(const int quality, // Samples a texture with Catmull-Rom filtering, using 9 texture fetches instead of 16. // https://therealmjp.github.io/ +// // Some optimizations from here: // http://vec3.ca/bicubic-filtering-in-fewer-taps/ for more details +// // Optimized to 5 taps by removing the corner samples -// And modified for mediump support -vec4 sampleTextureCatmullRom(const sampler2D tex, const highp vec2 uv, const highp vec2 texSize) { +// Jorge Jimenez, SIGGRAPH 2016, Filmic SMAA +// http://advances.realtimerendering.com/s2016/Filmic%20SMAA%20v7.pptx +// +// And modified for mediump support + deringing +// +vec4 sampleTextureCatmullRom(const sampler2D tex, const highp vec2 uv, const highp vec4 texSize) { // We're going to sample a a 4x4 grid of texels surrounding the target UV coordinate. // We'll do this by rounding down the sample location to get the exact center of our "starting" // texel. The starting texel will be at location [1, 1] in the grid, where [0, 0] is the // top left corner. - highp vec2 samplePos = uv * texSize; + highp vec2 samplePos = uv * texSize.xy; highp vec2 texPos1 = floor(samplePos - 0.5) + 0.5; // Compute the fractional offset from our starting texel to our original sample location, @@ -226,7 +281,7 @@ vec4 sampleTextureCatmullRom(const sampler2D tex, const highp vec2 uv, const hig highp vec2 texPos3 = texPos1 + vec2(2.0); highp vec2 texPos12 = texPos1 + w2 / w12; - highp vec2 invTexSize = 1.0 / texSize; + highp vec2 invTexSize = texSize.zw; texPos0 *= invTexSize; texPos3 *= invTexSize; texPos12 *= invTexSize; @@ -252,14 +307,28 @@ vec4 sampleTextureCatmullRom(const sampler2D tex, const highp vec2 uv, const hig result *= rcp(k0 + k1 + k2 + k3 + k4); - // we could end-up with negative values - result = max(vec4(0), result); - + // deringing... + vec4 boxmin = min(min(min(s[0], s[1]), min(s[2], s[3])), s[4]); + vec4 boxmax = max(max(max(s[0], s[1]), max(s[2], s[3])), s[4]); + result = clamp(result, boxmin, boxmax); return result; } +const vec2 kInputFilterSampleOffsets[9] = vec2[9]( + vec2( 0, 0), + vec2( 0, -1), vec2(-1, 0), vec2( 1, 0), vec2( 0, 1), + vec2(-1, -1), vec2( 1, -1), vec2(-1, 1), vec2( 1, 1) +); + void postProcess(inout PostProcessInputs postProcess) { - highp vec4 uv = variable_vertex.xyxy; // interpolated to pixel center + const bool isColorYCoCg = materialConstants_useYCoCg && + materialConstants_boxClipping != BOX_CLIPPING_NONE; + + const bool isHistoryYCoCg = materialConstants_useYCoCg && + materialConstants_boxClipping != BOX_CLIPPING_NONE; + + // destination uv interpolated to pixel center + highp vec4 uv = variable_vertex; if (materialConstants_historyReprojection) { // read the depth buffer center sample for reprojection @@ -274,133 +343,183 @@ void postProcess(inout PostProcessInputs postProcess) { // read center color and history samples vec4 history; if (materialConstants_filterHistory) { - history = sampleTextureCatmullRom(materialParams_history, uv.zw, - vec2(textureSize(materialParams_history, 0))); + history = sampleTextureCatmullRom(materialParams_history, uv.zw, materialParams.historyResolution); } else { history = textureLod(materialParams_history, uv.zw, 0.0); } - highp vec2 size = vec2(textureSize(materialParams_color, 0)); - highp vec2 p = (floor(uv.xy * size) + 0.5) / size; + highp vec2 inputSize = materialParams.colorResolution.xy; + highp vec2 inputPixelSize = materialParams.colorResolution.zw; + + // +----------+----------+-- + // | | | + // | o | o | + // | | | + // +----------p----------+-- + // | | | + // | o | o | + // | | x | + // +----------+----------+- + // | | | + // o: reconstructed sample + // x: jittered sample + // p: texture coordinate of the closest jittered sample from the reconstructed sample + + highp vec2 reconstructedFragmentPos = uv.xy * inputSize; + highp vec2 closestInputSamplePos = floor(reconstructedFragmentPos - materialParams.jitter) + 0.5; + highp vec2 p = closestInputSamplePos * inputPixelSize; + highp vec2 samplePos = closestInputSamplePos + materialParams.jitter; + vec2 dcenter = samplePos - reconstructedFragmentPos; + vec4 filtered = textureLod(materialParams_color, p, 0.0); - vec3 s[9]; - if (materialConstants_filterInput || materialConstants_boxClipping != BOX_CLIPPING_NONE) { - s[0] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, -1)).rgb; - s[1] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 0, -1)).rgb; - s[2] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, -1)).rgb; - s[3] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, 0)).rgb; - s[4] = filtered.rgb; - s[5] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, 0)).rgb; - s[6] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, 1)).rgb; - s[7] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 0, 1)).rgb; - s[8] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, 1)).rgb; - } - - int j = 0; float confidence = 1.0; - if (materialConstants_upscaling) { - highp vec2 subPixelOffset = (p - uv.xy) * size; // +/- [0.25, 0.25] - // we reduce the contribution of a sample based on the distance - // to the high resolution pixel center - const float cutoff = 0.5; - highp float l = length(materialParams.jitter - subPixelOffset) / cutoff; - confidence = saturate(1.0 - l * l); + if (materialConstants_filterInput || + materialConstants_boxClipping != BOX_CLIPPING_NONE) { + + vec3 s[9]; + s[0] = filtered.rgb; + s[1] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 0, -1)).rgb; + s[2] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, 0)).rgb; + s[3] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, 0)).rgb; + s[4] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 0, 1)).rgb; + s[5] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, -1)).rgb; + s[6] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, -1)).rgb; + s[7] = textureLodOffset(materialParams_color, p, 0.0, ivec2(-1, 1)).rgb; + s[8] = textureLodOffset(materialParams_color, p, 0.0, ivec2( 1, 1)).rgb; + + // the box can be used for deringing or history correction + vec3 box5min = min(s[0], min(min(s[1], s[2]), min(s[3], s[4]))); + vec3 box5max = max(s[0], max(max(s[1], s[2]), max(s[3], s[4]))); + vec3 box9min = min(box5min, min(min(s[5], s[6]), min(s[7], s[8]))); + vec3 box9max = max(box5max, max(max(s[5], s[6]), max(s[7], s[8]))); if (materialConstants_filterInput) { - int jxp = subPixelOffset.y > 0.0 ? 1 : 2; - int jxn = subPixelOffset.y > 0.0 ? 0 : 3; - j = subPixelOffset.x > 0.0 ? jxp : jxn; - } - } + // reconstruct the sample at the center of the current pixel from 3x3 input samples + // using Lanczos filter. The input sample positions are jittered, so this can also be + // thought of as "unjittering" the input sample. + // The distance (in input texels) from the reconstructed sample (uv) to the filter's center sample + // is given by: + // p: texture coordinate of the center input sample + // dcenter = (p * inputSize - jitter) - uv * inputSize - if (materialConstants_filterInput) { - // unjitter/filter input - filtered = vec4(vec3(0), filtered.a); - for (int i = 0; i < 9; i++) { - float w = materialParams.filterWeights[i][j]; - filtered.rgb += s[i] * w; - } - filtered.rgb = max(filtered.rgb, vec3(0)); - } + filtered = vec4(vec3(0), filtered.a); + if (materialConstants_upscaling) { + // the weigths act as a blending factor because they acount only for a part of the + // reconstruction filter (for this frame). If the max weight is 0, it doesn't mean we must + // accumulate black into the history, but it means that these samples have little effect on the + // final reconstruction. + confidence = 0.0; + } - // build the history clamping box - if (materialConstants_boxClipping != BOX_CLIPPING_NONE) { - if (materialConstants_useYCoCg) { - history.rgb = RGB_YCoCg(history.rgb); - filtered.rgb = RGB_YCoCg(filtered.rgb); - for (int i = 0; i < 9; i++) { - s[i] = RGB_YCoCg(s[i]); + // The 2D application of the lanczos filter as a separable filter, which is the correct application + // of lanczos, however it is not isotropic. + float w0 = lanczos2(dcenter.x - 1.0, materialParams.scale); + float w1 = lanczos2(dcenter.x, materialParams.scale); + float w2 = lanczos2(dcenter.x + 1.0, materialParams.scale); + float w3 = lanczos2(dcenter.y - 1.0, materialParams.scale); + float w4 = lanczos2(dcenter.y, materialParams.scale); + float w5 = lanczos2(dcenter.y + 1.0, materialParams.scale); + highp float wx = w0 + w1 + w2; + highp float wy = w3 + w4 + w5; + float sum = wx * wy; + vec3 f; + f = s[5] * w0; + f += s[1] * w1; + f += s[6] * w2; + filtered.rgb += f * w3; + f = s[2] * w0; + f += s[0] * w1; + f += s[3] * w2; + filtered.rgb += f * w4; + f = s[7] * w0; + f += s[4] * w1; + f += s[8] * w2; + filtered.rgb += f * w5; + + // Deringing must happen in RGB, because clamping in YCoCg can produce out-of-gamut colors + filtered.rgb *= (1.0 / sum); + filtered.rgb = clamp(filtered.rgb, box9min, box9max); + if (materialConstants_upscaling) { + confidence = sum; } } - vec3 boxmin; - vec3 boxmax; - if (materialConstants_boxType == BOX_TYPE_AABB || - materialConstants_boxType == BOX_TYPE_AABB_VARIANCE) { - boxmin = min(s[4], min(min(s[1], s[3]), min(s[5], s[7]))); - boxmax = max(s[4], max(max(s[1], s[3]), max(s[5], s[7]))); - vec3 box9min = min(boxmin, min(min(s[0], s[2]), min(s[6], s[8]))); - vec3 box9max = max(boxmax, max(max(s[0], s[2]), max(s[6], s[8]))); - // round the corners of the 3x3 box, giving less importance to the corner samples - boxmin = (boxmin + box9min) * 0.5; - boxmax = (boxmax + box9max) * 0.5; - } - - if (materialConstants_boxType == BOX_TYPE_VARIANCE || - materialConstants_boxType == BOX_TYPE_AABB_VARIANCE) { - // "An Excursion in Temporal Supersampling" by Marco Salvi - highp vec3 m0 = s[4];// conversion to highp - highp vec3 m1 = m0 * m0; - // we use only 5 samples instead of all 9 - for (int i = 1; i < 9; i+=2) { - highp vec3 c = s[i];// conversion to highp - m0 += c; - m1 += c * c; + // build the history clamping box + if (materialConstants_boxClipping != BOX_CLIPPING_NONE) { + if (materialConstants_useYCoCg) { + history.xyz = RGB_YCoCg(history.rgb); + filtered.xyz = RGB_YCoCg(filtered.rgb); + for (int i = 0; i < 9; i++) { + s[i] = RGB_YCoCg(s[i]); + } + // recompute the correction box in YCoCg + box5min = min(s[0], min(min(s[1], s[2]), min(s[3], s[4]))); + box5max = max(s[0], max(max(s[1], s[2]), max(s[3], s[4]))); + box9min = min(box5min, min(min(s[5], s[6]), min(s[7], s[8]))); + box9max = max(box5max, max(max(s[5], s[6]), max(s[7], s[8]))); } - highp vec3 a0 = m0 * (1.0 / 5.0); - highp vec3 a1 = m1 * (1.0 / 5.0); - highp vec3 sigma = sqrt(a1 - a0 * a0); - if (materialConstants_boxType == BOX_TYPE_VARIANCE) { - boxmin = a0 - materialConstants_varianceGamma * sigma; - boxmax = a0 + materialConstants_varianceGamma * sigma; - } else { + // give more or less importance to the corner samples by lerp'ing between the two boxes + vec3 boxmin = mix(box5min, box9min, 0.5); + vec3 boxmax = mix(box5max, box9max, 0.5); + + if (materialConstants_boxType == BOX_TYPE_AABB_VARIANCE) { + // "An Excursion in Temporal Supersampling" by Marco Salvi + highp vec3 m0 = s[0];// conversion to highp + highp vec3 m1 = m0 * m0; + // we use only 5 samples instead of all 9 + const int count = 5; // 5 or 9 + for (int i = 1; i < count; i++) { + highp vec3 c = s[i];// conversion to highp + m0 += c; + m1 += c * c; + } + highp vec3 a0 = m0 * (1.0 / float(count)); + highp vec3 a1 = m1 * (1.0 / float(count)); + // stdDev is never < 0, but we enforce it with abs(), which is free, to account for fp math + highp vec3 stdDev = sqrt(abs(a1 - a0 * a0)); + // intersect both bounding boxes - boxmin = max(boxmin, a0 - materialConstants_varianceGamma * sigma); - boxmax = min(boxmax, a0 + materialConstants_varianceGamma * sigma); + boxmin = max(boxmin, a0 - materialConstants_varianceGamma * stdDev); + boxmax = min(boxmax, a0 + materialConstants_varianceGamma * stdDev); } + // history clamping + history = clipToBox(materialConstants_boxClipping, boxmin, boxmax, filtered, history); } - // history clamping - history = clipToBox(materialConstants_boxClipping, boxmin, boxmax, filtered, history); } float alpha = materialParams.alpha * confidence; + if (materialConstants_preventFlickering) { // [Lottes] prevents flickering by modulating the blend weight by the difference in luma - float lumaColor = luma(filtered.rgb); - float lumaHistory = luma(history.rgb); + float lumaColor = luma(filtered.rgb, isColorYCoCg); + float lumaHistory = luma(history.rgb, isHistoryYCoCg); float diff = 1.0 - abs(lumaColor - lumaHistory) / (0.001 + max(lumaColor, lumaHistory)); alpha *= diff * diff; } // go back to RGB space before tonemapping - if (materialConstants_boxClipping != BOX_CLIPPING_NONE) { - if (materialConstants_useYCoCg) { - filtered.rgb = YCoCg_RGB(filtered.rgb); - history.rgb = YCoCg_RGB(history.rgb); - } + if (isColorYCoCg) { + filtered.rgb = YCoCg_RGB(filtered.rgb); + } + if (isHistoryYCoCg) { + history.rgb = YCoCg_RGB(history.rgb); } - // tonemap before mixing - filtered.rgb = tonemap(filtered.rgb); - history.rgb = tonemap(history.rgb); + if (materialConstants_hdr) { + // tonemap before mixing + filtered.rgb = tonemap(filtered.rgb); + history.rgb = tonemap(history.rgb); + } // combine history and current frame vec4 result = mix(history, filtered, alpha); - // untonemap result - result.rgb = untonemap(result.rgb); + if (materialConstants_hdr) { + // untonemap result + result.rgb = untonemap(result.rgb); + } #if POST_PROCESS_OPAQUE // kill the work performed above diff --git a/libs/viewer/src/Settings_generated.cpp b/libs/viewer/src/Settings_generated.cpp index 0dc7944e6b..af2682f327 100644 --- a/libs/viewer/src/Settings_generated.cpp +++ b/libs/viewer/src/Settings_generated.cpp @@ -717,7 +717,6 @@ std::ostream& operator<<(std::ostream& out, const MultiSampleAntiAliasingOptions int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, TemporalAntiAliasingOptions::BoxType* out) { if (0 == compare(tokens[i], jsonChunk, "AABB")) { *out = TemporalAntiAliasingOptions::BoxType::AABB; } - else if (0 == compare(tokens[i], jsonChunk, "VARIANCE")) { *out = TemporalAntiAliasingOptions::BoxType::VARIANCE; } else if (0 == compare(tokens[i], jsonChunk, "AABB_VARIANCE")) { *out = TemporalAntiAliasingOptions::BoxType::AABB_VARIANCE; } else { slog.w << "Invalid TemporalAntiAliasingOptions::BoxType: '" << STR(tokens[i], jsonChunk) << "'" << io::endl; @@ -728,7 +727,6 @@ int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, TemporalAntiAli std::ostream& operator<<(std::ostream& out, TemporalAntiAliasingOptions::BoxType in) { switch (in) { case TemporalAntiAliasingOptions::BoxType::AABB: return out << "\"AABB\""; - case TemporalAntiAliasingOptions::BoxType::VARIANCE: return out << "\"VARIANCE\""; case TemporalAntiAliasingOptions::BoxType::AABB_VARIANCE: return out << "\"AABB_VARIANCE\""; } return out << "\"INVALID\""; @@ -800,6 +798,8 @@ int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, TemporalAntiAli i = parse(tokens, i + 1, jsonChunk, &out->filterInput); } else if (compare(tok, jsonChunk, "useYCoCg") == 0) { i = parse(tokens, i + 1, jsonChunk, &out->useYCoCg); + } else if (compare(tok, jsonChunk, "hdr") == 0) { + i = parse(tokens, i + 1, jsonChunk, &out->hdr); } else if (compare(tok, jsonChunk, "boxType") == 0) { i = parse(tokens, i + 1, jsonChunk, &out->boxType); } else if (compare(tok, jsonChunk, "boxClipping") == 0) { @@ -831,10 +831,11 @@ std::ostream& operator<<(std::ostream& out, const TemporalAntiAliasingOptions& i << "\"lodBias\": " << (in.lodBias) << ",\n" << "\"sharpness\": " << (in.sharpness) << ",\n" << "\"enabled\": " << to_string(in.enabled) << ",\n" - << "\"upscaling\": " << to_string(in.upscaling) << ",\n" + << "\"upscaling\": " << (in.upscaling) << ",\n" << "\"filterHistory\": " << to_string(in.filterHistory) << ",\n" << "\"filterInput\": " << to_string(in.filterInput) << ",\n" << "\"useYCoCg\": " << to_string(in.useYCoCg) << ",\n" + << "\"hdr\": " << to_string(in.hdr) << ",\n" << "\"boxType\": " << (in.boxType) << ",\n" << "\"boxClipping\": " << (in.boxClipping) << ",\n" << "\"jitterPattern\": " << (in.jitterPattern) << ",\n" diff --git a/libs/viewer/src/ViewerGui.cpp b/libs/viewer/src/ViewerGui.cpp index d5531f91fe..a914d841c5 100644 --- a/libs/viewer/src/ViewerGui.cpp +++ b/libs/viewer/src/ViewerGui.cpp @@ -820,13 +820,13 @@ void ViewerGui::updateUserInterface() { } if (ImGui::CollapsingHeader("TAA Options")) { - ImGui::Checkbox("Upscaling", &mSettings.view.taa.upscaling); + ImGui::SliderFloat("Upscaling", &mSettings.view.taa.upscaling, 1.0f, 3.0f); ImGui::Checkbox("History Reprojection", &mSettings.view.taa.historyReprojection); ImGui::SliderFloat("Feedback", &mSettings.view.taa.feedback, 0.0f, 1.0f); ImGui::Checkbox("Filter History", &mSettings.view.taa.filterHistory); ImGui::Checkbox("Filter Input", &mSettings.view.taa.filterInput); - ImGui::SliderFloat("FilterWidth", &mSettings.view.taa.filterWidth, 0.2f, 2.0f); ImGui::SliderFloat("LOD bias", &mSettings.view.taa.lodBias, -8.0f, 0.0f); + ImGui::Checkbox("HDR", &mSettings.view.taa.hdr); ImGui::Checkbox("Use YCoCg", &mSettings.view.taa.useYCoCg); ImGui::Checkbox("Prevent Flickering", &mSettings.view.taa.preventFlickering); int jitterSequence = (int)mSettings.view.taa.jitterPattern; @@ -834,7 +834,7 @@ void ViewerGui::updateUserInterface() { int boxType = (int)mSettings.view.taa.boxType; ImGui::Combo("Jitter Pattern", &jitterSequence, "RGSS x4\0Uniform Helix x4\0Halton x8\0Halton x16\0Halton x32\0\0"); ImGui::Combo("Box Clipping", &boxClipping, "Accurate\0Clamp\0None\0\0"); - ImGui::Combo("Box Type", &boxType, "AABB\0Variance\0Both\0\0"); + ImGui::Combo("Box Type", &boxType, "AABB\0Variance\0\0"); ImGui::SliderFloat("Variance Gamma", &mSettings.view.taa.varianceGamma, 0.75f, 1.25f); ImGui::SliderFloat("RCAS", &mSettings.view.taa.sharpness, 0.0f, 1.0f); mSettings.view.taa.boxClipping = (TemporalAntiAliasingOptions::BoxClipping)boxClipping; diff --git a/samples/material_sandbox.cpp b/samples/material_sandbox.cpp index 62816cb67c..72ab6de5c3 100644 --- a/samples/material_sandbox.cpp +++ b/samples/material_sandbox.cpp @@ -717,7 +717,6 @@ static void gui(filament::Engine* engine, filament::View*) { if (params.taaOptions.enabled) { ImGui::Indent(); ImGui::SliderFloat("feedback", ¶ms.taaOptions.feedback, 0.0f, 1.0f); - ImGui::SliderFloat("filter", ¶ms.taaOptions.filterWidth, 0.02f, 2.0f); ImGui::Unindent(); } ImGui::Checkbox("FXAA", ¶ms.fxaa); diff --git a/web/filament-js/extensions_generated.js b/web/filament-js/extensions_generated.js index b92b549b25..636db97b14 100644 --- a/web/filament-js/extensions_generated.js +++ b/web/filament-js/extensions_generated.js @@ -157,10 +157,11 @@ Filament.loadGeneratedExtensions = function() { lodBias: -1.0, sharpness: 0.0, enabled: false, - upscaling: false, + upscaling: 1.0, filterHistory: true, filterInput: true, useYCoCg: false, + hdr: true, boxType: Filament.View$TemporalAntiAliasingOptions$BoxType.AABB, boxClipping: Filament.View$TemporalAntiAliasingOptions$BoxClipping.ACCURATE, jitterPattern: Filament.View$TemporalAntiAliasingOptions$JitterPattern.HALTON_23_X16, diff --git a/web/filament-js/filament.d.ts b/web/filament-js/filament.d.ts index fdcbecce8f..889683f556 100644 --- a/web/filament-js/filament.d.ts +++ b/web/filament-js/filament.d.ts @@ -1705,7 +1705,6 @@ export interface View$MultiSampleAntiAliasingOptions { export enum View$TemporalAntiAliasingOptions$BoxType { AABB, // use an AABB neighborhood - VARIANCE, // use the variance of the neighborhood (not recommended) AABB_VARIANCE, // use both AABB and variance } @@ -1736,7 +1735,7 @@ export enum View$TemporalAntiAliasingOptions$JitterPattern { */ export interface View$TemporalAntiAliasingOptions { /** - * reconstruction filter width typically between 1 (sharper) and 2 (smoother) + * @deprecated has no effect. */ filterWidth?: number; /** @@ -1756,9 +1755,9 @@ export interface View$TemporalAntiAliasingOptions { */ enabled?: boolean; /** - * 4x TAA upscaling. Disables Dynamic Resolution. [BETA] + * Upscaling factor. Disables Dynamic Resolution. [BETA] */ - upscaling?: boolean; + upscaling?: number; /** * whether to filter the history buffer */ @@ -1771,6 +1770,10 @@ export interface View$TemporalAntiAliasingOptions { * whether to use the YcoCg color-space for history rejection */ useYCoCg?: boolean; + /** + * set to true for HDR content + */ + hdr?: boolean; /** * type of color gamut box */ diff --git a/web/filament-js/jsbindings_generated.cpp b/web/filament-js/jsbindings_generated.cpp index 70c9ffdde2..e31e6c4db2 100644 --- a/web/filament-js/jsbindings_generated.cpp +++ b/web/filament-js/jsbindings_generated.cpp @@ -138,6 +138,7 @@ value_object("View$TemporalAntiAliasingOption .field("filterHistory", &View::TemporalAntiAliasingOptions::filterHistory) .field("filterInput", &View::TemporalAntiAliasingOptions::filterInput) .field("useYCoCg", &View::TemporalAntiAliasingOptions::useYCoCg) + .field("hdr", &View::TemporalAntiAliasingOptions::hdr) .field("boxType", &View::TemporalAntiAliasingOptions::boxType) .field("boxClipping", &View::TemporalAntiAliasingOptions::boxClipping) .field("jitterPattern", &View::TemporalAntiAliasingOptions::jitterPattern) diff --git a/web/filament-js/jsenums_generated.cpp b/web/filament-js/jsenums_generated.cpp index 638ac38ea3..682f439c0f 100644 --- a/web/filament-js/jsenums_generated.cpp +++ b/web/filament-js/jsenums_generated.cpp @@ -40,7 +40,6 @@ enum_("View$AmbientOcclusio enum_("View$TemporalAntiAliasingOptions$BoxType") .value("AABB", View::TemporalAntiAliasingOptions::BoxType::AABB) - .value("VARIANCE", View::TemporalAntiAliasingOptions::BoxType::VARIANCE) .value("AABB_VARIANCE", View::TemporalAntiAliasingOptions::BoxType::AABB_VARIANCE) ;