Add night adaptation control to ColorGrading (#4559)

In low-light conditions, peak luminance sensitivity of the eye shifts
toward the blue end of the visible spectrum. This effect called the
Purkinje effect occurs during the transition from photopic (cone-based)
vision to scotopic (rod-based) vision. Because the rods and cones use the
same neural pathways, a color shift is introduced as the rods take over to
improve low-light perception.

This function aims to (somewhat) replicate this color shift and peak
luminance sensitivity increase to more faithfully reproduce scenes in
low-light conditions as they would be perceived by a human observer
(as opposed to an artificial observer such as a camera sensor).

The night adaptation can be controlled using a 0..1 factor for artistic
reasons: `ColorGrading::Builder::nightAdaptation()`.

CMakeLists.txt

@@ -691,6 +691,7 @@ if (IS_HOST_PLATFORM)
     add_subdirectory(${TOOLS}/mipgen)
     add_subdirectory(${TOOLS}/normal-blending)
     add_subdirectory(${TOOLS}/resgen)
+    add_subdirectory(${TOOLS}/rgb-to-lmsr)
     add_subdirectory(${TOOLS}/roughness-prefilter)
     add_subdirectory(${TOOLS}/specular-color)
 endif()

README.md

@@ -134,9 +134,10 @@ steps:
 
 - HDR bloom
 - Depth of field bokeh
-- Multiple tone mappers: ACES, filmic, etc.
-- Color grading: white balance, channel mixer, shadows/mid-tones/highlights, ASC CDL,
-  contrast, saturation, etc.
+- Multiple tone mappers: generic (customizable), ACES, filmic, etc.
+- Color and tone management: luminance scaling, gamut mapping
+- Color grading: exposure, night adaptation, white balance, channel mixer,
+  shadows/mid-tones/highlights, ASC CDL, contrast, saturation, etc.
 - TAA, FXAA, MSAA
 - Screen-space lens flares
 

RELEASE_NOTES.md

@@ -5,6 +5,9 @@ A new header is inserted each time a *tag* is created.
 
 ## v1.12.4 (currently main branch)
 
+- engine: New night adaptation API on `ColorGrading`. This API can be used to create an effect that
+  that simulates color and brightness shits in human vision in low-light conditions.
+
 ## v1.12.3
 
 - engine: Support AMD FidelityFX Super Resolution for dynamic resolution scaling
@@ -22,8 +25,7 @@ A new header is inserted each time a *tag* is created.
 
 - engine: `double` precision translation support in TransformManager. Disabled by default.
   Augment model (and view) matrix on `Camera` to accept double precision matrices. When enabled,
-  double precision translations allow filament to handle a very large world space.
-  [**New API**].
+  double precision translations allow filament to handle a very large world space [**New API**].
 - engine: Fix, Views with custom render targets are now blendable.
 
 ## v1.12.0

android/filament-android/src/main/cpp/ColorGrading.cpp

@@ -90,6 +90,13 @@ Java_com_google_android_filament_ColorGrading_nBuilderExposure(JNIEnv*, jclass,
     builder->exposure(exposure);
 }
 
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_android_filament_ColorGrading_nBuilderNightAdaptation(JNIEnv*, jclass,
+        jlong nativeBuilder, jfloat adaptation) {
+    ColorGrading::Builder* builder = (ColorGrading::Builder*) nativeBuilder;
+    builder->nightAdaptation(adaptation);
+}
+
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_android_filament_ColorGrading_nBuilderWhiteBalance(JNIEnv*, jclass,
         jlong nativeBuilder, jfloat temperature, jfloat tint) {

android/filament-android/src/main/java/com/google/android/filament/ColorGrading.java

@@ -55,6 +55,7 @@ import static com.google.android.filament.Asserts.assertFloat4In;
  * The various transforms held by ColorGrading are applied in the following order:
  * <ul>
  * <li>Exposure</li>
+ * <li>Night adaptation</li>
  * <li>White balance</li>
  * <li>Channel mixer</li>
  * <li>Shadows/mid-tones/highlights</li>
@@ -73,6 +74,7 @@ import static com.google.android.filament.Asserts.assertFloat4In;
  * Here are the default color grading options:
  * <ul>
  * <li>Exposure: 0.0</li>
+ * <li>Night adaptation: 0.0</li>
  * <li>White balance: temperature <code>0.0</code>, and tint <code>0.0</code></li>
  * <li>Channel mixer: red <code>{1,0,0}</code>, green <code>{0,1,0}</code>, blue <code>{0,0,1}</code></li>
  * <li>Shadows/mid-tones/highlights: shadows <code>{1,1,1,0}</code>, mid-tones <code>{1,1,1,0}</code>,
@@ -247,6 +249,22 @@ public class ColorGrading {
             return this;
         }
 
+        /**
+         * Controls the amount of night adaptation to replicate a more natural representation of
+         * low-light conditions as perceived by the human vision system. In low-light conditions,
+         * peak luminance sensitivity of the eye shifts toward the blue end of the color spectrum:
+         * darker tones appear brighter, reducing contrast, and colors are blue shifted (the darker
+         * the more intense the effect).
+         *
+         * @param adaptation Amount of adaptation, between 0 (no adaptation) and 1 (full adaptation).
+         *
+         * @return This Builder, for chaining calls
+         */
+        public Builder nightAdaptation(float adaptation) {
+            nBuilderNightAdaptation(mNativeBuilder, adaptation);
+            return this;
+        }
+
         /**
          * Adjusts the while balance of the image. This can be used to remove color casts
          * and correct the appearance of the white point in the scene, or to alter the
@@ -544,6 +562,7 @@ public class ColorGrading {
     private static native void nBuilderLuminanceScaling(long nativeBuilder, boolean luminanceScaling);
     private static native void nBuilderGamutMapping(long nativeBuilder, boolean gamutMapping);
     private static native void nBuilderExposure(long nativeBuilder, float exposure);
+    private static native void nBuilderNightAdaptation(long nativeBuilder, float adaptation);
     private static native void nBuilderWhiteBalance(long nativeBuilder, float temperature, float tint);
     private static native void nBuilderChannelMixer(long nativeBuilder, float[] outRed, float[] outGreen, float[] outBlue);
     private static native void nBuilderShadowsMidtonesHighlights(long nativeBuilder, float[] shadows, float[] midtones, float[] highlights, float[] ranges);

filament/include/filament/ColorGrading.h

@@ -66,6 +66,7 @@ class FColorGrading;
  *
  * The various transforms held by ColorGrading are applied in the following order:
  * - Exposure
+ * - Night adaptation
  * - White balance
  * - Channel mixer
  * - Shadows/mid-tones/highlights
@@ -83,6 +84,7 @@ class FColorGrading;
  *
  * Here are the default color grading options:
  * - Exposure: 0.0
+ * - Night adaptation: 0.0
  * - White balance: temperature 0, and tint 0
  * - Channel mixer: red {1,0,0}, green {0,1,0}, blue {0,0,1}
  * - Shadows/mid-tones/highlights: shadows {1,1,1,0}, mid-tones {1,1,1,0}, highlights {1,1,1,0},
@@ -220,6 +222,19 @@ public:
          */
         Builder& exposure(float exposure) noexcept;
 
+        /**
+         * Controls the amount of night adaptation to replicate a more natural representation of
+         * low-light conditions as perceived by the human vision system. In low-light conditions,
+         * peak luminance sensitivity of the eye shifts toward the blue end of the color spectrum:
+         * darker tones appear brighter, reducing contrast, and colors are blue shifted (the darker
+         * the more intense the effect).
+         *
+         * @param adaptation Amount of adaptation, between 0 (no adaptation) and 1 (full adaptation).
+         *
+         * @return This Builder, for chaining calls
+         */
+        Builder& nightAdaptation(float adaptation) noexcept;
+
         /**
          * Adjusts the while balance of the image. This can be used to remove color casts
          * and correct the appearance of the white point in the scene, or to alter the

filament/src/ColorGrading.cpp

@@ -61,6 +61,8 @@ struct ColorGrading::BuilderDetails {
     bool   gamutMapping     = false;
     // Exposure
     float  exposure         = 0.0f;
+    // Night adaptation
+    float  nightAdaptation  = 0.0f;
     // White balance
     float2 whiteBalance     = {0.0f, 0.0f};
     // Channel mixer
@@ -95,6 +97,7 @@ struct ColorGrading::BuilderDetails {
                luminanceScaling == rhs.luminanceScaling &&
                gamutMapping == rhs.gamutMapping &&
                exposure == rhs.exposure &&
+               nightAdaptation == rhs.nightAdaptation &&
                whiteBalance == rhs.whiteBalance &&
                outRed == rhs.outRed &&
                outGreen == rhs.outGreen &&
@@ -153,6 +156,11 @@ ColorGrading::Builder& ColorGrading::Builder::exposure(float exposure) noexcept
     return *this;
 }
 
+ColorGrading::Builder& ColorGrading::Builder::nightAdaptation(float adaptation) noexcept {
+    mImpl->nightAdaptation = saturate(adaptation);
+    return *this;
+}
+
 ColorGrading::Builder& ColorGrading::Builder::whiteBalance(float temperature, float tint) noexcept {
     mImpl->whiteBalance = float2{
         clamp(temperature, -1.0f, 1.0f),
@@ -258,7 +266,7 @@ ColorGrading* ColorGrading::Builder::build(Engine& engine) {
 #pragma clang diagnostic pop
 
 //------------------------------------------------------------------------------
-// White balance
+// Exposure
 //------------------------------------------------------------------------------
 
 UTILS_ALWAYS_INLINE
@@ -266,6 +274,102 @@ inline float3 adjustExposure(float3 v, float exposure) {
     return v * std::exp2(exposure);
 }
 
+//------------------------------------------------------------------------------
+// Purkinje shift/scotopic vision
+//------------------------------------------------------------------------------
+
+// In low-light conditions, peak luminance sensitivity of the eye shifts toward
+// the blue end of the visible spectrum. This effect called the Purkinje effect
+// occurs during the transition from photopic (cone-based) vision to scotopic
+// (rod-based) vision. Because the rods and cones use the same neural pathways,
+// a color shift is introduced as the rods take over to improve low-light
+// perception.
+//
+// This function aims to (somewhat) replicate this color shift and peak luminance
+// sensitivity increase to more faithfully reproduce scenes in low-light conditions
+// as they would be perceived by a human observer (as opposed to an artificial
+// observer such as a camera sensor).
+//
+// The implementation below is based on two papers:
+// "Rod Contributions to Color Perception: Linear with Rod Contrast", Cao et al., 2008
+//     https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2630540/pdf/nihms80286.pdf
+// "Perceptually Based Tone Mapping for Low-Light Conditions", Kirk & O'Brien, 2011
+//     http://graphics.berkeley.edu/papers/Kirk-PBT-2011-08/Kirk-PBT-2011-08.pdf
+float3 scotopicAdaptation(float3 v, float nightAdaptation) noexcept {
+    // The 4 vectors below are generated by the command line tool rgb-to-lmsr.
+    // Together they form a 4x3 matrix that can be used to convert a Rec.709
+    // input color to the LMSR (long/medium/short cone + rod receptors) space.
+    // That matrix is computed using this formula:
+    //     Mij = \Integral Ei(lambda) I(lambda) Rj(lambda) d(lambda)
+    // Where:
+    //     i in {L, M, S, R}
+    //     j in {R, G, B}
+    //     lambda: wavelength
+    //     Ei(lambda): response curve of the corresponding receptor
+    //     I(lambda): relative spectral power of the CIE illuminant D65
+    //     Rj(lambda): spectral power of the corresponding Rec.709 color
+    constexpr float3 L{7.696847f, 18.424824f,  2.068096f};
+    constexpr float3 M{2.431137f, 18.697937f,  3.012463f};
+    constexpr float3 S{0.289117f,  1.401833f, 13.792292f};
+    constexpr float3 R{0.466386f, 15.564362f, 10.059963f};
+
+    constexpr mat3f LMS_to_RGB = inverse(transpose(mat3f{L, M, S}));
+
+    // Maximal LMS cone sensitivity, Cao et al. Table 1
+    constexpr float3 m{0.63721f, 0.39242f, 1.6064f};
+    // Strength of rod input, free parameters in Cao et al., manually tuned for our needs
+    // We follow Kirk & O'Brien who recommend constant values as opposed to Cao et al.
+    // who propose to adapt those values based on retinal illuminance. We instead offer
+    // artistic control at the end of the process
+    // The vector below is {k1, k1, k2} in Kirk & O'Brien, but {k5, k5, k6} in Cao et al.
+    constexpr float3 k{0.2f, 0.2f, 0.3f};
+
+    // Transform from opponent space back to LMS
+    constexpr mat3f opponent_to_LMS{
+        -0.5f, 0.5f, 0.0f,
+         0.0f, 0.0f, 1.0f,
+         0.5f, 0.5f, 1.0f
+    };
+
+    // The constants below follow Cao et al, using the KC pathway
+    // Scaling constant
+    constexpr float K_ = 45.0f;
+    // Static saturation
+    constexpr float S_ = 10.0f;
+    // Surround strength of opponent signal
+    constexpr float k3 = 0.6f;
+    // Radio of responses for white light
+    constexpr float rw = 0.139f;
+    // Relative weight of L cones
+    constexpr float p  = 0.6189f;
+
+    // Weighted cone response as described in Cao et al., section 3.3
+    // The approximately linear relation defined in the paper is represented here
+    // in matrix form to simplify the code
+    constexpr mat3f weightedRodResponse = (K_ / S_) * mat3f{
+       -(k3 + rw),       p * k3,          p * S_,
+        1.0f + k3 * rw, (1.0f - p) * k3, (1.0f - p) * S_,
+        0.0f,            1.0f,            0.0f
+    } * mat3f{k} * inverse(mat3f{m});
+
+    // Convert the scene color from Rec.709 to LMSR response
+    float4 q{dot(v, L), dot(v, M), dot(v, S), dot(v, R)};
+    // Regulated signal through the selected pathway (KC in Cao et al.)
+    float3 g = inversesqrt(1.0f + max(float3{0.0f}, (0.33f / m) * (q.rgb + k * q.w)));
+
+    // Manually tweaked constant to keep the effect plausible, combined with an
+    // artistic control value between 0.0 and 1.0.
+    float3 strength = float3{0.1f, 0.1f, 0.09f} * nightAdaptation;
+
+    // Compute the incremental effect that rods have in opponent space
+    float3 deltaOpponent = weightedRodResponse * g * q.w * strength;
+    // Photopic response in LMS space
+    float3 qHat = q.rgb + opponent_to_LMS * deltaOpponent;
+
+    // And finally, back to RGB
+    return LMS_to_RGB * qHat;
+}
+
 //------------------------------------------------------------------------------
 // White balance
 //------------------------------------------------------------------------------
@@ -552,13 +656,21 @@ FColorGrading::FColorGrading(FEngine& engine, const Builder& builder) {
                     // LogC encoding
                     v = LogC_to_linear(v);
 
-                    // Move to color grading color space
-                    v = c.colorGradingIn * v;
+                    // Kill negative values near 0.0f due to imprecision in the log conversion
+                    v = max(v, 0.0f);
 
                     if (builder->hasAdjustments) {
                         // Exposure
                         v = adjustExposure(v, builder->exposure);
 
+                        // Purkinje shift ("low-light" vision)
+                        v = scotopicAdaptation(v, builder->nightAdaptation);
+                    }
+
+                    // Move to color grading color space
+                    v = c.colorGradingIn * v;
+
+                    if (builder->hasAdjustments) {
                         // White balance
                         v = chromaticAdaptation(v, config.adaptationTransform);
 

libs/viewer/include/viewer/Settings.h

@@ -120,6 +120,7 @@ struct ColorGradingSettings {
     bool luminanceScaling = false;
     bool gamutMapping = false;
     float exposure = 0.0f;
+    float nightAdaptation = 0.0f;
     float temperature = 0.0f;
     float tint = 0.0f;
     math::float3 outRed{1.0f, 0.0f, 0.0f};

libs/viewer/src/Settings.cpp

@@ -314,6 +314,8 @@ static int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, ColorGra
             i = parse(tokens, i + 1, jsonChunk, &out->gamutMapping);
         } else if (compare(tok, jsonChunk, "exposure") == 0) {
             i = parse(tokens, i + 1, jsonChunk, &out->exposure);
+        } else if (compare(tok, jsonChunk, "nightAdaptation") == 0) {
+            i = parse(tokens, i + 1, jsonChunk, &out->nightAdaptation);
         } else if (compare(tok, jsonChunk, "temperature") == 0) {
             i = parse(tokens, i + 1, jsonChunk, &out->temperature);
         } else if (compare(tok, jsonChunk, "tint") == 0) {
@@ -1021,6 +1023,7 @@ ColorGrading* createColorGrading(const ColorGradingSettings& settings, Engine* e
     ColorGrading *colorGrading = ColorGrading::Builder()
             .quality(settings.quality)
             .exposure(settings.exposure)
+            .nightAdaptation(settings.nightAdaptation)
             .whiteBalance(settings.temperature, settings.tint)
             .channelMixer(settings.outRed, settings.outGreen, settings.outBlue)
             .shadowsMidtonesHighlights(
@@ -1161,6 +1164,7 @@ static std::ostream& operator<<(std::ostream& out, const ColorGradingSettings& i
         << "\"luminanceScaling\": " << to_string(in.luminanceScaling) << ",\n"
         << "\"gamutMapping\": " << to_string(in.gamutMapping) << ",\n"
         << "\"exposure\": " << (in.exposure) << ",\n"
+        << "\"nightAdaptation\": " << (in.nightAdaptation) << ",\n"
         << "\"temperature\": " << (in.temperature) << ",\n"
         << "\"tint\": " << (in.tint) << ",\n"
         << "\"outRed\": " << (in.outRed) << ",\n"
@@ -1425,7 +1429,7 @@ bool GenericToneMapperSettings::operator==(const GenericToneMapperSettings &rhs)
 bool ColorGradingSettings::operator==(const ColorGradingSettings &rhs) const {
     // If you had to fix the following codeline, then you likely also need to update the
     // implementation of operator==.
-    static_assert(sizeof(ColorGradingSettings) == 228, "Please update Settings.cpp");
+    static_assert(sizeof(ColorGradingSettings) == 232, "Please update Settings.cpp");
     return enabled == rhs.enabled &&
             quality == rhs.quality &&
             toneMapping == rhs.toneMapping &&
@@ -1433,6 +1437,7 @@ bool ColorGradingSettings::operator==(const ColorGradingSettings &rhs) const {
             luminanceScaling == rhs.luminanceScaling &&
             gamutMapping == rhs.gamutMapping &&
             exposure == rhs.exposure &&
+            nightAdaptation == rhs.nightAdaptation &&
             temperature == rhs.temperature &&
             tint == rhs.tint &&
             outRed == rhs.outRed &&

libs/viewer/src/SimpleViewer.cpp

@@ -214,6 +214,7 @@ static void colorGradingUI(Settings& settings, float* rangePlot, float* curvePlo
         ImGui::Checkbox("Gamut mapping", &colorGrading.gamutMapping);
 
         ImGui::SliderFloat("Exposure", &colorGrading.exposure, -10.0f, 10.0f);
+        ImGui::SliderFloat("Night adaptation", &colorGrading.nightAdaptation, 0.0f, 1.0f);
 
         if (ImGui::CollapsingHeader("White balance")) {
             int temperature = colorGrading.temperature * 100.0f;

libs/viewer/tests/test_settings.cpp

@@ -47,6 +47,7 @@ static const char* JSON_TEST_DEFAULTS = R"TXT(
             "luminanceScaling": false,
             "gamutMapping": false,
             "exposure": 0,
+            "nightAdaptation": 0,
             "temperature": 0,
             "tint": 0,
             "outRed": [1.0, 0.0, 0.0],

tools/rgb-to-lmsr/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.19)
+project(rgb-to-lmsr)
+
+set(TARGET rgb-to-lmsr)
+
+# ==================================================================================================
+# Sources and headers
+# ==================================================================================================
+set(SRCS src/main.cpp)
+
+# ==================================================================================================
+# Target definitions
+# ==================================================================================================
+add_executable(${TARGET} ${SRCS})
+
+target_link_libraries(${TARGET} getopt utils math)
+
+# =================================================================================================
+# Licenses
+# ==================================================================================================
+set(MODULE_LICENSES getopt)
+set(GENERATION_ROOT ${CMAKE_CURRENT_BINARY_DIR}/generated)
+list_licenses(${GENERATION_ROOT}/licenses/licenses.inc ${MODULE_LICENSES})
+target_include_directories(${TARGET} PRIVATE ${GENERATION_ROOT})