From deb071cc2cc974dcc37a5fb46429cb18747178d2 Mon Sep 17 00:00:00 2001 From: Ondrej Stava Date: Thu, 13 Feb 2020 14:21:24 -0800 Subject: [PATCH] Fixing undefined behavior in basisu_etc.cpp The value inside the static cast can be negative. Casting a negative floating point value to an unsigned integer is an undefined behavior and we were running into issues when executing the code with an address sanitizer turned on. Not sure if this is the proper way to fix this but it seems to work ok. Either way, the undefined behavior should be fixed. --- basisu_etc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/basisu_etc.cpp b/basisu_etc.cpp index 244f1d2..2a23afb 100644 --- a/basisu_etc.cpp +++ b/basisu_etc.cpp @@ -730,9 +730,9 @@ namespace basisu const float avg_delta_g_f = static_cast(delta_sum_g) / 8; const float avg_delta_b_f = static_cast(delta_sum_b) / 8; - const int br1 = clamp(static_cast((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit); - const int bg1 = clamp(static_cast((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit); - const int bb1 = clamp(static_cast((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit); + const int br1 = clamp(static_cast((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit); + const int bg1 = clamp(static_cast((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit); + const int bb1 = clamp(static_cast((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit); #if BASISU_DEBUG_ETC_ENCODER_DEEPER printf("Second refinement trial %u, avg_delta %f %f %f\n", i, avg_delta_r_f, avg_delta_g_f, avg_delta_b_f);