From d33586ca34d04ce41905ce432255a9b7b4d5dfaf Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 29 Sep 2026 00:55:56 +0200 Subject: [PATCH] Address the clang-tidy findings of the SIMD scan Hold the UTF-8 lookup tables in std::array, compute the length of a sequence without nested conditionals, and use std::array in the tests. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/simd.hpp | 63 +++++++++++++++------------ single_include/nlohmann/json_view.hpp | 63 +++++++++++++++------------ tests/src/unit-json_view_builder.cpp | 17 ++++---- 3 files changed, 79 insertions(+), 64 deletions(-) diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index 5ca46c0f6..40f2313b1 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -9,6 +9,7 @@ #pragma once +#include // array #include // size_t #include // uint8_t, uint64_t @@ -36,9 +37,9 @@ #endif #if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3) #include - #define NLOHMANN_VIEW_SSSE3 1 + #define NLOHMANN_VIEW_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #else - #define NLOHMANN_VIEW_SSSE3 0 + #define NLOHMANN_VIEW_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #endif #define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2) #define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3) @@ -101,37 +102,43 @@ struct utf8_lookup4 static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; - static const std::uint8_t byte_1_high[16]; - static const std::uint8_t byte_1_low[16]; - static const std::uint8_t byte_2_high[16]; + static const std::array byte_1_high; + static const std::array byte_1_low; + static const std::array byte_2_high; }; template -const std::uint8_t utf8_lookup4::byte_1_high[16] = +const std::array utf8_lookup4::byte_1_high = { - too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, - two_conts, two_conts, two_conts, two_conts, - too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + { + too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, + two_conts, two_conts, two_conts, two_conts, + too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + } }; template -const std::uint8_t utf8_lookup4::byte_1_low[16] = +const std::array utf8_lookup4::byte_1_low = { - carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, - carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, - carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, - carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + { + carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, + carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + } }; template -const std::uint8_t utf8_lookup4::byte_2_high[16] = +const std::array utf8_lookup4::byte_2_high = { - too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, - static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), - static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), - static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), - static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), - too_short, too_short, too_short, too_short + { + too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + too_short, too_short, too_short, too_short + } }; /// the end of scan_string_vector from block, where the vector loop stopped @@ -148,7 +155,7 @@ inline const unsigned char* scan_string_finish(const unsigned char* p, const uns } if (c >= 0xC0) { - const int len = c >= 0xF0 ? 4 : (c >= 0xE0 ? 3 : 2); + const int len = 2 + static_cast(c >= 0xE0) + static_cast(c >= 0xF0); if (len > i) { block -= i; @@ -191,9 +198,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi using lookup = utf8_lookup4<>; const unsigned char* block = p; #if NLOHMANN_VIEW_NEON - const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high); - const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low); - const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high); + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); uint8x16_t prev = vdupq_n_u8(0); while (e - block >= 16) { @@ -228,9 +235,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi #else // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit // shifts, as there are no byte shifts) - const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high))); - const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low))); - const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high))); + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); const __m128i nibble = _mm_set1_epi8(0x0F); const __m128i zero = _mm_setzero_si128(); __m128i prev = zero; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 33a6b88d1..8c06e0b89 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -407,6 +407,7 @@ NLOHMANN_JSON_NAMESPACE_END +#include // array #include // size_t #include // uint8_t, uint64_t @@ -435,9 +436,9 @@ NLOHMANN_JSON_NAMESPACE_END #endif #if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3) #include - #define NLOHMANN_VIEW_SSSE3 1 + #define NLOHMANN_VIEW_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #else - #define NLOHMANN_VIEW_SSSE3 0 + #define NLOHMANN_VIEW_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #endif #define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2) #define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3) @@ -500,37 +501,43 @@ struct utf8_lookup4 static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; - static const std::uint8_t byte_1_high[16]; - static const std::uint8_t byte_1_low[16]; - static const std::uint8_t byte_2_high[16]; + static const std::array byte_1_high; + static const std::array byte_1_low; + static const std::array byte_2_high; }; template -const std::uint8_t utf8_lookup4::byte_1_high[16] = +const std::array utf8_lookup4::byte_1_high = { - too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, - two_conts, two_conts, two_conts, two_conts, - too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + { + too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, + two_conts, two_conts, two_conts, two_conts, + too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + } }; template -const std::uint8_t utf8_lookup4::byte_1_low[16] = +const std::array utf8_lookup4::byte_1_low = { - carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, - carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, - carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, - carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + { + carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, + carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + } }; template -const std::uint8_t utf8_lookup4::byte_2_high[16] = +const std::array utf8_lookup4::byte_2_high = { - too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, - static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), - static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), - static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), - static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), - too_short, too_short, too_short, too_short + { + too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + too_short, too_short, too_short, too_short + } }; /// the end of scan_string_vector from block, where the vector loop stopped @@ -547,7 +554,7 @@ inline const unsigned char* scan_string_finish(const unsigned char* p, const uns } if (c >= 0xC0) { - const int len = c >= 0xF0 ? 4 : (c >= 0xE0 ? 3 : 2); + const int len = 2 + static_cast(c >= 0xE0) + static_cast(c >= 0xF0); if (len > i) { block -= i; @@ -590,9 +597,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi using lookup = utf8_lookup4<>; const unsigned char* block = p; #if NLOHMANN_VIEW_NEON - const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high); - const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low); - const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high); + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); uint8x16_t prev = vdupq_n_u8(0); while (e - block >= 16) { @@ -627,9 +634,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi #else // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit // shifts, as there are no byte shifts) - const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high))); - const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low))); - const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high))); + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); const __m128i nibble = _mm_set1_epi8(0x0F); const __m128i zero = _mm_setzero_si128(); __m128i prev = zero; diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 5a6f083ce..ba7448f8b 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -17,6 +17,7 @@ #endif using nlohmann::json; +#include #include #include #include @@ -435,8 +436,8 @@ TEST_CASE("json_view builder: strings across vector blocks") // at a time. Sequences are placed so that they start at every offset // around the block boundaries of keys (16, 32) and values (8, 24), with // text of several lengths after them. - const std::size_t prefixes[] = {0, 6, 7, 8, 13, 14, 15, 16, 21, 22, 23, 24, 29, 30, 31, 32}; - const std::size_t suffixes[] = {0, 3, 17}; + const std::array prefixes = {{0, 6, 7, 8, 13, 14, 15, 16, 21, 22, 23, 24, 29, 30, 31, 32}}; + const std::array suffixes = {{0, 3, 17}}; const auto around = [&](const std::string & seq, std::size_t prefix, std::size_t suffix) { return std::string(prefix, 'a') + seq + std::string(suffix, 'b'); @@ -460,7 +461,7 @@ TEST_CASE("json_view builder: strings across vector blocks") SECTION("three- and four-byte sequences") { - const unsigned conts[] = {0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0}; + const std::array conts = {{0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0}}; for (unsigned lead = 0xE0; lead <= 0xF7; ++lead) { for (const unsigned b2 : conts) @@ -485,21 +486,21 @@ TEST_CASE("json_view builder: strings across vector blocks") SECTION("long runs of text with one damaged byte") { - const char* const chars[] = {"a", "\xc3\xa9", "\xe3\x81\x82", "\xf0\x9f\x98\x80", "\xed\x9f\xbf", "\xef\xbf\xbf", "\xf4\x8f\xbf\xbf"}; - const char damage[] = {'\x80', '\xbf', '\xc0', '\xc1', '\xe0', '\xed', '\xf5', '\xff', '\x1f', '"', '\\'}; - std::mt19937 rng(5295); + const std::array chars = {{"a", "\xc3\xa9", "\xe3\x81\x82", "\xf0\x9f\x98\x80", "\xed\x9f\xbf", "\xef\xbf\xbf", "\xf4\x8f\xbf\xbf"}}; + const std::array damage = {{'\x80', '\xbf', '\xc0', '\xc1', '\xe0', '\xed', '\xf5', '\xff', '\x1f', '"', '\\'}}; + std::mt19937 rng(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible for (int i = 0; i < 4000; ++i) { std::string text; const auto n = rng() % 60; for (unsigned k = 0; k < n; ++k) { - text += chars[rng() % 7]; + text += chars[rng() % chars.size()]; } check_string(text); if (!text.empty()) { - text[rng() % text.size()] = damage[rng() % 11]; + text[rng() % text.size()] = damage[rng() % damage.size()]; check_string(text); } }