From c139eca9baf70025ebfd7bfd2b037277b6b8e4d1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 28 Sep 2026 21:15:40 +0200 Subject: [PATCH] Find the stop byte of a string run without a byte loop find_string_special() and find_ascii_copyable_run() test eight bytes at a time, but located the stopping byte inside a word with a byte loop. The lowest flagged byte of the SWAR tests is always a true hit (the borrows of the subtractions can only flag bytes above one), so its index is now the trailing-zero count of the mask; words are read in little-endian order on every platform, so this does not depend on the byte order. scalar_string_bulk_run() validates a run of multi-byte UTF-8 sequences one after another instead of searching for the next special byte in between, which helps text in non-Latin scripts. The kernels serve the lexer's contiguous fast path, the serializer, and the binary formats. New tests compare all three with byte-by-byte reference scans on 100,000 generated buffers at three alignments; the portable fallback of count_trailing_zeros() was checked against the builtin. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/bit_ops.hpp | 28 +++++- include/nlohmann/detail/input/string_scan.hpp | 51 +++++----- single_include/nlohmann/json.hpp | 80 ++++++++++------ tests/src/unit-class_lexer.cpp | 94 +++++++++++++++++++ 4 files changed, 193 insertions(+), 60 deletions(-) diff --git a/include/nlohmann/detail/bit_ops.hpp b/include/nlohmann/detail/bit_ops.hpp index 9655ff18b..48c5edaab 100644 --- a/include/nlohmann/detail/bit_ops.hpp +++ b/include/nlohmann/detail/bit_ops.hpp @@ -39,6 +39,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -68,14 +87,19 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc /// eight bytes as a little-endian word (compilers fold this into one load on /// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +inline std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); } +/// eight bytes as a little-endian word +inline std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 8a6f03122..403e12502 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -12,6 +12,7 @@ #include // uint64_t #include // memcpy +#include #include // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external @@ -69,18 +70,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -114,8 +109,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -126,7 +120,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -253,12 +249,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -273,8 +275,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -282,14 +283,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7b43d5e5b..22e919f39 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8547,6 +8547,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -8576,15 +8595,20 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc /// eight bytes as a little-endian word (compilers fold this into one load on /// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +inline std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); } +/// eight bytes as a little-endian word +inline std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -9982,6 +10006,8 @@ NLOHMANN_JSON_NAMESPACE_END #include // uint64_t #include // memcpy +// #include + // #include @@ -10040,18 +10066,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -10085,8 +10105,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -10097,7 +10116,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -10224,12 +10245,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -10244,8 +10271,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -10253,14 +10279,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index c5e303c5d..2923f6d93 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -1482,3 +1482,97 @@ TEST_CASE("float overflow and underflow in the parser") check_parse("-7.006492321624086e-46", std::uint32_t{0x80000001u}, std::uint32_t{0x7F800000u}); } } + +TEST_CASE("string scanning kernels") +{ + // the word-at-a-time kernels must stop exactly where a byte-by-byte scan + // stops, for any content, length, and alignment + const auto reference_special = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && !nlohmann::detail::is_string_special(data[i])) + { + ++i; + } + return i; + }; + const auto reference_copyable = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && nlohmann::detail::is_ascii_copyable(data[i])) + { + ++i; + } + return i; + }; + const auto reference_bulk_run = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n) + { + if (data[i] < 0x80u) + { + if (nlohmann::detail::is_string_special(data[i])) + { + break; + } + ++i; + continue; + } + const std::size_t seq = nlohmann::detail::validate_one_utf8(data + i, n - i); + if (seq == 0) + { + break; + } + i += seq; + } + return i; + }; + + // pieces: ordinary ASCII, stops, DEL, well-formed sequences of every + // length, and ill-formed or truncated ones + const std::vector pieces = + { + "a", "Z", " ", "~", "0123456789", "\"", "\\", std::string(1, '\0'), "\n", "\x1F", "\x7F", + "\xC3\xA4", "\xE2\x82\xAC", "\xE6\x97\xA5\xE6\x9C\xAC", "\xF0\x9F\x98\x80", "\xED\x9F\xBF", + "\x80", "\xC0\x80", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\xFF", + }; + std::uint64_t state = 5295; + const auto next = [&state]() + { + state ^= state << 13u; + state ^= state >> 7u; + state ^= state << 17u; + return state; + }; + for (int round = 0; round < 100000; ++round) + { + // mostly ordinary text, so that runs span several words + std::string text(static_cast(next() % 8), '.'); + const auto count = static_cast(next() % 12); + for (std::size_t k = 0; k < count; ++k) + { + const std::size_t p = (next() % 4 == 0) ? static_cast(next() % pieces.size()) : 0; + text += pieces[p]; + text += std::string(static_cast(next() % 10), 'x'); + } + const auto* data = reinterpret_cast(text.data()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + for (std::size_t offset = 0; offset < 3 && offset <= text.size(); ++offset) + { + const std::size_t n = text.size() - offset; + CAPTURE(text); + CAPTURE(offset); + CHECK(nlohmann::detail::find_string_special(data + offset, n) == reference_special(data + offset, n)); + CHECK(nlohmann::detail::find_ascii_copyable_run(data + offset, n) == reference_copyable(data + offset, n)); + CHECK(nlohmann::detail::scalar_string_bulk_run(data + offset, n) == reference_bulk_run(data + offset, n)); + } + } + + // the trailing-zero count, whichever implementation the compiler gets + for (int k = 0; k < 64; ++k) + { + const std::uint64_t bit = std::uint64_t{1} << k; + CHECK(nlohmann::detail::count_trailing_zeros(bit) == k); + CHECK(nlohmann::detail::count_trailing_zeros(bit | (bit << 1u) | 0x8000000000000000u) == k); + } +}