diff --git a/docs/mkdocs/docs/features/types/number_handling.md b/docs/mkdocs/docs/features/types/number_handling.md index 1bbf68e6d..83f7f68e6 100644 --- a/docs/mkdocs/docs/features/types/number_handling.md +++ b/docs/mkdocs/docs/features/types/number_handling.md @@ -74,7 +74,8 @@ otherwise, it uses unsigned integer storage. - The library converts integers and floating-point numbers itself, independent of the locale. Floating-point numbers are correctly rounded (to nearest, ties to even). Only a `#!c long double` that is not IEEE 754 binary64 (e.g., the 80-bit x87 format) is converted with `#!cpp std::from_chars` where available, or with - [`std::strtold`](https://en.cppreference.com/w/cpp/string/byte/strtof). + [`std::strtold`](https://en.cppreference.com/w/cpp/string/byte/strtof), which gets the decimal point of the + current locale, also one longer than one byte (e.g., in `fa_IR.UTF-8`). !!! example "Examples" diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index 5be5b7655..45ad04a91 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -19,6 +19,7 @@ #include // numeric_limits #include // string #include // conditional, integral_constant, true_type, false_type +#include // move #include #include @@ -917,12 +918,12 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @@ -936,22 +937,26 @@ has_native_float_format). These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -960,36 +965,46 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string copy(token.data(), token.size()); + copy.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, copy.c_str(), &endptr); + complete = endptr == copy.c_str() + copy.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 13ff3970d..a51cbde2d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8533,6 +8533,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // numeric_limits #include // string #include // conditional, integral_constant, true_type, false_type +#include // move // #include // __ _____ _____ _____ @@ -9888,12 +9889,12 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @@ -9907,22 +9908,26 @@ has_native_float_format). These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -9931,36 +9936,46 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string copy(token.data(), token.size()); + copy.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, copy.c_str(), &endptr); + complete = endptr == copy.c_str() + copy.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index eeddda358..e9c63411f 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -353,8 +353,14 @@ TEST_CASE("locale with a multi-byte decimal point") // Some locales use a decimal point that is not a single character, e.g. // U+066B ARABIC DECIMAL SEPARATOR (two bytes in UTF-8). It cannot be // substituted in place for '.', so the strtold fallback (only for long - // double formats other than binary64) stops early. The conversion must - // still terminate rather than retry forever. + // double formats other than binary64) converts a copy of the token with + // the whole decimal point instead (#5660). The values must be those of the + // "C" locale. + using long_double_json = nlohmann::basic_json; + const char* const long_double_numbers = "[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]"; + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + const long_double_json expected_long_double = long_double_json::parse(long_double_numbers); + const std::array names = {{"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}}; bool tested = false; for (const char* name : names) @@ -381,11 +387,10 @@ TEST_CASE("locale with a multi-byte decimal point") CHECK(j[2] == -0.000123456789012345678); CHECK(json::accept("3.14159265358979323846")); - // a long double that reaches the strtold fallback must still terminate - using long_double_json = nlohmann::basic_json; + // a long double that reaches the strtold fallback is not truncated long_double_json ld; - CHECK_NOTHROW(ld = long_double_json::parse("[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]")); - CHECK(ld.is_array()); + CHECK_NOTHROW(ld = long_double_json::parse(long_double_numbers)); + CHECK(ld == expected_long_double); // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5);