diff --git a/docs/mkdocs/docs/api/basic_json/swap.md b/docs/mkdocs/docs/api/basic_json/swap.md index 3a3d288fb..aa5aa6c4c 100644 --- a/docs/mkdocs/docs/api/basic_json/swap.md +++ b/docs/mkdocs/docs/api/basic_json/swap.md @@ -34,10 +34,14 @@ void swap(typename binary_t::container_type& other); ``` 1. Exchanges the contents of the JSON value with those of `other`. Does not invoke any move, copy, or swap operations on - individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. + individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 2. Exchanges the contents of the JSON value from `left` with those of `right`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is - invalidated. Implemented as a friend function callable via ADL. + invalidated. Implemented as a friend function callable via ADL. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 3. Exchanges the contents of a JSON array with those of `other`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. 4. Exchanges the contents of a JSON object with those of `other`. Does not invoke any move, copy, or swap operations on diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index ec3e462c1..5c9f72fc3 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -394,7 +394,14 @@ plugins: - http://nlohmann.github.io/json/* - https://nlohmann.github.io/json/* - mailto:* - - privacy + - privacy: + # repology.org refuses requests from GitHub Actions runners, which made + # the privacy plugin abort the whole build when it could not download the + # package badges (the fetch fails, then reading the missing cache entry + # raises FileNotFoundError). Readers' browsers are served normally, so + # leave these badges as external references instead of self-hosting them. + assets_exclude: + - repology.org/* - llmstxt: markdown_description: > JSON for Modern C++ is a C++11 header-only library implementing a JSON diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 1627d2326..37d0ab270 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,6 +8,7 @@ #pragma once +#include // min #include #include // string #include // enable_if_t @@ -17,6 +18,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -150,6 +152,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -305,6 +330,11 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } + return true; } @@ -683,6 +713,11 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } } return true; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 016a4dd27..741c08b53 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3576,6 +3576,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c602063ed..f3dcd3426 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7892,6 +7892,7 @@ NLOHMANN_JSON_NAMESPACE_END +#include // min #include #include // string #include // enable_if_t @@ -10729,6 +10730,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -10863,6 +10866,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -11018,6 +11044,11 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } + return true; } @@ -11396,6 +11427,11 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } } return true; @@ -27319,6 +27355,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 2d7242943..a53bd17ce 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3596,6 +3596,111 @@ TEST_CASE("BJData") } } +TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no + // element data at all. max_size() for a std::vector is far larger + // than this count, so it does not reject the header outright; the + // (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_bjdata(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_bjdata(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_bjdata(j); + CHECK(json::from_bjdata(packed_plain) == j); + + const auto packed_optimized = json::to_bjdata(j, true, true); + CHECK(json::from_bjdata(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_bjdata(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 032e6641b..4c9107517 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2174,6 +2174,92 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") } } +TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's detail::unknown_size() sentinel (SIZE_MAX), so the + // format-level size check rejects it outright (out_of_range.408, + // "excessive ... size") before the SAX consumer's own max_size() + // check would even run; on a 64-bit platform it passes both of + // those checks and is only found short of data once the (capped) + // reservation looks for element bytes that were never provided + // (parse_error.110). Either is an acceptable, bounded rejection of + // the hostile header -- the property under test is that no path + // attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_cbor(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_cbor(j); + CHECK(json::from_cbor(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_cbor(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor)); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 34ca5db2b..af76e93cc 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2259,6 +2259,86 @@ TEST_CASE("parser class") #endif } +#if JSON_DIAGNOSTIC_POSITIONS + +TEST_CASE("diagnostic positions: value lifetime") +{ + SECTION("copy constructor copies positions, recursively") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + const json a = json::parse(s); + const json b = a; // NOLINT(performance-unnecessary-copy-initialization) + + CHECK(b.start_pos() == a.start_pos()); + CHECK(b.end_pos() == a.end_pos()); + CHECK(b["b"].start_pos() == a["b"].start_pos()); + CHECK(b["b"].end_pos() == a["b"].end_pos()); + } + + SECTION("move constructor resets the moved-from value to npos") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + json a = json::parse(s); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + + const json b(std::move(a)); + + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + } + + SECTION("swap() exchanges positions along with the values") + { + // basic_json::swap() (and the friend swap() that forwards to it) used + // to swap only m_data.m_type/m_data.m_value, leaving + // start_position/end_position untouched -- unlike copy-assignment's + // operator=(basic_json), which swaps positions as part of its + // copy-and-swap implementation. After swap(a, b), each value ended up + // with the *other* value's content but its *own* original position. + // This is now fixed so that swap() is consistent with copy-assignment. + json a = json::parse(R"({"a":1})"); + json b = json::parse(R"([1,2,3,4,5])"); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + const auto b_start = b.start_pos(); + const auto b_end = b.end_pos(); + // lengths (and thus end positions) differ, which is enough to tell + // after the swap whether positions actually moved with the values + CHECK(a_end != b_end); + + using std::swap; + swap(a, b); + + CHECK(a == json::parse(R"([1,2,3,4,5])")); + CHECK(b == json::parse(R"({"a":1})")); + + CHECK(a.start_pos() == b_start); + CHECK(a.end_pos() == b_end); + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + // member swap() behaves the same as the free function + json c = json::parse(R"({"a":1})"); + json d = json::parse(R"([1,2,3,4,5])"); + const auto c_start = c.start_pos(); + const auto c_end = c.end_pos(); + const auto d_start = d.start_pos(); + const auto d_end = d.end_pos(); + + c.swap(d); + + CHECK(c.start_pos() == d_start); + CHECK(c.end_pos() == d_end); + CHECK(d.start_pos() == c_start); + CHECK(d.end_pos() == c_end); + } +} +#endif + // this test relies on parse errors being thrown, so it is skipped when // exceptions are disabled (json::parse aborts instead of throwing there) #if !defined(JSON_NOEXCEPTION) diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 75c2ae464..74f7f4969 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1597,6 +1597,91 @@ TEST_CASE("MessagePack") } } +TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's SIZE_MAX, which some size-narrowing checks treat the + // same as detail::unknown_size(); it may then be rejected before + // the SAX consumer's own max_size() check (out_of_range.408) rather + // than being accepted and only found short of data once the + // (capped) reservation looks for element bytes that were never + // provided (parse_error.110). Either is an acceptable, bounded + // rejection of the hostile header -- the property under test is + // that no path attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_msgpack(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_msgpack(j); + CHECK(json::from_msgpack(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_msgpack(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack)); + } +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("MessagePack nesting does not consume the call stack") { diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index cb2ed59a6..11c6a7da8 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -27,6 +27,7 @@ using ordered_json = nlohmann::ordered_json; #endif #include +#include #include #include #include @@ -896,4 +897,49 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with } +TEST_CASE("regression test #5476 - array type without reserve()") +{ + // the capacity reserved for definite-length arrays must not require the + // array type to have a reserve() member function + using deque_json = nlohmann::basic_json; + + SECTION("std::deque") + { + const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); + CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); + + // the binary formats pass a definite length to start_array() + CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); + CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); + + // parse() instantiates the callback parser as well, which reserves too + const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept + { + return true; + }); + CHECK(with_callback == deque_json({1, 2, 3})); + } + + SECTION("std::vector still reserves") + { + json array = json::array(); + for (int i = 0; i < 100; ++i) + { + array.push_back(i); + } + + const auto j = json::from_cbor(json::to_cbor(array)); + CHECK(j == array); + CHECK(j.get_ref().capacity() >= 100); + } + + SECTION("the reservation stays capped") + { + // CBOR array announcing 2^32-1 elements, but truncated right after the + // header: the input must be rejected without reserving that capacity + const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_cbor(truncated, true, false).is_discarded()); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 2ebb55b90..aafbbf5a4 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2315,6 +2315,112 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } +TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all. + // max_size() for a std::vector is far larger than this count, so it + // does not reject the header outright; the (capped) reservation must + // not attempt to allocate space for billions of elements before the + // missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_ubjson(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_ubjson(j); + CHECK(json::from_ubjson(packed_plain) == j); + + const auto packed_optimized = json::to_ubjson(j, true, true); + CHECK(json::from_ubjson(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_ubjson(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson)); + } +} + + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value")