diff --git a/docs/mkdocs/docs/api/basic_json/sax_parse.md b/docs/mkdocs/docs/api/basic_json/sax_parse.md index fc8ce07b6..bf61ea9eb 100644 --- a/docs/mkdocs/docs/api/basic_json/sax_parse.md +++ b/docs/mkdocs/docs/api/basic_json/sax_parse.md @@ -69,7 +69,9 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index [`input_format_t`](input_format_t.md) for more information `strict` (in) -: whether the input has to be consumed completely (optional, `#!cpp true` by default) +: whether the input has to be consumed completely (optional, `#!cpp true` by default); when `#!cpp false` and the + input is a `#!cpp std::istream`, the character that terminates a number is consumed unless + [`JSON_PRECISE_STREAM_POSITION`](../macros/json_precise_stream_position.md) is defined to `1`; see [`operator>>`](../operator_gtgt.md#notes) `ignore_comments` (in) : whether comments should be ignored and treated like whitespace (`#!cpp true`) or yield a parse error @@ -136,6 +138,8 @@ A UTF-8 byte order mark is silently ignored. - Added `ignore_trailing_commas` in version 3.13.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right + after the parsed value when `strict` is `#!cpp false`. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 70f02a7a6..bf773b5c4 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -16,6 +16,8 @@ header. See also the [macro overview page](../../features/macros.md). ## Parsing +- [**JSON_PRECISE_STREAM_POSITION**](json_precise_stream_position.md) - opt in to leaving an input stream positioned + right after a parsed number - [**JSON_STRICT_NUL_HANDLING**](json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input diff --git a/docs/mkdocs/docs/api/macros/json_precise_stream_position.md b/docs/mkdocs/docs/api/macros/json_precise_stream_position.md new file mode 100644 index 000000000..5710e9975 --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_precise_stream_position.md @@ -0,0 +1,131 @@ +# JSON_PRECISE_STREAM_POSITION + +```cpp +#define JSON_PRECISE_STREAM_POSITION /* value */ +``` + +When defined to `1`, [`operator>>`](../operator_gtgt.md) and [`sax_parse`](../basic_json/sax_parse.md) with +`strict = false` leave a `#!cpp std::istream` positioned right after the parsed value for every value type. By default, +the character that terminates a number is consumed as well. + +The macro only affects reading from a `#!cpp std::istream` when the rest of the stream is not required to be consumed. +[`parse`](../basic_json/parse.md), [`accept`](../basic_json/accept.md), and all other inputs (strings, iterators, +containers, `#!cpp FILE*`) are never affected. + +## Default definition + +The default value is `0` (disabled — existing behavior is preserved). + +```cpp +#define JSON_PRECISE_STREAM_POSITION 0 +``` + +## Notes + +!!! note "Background" + + A number is the only JSON value whose end can be detected solely by reading the character that follows it. By + default, that character is consumed and not put back, so the stream is left one byte too far after a number, and + only after a number: + + ```cpp + std::istringstream input("1true"); + json j; + input >> j; // j == 1, but the stream now starts at "rue" + ``` + + With this macro, the character is only looked at and left in the stream, so the stream starts at `true`. This + does not require the stream buffer to support putting a character back. + + This was not changed unconditionally, because code can depend on the consumed character, even unknowingly (see + [#5340](https://github.com/nlohmann/json/issues/5340)). Both of the following work by default only because the + character after each number is swallowed, and behave differently with this macro: + + ```cpp + std::istringstream input("1,2,3"); + json j1, j2, j3; + input >> j1 >> j2 >> j3; // default: 1, 2, 3 + // with the macro: throws parse_error.101 at the ',' + ``` + + ```cpp + std::istringstream input("42\nfoo"); + json j; + std::string line; + input >> j; + std::getline(input, line); // default: "foo" + // with the macro: "" (like after reading an int with >>) + ``` + + In both cases, the behavior with the macro is what you already get today when the value is not a number: `"a","b"` + fails at the `,`, and `std::getline` after `{}` returns an empty string. This macro offers an opt-in path to + the consistent behavior ahead of version 4.0.0, where it is planned to become the default. + +!!! warning "Opt-in only" + + This macro must be defined **before** including ``. Defining it after the include has no + effect. + +!!! note "ABI compatibility" + + The value of this macro is encoded in the [namespace](../../features/namespace.md) (tag `_psp`), resulting in + distinct symbol names. Translation units compiled with and without it can therefore be linked into the same program + without One Definition Rule (ODR) violations, but they cannot exchange instances of library types. + +!!! tip "Workaround without the macro" + + Separate the values in the stream with whitespace. The character consumed after a number is then the separator, + and whitespace before the next value is skipped anyway. + +## Examples + +??? example "Default behavior (macro not defined)" + + Without the macro, the character after a number is consumed: + + ```cpp + #include + #include + #include + + using json = nlohmann::json; + + int main() + { + std::istringstream input("1true"); + json j1, j2; + input >> j1; // j1 == 1 + input >> j2; // throws parse_error.101: the stream now starts at "rue" + } + ``` + +??? example "Opt-in precise stream position (macro defined to 1)" + + With the macro, the stream is positioned right after the number: + + ```cpp + #define JSON_PRECISE_STREAM_POSITION 1 + #include + #include + #include + + using json = nlohmann::json; + + int main() + { + std::istringstream input("1true"); + json j1, j2; + input >> j1; // j1 == 1 + input >> j2; // j2 == true + } + ``` + +## See also + +- [**operator>>**](../operator_gtgt.md) - deserialize from stream +- [**sax_parse**](../basic_json/sax_parse.md) - generate SAX events + +## Version history + +- Added in version 3.13.0. +- Planned to become the default (with the macro removed) in version 4.0.0. diff --git a/docs/mkdocs/docs/api/operator_gtgt.md b/docs/mkdocs/docs/api/operator_gtgt.md index 3e60d5236..0173b9fb3 100644 --- a/docs/mkdocs/docs/api/operator_gtgt.md +++ b/docs/mkdocs/docs/api/operator_gtgt.md @@ -67,7 +67,9 @@ input >> j2; // parses the next value Only numbers are affected. Values ending in a self-delimiting character do not read past themselves, so `truefalse`, `[1][2]`, `{"a":1}{"b":2}`, and `"a""b"` can be read back to back without a separator. - This is tracked in [#5340](https://github.com/nlohmann/json/issues/5340). + Define [`JSON_PRECISE_STREAM_POSITION`](macros/json_precise_stream_position.md) to `1` to leave the terminating character in the stream + instead, so that the stream is positioned right after the value for every value type and no separator is + needed. This is tracked in [#5340](https://github.com/nlohmann/json/issues/5340). Note that reading concatenated values does **not** work for [JSON Lines](../features/parsing/json_lines.md) (newline-delimited JSON) input -- see that page for why and for the recommended alternative. @@ -107,9 +109,12 @@ being read. - [parse](basic_json/parse.md) - deserialize from a compatible input - [`JSON_STRICT_NUL_HANDLING`](macros/json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input +- [`JSON_PRECISE_STREAM_POSITION`](macros/json_precise_stream_position.md) - opt in to leaving the stream positioned right after a number ## Version history - Added in version 1.0.0. - `JSON_STRICT_NUL_HANDLING` added in version 3.13.0 to optionally reject a NUL byte in the input instead of treating it as end of input; planned to become the default in version 4.0.0. +- `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave the character that terminates a number in + the stream; planned to become the default in version 4.0.0. diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index e7baba0ae..7d6d23148 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -98,6 +98,15 @@ rather than descending into a bounded number of levels first, which is slower bu See [full documentation of `JSON_NO_THREAD_LOCAL`](../api/macros/json_no_thread_local.md). +## `JSON_PRECISE_STREAM_POSITION` + +When defined to `1`, [`operator>>`](../api/operator_gtgt.md) and non-strict +[`sax_parse`](../api/basic_json/sax_parse.md) leave an input stream positioned right after the parsed value, instead of +also consuming the character that terminates a number. The default value is `0`, which preserves the existing behavior; +this is planned to become the default in version 4.0.0. + +See [full documentation of `JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md). + ## `JSON_SKIP_LIBRARY_VERSION_CHECK` When defined, the library will not create a compiler warning when a different version of the library was already diff --git a/docs/mkdocs/docs/features/namespace.md b/docs/mkdocs/docs/features/namespace.md index 09e53f3a2..5eb4a76a9 100644 --- a/docs/mkdocs/docs/features/namespace.md +++ b/docs/mkdocs/docs/features/namespace.md @@ -18,6 +18,7 @@ The complete default namespace name is derived as follows: - [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) defined non-zero appends `_dp`. - [`JSON_BRACE_INIT_COPY_SEMANTICS`](../api/macros/json_brace_init_copy_semantics.md) defined non-zero appends `_bics`. + - [`JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md) defined non-zero appends `_psp`. - The inline namespace ends with the suffix `_v` followed by the 3 components of the version number separated by underscores. To omit the version component, see [Disabling the version component](#disabling-the-version-component) below. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 17624b8a2..476f024fa 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -41,7 +41,8 @@ document followed by trailing bytes" is accepted rather than rejected. If you ar reject any input that is not exactly one JSON document, prefer `parse`. When using `operator>>` to read several concatenated values this way, a value that is a number must be followed by -whitespace, because `operator>>` consumes the character that terminates a number — see the +whitespace, because `operator>>` consumes the character that terminates a number, unless +[`JSON_PRECISE_STREAM_POSITION`](../../api/macros/json_precise_stream_position.md) is defined to `1` — see the [`operator>>` notes](../../api/operator_gtgt.md#notes) for details and examples. ## SAX vs. DOM parsing diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index ffb0fae80..a05ce2dff 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -293,6 +293,7 @@ nav: - 'JSON_NOEXCEPTION': api/macros/json_noexception.md - 'JSON_NO_IO': api/macros/json_no_io.md - 'JSON_NO_THREAD_LOCAL': api/macros/json_no_thread_local.md + - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md - 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md - 'JSON_STRICT_NUL_HANDLING': api/macros/json_strict_nul_handling.md diff --git a/include/nlohmann/detail/abi_macros.hpp b/include/nlohmann/detail/abi_macros.hpp index cca04e8ec..a6666c66e 100644 --- a/include/nlohmann/detail/abi_macros.hpp +++ b/include/nlohmann/detail/abi_macros.hpp @@ -38,6 +38,10 @@ #define JSON_BRACE_INIT_COPY_SEMANTICS 0 #endif +#ifndef JSON_PRECISE_STREAM_POSITION + #define JSON_PRECISE_STREAM_POSITION 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -62,21 +66,28 @@ #define NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS #endif +#if JSON_PRECISE_STREAM_POSITION + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION _psp +#else + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) json_abi ## a ## b ## c ## d -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) json_abi ## a ## b ## c ## d ## e +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ - NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS) + NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ + NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 775a8398c..317b38806 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -101,6 +101,11 @@ class input_stream_adapter // maintain ifstream flags, except eof if (is != nullptr) { +#if JSON_PRECISE_STREAM_POSITION + // consume the character last returned by get_character() unless it + // was given back with release_lookahead() + commit_lookahead(); +#endif is->clear(is->rdstate() & std::ios::eofbit); } } @@ -114,6 +119,58 @@ class input_stream_adapter input_stream_adapter& operator=(input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&&) = delete; +#if JSON_PRECISE_STREAM_POSITION + input_stream_adapter(input_stream_adapter&& rhs) noexcept + : is(rhs.is), sb(rhs.sb), lookahead(rhs.lookahead) + { + rhs.is = nullptr; + rhs.sb = nullptr; + rhs.lookahead = false; + } + + // Whether the character last returned by get_character() can be given back + // to the input with release_lookahead(). + static constexpr bool supports_lookahead = true; + + // std::istream/std::streambuf use std::char_traits::to_int_type, to + // ensure that std::char_traits::eof() and the character 0xFF do not + // end up as the same value, e.g., 0xFFFFFFFF. + // + // The character is peeked rather than consumed: it is only stepped over + // once the next character is requested, or when the adapter is destroyed. + // Until then, release_lookahead() can leave it in the input. + std::char_traits::int_type get_character() + { + if (lookahead) + { + // step over the character returned by the previous call + sb->sbumpc(); + } + + auto res = sb->sgetc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { + // there is nothing to step over next time + lookahead = false; + is->clear(is->rdstate() | std::ios::eofbit); + } + else + { + lookahead = true; + } + return res; + } + + // Leave the character last returned by get_character() in the input, so + // that the next read from the stream - by this adapter or by the caller + // once parsing is done - sees it again. Unlike putting a consumed + // character back, this cannot fail. + void release_lookahead() noexcept + { + lookahead = false; + } +#else input_stream_adapter(input_stream_adapter&& rhs) noexcept : is(rhs.is), sb(rhs.sb) { @@ -124,6 +181,9 @@ class input_stream_adapter // std::istream/std::streambuf use std::char_traits::to_int_type, to // ensure that std::char_traits::eof() and the character 0xFF do not // end up as the same value, e.g., 0xFFFFFFFF. + // + // The character is consumed, so the character that terminates a number + // stays consumed after parsing; see JSON_PRECISE_STREAM_POSITION. std::char_traits::int_type get_character() { auto res = sb->sbumpc(); @@ -134,10 +194,14 @@ class input_stream_adapter } return res; } +#endif template std::size_t get_elements(T* dest, std::size_t count = 1) { +#if JSON_PRECISE_STREAM_POSITION + commit_lookahead(); +#endif auto res = static_cast(sb->sgetn(reinterpret_cast(dest), static_cast(count * sizeof(T)))); if (JSON_HEDLEY_UNLIKELY(res < count * sizeof(T))) { @@ -147,9 +211,27 @@ class input_stream_adapter } private: +#if JSON_PRECISE_STREAM_POSITION + // Step over the character last returned by get_character(). The character + // has already been peeked successfully, so for every streambuf with a get + // area this is a pointer increment that cannot fail. + void commit_lookahead() + { + if (lookahead) + { + lookahead = false; + sb->sbumpc(); + } + } +#endif + /// the associated input stream std::istream* is = nullptr; std::streambuf* sb = nullptr; +#if JSON_PRECISE_STREAM_POSITION + /// whether get_character() peeked a character that is not consumed yet + bool lookahead = false; +#endif }; #endif // JSON_NO_IO diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index fe85cd53d..98c0fd76a 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -127,6 +127,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter reads with one character of lookahead that +// can be left in the input (see input_stream_adapter::supports_lookahead, +// which is only defined with JSON_PRECISE_STREAM_POSITION), detected like +// supports_seek above. +template +using detect_supports_lookahead = decltype(InputAdapterType::supports_lookahead); + +template +constexpr bool input_adapter_supports_lookahead(std::true_type /*detected*/) +{ + return InputAdapterType::supports_lookahead; +} + +template +constexpr bool input_adapter_supports_lookahead(std::false_type /*detected*/) +{ + return false; +} + // Detect whether an input adapter exposes a contiguous byte block that the // lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). // Adapters without the flag - file, stream, wide-string, user-defined - fall @@ -167,6 +186,12 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether a simulated unget can be passed on to the input adapter, which + /// then leaves the character in the input; see + /// input_adapter_supports_lookahead + static constexpr bool can_release_lookahead = + input_adapter_supports_lookahead(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing @@ -1898,6 +1923,21 @@ scan_number_done: uncapture_char(std::integral_constant {}); } + /// adapter without lookahead: nothing to do (see release_lookahead) + void release_lookahead_impl(std::false_type /*can_release*/) const noexcept {} + + /// adapter with lookahead: leave the character in the input instead + void release_lookahead_impl(std::true_type /*can_release*/) + { + if (next_unget) + { + // the character is read from the input again rather than replayed + // from current, so the adapter must not step over it + next_unget = false; + ia.release_lookahead(); + } + } + /// seekable adapter: nothing was captured, so nothing to undo void uncapture_char(std::true_type /*lazy*/) const noexcept {} @@ -1961,6 +2001,31 @@ scan_number_done: return position; } + /*! + @brief pass a pending simulated unget on to the input + + unget() only rewinds the lexer's own bookkeeping, so the character that + terminated the last token (e.g. the character after a number) would still + be stepped over when the input adapter is done. Callers that hand the + input back to the user afterwards - operator>> and non-strict sax_parse - + call this once when scanning is done, so that the input is positioned + right after the value. + + Adapters without lookahead (see input_adapter_supports_lookahead) are not + handed back to the user, so this is a no-op for them. Without + JSON_PRECISE_STREAM_POSITION, no adapter has lookahead, so this is always a + no-op and the terminating character stays consumed. + + Scanning may continue after this call: @a next_unget is cleared, and the + character is read from the input again instead of being replayed from + @a current. A pending unget of EOF needs no special case, because reaching + EOF leaves no lookahead to release. + */ + void release_lookahead() + { + release_lookahead_impl(std::integral_constant {}); + } + #if JSON_DIAGNOSTIC_POSITIONS /// return the offset of the first character of the last read token; unlike /// the token's parsed value, this accounts for escape sequences diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index a45ee4a0a..5fec57a70 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -100,13 +100,22 @@ class parser json_sax_dom_callback_parser sdp(result, callback, allow_exceptions, &m_lexer); sax_parse_internal(&sdp); - // in strict mode, input must be completely read - if (strict && (get_token() != token_type::end_of_input)) + if (strict) { - sdp.parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_of_input, "value"), nullptr)); + // in strict mode, input must be completely read + if (get_token() != token_type::end_of_input) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // the caller keeps using the input: position it right after + // the value by leaving the character that terminated it + m_lexer.release_lookahead(); } // in case of an error, return a discarded value @@ -128,12 +137,20 @@ class parser json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); sax_parse_internal(&sdp); - // in strict mode, input must be completely read - if (strict && (get_token() != token_type::end_of_input)) + if (strict) { - sdp.parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + // in strict mode, input must be completely read + if (get_token() != token_type::end_of_input) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // see above + m_lexer.release_lookahead(); } // in case of an error, return a discarded value @@ -166,12 +183,24 @@ class parser (void)detail::is_sax_static_asserts {}; const bool result = sax_parse_internal(sax); - // strict mode: next byte must be EOF - if (result && strict && (get_token() != token_type::end_of_input)) + if (result) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + if (strict) + { + // strict mode: next byte must be EOF + if (get_token() != token_type::end_of_input) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // the caller keeps using the input: position it right after + // the value by leaving the character that terminated it + m_lexer.release_lookahead(); + } } return result; diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index afcbfc38b..6ace4cf4a 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -45,6 +45,7 @@ #undef JSON_HAS_STATIC_RTTI #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON #undef JSON_BRACE_INIT_COPY_SEMANTICS + #undef JSON_PRECISE_STREAM_POSITION #endif #include diff --git a/nlohmann_json.natvis b/nlohmann_json.natvis index 2eccbe17c..8f4eec31a 100644 --- a/nlohmann_json.natvis +++ b/nlohmann_json.natvis @@ -335,6 +335,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -515,6 +575,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -635,6 +755,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -695,6 +875,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -815,6 +1115,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -875,6 +1235,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -935,6 +1415,186 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -995,4 +1655,304 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9091c1198..e3ca0f0d0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -95,6 +95,10 @@ #define JSON_BRACE_INIT_COPY_SEMANTICS 0 #endif +#ifndef JSON_PRECISE_STREAM_POSITION + #define JSON_PRECISE_STREAM_POSITION 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -119,21 +123,28 @@ #define NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS #endif +#if JSON_PRECISE_STREAM_POSITION + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION _psp +#else + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) json_abi ## a ## b ## c ## d -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) json_abi ## a ## b ## c ## d ## e +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ - NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS) + NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ + NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ @@ -7419,6 +7430,11 @@ class input_stream_adapter // maintain ifstream flags, except eof if (is != nullptr) { +#if JSON_PRECISE_STREAM_POSITION + // consume the character last returned by get_character() unless it + // was given back with release_lookahead() + commit_lookahead(); +#endif is->clear(is->rdstate() & std::ios::eofbit); } } @@ -7432,6 +7448,58 @@ class input_stream_adapter input_stream_adapter& operator=(input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&&) = delete; +#if JSON_PRECISE_STREAM_POSITION + input_stream_adapter(input_stream_adapter&& rhs) noexcept + : is(rhs.is), sb(rhs.sb), lookahead(rhs.lookahead) + { + rhs.is = nullptr; + rhs.sb = nullptr; + rhs.lookahead = false; + } + + // Whether the character last returned by get_character() can be given back + // to the input with release_lookahead(). + static constexpr bool supports_lookahead = true; + + // std::istream/std::streambuf use std::char_traits::to_int_type, to + // ensure that std::char_traits::eof() and the character 0xFF do not + // end up as the same value, e.g., 0xFFFFFFFF. + // + // The character is peeked rather than consumed: it is only stepped over + // once the next character is requested, or when the adapter is destroyed. + // Until then, release_lookahead() can leave it in the input. + std::char_traits::int_type get_character() + { + if (lookahead) + { + // step over the character returned by the previous call + sb->sbumpc(); + } + + auto res = sb->sgetc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { + // there is nothing to step over next time + lookahead = false; + is->clear(is->rdstate() | std::ios::eofbit); + } + else + { + lookahead = true; + } + return res; + } + + // Leave the character last returned by get_character() in the input, so + // that the next read from the stream - by this adapter or by the caller + // once parsing is done - sees it again. Unlike putting a consumed + // character back, this cannot fail. + void release_lookahead() noexcept + { + lookahead = false; + } +#else input_stream_adapter(input_stream_adapter&& rhs) noexcept : is(rhs.is), sb(rhs.sb) { @@ -7442,6 +7510,9 @@ class input_stream_adapter // std::istream/std::streambuf use std::char_traits::to_int_type, to // ensure that std::char_traits::eof() and the character 0xFF do not // end up as the same value, e.g., 0xFFFFFFFF. + // + // The character is consumed, so the character that terminates a number + // stays consumed after parsing; see JSON_PRECISE_STREAM_POSITION. std::char_traits::int_type get_character() { auto res = sb->sbumpc(); @@ -7452,10 +7523,14 @@ class input_stream_adapter } return res; } +#endif template std::size_t get_elements(T* dest, std::size_t count = 1) { +#if JSON_PRECISE_STREAM_POSITION + commit_lookahead(); +#endif auto res = static_cast(sb->sgetn(reinterpret_cast(dest), static_cast(count * sizeof(T)))); if (JSON_HEDLEY_UNLIKELY(res < count * sizeof(T))) { @@ -7465,9 +7540,27 @@ class input_stream_adapter } private: +#if JSON_PRECISE_STREAM_POSITION + // Step over the character last returned by get_character(). The character + // has already been peeked successfully, so for every streambuf with a get + // area this is a pointer increment that cannot fail. + void commit_lookahead() + { + if (lookahead) + { + lookahead = false; + sb->sbumpc(); + } + } +#endif + /// the associated input stream std::istream* is = nullptr; std::streambuf* sb = nullptr; +#if JSON_PRECISE_STREAM_POSITION + /// whether get_character() peeked a character that is not consumed yet + bool lookahead = false; +#endif }; #endif // JSON_NO_IO @@ -8879,6 +8972,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter reads with one character of lookahead that +// can be left in the input (see input_stream_adapter::supports_lookahead, +// which is only defined with JSON_PRECISE_STREAM_POSITION), detected like +// supports_seek above. +template +using detect_supports_lookahead = decltype(InputAdapterType::supports_lookahead); + +template +constexpr bool input_adapter_supports_lookahead(std::true_type /*detected*/) +{ + return InputAdapterType::supports_lookahead; +} + +template +constexpr bool input_adapter_supports_lookahead(std::false_type /*detected*/) +{ + return false; +} + // Detect whether an input adapter exposes a contiguous byte block that the // lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). // Adapters without the flag - file, stream, wide-string, user-defined - fall @@ -8919,6 +9031,12 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether a simulated unget can be passed on to the input adapter, which + /// then leaves the character in the input; see + /// input_adapter_supports_lookahead + static constexpr bool can_release_lookahead = + input_adapter_supports_lookahead(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing @@ -10650,6 +10768,21 @@ scan_number_done: uncapture_char(std::integral_constant {}); } + /// adapter without lookahead: nothing to do (see release_lookahead) + void release_lookahead_impl(std::false_type /*can_release*/) const noexcept {} + + /// adapter with lookahead: leave the character in the input instead + void release_lookahead_impl(std::true_type /*can_release*/) + { + if (next_unget) + { + // the character is read from the input again rather than replayed + // from current, so the adapter must not step over it + next_unget = false; + ia.release_lookahead(); + } + } + /// seekable adapter: nothing was captured, so nothing to undo void uncapture_char(std::true_type /*lazy*/) const noexcept {} @@ -10713,6 +10846,31 @@ scan_number_done: return position; } + /*! + @brief pass a pending simulated unget on to the input + + unget() only rewinds the lexer's own bookkeeping, so the character that + terminated the last token (e.g. the character after a number) would still + be stepped over when the input adapter is done. Callers that hand the + input back to the user afterwards - operator>> and non-strict sax_parse - + call this once when scanning is done, so that the input is positioned + right after the value. + + Adapters without lookahead (see input_adapter_supports_lookahead) are not + handed back to the user, so this is a no-op for them. Without + JSON_PRECISE_STREAM_POSITION, no adapter has lookahead, so this is always a + no-op and the terminating character stays consumed. + + Scanning may continue after this call: @a next_unget is cleared, and the + character is read from the input again instead of being replayed from + @a current. A pending unget of EOF needs no special case, because reaching + EOF leaves no lookahead to release. + */ + void release_lookahead() + { + release_lookahead_impl(std::integral_constant {}); + } + #if JSON_DIAGNOSTIC_POSITIONS /// return the offset of the first character of the last read token; unlike /// the token's parsed value, this accounts for escape sequences @@ -15960,13 +16118,22 @@ class parser json_sax_dom_callback_parser sdp(result, callback, allow_exceptions, &m_lexer); sax_parse_internal(&sdp); - // in strict mode, input must be completely read - if (strict && (get_token() != token_type::end_of_input)) + if (strict) { - sdp.parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_of_input, "value"), nullptr)); + // in strict mode, input must be completely read + if (get_token() != token_type::end_of_input) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // the caller keeps using the input: position it right after + // the value by leaving the character that terminated it + m_lexer.release_lookahead(); } // in case of an error, return a discarded value @@ -15988,12 +16155,20 @@ class parser json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); sax_parse_internal(&sdp); - // in strict mode, input must be completely read - if (strict && (get_token() != token_type::end_of_input)) + if (strict) { - sdp.parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + // in strict mode, input must be completely read + if (get_token() != token_type::end_of_input) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // see above + m_lexer.release_lookahead(); } // in case of an error, return a discarded value @@ -16026,12 +16201,24 @@ class parser (void)detail::is_sax_static_asserts {}; const bool result = sax_parse_internal(sax); - // strict mode: next byte must be EOF - if (result && strict && (get_token() != token_type::end_of_input)) + if (result) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + if (strict) + { + // strict mode: next byte must be EOF + if (get_token() != token_type::end_of_input) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + } + } + else + { + // the caller keeps using the input: position it right after + // the value by leaving the character that terminated it + m_lexer.release_lookahead(); + } } return result; @@ -30765,6 +30952,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_HAS_STATIC_RTTI #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON #undef JSON_BRACE_INIT_COPY_SEMANTICS + #undef JSON_PRECISE_STREAM_POSITION #endif // #include diff --git a/single_include/nlohmann/json_fwd.hpp b/single_include/nlohmann/json_fwd.hpp index 281c05efa..af776d652 100644 --- a/single_include/nlohmann/json_fwd.hpp +++ b/single_include/nlohmann/json_fwd.hpp @@ -56,6 +56,10 @@ #define JSON_BRACE_INIT_COPY_SEMANTICS 0 #endif +#ifndef JSON_PRECISE_STREAM_POSITION + #define JSON_PRECISE_STREAM_POSITION 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -80,21 +84,28 @@ #define NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS #endif +#if JSON_PRECISE_STREAM_POSITION + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION _psp +#else + #define NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) json_abi ## a ## b ## c ## d -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) json_abi ## a ## b ## c ## d ## e +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ - NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS) + NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ + NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ diff --git a/tests/abi/config/default.cpp b/tests/abi/config/default.cpp index d0b4ba54b..8f66dfbf1 100644 --- a/tests/abi/config/default.cpp +++ b/tests/abi/config/default.cpp @@ -36,6 +36,10 @@ TEST_CASE("default namespace") expected += "_bics"; #endif +#if JSON_PRECISE_STREAM_POSITION + expected += "_psp"; +#endif + expected += "_v" STRINGIZE(NLOHMANN_JSON_VERSION_MAJOR); expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_MINOR); expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_PATCH) "::basic_json"; diff --git a/tests/abi/config/noversion.cpp b/tests/abi/config/noversion.cpp index 789107181..12b7603b9 100644 --- a/tests/abi/config/noversion.cpp +++ b/tests/abi/config/noversion.cpp @@ -37,6 +37,10 @@ TEST_CASE("default namespace without version component") expected += "_bics"; #endif +#if JSON_PRECISE_STREAM_POSITION + expected += "_psp"; +#endif + expected += "::basic_json"; // fallback for Clang diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index 0dbfdd15c..28359804b 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -25,6 +25,7 @@ using nlohmann::json; #include #include #include +#include #include #if defined(_WIN32) @@ -1233,6 +1234,57 @@ TEST_CASE("deserialization") } } + SECTION("stream position after extraction without JSON_PRECISE_STREAM_POSITION (#5340)") + { + // By default, the character that terminates a number is consumed, so + // the stream is left one byte too far after a number (and only after a + // number). JSON_PRECISE_STREAM_POSITION changes this; see + // unit-precise-stream-position.cpp. These checks pin the default. + const auto remaining = [](std::istream & is) + { + return std::string(std::istreambuf_iterator(is), std::istreambuf_iterator()); + }; + + SECTION("the character after a number is consumed") + { + std::istringstream ss("1true"); + json j; + ss >> j; + CHECK(j == 1); + CHECK(remaining(ss) == "rue"); + } + + SECTION("the character after other values is not consumed") + { + std::istringstream ss("[1]true"); + json j; + ss >> j; + CHECK(j == json::parse("[1]")); + CHECK(remaining(ss) == "true"); + } + + SECTION("comma-separated numbers can be read one by one") + { + std::istringstream ss("1,2,3"); + json j1, j2, j3; + ss >> j1 >> j2 >> j3; + CHECK(j1 == 1); + CHECK(j2 == 2); + CHECK(j3 == 3); + } + + SECTION("std::getline after a number skips the line break") + { + std::istringstream ss("42\nfoo"); + json j; + std::string line; + ss >> j; + std::getline(ss, line); + CHECK(j == 42); + CHECK(line == "foo"); + } + } + // build with C++20 // JSON_HAS_CPP_20 #if defined(__cpp_char8_t) diff --git a/tests/src/unit-precise-stream-position.cpp b/tests/src/unit-precise-stream-position.cpp new file mode 100644 index 000000000..5b6bff682 --- /dev/null +++ b/tests/src/unit-precise-stream-position.cpp @@ -0,0 +1,237 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +// This file tests the opt-in JSON_PRECISE_STREAM_POSITION, so it defines the +// macro itself rather than relying on a -D flag, and runs in every build. The +// default behavior is pinned in unit-deserialization.cpp. +#ifdef JSON_PRECISE_STREAM_POSITION + #undef JSON_PRECISE_STREAM_POSITION +#endif + +#define JSON_PRECISE_STREAM_POSITION 1 + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include + +#define STRINGIZE_EX(x) #x +#define STRINGIZE(x) STRINGIZE_EX(x) + +namespace +{ +// A streambuf that keeps no get area at all and refuses every putback: with an +// empty get area, sungetc() always ends up in pbackfail(). Used to check that +// the character terminating a number is left in the input without relying on +// the streambuf being able to put a consumed character back. +class no_putback_streambuf : public std::streambuf +{ + public: + explicit no_putback_streambuf(std::string s) : m_data(std::move(s)) {} + + protected: + // peek at the next character without consuming it + int_type underflow() override + { + if (m_pos >= m_data.size()) + { + return traits_type::eof(); + } + return traits_type::to_int_type(m_data[m_pos]); + } + + // consume the next character + int_type uflow() override + { + if (m_pos >= m_data.size()) + { + return traits_type::eof(); + } + return traits_type::to_int_type(m_data[m_pos++]); + } + + int_type pbackfail(int_type /*c*/) override + { + return traits_type::eof(); + } + + private: + std::string m_data; + std::size_t m_pos = 0; +}; + +// read the characters that are left in a stream +std::string remaining(std::istream& is) +{ + std::string result; + char c = 0; + while (is.get(c)) + { + result += c; + } + return result; +} +} // namespace + +TEST_CASE("JSON_PRECISE_STREAM_POSITION") +{ + SECTION("the macro is part of the ABI tag") + { + const std::string ns = STRINGIZE(NLOHMANN_JSON_NAMESPACE); + // other tags may come before it, e.g. json_abi_diag_psp + CHECK(ns.find("_psp") != std::string::npos); + } + + SECTION("a number does not consume the character that terminates it") + { + // a number is only terminated by the character following it; that + // character must be given back so the stream is positioned right + // after the value + const std::vector> tests = + { + {"1true", "true"}, + {"1[2]", "[2]"}, + {"1{}", "{}"}, + {R"(1"a")", R"("a")"}, + {"1 true", " true"}, + {"12,", ","}, + {"-0.5e3x", "x"}, + {"1null", "null"} + }; + + for (const auto& test : tests) + { + CAPTURE(test.first); + std::istringstream ss(test.first); + json j; + ss >> j; + CHECK(j == json::parse(test.first.substr(0, test.first.size() - test.second.size()))); + CHECK(remaining(ss) == test.second); + } + } + + SECTION("values that are self-delimiting are unaffected") + { + const std::vector> tests = + { + {"truefalse", "false"}, + {"[1][2]", "[2]"}, + {R"({"a":1}{"b":2})", R"({"b":2})"}, + {R"("a""b")", R"("b")"}, + {"null null", " null"} + }; + + for (const auto& test : tests) + { + CAPTURE(test.first); + std::istringstream ss(test.first); + json j; + ss >> j; + CHECK(remaining(ss) == test.second); + } + } + + SECTION("a number at the end of the input leaves nothing behind") + { + for (const std::string s : + {"1", "12", "-3.5e2", " 7 " + }) + { + CAPTURE(s); + std::istringstream ss(s); + json j; + ss >> j; + CHECK(remaining(ss).find_first_not_of(" \t\n\r") == std::string::npos); + } + } + + SECTION("repeated extraction of concatenated values") + { + std::istringstream ss(R"(1true[2]3"x"{"a":4}5)"); + const std::vector expected = + { + json(1), json(true), json::parse("[2]"), json(3), + json("x"), json::parse(R"({"a":4})"), json(5) + }; + + for (const auto& e : expected) + { + json j; + ss >> j; + CHECK(j == e); + } + } + + SECTION("differences to the default behavior") + { + // both of these work by accident without the macro, because the + // character after a number is swallowed; see unit-deserialization.cpp + + SECTION("a separator after a number is not skipped") + { + std::istringstream ss("1,2"); + json j; + ss >> j; + CHECK(j == 1); + CHECK_THROWS_AS(ss >> j, json::parse_error&); + } + + SECTION("std::getline after a number sees the line break") + { + std::istringstream ss("42\nfoo"); + json j; + std::string line; + ss >> j; + std::getline(ss, line); + CHECK(j == 42); + CHECK(line.empty()); + std::getline(ss, line); + CHECK(line == "foo"); + } + } + + SECTION("sax_parse with strict == false") + { + std::istringstream ss("1true"); + json j; + nlohmann::detail::json_sax_dom_parser sdp(j, true); + CHECK(json::sax_parse(ss, &sdp, nlohmann::detail::input_format_t::json, false)); + CHECK(j == 1); + CHECK(remaining(ss) == "true"); + } + + SECTION("strict parsing still rejects trailing data") + { + std::istringstream ss("1true"); + json _; + CHECK_THROWS_WITH_AS(_ = json::parse(ss), + "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - unexpected true literal; expected end of input", json::parse_error&); + + std::istringstream ss2("1true"); + CHECK_FALSE(json::accept(ss2)); + } + + SECTION("a streambuf that cannot put back is not needed") + { + // the terminating character is never consumed, so no putback + // position is required + no_putback_streambuf buf("1true"); + std::istream is(&buf); + json j; + is >> j; + CHECK(j == json(1)); + CHECK(remaining(is) == "true"); + } +} diff --git a/tools/generate_natvis/generate_natvis.py b/tools/generate_natvis/generate_natvis.py index 968690abe..fb1210db1 100755 --- a/tools/generate_natvis/generate_natvis.py +++ b/tools/generate_natvis/generate_natvis.py @@ -20,7 +20,7 @@ if __name__ == '__main__': namespaces = ['nlohmann'] abi_prefix = 'json_abi' - abi_tags = ['_diag', '_ldvcmp', '_dp', '_bics'] + abi_tags = ['_diag', '_ldvcmp', '_dp', '_bics', '_psp'] version = '_v' + args.version.replace('.', '_') inline_namespaces = []