diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index 95c82e873..6f5603c8c 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -109,6 +109,15 @@ The library maps BSON record types to JSON value types as follows: If BSON input must be validated for strict specification compliance, validate it separately before passing it to `from_bson()`. +!!! warning "UTF-8 validation of string values" + + The BSON specification requires `string` values (type `0x02`) to be valid UTF-8. This library validates the + bytes of every such string at decode time and rejects ill-formed UTF-8 with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions` + set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. Element + (key) names and `binary` values (type `0x05`) are unaffected and are never validated, since they are read + byte-by-byte as a C string, or are not required to hold text, respectively. + ??? example ```cpp diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index 670a23455..e4c257e27 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -176,6 +176,16 @@ The library maps CBOR types to JSON value types as follows: CBOR allows map keys of any type, whereas JSON only allows strings as keys in object values. Therefore, CBOR maps with keys other than UTF-8 strings are rejected. +!!! warning "UTF-8 validation of text strings" + + [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings + (major type 3) to be valid UTF-8. This library validates the bytes of every text string (object keys included) at + decode time and rejects ill-formed UTF-8 with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with + `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting value is + dumped. Byte strings (major type 2) are unaffected and are never validated, since they are not required to hold + text. + !!! warning "Tagged items" Tagged items (0xC0..0xDB) will throw a parse error by default. They can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`, in which case the tag is skipped and the enclosed data item is parsed on its own. They can be stored by passing `cbor_tag_handler_t::store` to function `from_cbor`. Note that no tag is ever interpreted: for instance, a text string tagged with tag 0 (date/time) stays a string. diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index bd0c840f2..a434909c4 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -136,6 +136,14 @@ The library maps MessagePack types to JSON value types as follows: Any MessagePack output created by `to_msgpack` can be successfully parsed by `from_msgpack`. +!!! warning "UTF-8 validation of string values" + + The MessagePack specification requires `str` values (`fixstr`, `str 8`, `str 16`, `str 32`) to be valid UTF-8. + This library validates the bytes of every such string (object keys included) at decode time and rejects + ill-formed UTF-8 with a [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, + with `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting + value is dumped. `bin`/`ext`/`fixext` values are unaffected and are never validated, since they are not required + to hold text. ??? example diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 9a7698b2f..ee76596f6 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -340,7 +340,8 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde ### json.exception.parse_error.113 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a -string was read where one was required (for instance as a map key), or the string's length specification is invalid. +string was read where one was required (for instance as a map key), the string's length specification is invalid, or +the string's bytes are not valid UTF-8. !!! failure "Example messages" @@ -356,6 +357,9 @@ string was read where one was required (for instance as a map key), or the strin ``` [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative ``` + ``` + [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte + ``` ### json.exception.parse_error.114 diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index df46eea58..4132c03ca 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -432,7 +433,21 @@ class binary_reader exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); } - return get_string(input_format_t::bson, len - static_cast(1), result) && get() != char_traits::eof(); + if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast(1), result))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(get() != 0x00)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, + "BSON string is not null-terminated", + "string"), nullptr)); + } + + return true; } /*! @@ -550,8 +565,6 @@ class binary_reader } } - - ////////// // CBOR // ////////// @@ -3304,7 +3317,28 @@ class binary_reader const NumberType len, string_t& result) { - return get_bytes(format, len, "string", result); + // get_bytes() appends to result, and CBOR indefinite-length strings + // collect all their chunks in the same result; validating only the + // newly read bytes keeps the check linear in the input size + const std::size_t old_size = result.size(); + if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) + { + return false; + } + + // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications + // all require text strings to be valid UTF-8; reject anything else + // right here so malformed input is caught at decode time instead of + // only surfacing later as a type_error.316 when the value is dumped + // (which would defeat allow_exceptions=false / strict discarding). + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + } + + return true; } /*! diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 7c38276ce..f968b6001 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -58,8 +59,6 @@ class serializer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using binary_char_t = typename BasicJsonType::binary_t::value_type; - static constexpr std::uint8_t UTF8_ACCEPT = 0; - static constexpr std::uint8_t UTF8_REJECT = 1; public: /*! @@ -1592,62 +1591,6 @@ class serializer } } - /*! - @brief check whether a string is UTF-8 encoded - - The function checks each byte of a string whether it is UTF-8 encoded. The - result of the check is stored in the @a state parameter. The function must - be called initially with state 0 (accept). State 1 means the string must - be rejected, because the current byte is not allowed. If the string is - completely processed, but the state is non-zero, the string ended - prematurely; that is, the last byte indicated more bytes should have - followed. - - @param[in,out] state the state of the decoding - @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) - @param[in] byte next byte to decode - @return new state - - @note The function has been edited: a std::array is used. - - @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann - @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - */ - static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept - { - static const std::array utf8d = - { - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF - 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF - 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 - } - }; - - JSON_ASSERT(static_cast(byte) < utf8d.size()); - const std::uint8_t type = utf8d[byte]; - - codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6u) - : (0xFFu >> type) & (byte); - - const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); - JSON_ASSERT(index < utf8d.size()); - state = utf8d[index]; - return state; - } - /* * Overload to make the compiler happy while it is instantiating * dump_integer for number_unsigned_t. diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index fe2f9109d..142943cd6 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -8,10 +8,13 @@ #pragma once +#include // array #include // size_t +#include // uint8_t, uint32_t #include // string, to_string #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -33,5 +36,102 @@ StringType to_string(std::size_t value) return result; } +/////////////////// +// UTF-8 decoding // +/////////////////// + +// UTF-8 decoder states used by decode() below +static constexpr std::uint8_t UTF8_ACCEPT = 0; +static constexpr std::uint8_t UTF8_REJECT = 1; + +/*! +@brief process a byte of a UTF-8 sequence + +This is a single-byte step of a "shift-based" UTF-8 decoder originally +written by Björn Hoehrmann. See +http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. + +This decoder is the single source of truth for UTF-8 validation in this +library: it is used both by the serializer (to escape and, in strict mode, +reject ill-formed UTF-8 when dumping a string) and by the binary readers +(to reject ill-formed UTF-8 in CBOR/MessagePack/BSON/UBJSON text strings at +decode time; see @ref is_valid_utf8 below). + +@param[in,out] state the current decoder state +@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) +@param[in] byte next byte to decode +@return new state + +@note Original source: http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +*/ +inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept +{ + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(static_cast(byte) < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); + JSON_ASSERT(index < utf8d.size()); + state = utf8d[index]; + return state; +} + +/*! +@brief check whether a string consists solely of valid UTF-8 + +Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text +strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the +MessagePack/BSON specifications all require text strings to be UTF-8), so +that malformed input is caught immediately instead of only surfacing later +as a type_error.316 when the resulting value is dumped. + +@param[in] s the string to check +@param[in] first index of the first byte to check; the bytes before it are + assumed to have been validated already and to end on a + code point boundary +@return whether @a s (from index @a first on) is valid UTF-8 +*/ +template +inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept +{ + std::uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint = 0; + + for (std::size_t i = first; i < s.size(); ++i) + { + decode(state, codepoint, static_cast(s[i])); + if (state == UTF8_REJECT) + { + return false; + } + } + + return state == UTF8_ACCEPT; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index e6fe169ef..647312562 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6197,11 +6197,15 @@ NLOHMANN_JSON_NAMESPACE_END +#include // array #include // size_t +#include // uint8_t, uint32_t #include // string, to_string // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6223,6 +6227,103 @@ StringType to_string(std::size_t value) return result; } +/////////////////// +// UTF-8 decoding // +/////////////////// + +// UTF-8 decoder states used by decode() below +static constexpr std::uint8_t UTF8_ACCEPT = 0; +static constexpr std::uint8_t UTF8_REJECT = 1; + +/*! +@brief process a byte of a UTF-8 sequence + +This is a single-byte step of a "shift-based" UTF-8 decoder originally +written by Björn Hoehrmann. See +http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. + +This decoder is the single source of truth for UTF-8 validation in this +library: it is used both by the serializer (to escape and, in strict mode, +reject ill-formed UTF-8 when dumping a string) and by the binary readers +(to reject ill-formed UTF-8 in CBOR/MessagePack/BSON/UBJSON text strings at +decode time; see @ref is_valid_utf8 below). + +@param[in,out] state the current decoder state +@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) +@param[in] byte next byte to decode +@return new state + +@note Original source: http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +*/ +inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept +{ + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(static_cast(byte) < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); + JSON_ASSERT(index < utf8d.size()); + state = utf8d[index]; + return state; +} + +/*! +@brief check whether a string consists solely of valid UTF-8 + +Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text +strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the +MessagePack/BSON specifications all require text strings to be UTF-8), so +that malformed input is caught immediately instead of only surfacing later +as a type_error.316 when the resulting value is dumped. + +@param[in] s the string to check +@param[in] first index of the first byte to check; the bytes before it are + assumed to have been validated already and to end on a + code point boundary +@return whether @a s (from index @a first on) is valid UTF-8 +*/ +template +inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept +{ + std::uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint = 0; + + for (std::size_t i = first; i < s.size(); ++i) + { + decode(state, codepoint, static_cast(s[i])); + if (state == UTF8_REJECT) + { + return false; + } + } + + return state == UTF8_ACCEPT; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -12619,6 +12720,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include @@ -13020,7 +13123,21 @@ class binary_reader exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); } - return get_string(input_format_t::bson, len - static_cast(1), result) && get() != char_traits::eof(); + if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast(1), result))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(get() != 0x00)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, + "BSON string is not null-terminated", + "string"), nullptr)); + } + + return true; } /*! @@ -13138,8 +13255,6 @@ class binary_reader } } - - ////////// // CBOR // ////////// @@ -15892,7 +16007,28 @@ class binary_reader const NumberType len, string_t& result) { - return get_bytes(format, len, "string", result); + // get_bytes() appends to result, and CBOR indefinite-length strings + // collect all their chunks in the same result; validating only the + // newly read bytes keeps the check linear in the input size + const std::size_t old_size = result.size(); + if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) + { + return false; + } + + // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications + // all require text strings to be valid UTF-8; reject anything else + // right here so malformed input is caught at decode time instead of + // only surfacing later as a type_error.316 when the value is dumped + // (which would defeat allow_exceptions=false / strict discarding). + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + } + + return true; } /*! @@ -22726,6 +22862,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include @@ -22753,8 +22891,6 @@ class serializer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using binary_char_t = typename BasicJsonType::binary_t::value_type; - static constexpr std::uint8_t UTF8_ACCEPT = 0; - static constexpr std::uint8_t UTF8_REJECT = 1; public: /*! @@ -24287,62 +24423,6 @@ class serializer } } - /*! - @brief check whether a string is UTF-8 encoded - - The function checks each byte of a string whether it is UTF-8 encoded. The - result of the check is stored in the @a state parameter. The function must - be called initially with state 0 (accept). State 1 means the string must - be rejected, because the current byte is not allowed. If the string is - completely processed, but the state is non-zero, the string ended - prematurely; that is, the last byte indicated more bytes should have - followed. - - @param[in,out] state the state of the decoding - @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) - @param[in] byte next byte to decode - @return new state - - @note The function has been edited: a std::array is used. - - @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann - @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - */ - static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept - { - static const std::array utf8d = - { - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF - 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF - 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 - } - }; - - JSON_ASSERT(static_cast(byte) < utf8d.size()); - const std::uint8_t type = utf8d[byte]; - - codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6u) - : (0xFFu >> type) & (byte); - - const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); - JSON_ASSERT(index < utf8d.size()); - state = utf8d[index]; - return state; - } - /* * Overload to make the compiler happy while it is instantiating * dump_integer for number_unsigned_t. diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 669a4bfe1..4808c3166 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -1697,3 +1697,76 @@ TEST_CASE("BSON roundtrips" * doctest::skip()) } } } + +TEST_CASE("Invalid document size handling") +{ + SECTION("document size must be at least 5") + { + std::vector const v = {0x04, 0x00, 0x00, 0x00, 0x00}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 4 does not match the number of bytes read (5)", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + } + + SECTION("declared document size must match consumed bytes (extra trailing element)") + { + // Declares 5-byte empty document but appends an int32 element after the declared end. + std::vector const v = + { + 0x05, 0x00, 0x00, 0x00, + 0x10, 'a', 'd', 'm', 'i', 'n', 0x00, + 0x01, 0x00, 0x00, 0x00, + 0x00 + }; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 16: syntax error while parsing BSON document: document size 5 does not match the number of bytes read (16)", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + } + + SECTION("declared document size must match consumed bytes (premature terminator)") + { + // Declares 32-byte document but only contains the size field followed by an immediate terminator. + std::vector const v = + { + 0x20, 0x00, 0x00, 0x00, + 0x00 + }; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 32 does not match the number of bytes read (5)", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + } + + SECTION("array declared size must match consumed bytes") + { + // Outer object contains an array "a" that declares 5 bytes (empty) but + // actually contains an int32 element before its terminator. + std::vector const v = + { + 0x14, 0x00, 0x00, 0x00, // object size = 20 + 0x04, 'a', 0x00, // key "a", array type + 0x05, 0x00, 0x00, 0x00, // array declared size = 5 (empty) + 0x10, '0', 0x00, 0x01, 0x00, 0x00, 0x00, // extra int32 element "0" = 1 + 0x00, // array terminator + 0x00 // object terminator + }; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 19: syntax error while parsing BSON document: document size 5 does not match the number of bytes read (12)", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + } + + SECTION("BSON string must end with 0x00") + { + // Length-prefixed string whose terminator byte is 'X' (0x58), not 0x00. + std::vector const v = + { + 0x0F, 0x00, 0x00, 0x00, + 0x02, 's', 0x00, + 0x02, 0x00, 0x00, 0x00, + 'A', 'X', + 0x00 + }; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 13: syntax error while parsing BSON string: BSON string is not null-terminated", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + } +} diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 4c9107517..9c799b0eb 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1833,6 +1833,59 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0xa1, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a two-character text string (major type 3) whose bytes are not + // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be + // rejected at decode time, matching every other kind of + // malformed binary input, rather than only failing later when + // the resulting value is dumped + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x62, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_cbor(std::vector({0x62, 0xc0, 0xae}), true, false).is_discarded()); + + // a CBOR byte string (major type 2) with the very same bytes is + // NOT text and must still be accepted as-is + CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x42, 0xc0, 0xae}))); + CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_cbor(json::to_cbor(j)) == j); + } + + SECTION("invalid UTF-8 in indefinite-length string") + { + json _; + + // every chunk must be valid UTF-8 on its own (RFC 8949, Section + // 3.2.3), so a code point split across two chunks is rejected + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}), true, false).is_discarded()); + + // an ill-formed later chunk is rejected after valid ones + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + + // valid multi-byte chunks are accepted + CHECK(json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6"); + } + + SECTION("many chunks in indefinite-length string") + { + // only the newly read chunk is validated, not the whole string + // collected so far; validating the latter made this input take + // quadratic time (about ten seconds for 100000 chunks) + constexpr std::size_t chunks = 100000; + std::vector v{0x7f}; + for (std::size_t i = 0; i < chunks; ++i) + { + v.push_back(0x61); + v.push_back('a'); + } + v.push_back(0xff); + CHECK(json::from_cbor(v) == std::string(chunks, 'a')); + } + SECTION("strict mode") { std::vector const vec = {0xf6, 0xf6}; diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index a8892081d..805b03a13 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1554,6 +1554,27 @@ TEST_CASE("MessagePack") CHECK(json::from_msgpack(std::vector({0x81, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a fixstr of length 2 (0xA0 | 2) whose bytes are not valid UTF-8 + // (0xC0 0xAE is an overlong encoding of '.') must be rejected at + // decode time, matching every other kind of malformed binary + // input, rather than only failing later when the resulting + // value is dumped + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector({0xa2, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing MessagePack string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_msgpack(std::vector({0xa2, 0xc0, 0xae}), true, false).is_discarded()); + + // a MessagePack bin8 blob with the very same bytes is NOT text + // and must still be accepted as-is + CHECK_NOTHROW(_ = json::from_msgpack(std::vector({0xc4, 0x02, 0xc0, 0xae}))); + CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_msgpack(json::to_msgpack(j)) == j); + } + SECTION("strict mode") { std::vector const vec = {0xc0, 0xc0}; diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 00b305a75..45617c3b3 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -94,6 +94,16 @@ TEST_CASE("serialization") CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\""); } + SECTION("invalid character (regression guard for shared UTF-8 decoder, see #5529)") + { + // dump_escaped_impl() now calls the UTF-8 decoder shared with the + // binary readers (detail::decode() in string_utils.hpp) instead + // of a private copy; the exact type_error.316 message/behavior + // must stay byte-for-byte the same as before that extraction + const json j = "ä\xA9ü"; + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&); + } + SECTION("ending with incomplete character") { const json j = "123\xC2";