Scarfski/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-29 13:35:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add error_handler_t::keep to copy invalid UTF-8 bytes unchanged

Browse Source 
error_handler_t::ignore drops invalid bytes although its docs promised to copy them (#4552). Add keep, which copies each ill-formed subsequence byte-for-byte while still escaping valid characters.

Fixes #4552

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-26 07:37:05 +02:00
parent f422b753cc
commit 8aabb9981f
15 changed files with 283 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
include/nlohmann/detail/output/serializer.hpp
View File

@@ -48,7 +48,8 @@ enum class error_handler_t
{
strict, ///< throw a type_error exception in case of invalid UTF-8
replace, ///< replace invalid UTF-8 sequences with U+FFFD
ignore ///< ignore invalid UTF-8 sequences
ignore, ///< ignore invalid UTF-8 sequences
keep ///< keep invalid UTF-8 sequences; their bytes are copied unchanged
};
template<typename BasicJsonType>
@@ -1019,6 +1020,47 @@ class serializer
break;
}
case error_handler_t::keep:
{
// drop whatever the incomplete sequence left in
// the buffer (only copied if !EnsureAscii) and copy
// the ill-formed bytes from the input instead
bytes = bytes_after_last_accept;
if (undumped_chars > 0)
{
// the pending bytes of the incomplete sequence
// are ill-formed; the current byte may be OK for
// itself, so we would like to read it again
for (std::size_t j = i - undumped_chars; j < i; ++j)
{
string_buffer[bytes++] = s[j];
}
--i;
}
else
{
// the current byte cannot start any sequence
string_buffer[bytes++] = s[i];
}
// write buffer and reset index; there must be 13 bytes
// left, as this is the maximal number of bytes to be
// written ("\uxxxx\uxxxx\0") for one code point
if (string_buffer.size() - bytes < 13)
{
put_buffer(string_buffer, bytes);
bytes = 0;
}
bytes_after_last_accept = bytes;
undumped_chars = 0;
// continue processing the string
state = UTF8_ACCEPT;
break;
}
default: // LCOV_EXCL_LINE
JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
}
@@ -1064,6 +1106,15 @@ class serializer
break;
}
case error_handler_t::keep:
{
// write all accepted bytes
put_buffer(string_buffer, bytes_after_last_accept);
// copy the bytes of the incomplete sequence unchanged
put_string(s, s.size() - undumped_chars, s.size());
break;
}
case error_handler_t::replace:
{
// write all accepted bytes