From 2b1a720fc083694aff2055ab1b1951bc38de6389 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 29 Sep 2026 00:18:56 +0200 Subject: [PATCH] Walk the index of json_view through a navigation policy Preparation for editable documents, without a change in behavior: views and documents get a template parameter Editable (false by default), and every walk over the index (iterators, lookups, dump(), materialize()) goes through detail::view::navigation. For read-only documents it is the plain node array, as before, so they compile without any of the edit handling. For editable documents it also follows the representation of edits, which this commit defines: - node flags `edited` (a string or number token in the edit arena), `moved` (the elements of an array/object live in a separate sequence), and `is_new` (no source position), and link nodes (kind_link) that stand for a value stored elsewhere - document_data::edit_state: the moved sequences, the storage of new values, and the edit arena materialize() now keeps a frame per open container instead of returning to the end of a closed one, as the serializer does, so that it can follow moved sequences. Floats whose token lives in the edit arena (also "nan", "inf", "-inf") are converted out of line. dump() copies only strings of the source without escaping, and shrink_to_fit() leaves the node array in place once there are edits, as they link into it. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/view/document_data.hpp | 98 +++++- include/nlohmann/detail/view/iterator.hpp | 2 +- include/nlohmann/detail/view/lookup.hpp | 37 +- include/nlohmann/detail/view/materialize.hpp | 60 ++-- include/nlohmann/detail/view/node.hpp | 28 +- include/nlohmann/detail/view/number.hpp | 21 ++ include/nlohmann/detail/view/serializer.hpp | 20 +- include/nlohmann/json_view.hpp | 51 +-- single_include/nlohmann/json_view.hpp | 317 +++++++++++++----- 9 files changed, 484 insertions(+), 150 deletions(-) diff --git a/include/nlohmann/detail/view/document_data.hpp b/include/nlohmann/detail/view/document_data.hpp index d56c1a119..4b43953a6 100644 --- a/include/nlohmann/detail/view/document_data.hpp +++ b/include/nlohmann/detail/view/document_data.hpp @@ -12,6 +12,9 @@ #include // size_t #include // uint32_t #include // memcpy +#include // less +#include // map +#include // unique_ptr #include // operator new, placement new #include // string #include // vector @@ -50,9 +53,30 @@ struct document_data std::vector indexes{}; // NOLINT(readability-redundant-member-init) std::vector index_slots{}; // NOLINT(readability-redundant-member-init) std::vector large_objects{}; ///< positions of the objects to index (noted while parsing) // NOLINT(readability-redundant-member-init) - std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena (indexed by flags & node_flags::storage) + std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena (indexed by flags & node_flags::storage) bool discarded = true; + /// The storage of edits (editable documents only; see edit_storage.hpp). + /// Edits never move or resize the parsed index, so views stay valid: an + /// array/object whose elements change gets node_flags::moved, and its + /// elements then live in a separate sequence (a header node, then the + /// entries), whose entries link to the values. + struct edit_state + { + std::vector moved{}; ///< element sequences of moved arrays/objects (header node first) + std::vector moved_cap{}; ///< capacity in nodes of a growable block; 0: a fixed sequence (a new value) + std::vector> chunks{}; ///< storage of new values and blocks; never moved + std::map> regions{}; ///< new arrays/objects: root -> container that uses it as its element sequence (nullptr: linked from a block) + node* chunk_cur = nullptr; + node* chunk_end = nullptr; + std::size_t chunk_next = 64; + std::vector> texts{}; ///< edit arena, the current buffer last; earlier ones stay alive for string views + std::size_t text_used = 0; + std::size_t text_cap = 0; + std::size_t bytes = 0; ///< memory held by edits + }; + std::unique_ptr edits{}; ///< created by the first edit + /// one allocation for the header and room for `nodes` nodes; large /// documents get a separate node array instead (so it can be trimmed) static document_data* create(std::size_t nodes) @@ -136,6 +160,78 @@ struct document_data { return n + n->next; } + + /// (editable documents) first element or key, also of a moved container + NLOHMANN_VIEW_ALWAYS_INLINE const node* first_child_edited(const node* n) const noexcept + { + return NLOHMANN_VIEW_LIKELY((n->flags & node_flags::moved) == 0) ? n + 1 : edits->moved[n->off] + 1; + } + + /// (editable documents) end of the elements, also of a moved container + NLOHMANN_VIEW_ALWAYS_INLINE const node* child_end_edited(const node* n) const noexcept + { + if (NLOHMANN_VIEW_LIKELY((n->flags & node_flags::moved) == 0)) + { + return n + n->next; + } + const node* const h = edits->moved[n->off]; + return h + h->next; + } + + /// (editable documents) the value at an element position: entries of + /// moved sequences are links. The link case is out of line, so that this + /// compiles to a predicted branch rather than a select that delays the + /// following loads. + static NLOHMANN_VIEW_ALWAYS_INLINE const node* deref(const node* n) noexcept + { + return NLOHMANN_VIEW_LIKELY(n->kind != kind_link) ? n : follow_link(n); + } + + static NLOHMANN_VIEW_NOINLINE const node* follow_link(const node* n) noexcept + { + return link_target(*n); + } +}; + +/// How the index is walked: views of read-only documents follow the node +/// array alone and compile without any of the edit handling; views of +/// editable documents also follow moved element sequences and links. +template +struct navigation +{ + static NLOHMANN_VIEW_ALWAYS_INLINE const node* first(const document_data& /*d*/, const node* n) noexcept + { + return n + 1; + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* end(const document_data& /*d*/, const node* n) noexcept + { + return n + n->next; + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* value(const node* n) noexcept + { + return n; + } +}; + +template<> +struct navigation +{ + static NLOHMANN_VIEW_ALWAYS_INLINE const node* first(const document_data& d, const node* n) noexcept + { + return d.first_child_edited(n); + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* end(const document_data& d, const node* n) noexcept + { + return d.child_end_edited(n); + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* value(const node* n) noexcept + { + return document_data::deref(n); + } }; } // namespace view diff --git a/include/nlohmann/detail/view/iterator.hpp b/include/nlohmann/detail/view/iterator.hpp index dfeda4a34..12906e9c1 100644 --- a/include/nlohmann/detail/view/iterator.hpp +++ b/include/nlohmann/detail/view/iterator.hpp @@ -73,7 +73,7 @@ class view_iterator NLOHMANN_VIEW_ALWAYS_INLINE View operator*() const noexcept { - return View(m_doc, m_pos + m_value_offset); + return View(m_doc, View::navigation::value(m_pos + m_value_offset)); } pointer operator->() const noexcept diff --git a/include/nlohmann/detail/view/lookup.hpp b/include/nlohmann/detail/view/lookup.hpp index 5d50c0667..718acc077 100644 --- a/include/nlohmann/detail/view/lookup.hpp +++ b/include/nlohmann/detail/view/lookup.hpp @@ -84,18 +84,20 @@ class short_key /// the key node of the first member of an object with the given key, or /// nullptr; most keys are rejected by their length, from the index alone -inline const node* find_member(const document_data& d, const node* object, const char* key, std::size_t n) noexcept +template +const node* find_member(const document_data& d, const node* object, const char* key, std::size_t n) noexcept { - if (NLOHMANN_VIEW_UNLIKELY(object->extra != 0)) + using nav = navigation; + if (NLOHMANN_VIEW_UNLIKELY(object->extra != 0) && (!Editable || (object->flags & node_flags::moved) == 0)) { - return find_indexed(d, object, key, n); // a large object + return find_indexed(d, object, key, n); // a large object (whose members have not been edited) } - const node* const end = document_data::child_end(object); + const node* const end = nav::end(d, object); const auto* const k = reinterpret_cast(key); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) if (NLOHMANN_VIEW_LIKELY(n <= 16)) { const short_key probe(k, n); - for (const node* m = document_data::first_child(object); m != end; m = document_data::after(m + 1)) + for (const node* m = nav::first(d, object); m != end; m = document_data::after(m + 1)) { if (m->len == n && probe.matches(reinterpret_cast(d.str(*m)))) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) { @@ -104,7 +106,7 @@ inline const node* find_member(const document_data& d, const node* object, const } return nullptr; } - for (const node* m = document_data::first_child(object); m != end; m = document_data::after(m + 1)) + for (const node* m = nav::first(d, object); m != end; m = document_data::after(m + 1)) { if (m->len == n && std::memcmp(d.str(*m), key, n) == 0) { @@ -114,10 +116,16 @@ inline const node* find_member(const document_data& d, const node* object, const return nullptr; } -/// the element of an array at an index below its size -inline const node* element_at(const node* array, std::size_t idx) noexcept +/// the entry of the element of an array at an index below its size (a link +/// in the moved sequences of editable documents) +template +const node* element_at(const document_data& d, const node* array, std::size_t idx) noexcept { - const node* e = document_data::first_child(array); + const node* e = navigation::first(d, array); + if (Editable && (array->flags & node_flags::moved) != 0 && d.edits->moved_cap[array->off] != 0) + { + return e + idx; // a growable block: one link per element + } for (std::size_t i = 0; i < idx; ++i) { e = document_data::after(e); @@ -125,13 +133,14 @@ inline const node* element_at(const node* array, std::size_t idx) noexcept return e; } -/// the last element of a non-empty array, or the key of the last member of a -/// non-empty object -inline const node* last_child(const node* container) noexcept +/// the entry of the last element of a non-empty array, or the key of the +/// last member of a non-empty object +template +const node* last_child(const document_data& d, const node* container) noexcept { const std::size_t value_offset = container->kind == static_cast(value_t::object) ? 1 : 0; - const node* const end = document_data::child_end(container); - const node* last = document_data::first_child(container); + const node* const end = navigation::end(d, container); + const node* last = navigation::first(d, container); for (const node* c = document_data::after(last + value_offset); c != end; c = document_data::after(c + value_offset)) { last = c; diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index 79fafee1b..f9c381e1a 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -34,17 +34,24 @@ iterative, so the nesting depth is limited by memory only, as for parse(). Without a lexer the handler records no source positions (JSON_DIAGNOSTIC_POSITIONS). */ -template +template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; using sax_t = json_sax_dom_parser>; + using nav = navigation; + + struct frame + { + const node* pos; ///< next element, or key of the next member + const node* end; + bool object; + }; BasicJsonType result; sax_t sax(result, true); const string_t no_token{}; - // the ends of the open containers, and whether they are objects - std::vector> open; + std::vector open; for (;;) { switch (static_cast(n->kind)) @@ -61,67 +68,66 @@ BasicJsonType materialize(const document_data& d, const node* n) { sax.start_array(n->len); } - open.emplace_back(document_data::child_end(n), object); - n = document_data::first_child(n); + open.push_back(frame{nav::first(d, n), nav::end(d, n), object}); break; } case value_t::string: { string_t s(d.str(*n), n->len); sax.string(s); - ++n; break; } case value_t::number_integer: sax.number_integer(static_cast(static_cast(integer_bits(*n)))); - ++n; break; case value_t::number_unsigned: sax.number_unsigned(static_cast(integer_bits(*n))); - ++n; break; case value_t::number_float: sax.number_float(float_value(d, *n), no_token); - ++n; break; case value_t::boolean: sax.boolean((n->flags & node_flags::is_true) != 0); - ++n; break; case value_t::null: case value_t::binary: case value_t::discarded: default: sax.null(); - ++n; break; } + + // the next value: close finished containers, then read the key for (;;) { if (open.empty()) { return result; } - if (n != open.back().first) + frame& f = open.back(); + if (f.pos == f.end) { - break; + if (f.object) + { + sax.end_object(); + } + else + { + sax.end_array(); + } + open.pop_back(); + continue; } - if (open.back().second) + const node* entry = f.pos; + if (f.object) { - sax.end_object(); + string_t key(d.str(*entry), entry->len); + sax.key(key); + ++entry; } - else - { - sax.end_array(); - } - open.pop_back(); - } - if (open.back().second) - { - // the key of the next member - string_t key(d.str(*n), n->len); - sax.key(key); - ++n; + n = nav::value(entry); + f.pos = document_data::after(entry); + break; } } } diff --git a/include/nlohmann/detail/view/node.hpp b/include/nlohmann/detail/view/node.hpp index 471c1f844..33be4f001 100644 --- a/include/nlohmann/detail/view/node.hpp +++ b/include/nlohmann/detail/view/node.hpp @@ -33,20 +33,27 @@ static_assert(static_cast(value_t::null) == 0 && static_cast(n.kind) - 1u <= 1u; } +/// the value a link node stands for +NLOHMANN_VIEW_ALWAYS_INLINE const node* link_target(const node& n) noexcept +{ + const node* t = nullptr; + std::memcpy(&t, reinterpret_cast(&n) + 8, sizeof(t)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return t; +} + +inline void make_link(node& n, const node* target) noexcept +{ + n = node{}; + n.kind = kind_link; + std::memcpy(reinterpret_cast(&n) + 8, &target, sizeof(target)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +} + /// the converted value of an integer node (stored in len/next) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t integer_bits(const node& n) noexcept { diff --git a/include/nlohmann/detail/view/number.hpp b/include/nlohmann/detail/view/number.hpp index 359f4158e..4b34a7a6a 100644 --- a/include/nlohmann/detail/view/number.hpp +++ b/include/nlohmann/detail/view/number.hpp @@ -13,6 +13,7 @@ #include // size_t #include // int64_t, uint64_t #include // memcpy +#include // numeric_limits #include // string #include // integral_constant, is_same @@ -137,11 +138,31 @@ NLOHMANN_VIEW_ALWAYS_INLINE double layout_double(const unsigned char* p, const u return negative ? -result : result; } +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + /// the value of the float token of a node, as parse() converts it; doubles /// with at most 19 digits are converted from the digit layout template FloatType float_value(const document_data& d, const node& n) { + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } return float_value(d, n, std::is_same {}); } diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 26bfb3225..3c5addf11 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -115,9 +115,10 @@ the library's conversion; integers are copied from the source, where they are canonical (except "-0", which parse() reads as 0). The walk is iterative, so the nesting depth is limited by memory only. */ -template +template class view_serializer { + using nav = navigation; using string_t = typename BasicJsonType::string_t; using number_float_t = typename BasicJsonType::number_float_t; @@ -150,7 +151,7 @@ class view_serializer else { m_out.put(object ? '{' : '['); - stack.push_back(frame{document_data::first_child(n), document_data::child_end(n), object, true}); + stack.push_back(frame{nav::first(m_doc, n), nav::end(m_doc, n), object, true}); } } else @@ -192,13 +193,14 @@ class view_serializer { m_out.put(':'); } - n = f.pos + 1; + n = nav::value(f.pos + 1); + f.pos = document_data::after(f.pos + 1); } else { - n = f.pos; + n = nav::value(f.pos); + f.pos = document_data::after(f.pos); } - f.pos = document_data::after(n); break; } } @@ -250,9 +252,9 @@ class view_serializer break; } case value_t::number_float: - if (m_style.source_numbers) + if (m_style.source_numbers && (n.flags & node_flags::storage) != node_flags::edited) { - m_out.put(m_doc.str(n), n.len); + m_out.put(m_doc.str(n), n.len); // (a float set by an edit is written as with shortest) } else { @@ -299,9 +301,9 @@ class view_serializer { const char* const s = m_doc.str(n); m_out.put('"'); - if ((n.flags & node_flags::escaped) == 0 && !m_style.ensure_ascii) + if ((n.flags & node_flags::storage) == 0 && !m_style.ensure_ascii) { - // a string without escape sequences has nothing to escape + // a string of the source without escape sequences has nothing to escape m_out.put(s, n.len); } else if (m_style.ensure_ascii) diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 3c00d6d19..4fc738ab9 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -65,7 +65,7 @@ NLOHMANN_JSON_NAMESPACE_BEGIN -template +template class basic_json_document; /*! @@ -74,11 +74,13 @@ class basic_json_document; Trivially copyable (two pointers). Valid as long as the document is alive and has not been re-parsed, and as long as a borrowed source text is alive. */ -template +template class basic_json_view { using node = detail::view::node; using document_data = detail::view::document_data; + /// how the index is walked (with edits only for editable documents) + using navigation = detail::view::navigation; public: using value_t = detail::value_t; @@ -271,7 +273,7 @@ class basic_json_view { detail::view::throw_type_error(305, "cannot use operator[] with a numeric argument with ", type_name()); } - return idx < m_node->len ? basic_json_view(m_doc, detail::view::element_at(m_node, idx)) : basic_json_view(); + return idx < m_node->len ? basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))) : basic_json_view(); } /// (an int argument would be ambiguous between size_type and const char*) @@ -327,7 +329,7 @@ class basic_json_view { detail::view::throw_out_of_range(401, detail::concat("array index ", std::to_string(idx), " is out of range")); } - return basic_json_view(m_doc, detail::view::element_at(m_node, idx)); + return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); } basic_json_view at(int idx) const @@ -399,7 +401,7 @@ class basic_json_view { if (is_structured() && m_node->len != 0) { - return basic_json_view(m_doc, detail::view::last_child(m_node) + (is_object() ? 1 : 0)); + return basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); } return front(); } @@ -416,7 +418,7 @@ class basic_json_view { return end(); } - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); return k != nullptr ? iterator(m_doc, k, true) : end(); } @@ -433,7 +435,7 @@ class basic_json_view /// whether this is an object with a member with this key bool contains(string_view_t key) const { - return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; + return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; } bool contains(const char* key) const @@ -480,7 +482,7 @@ class basic_json_view { if (NLOHMANN_VIEW_LIKELY(is_structured())) { - return iterator(m_doc, document_data::first_child(m_node), is_object()); + return iterator(m_doc, navigation::first(*m_doc, m_node), is_object()); } return iterator(m_doc, m_node, false); } @@ -489,7 +491,7 @@ class basic_json_view { if (NLOHMANN_VIEW_LIKELY(is_structured())) { - return iterator(m_doc, document_data::child_end(m_node), is_object()); + return iterator(m_doc, navigation::end(*m_doc, m_node), is_object()); } return iterator(m_doc, (is_null() || is_discarded()) ? m_node : m_node + 1, false); } @@ -589,7 +591,7 @@ class basic_json_view style.source_numbers = numbers == number_format::source; // the compact text is about as long as the source text of the value const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 64; - detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); return out; } @@ -656,7 +658,7 @@ class basic_json_view { return BasicJsonType(value_t::discarded); } - return detail::view::materialize(*m_doc, m_node); + return detail::view::materialize(*m_doc, m_node); } /// byte offset of this value in the source text (for strings: of the @@ -664,12 +666,13 @@ class basic_json_view /// a discarded view and for strings with escapes, which are decoded std::size_t source_offset() const noexcept { - return m_node != nullptr && (m_node->flags & detail::view::node_flags::storage) == 0 + return m_node != nullptr && (m_node->flags & (detail::view::node_flags::storage | detail::view::node_flags::moved | detail::view::node_flags::is_new)) == 0 ? m_node->off : static_cast(-1); } private: - template friend class basic_json_document; + template friend class basic_json_document; + template friend class basic_json_view; friend iterator; basic_json_view(const document_data* d, const node* n) noexcept @@ -687,6 +690,11 @@ class basic_json_view /// decoded strings) std::size_t source_extent() const noexcept { + if (Editable && m_doc->edits != nullptr) + { + // positions of moved and new values are not source offsets + return m_node == m_doc->tape ? m_doc->size + m_doc->edits->text_used : 64; + } const node* const next = document_data::after(m_node); const bool in_source = (m_node->flags & detail::view::node_flags::storage) == 0; if (!in_source) @@ -704,8 +712,8 @@ class basic_json_view /// (object required) NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view lookup(string_view_t key) const noexcept { - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); - return k != nullptr ? basic_json_view(m_doc, k + 1) : basic_json_view(); + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + return k != nullptr ? basic_json_view(m_doc, navigation::value(k + 1)) : basic_json_view(); } // --- get() dispatch --- @@ -796,7 +804,7 @@ Borrowed parses keep a pointer to the caller's text, which must outlive the document. Owned parses (parse_copy, rvalue std::string, streams, and inputs that are not contiguous byte ranges) keep their own copy. */ -template +template class basic_json_document { using document_data = detail::view::document_data; @@ -805,7 +813,7 @@ class basic_json_document "json_view supports 64-bit integer types only"); public: - using view_type = basic_json_view; + using view_type = basic_json_view; using value_t = detail::value_t; /// an empty (discarded) document @@ -929,7 +937,8 @@ class basic_json_document + (m_data->tape != m_data->inline_tape ? m_data->tape_cap * sizeof(detail::view::node) : 0) + m_data->arena.capacity() + m_data->owned.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) - + (m_data->large_objects.capacity() * sizeof(std::uint32_t)); + + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + + (m_data->edits != nullptr ? m_data->edits->bytes : 0); } /// release unused capacity of the index and the decoded strings; like @@ -948,7 +957,8 @@ class basic_json_document // unchanged const bool shrink_arena = d.arena.capacity() > d.arena.size(); std::string arena(shrink_arena ? d.arena : std::string()); - const bool shrink_tape = d.tape != d.inline_tape && d.tape_size != d.tape_cap; + // (edits link to the nodes of the index, which then stays in place) + const bool shrink_tape = d.tape != d.inline_tape && d.tape_size != d.tape_cap && d.edits == nullptr; const bool into_header = d.tape_size <= d.inline_cap; node* fresh = (shrink_tape && !into_header) ? static_cast(::operator new (d.tape_size * sizeof(node))) : d.inline_tape; @@ -998,6 +1008,8 @@ class basic_json_document d.src = src; d.size = size; d.tape_size = 0; + d.edits.reset(); // (views of the previous text end here anyway) + d.base[2] = nullptr; d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); @@ -1133,7 +1145,6 @@ using json_view = basic_json_view; using ordered_json_document = basic_json_document; /// a value of an ordered_json_document using ordered_json_view = basic_json_view; - NLOHMANN_JSON_NAMESPACE_END // tuple protocol for the items of basic_json_view::items() (structured bindings) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 24b00a70f..c8f447a97 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -84,6 +84,9 @@ #include // size_t #include // uint32_t #include // memcpy +#include // less +#include // map +#include // unique_ptr #include // operator new, placement new #include // string #include // vector @@ -199,20 +202,27 @@ static_assert(static_cast(value_t::null) == 0 && static_cast(n.kind) - 1u <= 1u; } +/// the value a link node stands for +NLOHMANN_VIEW_ALWAYS_INLINE const node* link_target(const node& n) noexcept +{ + const node* t = nullptr; + std::memcpy(&t, reinterpret_cast(&n) + 8, sizeof(t)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return t; +} + +inline void make_link(node& n, const node* target) noexcept +{ + n = node{}; + n.kind = kind_link; + std::memcpy(reinterpret_cast(&n) + 8, &target, sizeof(target)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +} + /// the converted value of an integer node (stored in len/next) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t integer_bits(const node& n) noexcept { @@ -293,9 +318,30 @@ struct document_data std::vector indexes{}; // NOLINT(readability-redundant-member-init) std::vector index_slots{}; // NOLINT(readability-redundant-member-init) std::vector large_objects{}; ///< positions of the objects to index (noted while parsing) // NOLINT(readability-redundant-member-init) - std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena (indexed by flags & node_flags::storage) + std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena (indexed by flags & node_flags::storage) bool discarded = true; + /// The storage of edits (editable documents only; see edit_storage.hpp). + /// Edits never move or resize the parsed index, so views stay valid: an + /// array/object whose elements change gets node_flags::moved, and its + /// elements then live in a separate sequence (a header node, then the + /// entries), whose entries link to the values. + struct edit_state + { + std::vector moved{}; ///< element sequences of moved arrays/objects (header node first) + std::vector moved_cap{}; ///< capacity in nodes of a growable block; 0: a fixed sequence (a new value) + std::vector> chunks{}; ///< storage of new values and blocks; never moved + std::map> regions{}; ///< new arrays/objects: root -> container that uses it as its element sequence (nullptr: linked from a block) + node* chunk_cur = nullptr; + node* chunk_end = nullptr; + std::size_t chunk_next = 64; + std::vector> texts{}; ///< edit arena, the current buffer last; earlier ones stay alive for string views + std::size_t text_used = 0; + std::size_t text_cap = 0; + std::size_t bytes = 0; ///< memory held by edits + }; + std::unique_ptr edits{}; ///< created by the first edit + /// one allocation for the header and room for `nodes` nodes; large /// documents get a separate node array instead (so it can be trimmed) static document_data* create(std::size_t nodes) @@ -379,6 +425,78 @@ struct document_data { return n + n->next; } + + /// (editable documents) first element or key, also of a moved container + NLOHMANN_VIEW_ALWAYS_INLINE const node* first_child_edited(const node* n) const noexcept + { + return NLOHMANN_VIEW_LIKELY((n->flags & node_flags::moved) == 0) ? n + 1 : edits->moved[n->off] + 1; + } + + /// (editable documents) end of the elements, also of a moved container + NLOHMANN_VIEW_ALWAYS_INLINE const node* child_end_edited(const node* n) const noexcept + { + if (NLOHMANN_VIEW_LIKELY((n->flags & node_flags::moved) == 0)) + { + return n + n->next; + } + const node* const h = edits->moved[n->off]; + return h + h->next; + } + + /// (editable documents) the value at an element position: entries of + /// moved sequences are links. The link case is out of line, so that this + /// compiles to a predicted branch rather than a select that delays the + /// following loads. + static NLOHMANN_VIEW_ALWAYS_INLINE const node* deref(const node* n) noexcept + { + return NLOHMANN_VIEW_LIKELY(n->kind != kind_link) ? n : follow_link(n); + } + + static NLOHMANN_VIEW_NOINLINE const node* follow_link(const node* n) noexcept + { + return link_target(*n); + } +}; + +/// How the index is walked: views of read-only documents follow the node +/// array alone and compile without any of the edit handling; views of +/// editable documents also follow moved element sequences and links. +template +struct navigation +{ + static NLOHMANN_VIEW_ALWAYS_INLINE const node* first(const document_data& /*d*/, const node* n) noexcept + { + return n + 1; + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* end(const document_data& /*d*/, const node* n) noexcept + { + return n + n->next; + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* value(const node* n) noexcept + { + return n; + } +}; + +template<> +struct navigation +{ + static NLOHMANN_VIEW_ALWAYS_INLINE const node* first(const document_data& d, const node* n) noexcept + { + return d.first_child_edited(n); + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* end(const document_data& d, const node* n) noexcept + { + return d.child_end_edited(n); + } + + static NLOHMANN_VIEW_ALWAYS_INLINE const node* value(const node* n) noexcept + { + return document_data::deref(n); + } }; } // namespace view @@ -2492,7 +2610,7 @@ class view_iterator NLOHMANN_VIEW_ALWAYS_INLINE View operator*() const noexcept { - return View(m_doc, m_pos + m_value_offset); + return View(m_doc, View::navigation::value(m_pos + m_value_offset)); } pointer operator->() const noexcept @@ -2904,18 +3022,20 @@ class short_key /// the key node of the first member of an object with the given key, or /// nullptr; most keys are rejected by their length, from the index alone -inline const node* find_member(const document_data& d, const node* object, const char* key, std::size_t n) noexcept +template +const node* find_member(const document_data& d, const node* object, const char* key, std::size_t n) noexcept { - if (NLOHMANN_VIEW_UNLIKELY(object->extra != 0)) + using nav = navigation; + if (NLOHMANN_VIEW_UNLIKELY(object->extra != 0) && (!Editable || (object->flags & node_flags::moved) == 0)) { - return find_indexed(d, object, key, n); // a large object + return find_indexed(d, object, key, n); // a large object (whose members have not been edited) } - const node* const end = document_data::child_end(object); + const node* const end = nav::end(d, object); const auto* const k = reinterpret_cast(key); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) if (NLOHMANN_VIEW_LIKELY(n <= 16)) { const short_key probe(k, n); - for (const node* m = document_data::first_child(object); m != end; m = document_data::after(m + 1)) + for (const node* m = nav::first(d, object); m != end; m = document_data::after(m + 1)) { if (m->len == n && probe.matches(reinterpret_cast(d.str(*m)))) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) { @@ -2924,7 +3044,7 @@ inline const node* find_member(const document_data& d, const node* object, const } return nullptr; } - for (const node* m = document_data::first_child(object); m != end; m = document_data::after(m + 1)) + for (const node* m = nav::first(d, object); m != end; m = document_data::after(m + 1)) { if (m->len == n && std::memcmp(d.str(*m), key, n) == 0) { @@ -2934,10 +3054,16 @@ inline const node* find_member(const document_data& d, const node* object, const return nullptr; } -/// the element of an array at an index below its size -inline const node* element_at(const node* array, std::size_t idx) noexcept +/// the entry of the element of an array at an index below its size (a link +/// in the moved sequences of editable documents) +template +const node* element_at(const document_data& d, const node* array, std::size_t idx) noexcept { - const node* e = document_data::first_child(array); + const node* e = navigation::first(d, array); + if (Editable && (array->flags & node_flags::moved) != 0 && d.edits->moved_cap[array->off] != 0) + { + return e + idx; // a growable block: one link per element + } for (std::size_t i = 0; i < idx; ++i) { e = document_data::after(e); @@ -2945,13 +3071,14 @@ inline const node* element_at(const node* array, std::size_t idx) noexcept return e; } -/// the last element of a non-empty array, or the key of the last member of a -/// non-empty object -inline const node* last_child(const node* container) noexcept +/// the entry of the last element of a non-empty array, or the key of the +/// last member of a non-empty object +template +const node* last_child(const document_data& d, const node* container) noexcept { const std::size_t value_offset = container->kind == static_cast(value_t::object) ? 1 : 0; - const node* const end = document_data::child_end(container); - const node* last = document_data::first_child(container); + const node* const end = navigation::end(d, container); + const node* last = navigation::first(d, container); for (const node* c = document_data::after(last + value_offset); c != end; c = document_data::after(c + value_offset)) { last = c; @@ -3003,6 +3130,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t #include // int64_t, uint64_t #include // memcpy +#include // numeric_limits #include // string #include // integral_constant, is_same @@ -3131,11 +3259,31 @@ NLOHMANN_VIEW_ALWAYS_INLINE double layout_double(const unsigned char* p, const u return negative ? -result : result; } +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + /// the value of the float token of a node, as parse() converts it; doubles /// with at most 19 digits are converted from the digit layout template FloatType float_value(const document_data& d, const node& n) { + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } return float_value(d, n, std::is_same {}); } @@ -3180,17 +3328,24 @@ iterative, so the nesting depth is limited by memory only, as for parse(). Without a lexer the handler records no source positions (JSON_DIAGNOSTIC_POSITIONS). */ -template +template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; using sax_t = json_sax_dom_parser>; + using nav = navigation; + + struct frame + { + const node* pos; ///< next element, or key of the next member + const node* end; + bool object; + }; BasicJsonType result; sax_t sax(result, true); const string_t no_token{}; - // the ends of the open containers, and whether they are objects - std::vector> open; + std::vector open; for (;;) { switch (static_cast(n->kind)) @@ -3207,67 +3362,66 @@ BasicJsonType materialize(const document_data& d, const node* n) { sax.start_array(n->len); } - open.emplace_back(document_data::child_end(n), object); - n = document_data::first_child(n); + open.push_back(frame{nav::first(d, n), nav::end(d, n), object}); break; } case value_t::string: { string_t s(d.str(*n), n->len); sax.string(s); - ++n; break; } case value_t::number_integer: sax.number_integer(static_cast(static_cast(integer_bits(*n)))); - ++n; break; case value_t::number_unsigned: sax.number_unsigned(static_cast(integer_bits(*n))); - ++n; break; case value_t::number_float: sax.number_float(float_value(d, *n), no_token); - ++n; break; case value_t::boolean: sax.boolean((n->flags & node_flags::is_true) != 0); - ++n; break; case value_t::null: case value_t::binary: case value_t::discarded: default: sax.null(); - ++n; break; } + + // the next value: close finished containers, then read the key for (;;) { if (open.empty()) { return result; } - if (n != open.back().first) + frame& f = open.back(); + if (f.pos == f.end) { - break; + if (f.object) + { + sax.end_object(); + } + else + { + sax.end_array(); + } + open.pop_back(); + continue; } - if (open.back().second) + const node* entry = f.pos; + if (f.object) { - sax.end_object(); + string_t key(d.str(*entry), entry->len); + sax.key(key); + ++entry; } - else - { - sax.end_array(); - } - open.pop_back(); - } - if (open.back().second) - { - // the key of the next member - string_t key(d.str(*n), n->len); - sax.key(key); - ++n; + n = nav::value(entry); + f.pos = document_data::after(entry); + break; } } } @@ -3577,9 +3731,10 @@ the library's conversion; integers are copied from the source, where they are canonical (except "-0", which parse() reads as 0). The walk is iterative, so the nesting depth is limited by memory only. */ -template +template class view_serializer { + using nav = navigation; using string_t = typename BasicJsonType::string_t; using number_float_t = typename BasicJsonType::number_float_t; @@ -3612,7 +3767,7 @@ class view_serializer else { m_out.put(object ? '{' : '['); - stack.push_back(frame{document_data::first_child(n), document_data::child_end(n), object, true}); + stack.push_back(frame{nav::first(m_doc, n), nav::end(m_doc, n), object, true}); } } else @@ -3654,13 +3809,14 @@ class view_serializer { m_out.put(':'); } - n = f.pos + 1; + n = nav::value(f.pos + 1); + f.pos = document_data::after(f.pos + 1); } else { - n = f.pos; + n = nav::value(f.pos); + f.pos = document_data::after(f.pos); } - f.pos = document_data::after(n); break; } } @@ -3712,9 +3868,9 @@ class view_serializer break; } case value_t::number_float: - if (m_style.source_numbers) + if (m_style.source_numbers && (n.flags & node_flags::storage) != node_flags::edited) { - m_out.put(m_doc.str(n), n.len); + m_out.put(m_doc.str(n), n.len); // (a float set by an edit is written as with shortest) } else { @@ -3761,9 +3917,9 @@ class view_serializer { const char* const s = m_doc.str(n); m_out.put('"'); - if ((n.flags & node_flags::escaped) == 0 && !m_style.ensure_ascii) + if ((n.flags & node_flags::storage) == 0 && !m_style.ensure_ascii) { - // a string without escape sequences has nothing to escape + // a string of the source without escape sequences has nothing to escape m_out.put(s, n.len); } else if (m_style.ensure_ascii) @@ -4112,7 +4268,7 @@ NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_BEGIN -template +template class basic_json_document; /*! @@ -4121,11 +4277,13 @@ class basic_json_document; Trivially copyable (two pointers). Valid as long as the document is alive and has not been re-parsed, and as long as a borrowed source text is alive. */ -template +template class basic_json_view { using node = detail::view::node; using document_data = detail::view::document_data; + /// how the index is walked (with edits only for editable documents) + using navigation = detail::view::navigation; public: using value_t = detail::value_t; @@ -4318,7 +4476,7 @@ class basic_json_view { detail::view::throw_type_error(305, "cannot use operator[] with a numeric argument with ", type_name()); } - return idx < m_node->len ? basic_json_view(m_doc, detail::view::element_at(m_node, idx)) : basic_json_view(); + return idx < m_node->len ? basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))) : basic_json_view(); } /// (an int argument would be ambiguous between size_type and const char*) @@ -4374,7 +4532,7 @@ class basic_json_view { detail::view::throw_out_of_range(401, detail::concat("array index ", std::to_string(idx), " is out of range")); } - return basic_json_view(m_doc, detail::view::element_at(m_node, idx)); + return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); } basic_json_view at(int idx) const @@ -4446,7 +4604,7 @@ class basic_json_view { if (is_structured() && m_node->len != 0) { - return basic_json_view(m_doc, detail::view::last_child(m_node) + (is_object() ? 1 : 0)); + return basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); } return front(); } @@ -4463,7 +4621,7 @@ class basic_json_view { return end(); } - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); return k != nullptr ? iterator(m_doc, k, true) : end(); } @@ -4480,7 +4638,7 @@ class basic_json_view /// whether this is an object with a member with this key bool contains(string_view_t key) const { - return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; + return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; } bool contains(const char* key) const @@ -4527,7 +4685,7 @@ class basic_json_view { if (NLOHMANN_VIEW_LIKELY(is_structured())) { - return iterator(m_doc, document_data::first_child(m_node), is_object()); + return iterator(m_doc, navigation::first(*m_doc, m_node), is_object()); } return iterator(m_doc, m_node, false); } @@ -4536,7 +4694,7 @@ class basic_json_view { if (NLOHMANN_VIEW_LIKELY(is_structured())) { - return iterator(m_doc, document_data::child_end(m_node), is_object()); + return iterator(m_doc, navigation::end(*m_doc, m_node), is_object()); } return iterator(m_doc, (is_null() || is_discarded()) ? m_node : m_node + 1, false); } @@ -4636,7 +4794,7 @@ class basic_json_view style.source_numbers = numbers == number_format::source; // the compact text is about as long as the source text of the value const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 64; - detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); return out; } @@ -4703,7 +4861,7 @@ class basic_json_view { return BasicJsonType(value_t::discarded); } - return detail::view::materialize(*m_doc, m_node); + return detail::view::materialize(*m_doc, m_node); } /// byte offset of this value in the source text (for strings: of the @@ -4711,12 +4869,13 @@ class basic_json_view /// a discarded view and for strings with escapes, which are decoded std::size_t source_offset() const noexcept { - return m_node != nullptr && (m_node->flags & detail::view::node_flags::storage) == 0 + return m_node != nullptr && (m_node->flags & (detail::view::node_flags::storage | detail::view::node_flags::moved | detail::view::node_flags::is_new)) == 0 ? m_node->off : static_cast(-1); } private: - template friend class basic_json_document; + template friend class basic_json_document; + template friend class basic_json_view; friend iterator; basic_json_view(const document_data* d, const node* n) noexcept @@ -4734,6 +4893,11 @@ class basic_json_view /// decoded strings) std::size_t source_extent() const noexcept { + if (Editable && m_doc->edits != nullptr) + { + // positions of moved and new values are not source offsets + return m_node == m_doc->tape ? m_doc->size + m_doc->edits->text_used : 64; + } const node* const next = document_data::after(m_node); const bool in_source = (m_node->flags & detail::view::node_flags::storage) == 0; if (!in_source) @@ -4751,8 +4915,8 @@ class basic_json_view /// (object required) NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view lookup(string_view_t key) const noexcept { - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); - return k != nullptr ? basic_json_view(m_doc, k + 1) : basic_json_view(); + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + return k != nullptr ? basic_json_view(m_doc, navigation::value(k + 1)) : basic_json_view(); } // --- get() dispatch --- @@ -4843,7 +5007,7 @@ Borrowed parses keep a pointer to the caller's text, which must outlive the document. Owned parses (parse_copy, rvalue std::string, streams, and inputs that are not contiguous byte ranges) keep their own copy. */ -template +template class basic_json_document { using document_data = detail::view::document_data; @@ -4852,7 +5016,7 @@ class basic_json_document "json_view supports 64-bit integer types only"); public: - using view_type = basic_json_view; + using view_type = basic_json_view; using value_t = detail::value_t; /// an empty (discarded) document @@ -4976,7 +5140,8 @@ class basic_json_document + (m_data->tape != m_data->inline_tape ? m_data->tape_cap * sizeof(detail::view::node) : 0) + m_data->arena.capacity() + m_data->owned.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) - + (m_data->large_objects.capacity() * sizeof(std::uint32_t)); + + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + + (m_data->edits != nullptr ? m_data->edits->bytes : 0); } /// release unused capacity of the index and the decoded strings; like @@ -4995,7 +5160,8 @@ class basic_json_document // unchanged const bool shrink_arena = d.arena.capacity() > d.arena.size(); std::string arena(shrink_arena ? d.arena : std::string()); - const bool shrink_tape = d.tape != d.inline_tape && d.tape_size != d.tape_cap; + // (edits link to the nodes of the index, which then stays in place) + const bool shrink_tape = d.tape != d.inline_tape && d.tape_size != d.tape_cap && d.edits == nullptr; const bool into_header = d.tape_size <= d.inline_cap; node* fresh = (shrink_tape && !into_header) ? static_cast(::operator new (d.tape_size * sizeof(node))) : d.inline_tape; @@ -5045,6 +5211,8 @@ class basic_json_document d.src = src; d.size = size; d.tape_size = 0; + d.edits.reset(); // (views of the previous text end here anyway) + d.base[2] = nullptr; d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); @@ -5180,7 +5348,6 @@ using json_view = basic_json_view; using ordered_json_document = basic_json_document; /// a value of an ordered_json_document using ordered_json_view = basic_json_view; - NLOHMANN_JSON_NAMESPACE_END // tuple protocol for the items of basic_json_view::items() (structured bindings)