diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 798bcc598..daaf30b17 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -87,6 +87,58 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) return n; } +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '\"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + if (!is_ascii_copyable(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + // Validate one UTF-8 sequence at the front of [data, data+avail). Returns its // length (2..4) only when the bytes form a *well-formed* sequence using exactly // the same ranges as scan_string()'s per-byte switch, so the bulk path accepts diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index a81bab0d1..7d6b3d2ce 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -16,6 +16,7 @@ #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy #include // numeric_limits #include // string, char_traits #include // setfill, setw @@ -110,6 +111,25 @@ class serializer const bool ensure_ascii, const unsigned int indent_step, const unsigned int current_indent = 0) + { + dump_internal(val, pretty_print, ensure_ascii, indent_step, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief recursive worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + */ + void dump_internal(const BasicJsonType& val, + const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) { switch (val.m_data.m_type) { @@ -117,13 +137,13 @@ class serializer { if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_chars("{}", 2); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_chars("{\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -136,51 +156,51 @@ class serializer auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); + put_chars(indent_string.c_str(), new_indent); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_chars("\": ", 3); + dump_internal(i->second, true, ensure_ascii, indent_step, new_indent); + put_chars(",\n", 2); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); + put_chars(indent_string.c_str(), new_indent); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_chars("\": ", 3); + dump_internal(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_chars("\":", 2); + dump_internal(i->second, false, ensure_ascii, indent_step, current_indent); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_chars("\":", 2); + dump_internal(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character('}'); + put_char('}'); } return; @@ -190,13 +210,13 @@ class serializer { if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_chars("[]", 2); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_chars("[\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -209,37 +229,37 @@ class serializer for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_chars(indent_string.c_str(), new_indent); + dump_internal(*i, true, ensure_ascii, indent_step, new_indent); + put_chars(",\n", 2); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_chars(indent_string.c_str(), new_indent); + dump_internal(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, false, ensure_ascii, indent_step, current_indent); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); - o->write_character(']'); + put_char(']'); } return; @@ -247,9 +267,9 @@ class serializer case value_t::string: { - o->write_character('\"'); + put_char('\"'); dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('\"'); return; } @@ -257,7 +277,7 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_chars("{\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -266,9 +286,9 @@ class serializer indent_string.resize(indent_string.size() * 2, ' '); } - o->write_characters(indent_string.c_str(), new_indent); + put_chars(indent_string.c_str(), new_indent); - o->write_characters("\"bytes\": [", 10); + put_chars("\"bytes\": [", 10); if (!val.m_data.m_value.binary->empty()) { @@ -276,30 +296,30 @@ class serializer i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); - o->write_characters(", ", 2); + put_chars(", ", 2); } dump_integer(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_chars("],\n", 3); + put_chars(indent_string.c_str(), new_indent); - o->write_characters("\"subtype\": ", 11); + put_chars("\"subtype\": ", 11); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_chars("null", 4); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_chars("{\"bytes\":[", 10); if (!val.m_data.m_value.binary->empty()) { @@ -307,20 +327,20 @@ class serializer i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); - o->write_character(','); + put_char(','); } dump_integer(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_chars("],\"subtype\":", 12); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_chars("null}", 5); } } return; @@ -330,11 +350,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_chars("true", 4); } else { - o->write_characters("false", 5); + put_chars("false", 5); } return; } @@ -359,13 +379,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_chars("", 11); return; } case value_t::null: { - o->write_characters("null", 4); + put_chars("null", 4); return; } @@ -401,28 +421,35 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { - // Fast path: when not escaping non-ASCII characters and sitting on a - // character boundary (state == UTF8_ACCEPT), bulk-copy the longest - // run of bytes that need no escaping. string_bulk_run() (shared with - // the lexer's contiguous scanner) stops exactly at the first byte - // that dump_escaped would handle individually - a quote, a backslash, - // a control character (< 0x20), or an ill-formed/truncated UTF-8 - // sequence - so that byte is left to the byte-at-a-time path below, - // keeping error handling and diagnostics unchanged. - if (!ensure_ascii && state == UTF8_ACCEPT) + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - ensure_ascii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - ensure_ascii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) { const auto* const data = reinterpret_cast(s.data()); - const std::size_t run = string_bulk_run(data + i, s.size() - i); + const std::size_t run = ensure_ascii + ? find_ascii_copyable_run(data + i, s.size() - i) + : string_bulk_run(data + i, s.size() - i); if (run != 0) { // emit any bytes still pending in string_buffer first to // preserve output order, then write the run directly if (bytes != 0) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } - o->write_characters(s.data() + i, run); + put_chars(s.data() + i, run); bytes_after_last_accept = 0; undumped_chars = 0; i += run; @@ -521,7 +548,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } @@ -580,7 +607,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } @@ -619,7 +646,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); } } else @@ -635,22 +662,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_chars(string_buffer.data(), bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_chars(string_buffer.data(), bytes_after_last_accept); // add a replacement character if (ensure_ascii) { - o->write_characters("\\ufffd", 6); + put_chars("\\ufffd", 6); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_chars("\xEF\xBF\xBD", 3); } break; } @@ -662,6 +689,60 @@ class serializer } private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a length characters to the write buffer + + Runs that do not fit the buffer are written straight through the output + adapter (after flushing what is pending), so large string/number payloads + are not copied an extra time. + */ + JSON_HEDLEY_NON_NULL(2) + void put_chars(const char* s, std::size_t length) + { + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + /*! @brief count digits @@ -785,7 +866,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -838,7 +919,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_chars(number_buffer.data(), n_chars); } /*! @@ -854,7 +935,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_chars("null", 4); return; } @@ -875,7 +956,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_chars(begin, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -926,7 +1007,7 @@ class serializer } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_chars(number_buffer.data(), static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -938,7 +1019,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_chars(".0", 2); } } @@ -1048,6 +1129,12 @@ class serializer /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a0f35195e..a08320d33 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8177,6 +8177,58 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) return n; } +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '\"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + if (!is_ascii_copyable(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + // Validate one UTF-8 sequence at the front of [data, data+avail). Returns its // length (2..4) only when the bytes form a *well-formed* sequence using exactly // the same ranges as scan_string()'s per-byte switch, so the bulk path accepts @@ -19419,6 +19471,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy #include // numeric_limits #include // string, char_traits #include // setfill, setw @@ -20641,6 +20694,25 @@ class serializer const bool ensure_ascii, const unsigned int indent_step, const unsigned int current_indent = 0) + { + dump_internal(val, pretty_print, ensure_ascii, indent_step, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief recursive worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + */ + void dump_internal(const BasicJsonType& val, + const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) { switch (val.m_data.m_type) { @@ -20648,13 +20720,13 @@ class serializer { if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_chars("{}", 2); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_chars("{\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -20667,51 +20739,51 @@ class serializer auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); + put_chars(indent_string.c_str(), new_indent); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_chars("\": ", 3); + dump_internal(i->second, true, ensure_ascii, indent_step, new_indent); + put_chars(",\n", 2); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); + put_chars(indent_string.c_str(), new_indent); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_chars("\": ", 3); + dump_internal(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_chars("\":", 2); + dump_internal(i->second, false, ensure_ascii, indent_step, current_indent); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); + put_char('\"'); dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_chars("\":", 2); + dump_internal(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character('}'); + put_char('}'); } return; @@ -20721,13 +20793,13 @@ class serializer { if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_chars("[]", 2); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_chars("[\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -20740,37 +20812,37 @@ class serializer for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_chars(indent_string.c_str(), new_indent); + dump_internal(*i, true, ensure_ascii, indent_step, new_indent); + put_chars(",\n", 2); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_chars(indent_string.c_str(), new_indent); + dump_internal(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, false, ensure_ascii, indent_step, current_indent); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); - o->write_character(']'); + put_char(']'); } return; @@ -20778,9 +20850,9 @@ class serializer case value_t::string: { - o->write_character('\"'); + put_char('\"'); dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('\"'); return; } @@ -20788,7 +20860,7 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_chars("{\n", 2); // variable to hold indentation for recursive calls const auto new_indent = current_indent + indent_step; @@ -20797,9 +20869,9 @@ class serializer indent_string.resize(indent_string.size() * 2, ' '); } - o->write_characters(indent_string.c_str(), new_indent); + put_chars(indent_string.c_str(), new_indent); - o->write_characters("\"bytes\": [", 10); + put_chars("\"bytes\": [", 10); if (!val.m_data.m_value.binary->empty()) { @@ -20807,30 +20879,30 @@ class serializer i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); - o->write_characters(", ", 2); + put_chars(", ", 2); } dump_integer(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_chars("],\n", 3); + put_chars(indent_string.c_str(), new_indent); - o->write_characters("\"subtype\": ", 11); + put_chars("\"subtype\": ", 11); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_chars("null", 4); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_chars(indent_string.c_str(), current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_chars("{\"bytes\":[", 10); if (!val.m_data.m_value.binary->empty()) { @@ -20838,20 +20910,20 @@ class serializer i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); - o->write_character(','); + put_char(','); } dump_integer(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_chars("],\"subtype\":", 12); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_chars("null}", 5); } } return; @@ -20861,11 +20933,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_chars("true", 4); } else { - o->write_characters("false", 5); + put_chars("false", 5); } return; } @@ -20890,13 +20962,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_chars("", 11); return; } case value_t::null: { - o->write_characters("null", 4); + put_chars("null", 4); return; } @@ -20932,28 +21004,35 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { - // Fast path: when not escaping non-ASCII characters and sitting on a - // character boundary (state == UTF8_ACCEPT), bulk-copy the longest - // run of bytes that need no escaping. string_bulk_run() (shared with - // the lexer's contiguous scanner) stops exactly at the first byte - // that dump_escaped would handle individually - a quote, a backslash, - // a control character (< 0x20), or an ill-formed/truncated UTF-8 - // sequence - so that byte is left to the byte-at-a-time path below, - // keeping error handling and diagnostics unchanged. - if (!ensure_ascii && state == UTF8_ACCEPT) + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - ensure_ascii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - ensure_ascii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) { const auto* const data = reinterpret_cast(s.data()); - const std::size_t run = string_bulk_run(data + i, s.size() - i); + const std::size_t run = ensure_ascii + ? find_ascii_copyable_run(data + i, s.size() - i) + : string_bulk_run(data + i, s.size() - i); if (run != 0) { // emit any bytes still pending in string_buffer first to // preserve output order, then write the run directly if (bytes != 0) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } - o->write_characters(s.data() + i, run); + put_chars(s.data() + i, run); bytes_after_last_accept = 0; undumped_chars = 0; i += run; @@ -21052,7 +21131,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } @@ -21111,7 +21190,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); bytes = 0; } @@ -21150,7 +21229,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_chars(string_buffer.data(), bytes); } } else @@ -21166,22 +21245,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_chars(string_buffer.data(), bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_chars(string_buffer.data(), bytes_after_last_accept); // add a replacement character if (ensure_ascii) { - o->write_characters("\\ufffd", 6); + put_chars("\\ufffd", 6); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_chars("\xEF\xBF\xBD", 3); } break; } @@ -21193,6 +21272,60 @@ class serializer } private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a length characters to the write buffer + + Runs that do not fit the buffer are written straight through the output + adapter (after flushing what is pending), so large string/number payloads + are not copied an extra time. + */ + JSON_HEDLEY_NON_NULL(2) + void put_chars(const char* s, std::size_t length) + { + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + /*! @brief count digits @@ -21316,7 +21449,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -21369,7 +21502,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_chars(number_buffer.data(), n_chars); } /*! @@ -21385,7 +21518,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_chars("null", 4); return; } @@ -21406,7 +21539,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_chars(begin, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -21457,7 +21590,7 @@ class serializer } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_chars(number_buffer.data(), static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -21469,7 +21602,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_chars(".0", 2); } } @@ -21579,6 +21712,12 @@ class serializer /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index f55ed8470..983782dcf 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -382,3 +382,87 @@ TEST_CASE("dump for basic_json with long double number_float_t") check_same(100.0L, 100.0); } } + +TEST_CASE("serialization of strings (bulk fast path)") +{ + // These cases exercise the SWAR bulk-copy fast path in dump_escaped and the + // internal write buffer: long runs, escapes interrupting runs, 0x7F/DEL, + // multibyte UTF-8 under both ensure_ascii settings, and payloads larger than + // the write buffer. + + SECTION("long unescaped ASCII exceeds the write buffer") + { + const std::string big(3000, 'a'); + const json j = big; + CHECK(j.dump() == '"' + big + '"'); + CHECK(j.dump(-1, ' ', true) == '"' + big + '"'); + // round-trips + CHECK(json::parse(j.dump()) == j); + } + + SECTION("runs interrupted by escapes") + { + const json j = std::string(500, 'x') + "\n\"\\" + std::string(500, 'y'); + const std::string out = j.dump(); + CHECK(out == '"' + std::string(500, 'x') + "\\n\\\"\\\\" + std::string(500, 'y') + '"'); + CHECK(json::parse(out) == j); + } + + SECTION("DEL (0x7F) depends on ensure_ascii") + { + const json j = std::string("a\x7f" "b"); + CHECK(j.dump(-1, ' ', false) == "\"a\x7f" "b\""); // copied verbatim + CHECK(j.dump(-1, ' ', true) == "\"a\\u007fb\""); // escaped + } + + SECTION("multibyte UTF-8 under both ensure_ascii settings") + { + const json j = std::string("A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z"); // A é 你 😀 Z + // not escaping non-ASCII: bytes are copied through the bulk validator + CHECK(j.dump(-1, ' ', false) == "\"A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z\""); + // ensure_ascii: escaped (with a surrogate pair for the emoji) + CHECK(j.dump(-1, ' ', true) == "\"A\\u00e9\\u4f60\\ud83d\\ude00Z\""); + CHECK(json::parse(j.dump(-1, ' ', true)) == j); + } + + SECTION("many small structural writes exceed the write buffer") + { + json arr = json::array(); + for (int i = 0; i < 2000; ++i) + { + arr.push_back(i); + } + const std::string out = arr.dump(); + CHECK(out.front() == '['); + CHECK(out.back() == ']'); + CHECK(json::parse(out) == arr); + + json obj = json::object(); + for (int i = 0; i < 500; ++i) + { + obj["key" + std::to_string(i)] = i; + } + CHECK(json::parse(obj.dump()) == obj); + CHECK(json::parse(obj.dump(2)) == obj); + + // deep nesting emits >1024 consecutive single-character writes, forcing + // the write buffer to flush mid-run + json nested = json::array(); + for (int i = 0; i < 1100; ++i) + { + nested = json::array({nested}); + } + const std::string out2 = nested.dump(); + CHECK(out2.substr(0, 1100) == std::string(1100, '[')); + CHECK(json::parse(out2) == nested); + } + + SECTION("invalid UTF-8 handling is unaffected by the fast path") + { + const json j = std::string("valid\xff" "more"); + CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\""); + } +}