mirror of
https://github.com/nlohmann/json.git
synced 2026-07-25 21:23:02 +04:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Claude
|
4190ebbee3 | ||
|
Claude
|
49ee76d891 |
@@ -15,14 +15,6 @@ guidance.
|
||||
|
||||
For vulnerabilities in third-party dependencies or modules, please report them directly to the respective maintainers.
|
||||
|
||||
## Unofficial packages
|
||||
|
||||
This project does not publish an official npm package. The npm package
|
||||
[`nlohmann-json`](https://www.npmjs.com/package/nlohmann-json) (or similarly named packages) is not maintained or
|
||||
endorsed by this project. See the
|
||||
[package managers documentation](https://json.nlohmann.me/integration/package_managers/#npm) for supported
|
||||
integration options.
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- Explore security-related topics and contribute to tools and projects through
|
||||
|
||||
@@ -930,12 +930,6 @@ If you are using [CocoaPods](https://cocoapods.org), you can use the library by
|
||||
to your podfile (see [an example](https://bitbucket.org/benman/nlohmann_json-cocoapod/src/master/)). Please file issues
|
||||
[here](https://bitbucket.org/benman/nlohmann_json-cocoapod/issues?status=new&status=open).
|
||||
|
||||
## npm
|
||||
|
||||
This project does not publish an official [npm](https://www.npmjs.com) package. The npm package
|
||||
[`nlohmann-json`](https://www.npmjs.com/package/nlohmann-json) (or similarly named packages) is not maintained or
|
||||
endorsed by this project. Use one of the package managers listed above, or integrate the single header directly.
|
||||
|
||||
## ESP-IDF and PlatformIO
|
||||
|
||||
There is no official package published to the [ESP-IDF Component Registry](https://components.espressif.com) or the
|
||||
|
||||
@@ -163,39 +163,12 @@ class binary_reader
|
||||
// BSON //
|
||||
//////////
|
||||
|
||||
/*!
|
||||
@brief Validate a BSON document's declared size against the bytes read.
|
||||
|
||||
A BSON document starts with an int32 that counts its own total length in
|
||||
bytes, including that prefix and the trailing 0x00. The reader is driven
|
||||
by the terminator rather than the declared length, so without this check a
|
||||
nested document could declare a length that disagrees with where its
|
||||
terminator actually falls and quietly hand the bytes in between to the
|
||||
enclosing document. A well-formed document is at least 5 bytes (the prefix
|
||||
plus the terminator); the equality also rejects those impossible sizes,
|
||||
since at least 5 bytes are always consumed.
|
||||
|
||||
@param[in] document_start value of chars_read before the size prefix
|
||||
@param[in] document_size the declared document size
|
||||
@return whether the declared size matches the number of bytes read
|
||||
*/
|
||||
bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
|
||||
{
|
||||
return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,
|
||||
exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief Reads in a BSON-object and passes it to the SAX-parser.
|
||||
@return whether a valid BSON-value was passed to the SAX parser
|
||||
*/
|
||||
bool parse_bson_internal()
|
||||
{
|
||||
const std::size_t document_start = chars_read;
|
||||
std::int32_t document_size{};
|
||||
get_number<std::int32_t, true>(input_format_t::bson, document_size);
|
||||
|
||||
@@ -209,11 +182,6 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return sax->end_object();
|
||||
}
|
||||
|
||||
@@ -429,7 +397,6 @@ class binary_reader
|
||||
*/
|
||||
bool parse_bson_array()
|
||||
{
|
||||
const std::size_t document_start = chars_read;
|
||||
std::int32_t document_size{};
|
||||
get_number<std::int32_t, true>(input_format_t::bson, document_size);
|
||||
|
||||
@@ -443,11 +410,6 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return sax->end_array();
|
||||
}
|
||||
|
||||
@@ -1884,29 +1846,6 @@ class binary_reader
|
||||
return get_ubjson_value(get_char ? get_ignore_noop() : current);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reject a negative UBJSON/BJData string length
|
||||
|
||||
String and key lengths are written with signed integer markers (i, I, l,
|
||||
L). A negative value is malformed; without this check get_string() would
|
||||
silently treat it as an empty string and leave the following bytes to be
|
||||
misread as the next value. This mirrors the non-negative check the
|
||||
optimized-container count path already performs in get_ubjson_size_value.
|
||||
|
||||
@param[in] len the string length read from the input
|
||||
@return whether the length is valid (non-negative)
|
||||
*/
|
||||
template<typename NumberType>
|
||||
bool check_ubjson_string_length(const NumberType len)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(len < 0))
|
||||
{
|
||||
return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
|
||||
exception_message(input_format, "string length must not be negative", "string"), nullptr));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reads a UBJSON string
|
||||
|
||||
@@ -1944,25 +1883,25 @@ class binary_reader
|
||||
case 'i':
|
||||
{
|
||||
std::int8_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'I':
|
||||
{
|
||||
std::int16_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'l':
|
||||
{
|
||||
std::int32_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'L':
|
||||
{
|
||||
std::int64_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'u':
|
||||
|
||||
@@ -231,6 +231,33 @@ class iterator_input_adapter
|
||||
std::is_same<IteratorType, SentinelType>::value && std::is_pointer<IteratorType>::value;
|
||||
#endif
|
||||
|
||||
public:
|
||||
// Whether the remaining input is a single contiguous block of 1-byte
|
||||
// elements that the lexer can inspect directly (used for the SWAR string
|
||||
// fast path). Restricted to same-type iterator/sentinel pairs so that plain
|
||||
// std::distance/std::advance are well-defined in all standards.
|
||||
static constexpr bool supports_bulk_scan =
|
||||
iterator_is_contiguous && std::is_same<IteratorType, SentinelType>::value && sizeof(char_type) == 1;
|
||||
|
||||
// Pointer to the next unread element; only valid when bulk_remaining() > 0.
|
||||
const char_type* bulk_data() const
|
||||
{
|
||||
return &*current;
|
||||
}
|
||||
|
||||
// Number of unread elements available as one contiguous block.
|
||||
std::size_t bulk_remaining() const
|
||||
{
|
||||
return static_cast<std::size_t>(std::distance(current, end));
|
||||
}
|
||||
|
||||
// Consume @a n elements previously inspected via bulk_data().
|
||||
void bulk_skip(std::size_t n)
|
||||
{
|
||||
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(n));
|
||||
}
|
||||
|
||||
private:
|
||||
// contiguous fast path: bulk copy the remaining range with std::memcpy
|
||||
template<class T>
|
||||
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
|
||||
|
||||
@@ -11,9 +11,12 @@
|
||||
#include <array> // array
|
||||
#include <clocale> // localeconv
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstdio> // snprintf
|
||||
#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
|
||||
#include <cstring> // memcpy
|
||||
#include <initializer_list> // initializer_list
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <utility> // move
|
||||
#include <vector> // vector
|
||||
@@ -125,6 +128,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Detect whether an input adapter exposes a contiguous byte block that the
|
||||
// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan).
|
||||
// Adapters without the flag - file, stream, wide-string, user-defined - fall
|
||||
// back to the character-at-a-time string scanner.
|
||||
template<typename InputAdapterType>
|
||||
using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan);
|
||||
|
||||
template<typename InputAdapterType>
|
||||
constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/)
|
||||
{
|
||||
return InputAdapterType::supports_bulk_scan;
|
||||
}
|
||||
|
||||
template<typename InputAdapterType>
|
||||
constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief lexical analysis
|
||||
|
||||
@@ -146,6 +168,14 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
static constexpr bool lazy_token_string =
|
||||
input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
|
||||
|
||||
/// whether string scanning may bulk-consume runs of ordinary characters
|
||||
/// directly from a contiguous input buffer (SWAR fast path). This requires
|
||||
/// the token to be reconstructible lazily (lazy_token_string), so bypassing
|
||||
/// the per-character capture in get() cannot lose error diagnostics.
|
||||
static constexpr bool bulk_scan =
|
||||
lazy_token_string
|
||||
&& input_adapter_supports_bulk_scan<InputAdapterType>(is_detected<detect_supports_bulk_scan, InputAdapterType> {});
|
||||
|
||||
public:
|
||||
using token_type = typename lexer_base<BasicJsonType>::token_type;
|
||||
|
||||
@@ -265,6 +295,92 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
return true;
|
||||
}
|
||||
|
||||
// classify a single byte as needing individual string handling: the
|
||||
// closing quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\')
|
||||
// are copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
static bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v
|
||||
// that is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
static std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n),
|
||||
// or n if every byte is ordinary; scans 8 bytes at a time
|
||||
static std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/// contiguous input: bulk-append the run of ordinary characters starting at
|
||||
/// the current read position, leaving the first special byte for get()
|
||||
void scan_string_bulk(std::true_type /*bulk*/)
|
||||
{
|
||||
// a pending unget must be consumed through the normal path first
|
||||
if (next_unget)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (remaining == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||
const std::size_t run = find_string_special(data, remaining);
|
||||
if (run == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), run);
|
||||
ia.bulk_skip(run);
|
||||
// the run contains no newline (all bytes < 0x20 are treated as special),
|
||||
// so only the flat character counters advance
|
||||
position.chars_read_total += run;
|
||||
position.chars_read_current_line += run;
|
||||
}
|
||||
|
||||
/// streaming input: no bulk fast path
|
||||
void scan_string_bulk(std::false_type /*bulk*/) const noexcept {}
|
||||
|
||||
/*!
|
||||
@brief scan a string literal
|
||||
|
||||
@@ -290,6 +406,10 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
|
||||
while (true)
|
||||
{
|
||||
// bulk-consume ordinary characters from contiguous input, then
|
||||
// handle the next special byte through the switch below
|
||||
scan_string_bulk(std::integral_constant<bool, bulk_scan> {});
|
||||
|
||||
// get the next character
|
||||
switch (get())
|
||||
{
|
||||
@@ -959,6 +1079,216 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
f = std::strtold(str, endptr);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated digit sequence
|
||||
|
||||
The scan_number() state machine has already checked that [first, last) is a
|
||||
valid JSON integer, so this only needs to accumulate the digits and detect
|
||||
overflow. This avoids the locale/errno machinery of std::strtoull, which
|
||||
dominates integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into number_unsigned_t; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching
|
||||
the previous std::strtoull behavior)
|
||||
*/
|
||||
static bool parse_integer_unsigned(const char* first, const char* last, number_unsigned_t& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<number_unsigned_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_unsigned_t
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
static bool parse_integer_signed(const char* first, const char* last, number_integer_t& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<number_integer_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_integer_t
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float
|
||||
and long double keep the std::strtof/std::strtold paths (see the templated
|
||||
overload below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
bool parse_float_fast(const char* first, const char* last, double& out) const noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<char_int_type>(c) == decimal_point_char)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief scan a number literal
|
||||
|
||||
@@ -1279,45 +1609,36 @@ scan_number_done:
|
||||
// we are done scanning a number)
|
||||
unget();
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
errno = 0;
|
||||
const char* const num_begin = token_buffer.data();
|
||||
const char* const num_end = num_begin + token_buffer.size();
|
||||
|
||||
// try to parse integers first and fall back to floats
|
||||
// try to parse integers first and fall back to floats; the digit
|
||||
// sequence has already been validated by the state machine above, so
|
||||
// a dedicated parser can avoid the locale/errno overhead of strtoull
|
||||
if (number_type == token_type::value_unsigned)
|
||||
{
|
||||
const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
if (errno != ERANGE)
|
||||
if (parse_integer_unsigned(num_begin, num_end, value_unsigned))
|
||||
{
|
||||
value_unsigned = static_cast<number_unsigned_t>(x);
|
||||
if (value_unsigned == x)
|
||||
{
|
||||
return token_type::value_unsigned;
|
||||
}
|
||||
return token_type::value_unsigned;
|
||||
}
|
||||
}
|
||||
else if (number_type == token_type::value_integer)
|
||||
{
|
||||
const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
if (errno != ERANGE)
|
||||
if (parse_integer_signed(num_begin, num_end, value_integer))
|
||||
{
|
||||
value_integer = static_cast<number_integer_t>(x);
|
||||
if (value_integer == x)
|
||||
{
|
||||
return token_type::value_integer;
|
||||
}
|
||||
return token_type::value_integer;
|
||||
}
|
||||
}
|
||||
|
||||
// this code is reached if we parse a floating-point number or if an
|
||||
// integer conversion above failed
|
||||
// integer conversion above overflowed. Try the exact fast path (double
|
||||
// only) before falling back to the locale-independent strtof/strtod.
|
||||
if (parse_float_fast(num_begin, num_end, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
// we checked the number format before
|
||||
|
||||
@@ -7218,6 +7218,33 @@ class iterator_input_adapter
|
||||
std::is_same<IteratorType, SentinelType>::value && std::is_pointer<IteratorType>::value;
|
||||
#endif
|
||||
|
||||
public:
|
||||
// Whether the remaining input is a single contiguous block of 1-byte
|
||||
// elements that the lexer can inspect directly (used for the SWAR string
|
||||
// fast path). Restricted to same-type iterator/sentinel pairs so that plain
|
||||
// std::distance/std::advance are well-defined in all standards.
|
||||
static constexpr bool supports_bulk_scan =
|
||||
iterator_is_contiguous && std::is_same<IteratorType, SentinelType>::value && sizeof(char_type) == 1;
|
||||
|
||||
// Pointer to the next unread element; only valid when bulk_remaining() > 0.
|
||||
const char_type* bulk_data() const
|
||||
{
|
||||
return &*current;
|
||||
}
|
||||
|
||||
// Number of unread elements available as one contiguous block.
|
||||
std::size_t bulk_remaining() const
|
||||
{
|
||||
return static_cast<std::size_t>(std::distance(current, end));
|
||||
}
|
||||
|
||||
// Consume @a n elements previously inspected via bulk_data().
|
||||
void bulk_skip(std::size_t n)
|
||||
{
|
||||
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(n));
|
||||
}
|
||||
|
||||
private:
|
||||
// contiguous fast path: bulk copy the remaining range with std::memcpy
|
||||
template<class T>
|
||||
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
|
||||
@@ -7705,9 +7732,12 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
#include <array> // array
|
||||
#include <clocale> // localeconv
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstdio> // snprintf
|
||||
#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
|
||||
#include <cstring> // memcpy
|
||||
#include <initializer_list> // initializer_list
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <utility> // move
|
||||
#include <vector> // vector
|
||||
@@ -7823,6 +7853,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Detect whether an input adapter exposes a contiguous byte block that the
|
||||
// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan).
|
||||
// Adapters without the flag - file, stream, wide-string, user-defined - fall
|
||||
// back to the character-at-a-time string scanner.
|
||||
template<typename InputAdapterType>
|
||||
using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan);
|
||||
|
||||
template<typename InputAdapterType>
|
||||
constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/)
|
||||
{
|
||||
return InputAdapterType::supports_bulk_scan;
|
||||
}
|
||||
|
||||
template<typename InputAdapterType>
|
||||
constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief lexical analysis
|
||||
|
||||
@@ -7844,6 +7893,14 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
static constexpr bool lazy_token_string =
|
||||
input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
|
||||
|
||||
/// whether string scanning may bulk-consume runs of ordinary characters
|
||||
/// directly from a contiguous input buffer (SWAR fast path). This requires
|
||||
/// the token to be reconstructible lazily (lazy_token_string), so bypassing
|
||||
/// the per-character capture in get() cannot lose error diagnostics.
|
||||
static constexpr bool bulk_scan =
|
||||
lazy_token_string
|
||||
&& input_adapter_supports_bulk_scan<InputAdapterType>(is_detected<detect_supports_bulk_scan, InputAdapterType> {});
|
||||
|
||||
public:
|
||||
using token_type = typename lexer_base<BasicJsonType>::token_type;
|
||||
|
||||
@@ -7963,6 +8020,92 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
return true;
|
||||
}
|
||||
|
||||
// classify a single byte as needing individual string handling: the
|
||||
// closing quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\')
|
||||
// are copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
static bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v
|
||||
// that is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
static std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n),
|
||||
// or n if every byte is ordinary; scans 8 bytes at a time
|
||||
static std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/// contiguous input: bulk-append the run of ordinary characters starting at
|
||||
/// the current read position, leaving the first special byte for get()
|
||||
void scan_string_bulk(std::true_type /*bulk*/)
|
||||
{
|
||||
// a pending unget must be consumed through the normal path first
|
||||
if (next_unget)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (remaining == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||
const std::size_t run = find_string_special(data, remaining);
|
||||
if (run == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), run);
|
||||
ia.bulk_skip(run);
|
||||
// the run contains no newline (all bytes < 0x20 are treated as special),
|
||||
// so only the flat character counters advance
|
||||
position.chars_read_total += run;
|
||||
position.chars_read_current_line += run;
|
||||
}
|
||||
|
||||
/// streaming input: no bulk fast path
|
||||
void scan_string_bulk(std::false_type /*bulk*/) const noexcept {}
|
||||
|
||||
/*!
|
||||
@brief scan a string literal
|
||||
|
||||
@@ -7988,6 +8131,10 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
|
||||
while (true)
|
||||
{
|
||||
// bulk-consume ordinary characters from contiguous input, then
|
||||
// handle the next special byte through the switch below
|
||||
scan_string_bulk(std::integral_constant<bool, bulk_scan> {});
|
||||
|
||||
// get the next character
|
||||
switch (get())
|
||||
{
|
||||
@@ -8657,6 +8804,216 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
f = std::strtold(str, endptr);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated digit sequence
|
||||
|
||||
The scan_number() state machine has already checked that [first, last) is a
|
||||
valid JSON integer, so this only needs to accumulate the digits and detect
|
||||
overflow. This avoids the locale/errno machinery of std::strtoull, which
|
||||
dominates integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into number_unsigned_t; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching
|
||||
the previous std::strtoull behavior)
|
||||
*/
|
||||
static bool parse_integer_unsigned(const char* first, const char* last, number_unsigned_t& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<number_unsigned_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_unsigned_t
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
static bool parse_integer_signed(const char* first, const char* last, number_integer_t& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<number_integer_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_integer_t
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float
|
||||
and long double keep the std::strtof/std::strtold paths (see the templated
|
||||
overload below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
bool parse_float_fast(const char* first, const char* last, double& out) const noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<char_int_type>(c) == decimal_point_char)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief scan a number literal
|
||||
|
||||
@@ -8977,45 +9334,36 @@ scan_number_done:
|
||||
// we are done scanning a number)
|
||||
unget();
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
errno = 0;
|
||||
const char* const num_begin = token_buffer.data();
|
||||
const char* const num_end = num_begin + token_buffer.size();
|
||||
|
||||
// try to parse integers first and fall back to floats
|
||||
// try to parse integers first and fall back to floats; the digit
|
||||
// sequence has already been validated by the state machine above, so
|
||||
// a dedicated parser can avoid the locale/errno overhead of strtoull
|
||||
if (number_type == token_type::value_unsigned)
|
||||
{
|
||||
const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
if (errno != ERANGE)
|
||||
if (parse_integer_unsigned(num_begin, num_end, value_unsigned))
|
||||
{
|
||||
value_unsigned = static_cast<number_unsigned_t>(x);
|
||||
if (value_unsigned == x)
|
||||
{
|
||||
return token_type::value_unsigned;
|
||||
}
|
||||
return token_type::value_unsigned;
|
||||
}
|
||||
}
|
||||
else if (number_type == token_type::value_integer)
|
||||
{
|
||||
const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
if (errno != ERANGE)
|
||||
if (parse_integer_signed(num_begin, num_end, value_integer))
|
||||
{
|
||||
value_integer = static_cast<number_integer_t>(x);
|
||||
if (value_integer == x)
|
||||
{
|
||||
return token_type::value_integer;
|
||||
}
|
||||
return token_type::value_integer;
|
||||
}
|
||||
}
|
||||
|
||||
// this code is reached if we parse a floating-point number or if an
|
||||
// integer conversion above failed
|
||||
// integer conversion above overflowed. Try the exact fast path (double
|
||||
// only) before falling back to the locale-independent strtof/strtod.
|
||||
if (parse_float_fast(num_begin, num_end, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
// we checked the number format before
|
||||
@@ -10712,39 +11060,12 @@ class binary_reader
|
||||
// BSON //
|
||||
//////////
|
||||
|
||||
/*!
|
||||
@brief Validate a BSON document's declared size against the bytes read.
|
||||
|
||||
A BSON document starts with an int32 that counts its own total length in
|
||||
bytes, including that prefix and the trailing 0x00. The reader is driven
|
||||
by the terminator rather than the declared length, so without this check a
|
||||
nested document could declare a length that disagrees with where its
|
||||
terminator actually falls and quietly hand the bytes in between to the
|
||||
enclosing document. A well-formed document is at least 5 bytes (the prefix
|
||||
plus the terminator); the equality also rejects those impossible sizes,
|
||||
since at least 5 bytes are always consumed.
|
||||
|
||||
@param[in] document_start value of chars_read before the size prefix
|
||||
@param[in] document_size the declared document size
|
||||
@return whether the declared size matches the number of bytes read
|
||||
*/
|
||||
bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
|
||||
{
|
||||
return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,
|
||||
exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief Reads in a BSON-object and passes it to the SAX-parser.
|
||||
@return whether a valid BSON-value was passed to the SAX parser
|
||||
*/
|
||||
bool parse_bson_internal()
|
||||
{
|
||||
const std::size_t document_start = chars_read;
|
||||
std::int32_t document_size{};
|
||||
get_number<std::int32_t, true>(input_format_t::bson, document_size);
|
||||
|
||||
@@ -10758,11 +11079,6 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return sax->end_object();
|
||||
}
|
||||
|
||||
@@ -10978,7 +11294,6 @@ class binary_reader
|
||||
*/
|
||||
bool parse_bson_array()
|
||||
{
|
||||
const std::size_t document_start = chars_read;
|
||||
std::int32_t document_size{};
|
||||
get_number<std::int32_t, true>(input_format_t::bson, document_size);
|
||||
|
||||
@@ -10992,11 +11307,6 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return sax->end_array();
|
||||
}
|
||||
|
||||
@@ -12433,29 +12743,6 @@ class binary_reader
|
||||
return get_ubjson_value(get_char ? get_ignore_noop() : current);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reject a negative UBJSON/BJData string length
|
||||
|
||||
String and key lengths are written with signed integer markers (i, I, l,
|
||||
L). A negative value is malformed; without this check get_string() would
|
||||
silently treat it as an empty string and leave the following bytes to be
|
||||
misread as the next value. This mirrors the non-negative check the
|
||||
optimized-container count path already performs in get_ubjson_size_value.
|
||||
|
||||
@param[in] len the string length read from the input
|
||||
@return whether the length is valid (non-negative)
|
||||
*/
|
||||
template<typename NumberType>
|
||||
bool check_ubjson_string_length(const NumberType len)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(len < 0))
|
||||
{
|
||||
return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
|
||||
exception_message(input_format, "string length must not be negative", "string"), nullptr));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reads a UBJSON string
|
||||
|
||||
@@ -12493,25 +12780,25 @@ class binary_reader
|
||||
case 'i':
|
||||
{
|
||||
std::int8_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'I':
|
||||
{
|
||||
std::int16_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'l':
|
||||
{
|
||||
std::int32_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'L':
|
||||
{
|
||||
std::int64_t len{};
|
||||
return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result);
|
||||
return get_number(input_format, len) && get_string(input_format, len, result);
|
||||
}
|
||||
|
||||
case 'u':
|
||||
|
||||
@@ -2721,19 +2721,6 @@ TEST_CASE("BJData")
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData string: expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x31", json::parse_error&);
|
||||
}
|
||||
|
||||
SECTION("negative length")
|
||||
{
|
||||
json _;
|
||||
|
||||
std::vector<uint8_t> const vi = {'S', 'i', 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vi), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_bjdata(vi, true, false).is_discarded());
|
||||
|
||||
std::vector<uint8_t> const vl = {'S', 'l', 0xFF, 0xFF, 0xFF, 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vl), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing BJData string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_bjdata(vl, true, false).is_discarded());
|
||||
}
|
||||
|
||||
SECTION("parse bjdata markers in ubjson")
|
||||
{
|
||||
// create a single-character string for all number types
|
||||
|
||||
@@ -854,62 +854,6 @@ TEST_CASE("Unsupported BSON input")
|
||||
CHECK(!json::sax_parse(bson, &scp, json::input_format_t::bson));
|
||||
}
|
||||
|
||||
TEST_CASE("BSON document size mismatch")
|
||||
{
|
||||
json _;
|
||||
|
||||
SECTION("top-level document declaring more bytes than it contains")
|
||||
{
|
||||
// empty object, but the length prefix claims 6 bytes instead of 5
|
||||
std::vector<std::uint8_t> const input = {0x06, 0x00, 0x00, 0x00, 0x00};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 6 does not match the number of bytes read (5)", json::parse_error&);
|
||||
CHECK(json::from_bson(input, true, false).is_discarded());
|
||||
}
|
||||
|
||||
SECTION("top-level document with a negative size")
|
||||
{
|
||||
std::vector<std::uint8_t> const input = {0xFF, 0xFF, 0xFF, 0xFF, 0x00};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size -1 does not match the number of bytes read (5)", json::parse_error&);
|
||||
CHECK(json::from_bson(input, true, false).is_discarded());
|
||||
}
|
||||
|
||||
SECTION("embedded document whose size disagrees with its terminator")
|
||||
{
|
||||
// the embedded document "d" declares 0x7FFFFFFF bytes but its 0x00
|
||||
// terminator falls right after {"a":null}; the length prefix would
|
||||
// otherwise let the following "h" element be read as a member of the
|
||||
// enclosing document instead of "d"
|
||||
std::vector<std::uint8_t> const input =
|
||||
{
|
||||
0x00, 0x00, 0x00, 0x00, // outer size
|
||||
0x03, 'd', 0x00, // entry: embedded document "d"
|
||||
0xFF, 0xFF, 0xFF, 0x7F, // embedded size 0x7FFFFFFF
|
||||
0x0A, 'a', 0x00, // entry: null "a"
|
||||
0x00, // embedded end marker
|
||||
0x08, 'h', 0x00, 0x01, // entry: bool "h" = true
|
||||
0x00 // outer end marker
|
||||
};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.112] parse error at byte 15: syntax error while parsing BSON document: document size 2147483647 does not match the number of bytes read (8)", json::parse_error&);
|
||||
CHECK(json::from_bson(input, true, false).is_discarded());
|
||||
}
|
||||
|
||||
SECTION("embedded array whose size disagrees with its terminator")
|
||||
{
|
||||
// array [42] is 12 bytes, but the length prefix claims 13
|
||||
std::vector<std::uint8_t> const input =
|
||||
{
|
||||
0x00, 0x00, 0x00, 0x00, // outer size
|
||||
0x04, 'a', 0x00, // entry: array "a"
|
||||
0x0D, 0x00, 0x00, 0x00, // array size 13 (real is 12)
|
||||
0x10, '0', 0x00, 0x2A, 0x00, 0x00, 0x00, // entry: int32 "0" = 42
|
||||
0x00, // array end marker
|
||||
0x00 // outer end marker
|
||||
};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.112] parse error at byte 19: syntax error while parsing BSON document: document size 13 does not match the number of bytes read (12)", json::parse_error&);
|
||||
CHECK(json::from_bson(input, true, false).is_discarded());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("BSON numerical data")
|
||||
{
|
||||
SECTION("number")
|
||||
|
||||
@@ -1862,31 +1862,6 @@ TEST_CASE("UBJSON")
|
||||
json _;
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x31", json::parse_error&);
|
||||
}
|
||||
|
||||
SECTION("negative length")
|
||||
{
|
||||
json _;
|
||||
|
||||
std::vector<uint8_t> const vi = {'S', 'i', 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vi), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing UBJSON string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_ubjson(vi, true, false).is_discarded());
|
||||
|
||||
std::vector<uint8_t> const vI = {'S', 'I', 0xFF, 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vI), "[json.exception.parse_error.113] parse error at byte 4: syntax error while parsing UBJSON string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_ubjson(vI, true, false).is_discarded());
|
||||
|
||||
std::vector<uint8_t> const vl = {'S', 'l', 0xFF, 0xFF, 0xFF, 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vl), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing UBJSON string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_ubjson(vl, true, false).is_discarded());
|
||||
|
||||
std::vector<uint8_t> const vL = {'S', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vL), "[json.exception.parse_error.113] parse error at byte 10: syntax error while parsing UBJSON string: string length must not be negative", json::parse_error&);
|
||||
CHECK(json::from_ubjson(vL, true, false).is_discarded());
|
||||
|
||||
// a length of zero remains valid and yields an empty string
|
||||
std::vector<uint8_t> const v0 = {'S', 'i', 0};
|
||||
CHECK(json::from_ubjson(v0) == json(""));
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("array")
|
||||
|
||||
Reference in New Issue
Block a user