mirror of
https://github.com/nlohmann/json.git
synced 2026-07-21 19:23:03 +04:00
Move byte-level scan/parse helpers out of lexer.hpp
lexer.hpp had grown by ~600 lines of byte-level helpers that have no dependency on the lexer's template parameters and clutter the state machine. Move them, unchanged, into two focused headers as free functions in namespace detail: - number_parse.hpp: parse_integer_unsigned/parse_integer_signed (now templated on the number type) and parse_float_fast (Clinger's exact double fast path, with the decimal point passed as an argument instead of read from a lexer member). - string_scan.hpp: the SWAR string helpers (is_string_special, swar_string_special, find_string_special, validate_one_utf8, scalar_string_bulk_run) and the backend-dispatched string_bulk_run, including the optional simdutf include and find_string_delimiter. lexer.hpp now includes these and calls the free functions; the methods that touch lexer state (scan_string, scan_number, scan_string_bulk, scan_number_bulk_contiguous, convert_number) stay put. This is a pure code move with no behavior change: lexer.hpp drops from 2357 to 1934 lines, the now-unused <cstdint>/<cstring>/<limits> includes are removed, and the free-function form makes the SWAR helpers reusable elsewhere (e.g. the serializer's string escaping). Verified: default and JSON_USE_SIMDUTF builds compile; 2,000,000 number and 2,000,000 arbitrary-byte-string differential-fuzz documents parse identically to before; lexer/parser/conversions/deserialization/locale/ diagnostic-position suites pass (20,576 assertions); warning-clean on g++ and clang in C++11/17/20; the amalgamation regenerates and passes check-amalgamation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
@@ -11,27 +11,17 @@
|
||||
#include <array> // array
|
||||
#include <clocale> // localeconv
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstdio> // snprintf
|
||||
#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
|
||||
#include <cstring> // memcpy
|
||||
#include <initializer_list> // initializer_list
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <utility> // move
|
||||
#include <vector> // vector
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in
|
||||
// external dependency: nlohmann/json itself stays header-only and the C++11
|
||||
// scalar validator below is always available; defining JSON_USE_SIMDUTF
|
||||
// additionally requires the simdutf headers on the include path and linking
|
||||
// the simdutf library. See scan_string_bulk().
|
||||
#include <simdutf.h>
|
||||
#endif
|
||||
|
||||
#include <nlohmann/detail/input/input_adapters.hpp>
|
||||
#include <nlohmann/detail/input/number_parse.hpp>
|
||||
#include <nlohmann/detail/input/position_t.hpp>
|
||||
#include <nlohmann/detail/input/string_scan.hpp>
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
#include <nlohmann/detail/meta/type_traits.hpp>
|
||||
|
||||
@@ -304,209 +294,6 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
return true;
|
||||
}
|
||||
|
||||
// classify a single byte as needing individual string handling: the
|
||||
// closing quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\')
|
||||
// are copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
static bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v
|
||||
// that is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
static std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n),
|
||||
// or n if every byte is ordinary; scans 8 bytes at a time
|
||||
static std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// Validate one UTF-8 sequence at the front of [data, data+avail). Returns
|
||||
// its length (2..4) only when the bytes form a *well-formed* sequence using
|
||||
// exactly the same ranges as scan_string()'s per-byte switch, so the bulk
|
||||
// path accepts precisely what the byte path accepts. Returns 0 for anything
|
||||
// that is invalid, incomplete, or that the byte path must diagnose (the
|
||||
// caller then defers to that path, keeping error messages unchanged). Lead
|
||||
// bytes < 0x80 are handled by the caller and never passed here.
|
||||
static std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept
|
||||
{
|
||||
const unsigned char c0 = data[0];
|
||||
// helper: a continuation byte within [lo, hi]
|
||||
// (kept inline to stay noexcept and C++11-friendly)
|
||||
if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF
|
||||
{
|
||||
if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xE0) // U+0800..U+0FFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates)
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF0) // U+10000..U+3FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF4) // U+100000..U+10FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return 0; // invalid, incomplete, or must be diagnosed by the byte path
|
||||
}
|
||||
|
||||
// Scalar (C++11) computation of the bulk run length: the number of leading
|
||||
// bytes in [data, data+n) that are ordinary ASCII or complete well-formed
|
||||
// UTF-8 sequences, stopping before the first byte that needs individual
|
||||
// handling (the closing quote, an escape, a control character, or an
|
||||
// ill-formed/truncated sequence). ASCII is skipped 8 bytes at a time.
|
||||
static std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t pos = 0;
|
||||
while (pos < n)
|
||||
{
|
||||
pos += find_string_special(data + pos, n - pos);
|
||||
if (pos >= n || data[pos] < 0x80u)
|
||||
{
|
||||
break; // end of buffer, or a quote/escape/control byte
|
||||
}
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
{
|
||||
break; // ill-formed or truncated: let the byte path diagnose it
|
||||
}
|
||||
pos += seq;
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII
|
||||
// bytes are *not* stops here - the whole run is handed to simdutf), or n.
|
||||
static std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull;
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull;
|
||||
const std::uint64_t hit = ((q - ones) & ~q & high)
|
||||
| ((b - ones) & ~b & high)
|
||||
| ((v - 0x2020202020202020ull) & ~v & high);
|
||||
if (hit != 0)
|
||||
{
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
const unsigned char c = data[i + j];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
const unsigned char c = data[i];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the
|
||||
// next delimiter is validated in one shot by simdutf; on the rare failure the
|
||||
// scalar helper recomputes the exact valid prefix so the byte path still
|
||||
// produces the precise diagnostic. Without it, the pure scalar path is used.
|
||||
static std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
const std::size_t run = find_string_delimiter(data, n);
|
||||
if (run != 0 && simdutf::validate_utf8(reinterpret_cast<const char*>(data), run))
|
||||
{
|
||||
return run;
|
||||
}
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#else
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
/// contiguous input: bulk-append the run of ordinary characters and complete
|
||||
/// well-formed UTF-8 sequences starting at the current read position, leaving
|
||||
/// the first byte that needs individual handling (the closing quote, an
|
||||
@@ -1239,216 +1026,6 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
f = std::strtold(str, endptr);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated digit sequence
|
||||
|
||||
The scan_number() state machine has already checked that [first, last) is a
|
||||
valid JSON integer, so this only needs to accumulate the digits and detect
|
||||
overflow. This avoids the locale/errno machinery of std::strtoull, which
|
||||
dominates integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into number_unsigned_t; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching
|
||||
the previous std::strtoull behavior)
|
||||
*/
|
||||
static bool parse_integer_unsigned(const char* first, const char* last, number_unsigned_t& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<number_unsigned_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_unsigned_t
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
static bool parse_integer_signed(const char* first, const char* last, number_integer_t& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<number_integer_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_integer_t
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float
|
||||
and long double keep the std::strtof/std::strtold paths (see the templated
|
||||
overload below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
bool parse_float_fast(const char* first, const char* last, double& out) const noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<char_int_type>(c) == decimal_point_char)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief scan a number literal
|
||||
|
||||
@@ -1807,7 +1384,7 @@ scan_number_done:
|
||||
// this code is reached if we parse a floating-point number or if an
|
||||
// integer conversion above overflowed. Try the exact fast path (double
|
||||
// only) before falling back to the locale-independent strtof/strtod.
|
||||
if (parse_float_fast(num_begin, num_end, value_float))
|
||||
if (parse_float_fast(num_begin, num_end, decimal_point_char, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint> // int64_t, uint64_t
|
||||
#include <limits> // numeric_limits
|
||||
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
// This file contains the value-conversion helpers used by the lexer to turn an
|
||||
// already-validated number token into a value, without the locale/errno
|
||||
// overhead of std::strtoull/std::strtod. They are free functions so the lexer
|
||||
// stays focused on scanning; see lexer::convert_number().
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated unsigned integer
|
||||
|
||||
The number scanner has already checked that [first, last) is a valid JSON
|
||||
integer, so this only needs to accumulate the digits and detect overflow. This
|
||||
avoids the locale/errno machinery of std::strtoull, which dominates
|
||||
integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into @a NumberUnsignedType; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching the
|
||||
previous std::strtoull behavior)
|
||||
*/
|
||||
template<typename NumberUnsignedType>
|
||||
bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<NumberUnsignedType>(x);
|
||||
// reject values that do not round-trip into a narrower NumberUnsignedType
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
template<typename NumberIntegerType>
|
||||
bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<NumberIntegerType>(x);
|
||||
// reject values that do not round-trip into a narrower NumberIntegerType
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float and
|
||||
long double keep the std::strtof/std::strtold paths (see the templated overload
|
||||
below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[in] decimal_point the (locale-dependent) decimal point character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
template<typename DecimalPointType>
|
||||
bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<DecimalPointType>(c) == decimal_point)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename DecimalPointType, typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
@@ -0,0 +1,237 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstring> // memcpy
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in
|
||||
// external dependency: nlohmann/json itself stays header-only and the C++11
|
||||
// scalar validator below is always available; defining JSON_USE_SIMDUTF
|
||||
// additionally requires the simdutf headers on the include path and linking
|
||||
// the simdutf library. See string_bulk_run().
|
||||
#include <simdutf.h>
|
||||
#endif
|
||||
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
// This file contains the byte-level string-scanning helpers used by the lexer's
|
||||
// contiguous fast path. They operate purely on raw bytes (no dependency on the
|
||||
// lexer's template parameters) so they are free functions, keeping the lexer
|
||||
// itself focused on the state machine; see lexer::scan_string_bulk().
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
|
||||
// classify a single byte as needing individual string handling: the closing
|
||||
// quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are
|
||||
// copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
inline bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v that
|
||||
// is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
inline std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n), or
|
||||
// n if every byte is ordinary; scans 8 bytes at a time
|
||||
inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its
|
||||
// length (2..4) only when the bytes form a *well-formed* sequence using exactly
|
||||
// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts
|
||||
// precisely what the byte path accepts. Returns 0 for anything that is invalid,
|
||||
// incomplete, or that the byte path must diagnose (the caller then defers to
|
||||
// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled
|
||||
// by the caller and never passed here.
|
||||
inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept
|
||||
{
|
||||
const unsigned char c0 = data[0];
|
||||
if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF
|
||||
{
|
||||
if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xE0) // U+0800..U+0FFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates)
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF0) // U+10000..U+3FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF4) // U+100000..U+10FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return 0; // invalid, incomplete, or must be diagnosed by the byte path
|
||||
}
|
||||
|
||||
// Scalar (C++11) computation of the bulk run length: the number of leading
|
||||
// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8
|
||||
// sequences, stopping before the first byte that needs individual handling (the
|
||||
// closing quote, an escape, a control character, or an ill-formed/truncated
|
||||
// sequence). ASCII is skipped 8 bytes at a time.
|
||||
inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t pos = 0;
|
||||
while (pos < n)
|
||||
{
|
||||
pos += find_string_special(data + pos, n - pos);
|
||||
if (pos >= n || data[pos] < 0x80u)
|
||||
{
|
||||
break; // end of buffer, or a quote/escape/control byte
|
||||
}
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
{
|
||||
break; // ill-formed or truncated: let the byte path diagnose it
|
||||
}
|
||||
pos += seq;
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII
|
||||
// bytes are *not* stops here - the whole run is handed to simdutf), or n.
|
||||
inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull;
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull;
|
||||
const std::uint64_t hit = ((q - ones) & ~q & high)
|
||||
| ((b - ones) & ~b & high)
|
||||
| ((v - 0x2020202020202020ull) & ~v & high);
|
||||
if (hit != 0)
|
||||
{
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
const unsigned char c = data[i + j];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
const unsigned char c = data[i];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the
|
||||
// next delimiter is validated in one shot by simdutf; on the rare failure the
|
||||
// scalar helper recomputes the exact valid prefix so the byte path still
|
||||
// produces the precise diagnostic. Without it, the pure scalar path is used.
|
||||
inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
const std::size_t run = find_string_delimiter(data, n);
|
||||
if (run != 0 && simdutf::validate_utf8(reinterpret_cast<const char*>(data), run))
|
||||
{
|
||||
return run;
|
||||
}
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#else
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
+478
-420
@@ -7770,28 +7770,499 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
#include <array> // array
|
||||
#include <clocale> // localeconv
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstdio> // snprintf
|
||||
#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
|
||||
#include <cstring> // memcpy
|
||||
#include <initializer_list> // initializer_list
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <utility> // move
|
||||
#include <vector> // vector
|
||||
|
||||
// #include <nlohmann/detail/input/input_adapters.hpp>
|
||||
|
||||
// #include <nlohmann/detail/input/number_parse.hpp>
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
|
||||
#include <cstdint> // int64_t, uint64_t
|
||||
#include <limits> // numeric_limits
|
||||
|
||||
// #include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
|
||||
// This file contains the value-conversion helpers used by the lexer to turn an
|
||||
// already-validated number token into a value, without the locale/errno
|
||||
// overhead of std::strtoull/std::strtod. They are free functions so the lexer
|
||||
// stays focused on scanning; see lexer::convert_number().
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated unsigned integer
|
||||
|
||||
The number scanner has already checked that [first, last) is a valid JSON
|
||||
integer, so this only needs to accumulate the digits and detect overflow. This
|
||||
avoids the locale/errno machinery of std::strtoull, which dominates
|
||||
integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into @a NumberUnsignedType; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching the
|
||||
previous std::strtoull behavior)
|
||||
*/
|
||||
template<typename NumberUnsignedType>
|
||||
bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<NumberUnsignedType>(x);
|
||||
// reject values that do not round-trip into a narrower NumberUnsignedType
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
template<typename NumberIntegerType>
|
||||
bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<NumberIntegerType>(x);
|
||||
// reject values that do not round-trip into a narrower NumberIntegerType
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float and
|
||||
long double keep the std::strtof/std::strtold paths (see the templated overload
|
||||
below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[in] decimal_point the (locale-dependent) decimal point character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
template<typename DecimalPointType>
|
||||
bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<DecimalPointType>(c) == decimal_point)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename DecimalPointType, typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
// #include <nlohmann/detail/input/position_t.hpp>
|
||||
|
||||
// #include <nlohmann/detail/input/string_scan.hpp>
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstring> // memcpy
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in
|
||||
// external dependency: nlohmann/json itself stays header-only and the C++11
|
||||
// scalar validator below is always available; defining JSON_USE_SIMDUTF
|
||||
// additionally requires the simdutf headers on the include path and linking
|
||||
// the simdutf library. See scan_string_bulk().
|
||||
// the simdutf library. See string_bulk_run().
|
||||
#include <simdutf.h>
|
||||
#endif
|
||||
|
||||
// #include <nlohmann/detail/input/input_adapters.hpp>
|
||||
// #include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
// #include <nlohmann/detail/input/position_t.hpp>
|
||||
|
||||
// This file contains the byte-level string-scanning helpers used by the lexer's
|
||||
// contiguous fast path. They operate purely on raw bytes (no dependency on the
|
||||
// lexer's template parameters) so they are free functions, keeping the lexer
|
||||
// itself focused on the state machine; see lexer::scan_string_bulk().
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
|
||||
// classify a single byte as needing individual string handling: the closing
|
||||
// quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are
|
||||
// copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
inline bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v that
|
||||
// is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
inline std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n), or
|
||||
// n if every byte is ordinary; scans 8 bytes at a time
|
||||
inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its
|
||||
// length (2..4) only when the bytes form a *well-formed* sequence using exactly
|
||||
// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts
|
||||
// precisely what the byte path accepts. Returns 0 for anything that is invalid,
|
||||
// incomplete, or that the byte path must diagnose (the caller then defers to
|
||||
// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled
|
||||
// by the caller and never passed here.
|
||||
inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept
|
||||
{
|
||||
const unsigned char c0 = data[0];
|
||||
if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF
|
||||
{
|
||||
if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xE0) // U+0800..U+0FFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates)
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF0) // U+10000..U+3FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF4) // U+100000..U+10FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return 0; // invalid, incomplete, or must be diagnosed by the byte path
|
||||
}
|
||||
|
||||
// Scalar (C++11) computation of the bulk run length: the number of leading
|
||||
// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8
|
||||
// sequences, stopping before the first byte that needs individual handling (the
|
||||
// closing quote, an escape, a control character, or an ill-formed/truncated
|
||||
// sequence). ASCII is skipped 8 bytes at a time.
|
||||
inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t pos = 0;
|
||||
while (pos < n)
|
||||
{
|
||||
pos += find_string_special(data + pos, n - pos);
|
||||
if (pos >= n || data[pos] < 0x80u)
|
||||
{
|
||||
break; // end of buffer, or a quote/escape/control byte
|
||||
}
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
{
|
||||
break; // ill-formed or truncated: let the byte path diagnose it
|
||||
}
|
||||
pos += seq;
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII
|
||||
// bytes are *not* stops here - the whole run is handed to simdutf), or n.
|
||||
inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull;
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull;
|
||||
const std::uint64_t hit = ((q - ones) & ~q & high)
|
||||
| ((b - ones) & ~b & high)
|
||||
| ((v - 0x2020202020202020ull) & ~v & high);
|
||||
if (hit != 0)
|
||||
{
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
const unsigned char c = data[i + j];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
const unsigned char c = data[i];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the
|
||||
// next delimiter is validated in one shot by simdutf; on the rare failure the
|
||||
// scalar helper recomputes the exact valid prefix so the byte path still
|
||||
// produces the precise diagnostic. Without it, the pure scalar path is used.
|
||||
inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
const std::size_t run = find_string_delimiter(data, n);
|
||||
if (run != 0 && simdutf::validate_utf8(reinterpret_cast<const char*>(data), run))
|
||||
{
|
||||
return run;
|
||||
}
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#else
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
// #include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
@@ -8067,209 +8538,6 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
return true;
|
||||
}
|
||||
|
||||
// classify a single byte as needing individual string handling: the
|
||||
// closing quote, an escape, a control character, or a non-ASCII (UTF-8)
|
||||
// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\')
|
||||
// are copied verbatim, which the bulk scanner does 8 bytes at a time.
|
||||
static bool is_string_special(unsigned char c) noexcept
|
||||
{
|
||||
return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u;
|
||||
}
|
||||
|
||||
// SWAR helper: return a word whose high bit is set in every byte of @a v
|
||||
// that is_string_special(); zero if the 8 bytes are all ordinary.
|
||||
static std::uint64_t swar_string_special(std::uint64_t v) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t has_quote = (q - ones) & ~q & high;
|
||||
const std::uint64_t has_backslash = (b - ones) & ~b & high;
|
||||
const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20
|
||||
const std::uint64_t has_non_ascii = v & high; // >= 0x80
|
||||
return has_quote | has_backslash | has_control | has_non_ascii;
|
||||
}
|
||||
|
||||
// return the index of the first is_string_special() byte in [data, data+n),
|
||||
// or n if every byte is ordinary; scans 8 bytes at a time
|
||||
static std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (is_string_special(data[i]))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// Validate one UTF-8 sequence at the front of [data, data+avail). Returns
|
||||
// its length (2..4) only when the bytes form a *well-formed* sequence using
|
||||
// exactly the same ranges as scan_string()'s per-byte switch, so the bulk
|
||||
// path accepts precisely what the byte path accepts. Returns 0 for anything
|
||||
// that is invalid, incomplete, or that the byte path must diagnose (the
|
||||
// caller then defers to that path, keeping error messages unchanged). Lead
|
||||
// bytes < 0x80 are handled by the caller and never passed here.
|
||||
static std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept
|
||||
{
|
||||
const unsigned char c0 = data[0];
|
||||
// helper: a continuation byte within [lo, hi]
|
||||
// (kept inline to stay noexcept and C++11-friendly)
|
||||
if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF
|
||||
{
|
||||
if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xE0) // U+0800..U+0FFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates)
|
||||
{
|
||||
if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF)
|
||||
{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF0) // U+10000..U+3FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
else if (c0 == 0xF4) // U+100000..U+10FFFF
|
||||
{
|
||||
if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF)
|
||||
{
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return 0; // invalid, incomplete, or must be diagnosed by the byte path
|
||||
}
|
||||
|
||||
// Scalar (C++11) computation of the bulk run length: the number of leading
|
||||
// bytes in [data, data+n) that are ordinary ASCII or complete well-formed
|
||||
// UTF-8 sequences, stopping before the first byte that needs individual
|
||||
// handling (the closing quote, an escape, a control character, or an
|
||||
// ill-formed/truncated sequence). ASCII is skipped 8 bytes at a time.
|
||||
static std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
std::size_t pos = 0;
|
||||
while (pos < n)
|
||||
{
|
||||
pos += find_string_special(data + pos, n - pos);
|
||||
if (pos >= n || data[pos] < 0x80u)
|
||||
{
|
||||
break; // end of buffer, or a quote/escape/control byte
|
||||
}
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
{
|
||||
break; // ill-formed or truncated: let the byte path diagnose it
|
||||
}
|
||||
pos += seq;
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII
|
||||
// bytes are *not* stops here - the whole run is handed to simdutf), or n.
|
||||
static std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull;
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull;
|
||||
const std::uint64_t hit = ((q - ones) & ~q & high)
|
||||
| ((b - ones) & ~b & high)
|
||||
| ((v - 0x2020202020202020ull) & ~v & high);
|
||||
if (hit != 0)
|
||||
{
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
const unsigned char c = data[i + j];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
const unsigned char c = data[i];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the
|
||||
// next delimiter is validated in one shot by simdutf; on the rare failure the
|
||||
// scalar helper recomputes the exact valid prefix so the byte path still
|
||||
// produces the precise diagnostic. Without it, the pure scalar path is used.
|
||||
static std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
#if defined(JSON_USE_SIMDUTF)
|
||||
const std::size_t run = find_string_delimiter(data, n);
|
||||
if (run != 0 && simdutf::validate_utf8(reinterpret_cast<const char*>(data), run))
|
||||
{
|
||||
return run;
|
||||
}
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#else
|
||||
return scalar_string_bulk_run(data, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
/// contiguous input: bulk-append the run of ordinary characters and complete
|
||||
/// well-formed UTF-8 sequences starting at the current read position, leaving
|
||||
/// the first byte that needs individual handling (the closing quote, an
|
||||
@@ -9002,216 +9270,6 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
f = std::strtold(str, endptr);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated digit sequence
|
||||
|
||||
The scan_number() state machine has already checked that [first, last) is a
|
||||
valid JSON integer, so this only needs to accumulate the digits and detect
|
||||
overflow. This avoids the locale/errno machinery of std::strtoull, which
|
||||
dominates integer-heavy inputs.
|
||||
|
||||
@param[in] first pointer to the first character (a digit)
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed value on success
|
||||
@return true if the value fit into number_unsigned_t; false on overflow, in
|
||||
which case the caller falls back to floating-point parsing (matching
|
||||
the previous std::strtoull behavior)
|
||||
*/
|
||||
static bool parse_integer_unsigned(const char* first, const char* last, number_unsigned_t& value) noexcept
|
||||
{
|
||||
// accumulate in the widest unsigned type used by the previous strtoull
|
||||
// path so the overflow behavior is unchanged for custom number types
|
||||
std::uint64_t x = 0;
|
||||
constexpr std::uint64_t cutoff = (std::numeric_limits<std::uint64_t>::max)() / 10u;
|
||||
constexpr std::uint64_t cutlim = (std::numeric_limits<std::uint64_t>::max)() % 10u;
|
||||
for (const char* p = first; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
x = x * 10u + digit;
|
||||
}
|
||||
value = static_cast<number_unsigned_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_unsigned_t
|
||||
return static_cast<std::uint64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief fast integer parser for an already-validated negative integer
|
||||
|
||||
@param[in] first pointer to the leading '-'
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] value the parsed (negative) value on success
|
||||
@return true on success; false on overflow (caller falls back to float)
|
||||
*/
|
||||
static bool parse_integer_signed(const char* first, const char* last, number_integer_t& value) noexcept
|
||||
{
|
||||
// the state machine only reaches the signed path via a leading '-'
|
||||
JSON_ASSERT(first != last && *first == '-');
|
||||
std::uint64_t magnitude = 0;
|
||||
// |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude
|
||||
constexpr std::uint64_t limit = static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1u;
|
||||
for (const char* p = first + 1; p != last; ++p)
|
||||
{
|
||||
const auto digit = static_cast<std::uint64_t>(static_cast<unsigned char>(*p) - static_cast<unsigned char>('0'));
|
||||
if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
magnitude = magnitude * 10u + digit;
|
||||
}
|
||||
const std::int64_t x = (magnitude == limit)
|
||||
? (std::numeric_limits<std::int64_t>::min)()
|
||||
: -static_cast<std::int64_t>(magnitude);
|
||||
value = static_cast<number_integer_t>(x);
|
||||
// reject values that do not round-trip into a narrower number_integer_t
|
||||
return static_cast<std::int64_t>(value) == x;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief exact fast path for parsing a `double` (Clinger's algorithm)
|
||||
|
||||
For the common case - at most 19 significant digits, a decimal exponent in
|
||||
[-22, 22], and a significand below 2^53 - the value equals significand *
|
||||
10^exp computed in IEEE-754 double arithmetic, which is exact under
|
||||
round-to-nearest because both operands are exactly representable. This is the
|
||||
same fast path used by fast_float/simdjson; the general cases are left to
|
||||
std::strtod. The parser only activates for number_float_t == double; float
|
||||
and long double keep the std::strtof/std::strtold paths (see the templated
|
||||
overload below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
bool parse_float_fast(const char* first, const char* last, double& out) const noexcept
|
||||
{
|
||||
static const double powers_of_ten[] =
|
||||
{
|
||||
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
|
||||
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22
|
||||
};
|
||||
|
||||
const char* p = first;
|
||||
bool negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
|
||||
std::uint64_t significand = 0;
|
||||
int num_digits = 0;
|
||||
int fractional_digits = 0;
|
||||
bool seen_dot = false;
|
||||
bool any_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
const char c = *p;
|
||||
if (c >= '0' && c <= '9')
|
||||
{
|
||||
any_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(num_digits >= 19))
|
||||
{
|
||||
return false; // significand may not fit into uint64_t
|
||||
}
|
||||
significand = significand * 10u + static_cast<std::uint64_t>(c - '0');
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (static_cast<char_int_type>(c) == decimal_point_char)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
seen_dot = true;
|
||||
}
|
||||
else if (c == 'e' || c == 'E')
|
||||
{
|
||||
++p;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
if (p != last) // an exponent part remains
|
||||
{
|
||||
bool exp_negative = false;
|
||||
if (p != last && (*p == '-' || *p == '+'))
|
||||
{
|
||||
exp_negative = (*p == '-');
|
||||
++p;
|
||||
}
|
||||
bool any_exp_digit = false;
|
||||
for (; p != last; ++p)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
exponent = exponent * 10 + (*p - '0');
|
||||
any_exp_digit = true;
|
||||
if (JSON_HEDLEY_UNLIKELY(exponent > 9999))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!any_exp_digit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exponent = -exponent;
|
||||
}
|
||||
}
|
||||
|
||||
const int scale = exponent - fractional_digits;
|
||||
if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast<std::uint64_t>(1) << 53)))
|
||||
{
|
||||
return false; // significand not exactly representable as double
|
||||
}
|
||||
|
||||
double result = static_cast<double>(significand);
|
||||
if (scale >= 0)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result *= powers_of_ten[scale];
|
||||
}
|
||||
else
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(-scale > 22))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result /= powers_of_ten[-scale];
|
||||
}
|
||||
out = negative ? -result : result;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief scan a number literal
|
||||
|
||||
@@ -9570,7 +9628,7 @@ scan_number_done:
|
||||
// this code is reached if we parse a floating-point number or if an
|
||||
// integer conversion above overflowed. Try the exact fast path (double
|
||||
// only) before falling back to the locale-independent strtof/strtod.
|
||||
if (parse_float_fast(num_begin, num_end, value_float))
|
||||
if (parse_float_fast(num_begin, num_end, decimal_point_char, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user