From 2aa570f8f3c12bca7edcf23be5b968ad0e1d609e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 20 Jul 2026 19:43:31 +0000 Subject: [PATCH] Use std::from_chars (Eisel-Lemire) for float conversion when available The Clinger fast path is exact only for the "easy" subset (<=19 significant digits, |exp10| <= 22); high-precision and scientific floats fall through to strtod, where the failed Clinger attempt actually makes parsing a net loss. std::from_chars implements the Eisel-Lemire algorithm in modern standard libraries: locale-independent, correctly rounded, and fast over the whole value range. convert_number() now tries parse_float_from_chars() first (guarded by __cpp_lib_to_chars, so C++11 and libc++-without-float-support keep the Clinger + strtod path unchanged), then Clinger, then strtof. from_chars is used only when it consumes the entire token; a partial parse means a non-'.' locale decimal point, and an under-/overflow (result_out_of_range) also declines - in both cases the existing strtod fallback supplies the exact value and the well-defined +/-inf/0 the parser expects, side-stepping the P4168 divergence between implementations. float and long double now get the fast path too (Clinger was double-only). Measured, C++17, g++ 13 -O3, json::parse/accept: - canada-style floats: ~unchanged (Clinger already covered them) - high-precision (17 digits): parse 2.1x, accept 2.5x - scientific (17 digits + exp): parse 3.6x, accept 4.1x Verified: C++11 (Clinger/strtod) and C++17 (from_chars) parse every value - including subnormals, boundary values, and 1e9999/1e-9999 over-/underflow - to bit-identical results; 2M number-fuzz clean; conversions/deserialization/ locale/number-fast-path suites pass in both C++11 and C++17; clang-tidy clean; warning-clean on g++ and clang in C++11/17/20. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 10 ++++- .../nlohmann/detail/input/number_parse.hpp | 34 ++++++++++++++ single_include/nlohmann/json.hpp | 44 ++++++++++++++++++- tests/src/unit-class_lexer.cpp | 6 ++- 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 5776afb67..ffd820f87 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -1382,8 +1382,14 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Try the exact fast path (double - // only) before falling back to the locale-independent strtof/strtod. + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } if (parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) { return token_type::value_float; diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index 83a7f5237..1a2d6d489 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -15,6 +15,11 @@ #include +#if defined(JSON_HAS_CPP_17) + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc +#endif + // This file contains the value-conversion helpers used by the lexer to turn an // already-validated number token into a value, without the locale/errno // overhead of std::strtoull/std::strtod. They are free functions so the lexer @@ -240,5 +245,34 @@ bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointT return false; } +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ +#if defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9ec86686d..240ef7db4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7798,6 +7798,11 @@ NLOHMANN_JSON_NAMESPACE_END // #include +#if defined(JSON_HAS_CPP_17) + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc +#endif + // This file contains the value-conversion helpers used by the lexer to turn an // already-validated number token into a value, without the locale/errno // overhead of std::strtoull/std::strtod. They are free functions so the lexer @@ -8023,6 +8028,35 @@ bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointT return false; } +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ +#if defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -9630,8 +9664,14 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Try the exact fast path (double - // only) before falling back to the locale-independent strtof/strtod. + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } if (parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) { return token_type::value_float; diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 3d09267c1..490397740 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -249,7 +249,11 @@ TEST_CASE("lexer number fast path") "-9223372036854775808", // INT64_MIN -> integer "-9223372036854775809", // INT64_MIN - 1 -> float "123456789012345678901234567890", // huge -> float - "0.30000000000000004", "2.2250738585072014e-308", "1e308" + "0.30000000000000004", "2.2250738585072014e-308", "1e308", + // high-precision / wide-exponent values that exercise the + // std::from_chars (Eisel-Lemire) path beyond the Clinger subset + "1.7976931348623157e308", "1.2345678901234567e-250", + "9007199254740993", "5e-324", "1e-320" }; for (const auto& n : numbers)