From 1c5a953de5087280111531da50f4ba61f9c07515 Mon Sep 17 00:00:00 2001 From: Angadi56 Date: Sun, 19 Jul 2026 22:03:42 +0530 Subject: [PATCH] reject unpaired UTF-16 surrogates in wide-string input (#5276) --- .../nlohmann/detail/input/input_adapters.hpp | 29 +++++++++++----- single_include/nlohmann/json.hpp | 29 +++++++++++----- tests/src/unit-wstring.cpp | 34 ++++++++++++++++++- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index ab9173613..2c3561cbc 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -395,17 +395,30 @@ struct wide_string_input_helper } else { - if (JSON_HEDLEY_UNLIKELY(!input.empty())) + // A supplementary code point is a high surrogate (0xD800..0xDBFF) + // followed by a low surrogate (0xDC00..0xDFFF). A lone low + // surrogate, a high surrogate at the end of the input, or a high + // surrogate followed by any other unit is malformed UTF-16. In + // that case the offending unit is passed through unchanged so the + // UTF-8 decoder rejects it, matching how \uXXXX surrogate escapes + // are handled in the lexer. + bool valid_pair = false; + if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty())) { const auto wc2 = static_cast(input.get_character()); - const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); - utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); - utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); - utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); - utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); - utf8_bytes_filled = 4; + if (0xDC00 <= wc2 && wc2 <= 0xDFFF) + { + const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); + utf8_bytes_filled = 4; + valid_pair = true; + } } - else + + if (!valid_pair) { utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 03d235173..e4c19c2c3 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7382,17 +7382,30 @@ struct wide_string_input_helper } else { - if (JSON_HEDLEY_UNLIKELY(!input.empty())) + // A supplementary code point is a high surrogate (0xD800..0xDBFF) + // followed by a low surrogate (0xDC00..0xDFFF). A lone low + // surrogate, a high surrogate at the end of the input, or a high + // surrogate followed by any other unit is malformed UTF-16. In + // that case the offending unit is passed through unchanged so the + // UTF-8 decoder rejects it, matching how \uXXXX surrogate escapes + // are handled in the lexer. + bool valid_pair = false; + if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty())) { const auto wc2 = static_cast(input.get_character()); - const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); - utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); - utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); - utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); - utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); - utf8_bytes_filled = 4; + if (0xDC00 <= wc2 && wc2 <= 0xDFFF) + { + const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); + utf8_bytes_filled = 4; + valid_pair = true; + } } - else + + if (!valid_pair) { utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; diff --git a/tests/src/unit-wstring.cpp b/tests/src/unit-wstring.cpp index dec9c1f9a..ffbe70e7e 100644 --- a/tests/src/unit-wstring.cpp +++ b/tests/src/unit-wstring.cpp @@ -53,6 +53,27 @@ TEST_CASE("wide strings") std::wstring const w = L"\"\xDBFF"; json _; CHECK_THROWS_AS(_ = json::parse(w), json::parse_error&); + + // the exact message depends on the width of wchar_t: a 16-bit + // wchar_t passes the lone surrogate to the UTF-8 decoder unchanged + // (rejected as a single ill-formed byte at column 2), while a + // 32-bit wchar_t first encodes it as an ill-formed three-byte + // sequence (rejected one byte later, at column 3) + const char* const error_low_surrogate = sizeof(wchar_t) == 2 + ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'" + : "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xED\xB0'"; + const char* const error_high_surrogate = sizeof(wchar_t) == 2 + ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'" + : "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xED\xA0'"; + + // a lone low surrogate cannot start a pair + CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L'"', static_cast(0xDC00), L'"'}), error_low_surrogate, json::parse_error&); + // a high surrogate followed by a non-low-surrogate unit is invalid + CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L'"', static_cast(0xD800), L'a', L'"'}), error_high_surrogate, json::parse_error&); + // a lone low surrogate must not swallow the following unit: pairing + // it with any second unit would produce valid UTF-8, so the error + // has to report an ill-formed byte at the surrogate's own position + CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L'"', static_cast(0xDC00), L'a', L'"'}), error_low_surrogate, json::parse_error&); } } @@ -68,11 +89,22 @@ TEST_CASE("wide strings") SECTION("invalid std::u16string") { - if (wstring_is_utf16()) + if (u16string_is_utf16()) { std::u16string const w = u"\"\xDBFF"; json _; CHECK_THROWS_AS(_ = json::parse(w), json::parse_error&); + + // a lone low surrogate cannot start a pair + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + // a high surrogate followed by a non-low-surrogate unit is invalid + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + // a lone low surrogate must not swallow the following unit: pairing + // it with any second unit would produce valid UTF-8, so the error + // has to report an ill-formed byte at the surrogate's own position + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + // a valid surrogate pair is still decoded (U+1F600) + CHECK(json::parse(std::u16string{u'"', 0xD83D, 0xDE00, u'"'}).get() == "\xF0\x9F\x98\x80"); } }