🐛 fix BSON conformance issue

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -170,7 +170,24 @@ class binary_reader
     bool parse_bson_internal()
     {
         std::int32_t document_size{};
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+        if (JSON_HEDLEY_UNLIKELY((!get_number<std::int32_t, true>(input_format_t::bson, document_size))))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(document_size < 5))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            concat("BSON document size must be at least 5, is ", std::to_string(document_size)),
+                                            "document size"), nullptr));
+        }
+
+        // The document begins at the size field and ends document_size bytes later
+        // (including the size field itself and the trailing 0x00 terminator).
+        const std::size_t document_start = chars_read - sizeof(std::int32_t);
+        const std::size_t expected_end = document_start + static_cast<std::size_t>(document_size);
 
         if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
         {
@@ -182,6 +199,15 @@ class binary_reader
             return false;
         }
 
+        if (JSON_HEDLEY_UNLIKELY(chars_read != expected_end))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON document terminator did not land at declared document size",
+                                            "document"), nullptr));
+        }
+
         return sax->end_object();
     }
 
@@ -231,7 +257,21 @@ class binary_reader
                                     exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr));
         }
 
-        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != char_traits<char_type>::eof();
+        if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast<NumberType>(1), result)))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(get() != 0x00))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON string is not null-terminated",
+                                            "string"), nullptr));
+        }
+
+        return true;
     }
 
     /*!
@@ -398,7 +438,24 @@ class binary_reader
     bool parse_bson_array()
     {
         std::int32_t document_size{};
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+        if (JSON_HEDLEY_UNLIKELY((!get_number<std::int32_t, true>(input_format_t::bson, document_size))))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(document_size < 5))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            concat("BSON document size must be at least 5, is ", std::to_string(document_size)),
+                                            "document size"), nullptr));
+        }
+
+        // The document begins at the size field and ends document_size bytes later
+        // (including the size field itself and the trailing 0x00 terminator).
+        const std::size_t document_start = chars_read - sizeof(std::int32_t);
+        const std::size_t expected_end = document_start + static_cast<std::size_t>(document_size);
 
         if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
         {
@@ -410,6 +467,15 @@ class binary_reader
             return false;
         }
 
+        if (JSON_HEDLEY_UNLIKELY(chars_read != expected_end))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON array terminator did not land at declared array size",
+                                            "array"), nullptr));
+        }
+
         return sax->end_array();
     }
 

include/nlohmann/detail/output/serializer.hpp

@@ -125,7 +125,11 @@ class serializer
                     o->write_characters("{\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     // first n-1 elements
                     auto i = val.m_data.m_value.object->cbegin();
@@ -194,7 +198,11 @@ class serializer
                     o->write_characters("[\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     // first n-1 elements
                     for (auto i = val.m_data.m_value.array->cbegin();
@@ -251,7 +259,11 @@ class serializer
                     o->write_characters("{\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     o->write_characters(indent_string.c_str(), new_indent);
 
@@ -361,33 +373,6 @@ class serializer
         }
     }
 
-    /*!
-    @brief compute the next indentation level and grow the indentation string
-
-    Computes the new indentation level @a current_indent + @a indent_step and
-    grows the indentation string (by doubling its size, but at least to the new
-    level) so that it can be used as a source for writing that many indentation
-    characters. The newly added characters use the configured @ref indent_char.
-
-    @param[in] current_indent  the current indentation level
-    @param[in] indent_step     the number of characters to indent per level
-
-    @return the new indentation level @a current_indent + @a indent_step
-    */
-    unsigned int reserve_indent(const unsigned int current_indent, const unsigned int indent_step)
-    {
-        const unsigned int new_indent = current_indent + indent_step;
-        // a very large indent_step can wrap the unsigned accumulation on deep
-        // nesting, which would silently truncate the indentation
-        JSON_ASSERT(new_indent >= current_indent);
-        if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
-        {
-            indent_string.resize((std::max)(indent_string.size() * 2, static_cast<std::size_t>(new_indent)), indent_char);
-        }
-        JSON_ASSERT(indent_string.size() >= new_indent);
-        return new_indent;
-    }
-
   JSON_PRIVATE_UNLESS_TESTED:
     /*!
     @brief dump escaped string

single_include/nlohmann/json.hpp

@@ -10312,7 +10312,24 @@ class binary_reader
     bool parse_bson_internal()
     {
         std::int32_t document_size{};
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+        if (JSON_HEDLEY_UNLIKELY((!get_number<std::int32_t, true>(input_format_t::bson, document_size))))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(document_size < 5))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            concat("BSON document size must be at least 5, is ", std::to_string(document_size)),
+                                            "document size"), nullptr));
+        }
+
+        // The document begins at the size field and ends document_size bytes later
+        // (including the size field itself and the trailing 0x00 terminator).
+        const std::size_t document_start = chars_read - sizeof(std::int32_t);
+        const std::size_t expected_end = document_start + static_cast<std::size_t>(document_size);
 
         if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
         {
@@ -10324,6 +10341,15 @@ class binary_reader
             return false;
         }
 
+        if (JSON_HEDLEY_UNLIKELY(chars_read != expected_end))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON document terminator did not land at declared document size",
+                                            "document"), nullptr));
+        }
+
         return sax->end_object();
     }
 
@@ -10373,7 +10399,21 @@ class binary_reader
                                     exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr));
         }
 
-        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != char_traits<char_type>::eof();
+        if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast<NumberType>(1), result)))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(get() != 0x00))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON string is not null-terminated",
+                                            "string"), nullptr));
+        }
+
+        return true;
     }
 
     /*!
@@ -10540,7 +10580,24 @@ class binary_reader
     bool parse_bson_array()
     {
         std::int32_t document_size{};
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+        if (JSON_HEDLEY_UNLIKELY((!get_number<std::int32_t, true>(input_format_t::bson, document_size))))
+        {
+            return false;
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(document_size < 5))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            concat("BSON document size must be at least 5, is ", std::to_string(document_size)),
+                                            "document size"), nullptr));
+        }
+
+        // The document begins at the size field and ends document_size bytes later
+        // (including the size field itself and the trailing 0x00 terminator).
+        const std::size_t document_start = chars_read - sizeof(std::int32_t);
+        const std::size_t expected_end = document_start + static_cast<std::size_t>(document_size);
 
         if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
         {
@@ -10552,6 +10609,15 @@ class binary_reader
             return false;
         }
 
+        if (JSON_HEDLEY_UNLIKELY(chars_read != expected_end))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+                                    exception_message(input_format_t::bson,
+                                            "BSON array terminator did not land at declared array size",
+                                            "array"), nullptr));
+        }
+
         return sax->end_array();
     }
 
@@ -19299,7 +19365,11 @@ class serializer
                     o->write_characters("{\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     // first n-1 elements
                     auto i = val.m_data.m_value.object->cbegin();
@@ -19368,7 +19438,11 @@ class serializer
                     o->write_characters("[\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     // first n-1 elements
                     for (auto i = val.m_data.m_value.array->cbegin();
@@ -19425,7 +19499,11 @@ class serializer
                     o->write_characters("{\n", 2);
 
                     // variable to hold indentation for recursive calls
-                    const auto new_indent = reserve_indent(current_indent, indent_step);
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
 
                     o->write_characters(indent_string.c_str(), new_indent);
 
@@ -19535,33 +19613,6 @@ class serializer
         }
     }
 
-    /*!
-    @brief compute the next indentation level and grow the indentation string
-
-    Computes the new indentation level @a current_indent + @a indent_step and
-    grows the indentation string (by doubling its size, but at least to the new
-    level) so that it can be used as a source for writing that many indentation
-    characters. The newly added characters use the configured @ref indent_char.
-
-    @param[in] current_indent  the current indentation level
-    @param[in] indent_step     the number of characters to indent per level
-
-    @return the new indentation level @a current_indent + @a indent_step
-    */
-    unsigned int reserve_indent(const unsigned int current_indent, const unsigned int indent_step)
-    {
-        const unsigned int new_indent = current_indent + indent_step;
-        // a very large indent_step can wrap the unsigned accumulation on deep
-        // nesting, which would silently truncate the indentation
-        JSON_ASSERT(new_indent >= current_indent);
-        if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
-        {
-            indent_string.resize((std::max)(indent_string.size() * 2, static_cast<std::size_t>(new_indent)), indent_char);
-        }
-        JSON_ASSERT(indent_string.size() >= new_indent);
-        return new_indent;
-    }
-
   JSON_PRIVATE_UNLESS_TESTED:
     /*!
     @brief dump escaped string

tests/src/unit-bson.cpp

@@ -1294,3 +1294,76 @@ TEST_CASE("BSON roundtrips" * doctest::skip())
         }
     }
 }
+
+TEST_CASE("Invalid document size handling")
+{
+    SECTION("document size must be at least 5")
+    {
+        std::vector<std::uint8_t> const v = {0x04, 0x00, 0x00, 0x00, 0x00};
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 4: syntax error while parsing BSON document size: BSON document size must be at least 5, is 4", json::parse_error&);
+        CHECK(json::from_bson(v, true, false).is_discarded());
+    }
+
+    SECTION("declared document size must match consumed bytes (extra trailing element)")
+    {
+        // Declares 5-byte empty document but appends an int32 element after the declared end.
+        std::vector<std::uint8_t> const v =
+        {
+            0x05, 0x00, 0x00, 0x00,
+            0x10, 'a', 'd', 'm', 'i', 'n', 0x00,
+            0x01, 0x00, 0x00, 0x00,
+            0x00
+        };
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 16: syntax error while parsing BSON document: BSON document terminator did not land at declared document size", json::parse_error&);
+        CHECK(json::from_bson(v, true, false).is_discarded());
+    }
+
+    SECTION("declared document size must match consumed bytes (premature terminator)")
+    {
+        // Declares 32-byte document but only contains the size field followed by an immediate terminator.
+        std::vector<std::uint8_t> const v =
+        {
+            0x20, 0x00, 0x00, 0x00,
+            0x00
+        };
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: BSON document terminator did not land at declared document size", json::parse_error&);
+        CHECK(json::from_bson(v, true, false).is_discarded());
+    }
+
+    SECTION("array declared size must match consumed bytes")
+    {
+        // Outer object contains an array "a" that declares 5 bytes (empty) but
+        // actually contains an int32 element before its terminator.
+        std::vector<std::uint8_t> const v =
+        {
+            0x14, 0x00, 0x00, 0x00,                         // object size = 20
+            0x04, 'a', 0x00,                                // key "a", array type
+            0x05, 0x00, 0x00, 0x00,                         // array declared size = 5 (empty)
+            0x10, '0', 0x00, 0x01, 0x00, 0x00, 0x00,        // extra int32 element "0" = 1
+            0x00,                                           // array terminator
+            0x00                                            // object terminator
+        };
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 19: syntax error while parsing BSON array: BSON array terminator did not land at declared array size", json::parse_error&);
+        CHECK(json::from_bson(v, true, false).is_discarded());
+    }
+
+    SECTION("BSON string must end with 0x00")
+    {
+        // Length-prefixed string whose terminator byte is 'X' (0x58), not 0x00.
+        std::vector<std::uint8_t> const v =
+        {
+            0x0F, 0x00, 0x00, 0x00,
+            0x02, 's', 0x00,
+            0x02, 0x00, 0x00, 0x00,
+            'A', 'X',
+            0x00
+        };
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 13: syntax error while parsing BSON string: BSON string is not null-terminated", json::parse_error&);
+        CHECK(json::from_bson(v, true, false).is_discarded());
+    }
+}

tests/src/unit-inspection.cpp

@@ -245,71 +245,6 @@ TEST_CASE("object inspection")
             CHECK(binary.dump(1024).size() == 2086);
         }
 
-        SECTION("indentation and resize")
-        {
-            SECTION("array")
-            {
-                const auto j_array = json::parse("[[[[[[]]]]]]");
-                // check right size after indentation triggering a resize
-                CHECK(j_array.dump(1024).size() == 25622);
-                // check if right indentation symbol is used
-                CHECK(j_array.dump(1024, '\t')[4096] == '\t');
-                // check resize is large enough
-                CHECK(j_array.dump(10000).size() == 250022);
-            }
-
-            SECTION("object")
-            {
-                const auto j_object = json::parse(R"({"":{"":{"":{"":{"":{}}}}}})");
-                // check right size after indentation triggering a resize
-                CHECK(j_object.dump(1024).size() == 25642);
-                // check if right indentation symbol is used
-                CHECK(j_object.dump(1024, '\t')[4096] == '\t');
-                // check resize is large enough
-                CHECK(j_object.dump(10000).size() == 250042);
-            }
-
-            SECTION("binary")
-            {
-                const auto j_binary = json::binary({1, 2, 3}, 128);
-                // check right size after indentation triggering a resize
-                CHECK(j_binary.dump(1024).size() == 2086);
-                CHECK(j_binary.dump(1024, '\t')[1024] == '\t');
-                // check resize is large enough
-                CHECK(j_binary.dump(10000).size() == 20038);
-            }
-
-            SECTION("full-width run integrity")
-            {
-                // the size/single-index checks above can't catch a corrupted byte
-                // in the middle of an indentation run, so also verify each level's
-                // indentation is an intact, uninterrupted run of the indent character
-                const std::string::size_type width = 1100;
-
-                SECTION("object")
-                {
-                    const json j_object = {{"outer", {{"inner", 1}}}};
-                    const std::string s = j_object.dump(static_cast<int>(width));
-                    CHECK(s.find('\n' + std::string(width, ' ') + "\"outer\"") != std::string::npos);
-                    CHECK(s.find('\n' + std::string(2 * width, ' ') + "\"inner\"") != std::string::npos);
-                }
-
-                SECTION("array")
-                {
-                    const json j_array = json::array({json::array({1})});
-                    const std::string s = j_array.dump(static_cast<int>(width));
-                    CHECK(s.find('\n' + std::string(2 * width, ' ') + '1') != std::string::npos);
-                }
-
-                SECTION("binary")
-                {
-                    const json j_binary = json::binary({1, 2, 3});
-                    const std::string s = j_binary.dump(static_cast<int>(width));
-                    CHECK(s.find('\n' + std::string(width, ' ') + "\"bytes\"") != std::string::npos);
-                }
-            }
-        }
-
         SECTION("dump and floating-point numbers")
         {
             auto s = json(42.23).dump();