From 104d987ca2b050fcfb6b6ce0a9718b02a7648b07 Mon Sep 17 00:00:00 2001 From: Varun Jaiswal <96684656+varun-jaiswal17@users.noreply.github.com> Date: Fri, 22 May 2026 22:26:51 +0530 Subject: [PATCH] Merge pull request #29093 from varun-jaiswal17:dnn-overflow-large-image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix int32 overflow in shape_utils::total() for large tensors #29093 Fixes https://github.com/opencv/opencv/issues/24914 ### Problem When running inference with a ConvTranspose (deconvolution) layer on large inputs (e.g. 4864×4864 with 30 channels), the DNN module crashes with: OpenCV net_impl.cpp: error: (expected: 'total(ints[i]) > 0'), where 'total(ints[i])' is -1455947776 must be greater than '0' is 0 The root cause is `shape_utils::total()` which returns `int` (32-bit signed). `ENGINE_CLASSIC` catches this via `CV_CheckGT` and throws. `ENGINE_NEW` was silently bypassing the check — the overflow in `total()` itself was never addressed. ### Changes **`modules/dnn/include/opencv2/dnn/shape_utils.hpp`** — root fix - Changed return type of both `total()` overloads from `int` to `size_t` - Changed accumulator from `int elems = 1` to `size_t elems = 1` **`modules/dnn/src/net_impl.cpp`** - Updated `CV_CheckGT(total(...), 0)` to `CV_CheckGT(total(...), (size_t)0)` to match the new return type **`modules/dnn/src/net_impl2.cpp`** - Added the same `CV_CheckGT` shape validation that `ENGINE_CLASSIC` has in `net_impl.cpp:1333-1337` — `ENGINE_NEW` was missing this check entirely **`modules/dnn/src/legacy_backend.hpp`** - Removed the now-incorrect `(int)` cast in `CV_CheckEQ` — both sides are now `size_t` ### Test Added `Net.ShapeUtils_total_no_int32_overflow` in `modules/dnn/test/test_misc.cpp`: - The shape [1920 × 1,478,656] is the exact im2col buffer from the bug report. EXPECT_EQ verifies total() returns the correct size_t value 2,839,019,520. EXPECT_LT documents that casting it to int wraps to -1,455,947,776 — the value that caused the original crash. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/include/opencv2/dnn/shape_utils.hpp | 12 ++++++------ modules/dnn/src/layers/concat_layer.cpp | 4 ++-- modules/dnn/src/layers/max_unpooling_layer.cpp | 2 +- modules/dnn/src/layers/prior_box_layer.cpp | 2 +- modules/dnn/src/legacy_backend.hpp | 4 ++-- modules/dnn/src/net_impl.cpp | 8 ++++---- modules/dnn/src/net_impl2.cpp | 6 ++++++ modules/dnn/test/test_misc.cpp | 9 +++++++++ 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/shape_utils.hpp b/modules/dnn/include/opencv2/dnn/shape_utils.hpp index 9b88b07b2d..c5e4aec627 100644 --- a/modules/dnn/include/opencv2/dnn/shape_utils.hpp +++ b/modules/dnn/include/opencv2/dnn/shape_utils.hpp @@ -152,7 +152,7 @@ static inline MatShape shape(int a0, int a1=-1, int a2=-1, int a3=-1) return shape(shape_, dims); } -static inline int total(const MatShape& shape, int start = -1, int end = -1) +static inline size_t total(const MatShape& shape, int start = -1, int end = -1) { //if (shape.empty()) // return 0; @@ -166,16 +166,16 @@ static inline int total(const MatShape& shape, int start = -1, int end = -1) CV_CheckLE(start, end, ""); CV_CheckLE(end, dims, ""); - int elems = 1; + size_t elems = 1; for (int i = start; i < end; i++) { - elems *= shape[i]; + elems *= (size_t)shape[i]; } return elems; } // TODO: rename to countDimsElements() -static inline int total(const Mat& mat, int start = -1, int end = -1) +static inline size_t total(const Mat& mat, int start = -1, int end = -1) { if (mat.empty()) return 0; @@ -189,10 +189,10 @@ static inline int total(const Mat& mat, int start = -1, int end = -1) CV_CheckLE(start, end, ""); CV_CheckLE(end, dims, ""); - int elems = 1; + size_t elems = 1; for (int i = start; i < end; i++) { - elems *= mat.size[i]; + elems *= (size_t)mat.size[i]; } return elems; } diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index 34e5aac174..dd4f3e0fc6 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -244,9 +244,9 @@ public: return false; int bottom_concat_axis; - int concat_size = total(shape(inputs[0]), cAxis + 1); + int concat_size = (int)total(shape(inputs[0]), cAxis + 1); int top_concat_axis = outputs[0].size[cAxis]; - int num_concats = total(shape(inputs[0]), 0, cAxis); + int num_concats = (int)total(shape(inputs[0]), 0, cAxis); int offset_concat_axis = 0; UMat& outMat = outputs[0]; String matType = matTypeToOclType(inputs[0].type()); diff --git a/modules/dnn/src/layers/max_unpooling_layer.cpp b/modules/dnn/src/layers/max_unpooling_layer.cpp index 66e3b13e52..0c0218cd2a 100644 --- a/modules/dnn/src/layers/max_unpooling_layer.cpp +++ b/modules/dnn/src/layers/max_unpooling_layer.cpp @@ -225,7 +225,7 @@ public: } getMemoryShapes(inpShapes, 1, outShapes, internals); - Mat zeros = Mat::zeros(1, total(outShapes[0]), CV_32F); + Mat zeros = Mat::zeros(1, (int)total(outShapes[0]), CV_32F); auto zeroInp = std::make_shared(ov::element::f32, ov::Shape{zeros.total()}, zeros.data); int newShape = -1; diff --git a/modules/dnn/src/layers/prior_box_layer.cpp b/modules/dnn/src/layers/prior_box_layer.cpp index 967aa7d1fa..4341bfa1df 100644 --- a/modules/dnn/src/layers/prior_box_layer.cpp +++ b/modules/dnn/src/layers/prior_box_layer.cpp @@ -419,7 +419,7 @@ public: // set the variance. { ocl::Kernel kernel("set_variance", ocl::dnn::prior_box_oclsrc, opts); - int offset = total(shape(outputs[0]), 2); + int offset = (int)total(shape(outputs[0]), 2); size_t nthreads = _layerHeight * _layerWidth * _numPriors; kernel.set(0, (int)nthreads); kernel.set(1, (int)offset); diff --git a/modules/dnn/src/legacy_backend.hpp b/modules/dnn/src/legacy_backend.hpp index 4b5b5f80ae..f85ec5c5e3 100644 --- a/modules/dnn/src/legacy_backend.hpp +++ b/modules/dnn/src/legacy_backend.hpp @@ -190,7 +190,7 @@ public: std::map::const_iterator hostIt; std::map::const_iterator refIt; - const int targetTotal = total(shape); + const int targetTotal = (int)total(shape); size_t bestBlobTotal = INT_MAX; for (hostIt = memHosts.begin(); hostIt != memHosts.end(); ++hostIt) @@ -299,7 +299,7 @@ public: LayerPin blobPin(ld.id, index); if (index < outShapes.size() && inPlace) { - CV_CheckEQ((int)ld.inputBlobs[0]->total(), total(shapes[index]), ""); + CV_CheckEQ(ld.inputBlobs[0]->total(), total(shapes[index]), ""); CV_CheckTypeEQ(ld.inputBlobs[0]->type(), types[index], "blob can't be reused if it has different type"); ld.outputBlobs[index] = ld.inputBlobs[0]->reshape(1, shapes[index]); reuse(ld.inputBlobsId[0], blobPin); diff --git a/modules/dnn/src/net_impl.cpp b/modules/dnn/src/net_impl.cpp index c1f1ec07d0..7f05ddab98 100644 --- a/modules/dnn/src/net_impl.cpp +++ b/modules/dnn/src/net_impl.cpp @@ -1330,11 +1330,11 @@ void Net::Impl::getLayerShapesRecursively(int id, LayersShapesMap& inOutShapes) try { - for (int i = 0; i < ints.size(); i++) - CV_CheckGT(total(ints[i]), 0, ""); + for (int i = 0; i < (int)ints.size(); i++) + CV_CheckGT(total(ints[i]), (size_t)0, ""); - for (int i = 0; i < os.size(); i++) - CV_CheckGT(total(os[i]), 0, ""); + for (int i = 0; i < (int)os.size(); i++) + CV_CheckGT(total(os[i]), (size_t)0, ""); } catch (const cv::Exception& e) { diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 52dd7ced79..bc09c4507c 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -598,6 +598,9 @@ void Net::Impl::allocateLayerOutputs( CV_Assert(tempShapes.size() == tempTypes.size()); CV_Assert(outShapes.size() == outTypes.size()); CV_Assert(outShapes.size() == noutputs); + + for (int i = 0; i < (int)tempShapes.size(); i++) + CV_CheckGT(total(tempShapes[i]), (size_t)0, ""); outputs.assign(noutputs, Mat()); outOrigData.resize(noutputs); for (size_t i = 0; i < noutputs; i++) { @@ -1628,6 +1631,9 @@ bool Net::Impl::tryInferGraphShapes(const Ptr& graph, layer->getTypes(inpTypes, noutputs, (int)tempShapes.size(), outTypes, tempTypes); CV_Assert((int)outTypes.size() == noutputs); + for (int i = 0; i < (int)tempShapes.size(); i++) + CV_CheckGT(total(tempShapes[i]), (size_t)0, ""); + for (int i = 0; i < noutputs; i++) { Arg out = outputs[i]; if (out.idx == 0) diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 6bd817ca10..45072cbd90 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -10,6 +10,7 @@ #include #include #include // CV_DNN_REGISTER_LAYER_CLASS +#include namespace opencv_test { namespace { @@ -1071,4 +1072,12 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_two_inputs, Combine( dnnBackendsAndTargets() )); +TEST(Net, ShapeUtils_total_no_int32_overflow) +{ + cv::MatShape shape = cv::dnn::shape(1920, 1478656); + + size_t t = cv::dnn::total(shape); + EXPECT_EQ(t, 2839019520llu); +} + }} // namespace