From 40ce5b4132b9e4417ef5de2b35faaaa4fecb32b5 Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Thu, 26 Mar 2026 17:37:56 +0530 Subject: [PATCH] Merge pull request #28637 from abhishek-gola:old_dnn_tickets_cleanup Added Output Tensor Names support in new DNN engine #28637 closes: https://github.com/opencv/opencv/issues/26201 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/include/opencv2/dnn/dnn.hpp | 7 ++- modules/dnn/src/layers/batch_norm_layer.cpp | 4 +- modules/dnn/src/net.cpp | 6 ++ modules/dnn/src/net_impl.cpp | 57 +++++++++++++++++-- modules/dnn/src/net_impl.hpp | 3 +- modules/dnn/src/net_impl2.cpp | 53 ++++++++++++++--- modules/dnn/src/net_impl_fuse.cpp | 15 ----- modules/dnn/test/test_model.cpp | 3 +- modules/dnn/test/test_onnx_importer.cpp | 5 +- .../video/src/tracking/tracker_dasiamrpn.cpp | 14 ++--- 10 files changed, 122 insertions(+), 45 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index b2b5a7e38f..f0bd0bb0c0 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -815,7 +815,12 @@ CV__DNN_INLINE_NS_BEGIN * then the following forward pass may fail. */ CV_WRAP void setParam(int layer, int numParam, CV_ND const Mat &blob); - CV_WRAP inline void setParam(const String& layerName, int numParam, CV_ND const Mat &blob) { return setParam(getLayerId(layerName), numParam, blob); } + /** @brief Sets the parameter blob of a layer identified by its name or output tensor name. + * @param layerName layer name (classic engine) or raw ONNX output tensor name (ENGINE_NEW). + * @param numParam index of the constant weight input to update (0 = kernel, 1 = bias, etc.). + * @param blob the new parameter value. + */ + CV_WRAP void setParam(const String& layerName, int numParam, CV_ND const Mat &blob); /** @brief Returns parameter blob of the layer. * @param layer name or id of the layer. diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 7939eaf18b..1167fbf488 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -163,8 +163,8 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - if (inputs[0].empty()) { // Support for 0D input - outputs.push_back(MatShape()); // Output is also a scalar. + if (inputs[0].size() == 0) { // Support for 0D input + outputs.push_back(MatShape::scalar()); // Output is also a scalar. return true; } diff --git a/modules/dnn/src/net.cpp b/modules/dnn/src/net.cpp index fdd51e6180..c7f7a0fb21 100644 --- a/modules/dnn/src/net.cpp +++ b/modules/dnn/src/net.cpp @@ -165,6 +165,12 @@ void Net::setParam(int layer, int numParam, const Mat& blob) return impl->setParam(layer, numParam, blob); } +void Net::setParam(const String& name, int numParam, const Mat& blob) +{ + CV_Assert(impl); + return impl->setParam(name, numParam, blob); +} + int Net::getLayerId(const String& layer) const { CV_Assert(impl); diff --git a/modules/dnn/src/net_impl.cpp b/modules/dnn/src/net_impl.cpp index b4039a7541..8a759a6d27 100644 --- a/modules/dnn/src/net_impl.cpp +++ b/modules/dnn/src/net_impl.cpp @@ -992,9 +992,9 @@ Mat Net::Impl::forward(const String& outputName) FPDenormalsIgnoreHintScope fp_denormals_ignore_scope; if (mainGraph) { - if (!outputName.empty()) - CV_Error(Error::StsNotImplemented, "The new dnn engine doesn't support inference until a specified layer. If you want to run the whole model, please don't set the outputName argument in the forward() call. If you want to run the model until a specified layer, please use the old dnn engine"); - return forwardWithSingleOutput(outputName); + Mat result; + forwardWithSingleOutput(outputName, result); + return result; } String layerName = outputName; @@ -1051,8 +1051,10 @@ void Net::Impl::forward(OutputArrayOfArrays outputBlobs, const String& outputNam FPDenormalsIgnoreHintScope fp_denormals_ignore_scope; if (mainGraph) { - if (!outputName.empty()) - CV_Error(Error::StsNotImplemented, "The new dnn engine doesn't support inference until a specified layer. If you want to run the whole model, please don't set the outputName argument in the forward() call. If you want to run the model until a specified layer, please use the old dnn engine"); + if (!outputName.empty()) { + forwardWithSingleOutput(outputName, outputBlobs); + return; + } forwardWithMultipleOutputs(outputBlobs, {}); return; } @@ -1636,6 +1638,51 @@ void Net::Impl::setParam(int layer, int numParam, const Mat& blob) layerBlobs[numParam] = blob; } +void Net::Impl::setParam(const std::string& outputTensorName, int numParam, const Mat& blob) +{ + if (mainGraph) { + auto it = argnames.find(outputTensorName); + if (it == argnames.end()) { + size_t excl = outputTensorName.rfind('!'); + if (excl != std::string::npos) + it = argnames.find(outputTensorName.substr(excl + 1)); + } + if (it == argnames.end()) + CV_Error_(Error::StsObjectNotFound, + ("DNN: tensor '%s' not found in the graph", outputTensorName.c_str())); + + int targetIdx = (int)it->second; + const std::vector>& prog = mainGraph->prog(); + for (const auto& layer : prog) { + bool produces = false; + for (const Arg& out : layer->outputs) + if (out.idx == targetIdx) { produces = true; break; } + if (!produces) + continue; + + if (numParam < (int)layer->blobs.size()) { + layer->blobs[numParam] = blob; + finalizeLayers = true; + return; + } + + Conv2Layer* conv = dynamic_cast(layer.get()); + if (conv && numParam == 0) { + conv->setWeights(blob, Mat(), defaultC0, accuracy); + finalizeLayers = true; + return; + } + + CV_Error_(Error::StsOutOfRange, + ("DNN: op producing '%s' has fewer than %d params", + outputTensorName.c_str(), numParam + 1)); + } + CV_Error_(Error::StsObjectNotFound, + ("DNN: no op found in graph producing tensor '%s'", outputTensorName.c_str())); + } + setParam(getLayerId(outputTensorName), numParam, blob); +} + static string dumpLayerParameterSize(const string& name, const LayerParams& lp) diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index a9d961fb02..6866b0301d 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -203,6 +203,7 @@ struct Net::Impl : public detail::NetImplBase virtual void setInput(InputArray blob, const String& name, double scalefactor, const Scalar& mean); Mat getParam(int layer, int numParam) const; void setParam(int layer, int numParam, const Mat& blob); + void setParam(const std::string& outputTensorName, int numParam, const Mat& blob); std::vector> getLayerInputs(int layerId) const; std::vector getLayerNames() const; @@ -406,7 +407,7 @@ struct Net::Impl : public detail::NetImplBase std::vector runOrtSession(std::vector inputBlobs, const std::vector& outIdxs); #endif // run the whole model, convenience wrapper - Mat forwardWithSingleOutput(const std::string& outname); + void forwardWithSingleOutput(const std::string& outname, OutputArrayOfArrays outputBlobs); // run the whole model, convenience wrapper void forwardWithMultipleOutputs(OutputArrayOfArrays outputBlobs, const std::vector& outBlobNames); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 3f5b74fdea..7860bf6dc3 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -624,7 +624,7 @@ void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays finalizeLayers = false; } -Mat Net::Impl::forwardWithSingleOutput(const std::string& outname) +void Net::Impl::forwardWithSingleOutput(const std::string& outname, OutputArrayOfArrays outputBlobs) { #ifdef HAVE_ONNXRUNTIME if (this->ort_session) @@ -648,25 +648,63 @@ Mat Net::Impl::forwardWithSingleOutput(const std::string& outname) std::vector outIdxs(1, outIdx); std::vector outs = runOrtSession(netInputLayer->blobs, outIdxs); CV_Assert(outs.size() == 1); - return outs[0]; + outputBlobs.assign(outs[0]); + return; } #endif { if (!mainGraph) { CV_Error(Error::StsNullPtr, "the model was not loaded"); } - const std::vector& outargs = mainGraph->outputs(); - CV_Assert(outargs.size() > 0); if (!outname.empty()) { - const ArgData& outdata = args.at(outargs[0].idx); - CV_Assert(outdata.name == outname); + auto it = argnames.find(outname); + if (it == argnames.end()) { + size_t excl = outname.rfind('!'); + if (excl != std::string::npos) { + it = argnames.find(outname.substr(excl + 1)); + } + } + if (it == argnames.end()) + CV_Error_(Error::StsObjectNotFound, + ("DNN: tensor '%s' is not found in the graph", outname.c_str())); + + Arg targetArg((int)it->second); + + std::vector inps, outs; + forwardMainGraph(inps, outs); + + const std::vector& gr_outputs = mainGraph->outputs(); + for (size_t i = 0; i < gr_outputs.size(); i++) { + if (gr_outputs[i].idx == targetArg.idx) { + outputBlobs.assign(outs[i]); + return; + } + } + + const ArgData& adata = args.at(targetArg.idx); + Mat result; + if (adata.kind == DNN_ARG_TEMP) { + int bufidx = bufidxs.at(targetArg.idx); + CV_Assert(bufidx >= 0 && bufidx < (int)buffers.size()); + result = buffers[bufidx]; + } else { + result = __tensors__.at(targetArg.idx); + } + if (result.shape().layout == DATA_LAYOUT_BLOCK) { + Mat converted; + transformLayout(result, converted, originalLayout, originalLayout, defaultC0); + outputBlobs.assign(converted); + } else { + outputBlobs.assign(result.clone()); + } + return; } } std::vector inps, outs; forwardMainGraph(inps, outs); CV_Assert(!outs.empty()); - return outs[0]; + outputBlobs.assign(outs[0]); } void Net::Impl::forwardWithMultipleOutputs(OutputArrayOfArrays outblobs, const std::vector& outnames) @@ -1037,7 +1075,6 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, for (i = 0; i < ninputs; i++) { Arg inp = inputs[i]; - //const ArgData& adata = args[inp.idx]; const Mat& m = argTensor(inp); inpMats[i] = m; inpTypes[i] = m.type(); diff --git a/modules/dnn/src/net_impl_fuse.cpp b/modules/dnn/src/net_impl_fuse.cpp index 19767d3185..fb386e4b74 100644 --- a/modules/dnn/src/net_impl_fuse.cpp +++ b/modules/dnn/src/net_impl_fuse.cpp @@ -672,20 +672,6 @@ void Net::Impl::fuseLayers(const std::vector& blobsToKeep_) if (preferableBackend != DNN_BACKEND_OPENCV && preferableBackend != DNN_BACKEND_CUDA) continue; // Go to the next layer. - // [TODO] temporarily disabled Concat optimization. - // - // Ticket: https://github.com/opencv/opencv/issues/26195 - // - // It's not quite compatible with dynamic shapes, - // so we need to make sure that we correctly predicted shapes - // of all the concatenated tensors and their offsets inside the result - // and also properly allocated that concatenated tensor in advance -#if 0 - // the optimization #2. if there is concat layer that concatenates channels - // from the inputs together (i.e. axis == 1) then we make the inputs of - // the concat layer to write to the concatenation output buffer - // (and so we eliminate the concatenation layer, because the channels - // are concatenated implicitly). Ptr concatLayer = ld.layerInstance.dynamicCast(); if( !concatLayer.empty() && !concatLayer->padding && ld.outputBlobs.size() == 1 ) { @@ -851,7 +837,6 @@ void Net::Impl::fuseLayers(const std::vector& blobsToKeep_) } } } -#endif } } diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 884b2efe0f..60a72377d9 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -753,8 +753,7 @@ TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); } -// BUG: https://github.com/opencv/opencv/issues/26246 -TEST_P(Test_Model, DISABLED_TextDetectionByDB) +TEST_P(Test_Model, TextDetectionByDB) { applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 7e1e23067c..049e011c12 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -1138,12 +1138,13 @@ TEST_P(Test_ONNX_layers, ResizeUnfusedTwoInputs) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2023000000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); +#endif +#ifdef HAVE_INF_ENGINE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); #endif testONNXModels("upsample_unfused_two_inputs_opset9_torch1.4", npy, 0, 0, false, true, 2); - // BUG: https://github.com/opencv/opencv/issues/26291 - // testONNXModels("upsample_unfused_two_inputs_opset11_torch1.4", npy, 0, 0, false, true, 2); + testONNXModels("upsample_unfused_two_inputs_opset11_torch1.4", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, MultyInputs) diff --git a/modules/video/src/tracking/tracker_dasiamrpn.cpp b/modules/video/src/tracking/tracker_dasiamrpn.cpp index 4cffcc1b24..dfb8a41009 100644 --- a/modules/video/src/tracking/tracker_dasiamrpn.cpp +++ b/modules/video/src/tracking/tracker_dasiamrpn.cpp @@ -59,13 +59,9 @@ class TrackerDaSiamRPNImpl : public TrackerDaSiamRPN public: TrackerDaSiamRPNImpl(const TrackerDaSiamRPN::Params& parameters) { - // the tracker uses DNN models in quite sophisticated way, - // so it's not supported yet by the new engine. - // BUG: https://github.com/opencv/opencv/issues/26201 - dnn::EngineType engine = dnn::ENGINE_CLASSIC; - siamRPN = dnn::readNet(parameters.model, "", "", engine); - siamKernelCL1 = dnn::readNet(parameters.kernel_cls1, "", "", engine); - siamKernelR1 = dnn::readNet(parameters.kernel_r1, "", "", engine); + siamRPN = dnn::readNet(parameters.model); + siamKernelCL1 = dnn::readNet(parameters.kernel_cls1); + siamKernelR1 = dnn::readNet(parameters.kernel_r1); CV_Assert(!siamRPN.empty()); CV_Assert(!siamKernelCL1.empty()); @@ -180,8 +176,8 @@ void TrackerDaSiamRPNImpl::trackerInit(Mat img) Mat r1 = siamKernelR1.forward(); std::vector r1_shape = { 20, 256, 4, 4 }, cls1_shape = { 10, 256, 4, 4 }; - siamRPN.setParam(siamRPN.getLayerId("onnx_node_output_0!65"), 0, r1.reshape(0, r1_shape)); - siamRPN.setParam(siamRPN.getLayerId("onnx_node_output_0!68"), 0, cls1.reshape(0, cls1_shape)); + siamRPN.setParam("onnx_node_output_0!65", 0, r1.reshape(0, r1_shape)); + siamRPN.setParam("onnx_node_output_0!68", 0, cls1.reshape(0, cls1_shape)); } bool TrackerDaSiamRPNImpl::update(InputArray image, Rect& boundingBox)