From 58c28e1e825a5ebda6a90537cce87f2482b9a5b9 Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Thu, 9 Jul 2026 18:59:44 +0530 Subject: [PATCH] wrapper-free GpuMatND forward path --- modules/dnn/include/opencv2/dnn/dnn.hpp | 8 +-- .../src/cuda4dnn/primitives/max_unpooling.hpp | 16 ++++++ modules/dnn/src/layer.cpp | 4 +- modules/dnn/src/layers/batch_norm2_layer.cpp | 4 +- modules/dnn/src/layers/batch_norm_layer.cpp | 4 +- modules/dnn/src/net_impl2.cpp | 49 +++++++++++++------ modules/dnn/src/op_cuda.hpp | 32 ++++++------ 7 files changed, 76 insertions(+), 41 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 82b177c11f..db8bcff474 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -274,7 +274,7 @@ CV__DNN_INLINE_NS_BEGIN * Each operation type registers a `static Ptr create(const LayerParams&)` factory * via @ref CV_DNN_REGISTER_OP_CLASS_STATIC. */ - class CV_EXPORTS_W LayerInfo : public Algorithm + class CV_EXPORTS_W LayerInfo { public: LayerInfo(); @@ -335,10 +335,10 @@ CV__DNN_INLINE_NS_BEGIN /** @brief This interface class allows to build new Layers - are building blocks of networks. * - * A %Layer is the *executable*, backend-specific counterpart of an LayerInfo node: it + * A %Layer is the *executable*, backend-specific counterpart of a LayerInfo node: it * implements forward() (and finalize()) for a particular backend/target. In the new graph - * engine a %Layer is created from an LayerInfo (held in #data) by Net::finalizeNet(); its - * inference methods (getMemoryShapes() etc., inherited from LayerInfo) delegate to #data. + * engine a %Layer is created from a LayerInfo by the executor factory; shape/type inference + * stays on the LayerInfo node. * * Each class, derived from Layer, must implement forward() method to compute outputs. * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros. diff --git a/modules/dnn/src/cuda4dnn/primitives/max_unpooling.hpp b/modules/dnn/src/cuda4dnn/primitives/max_unpooling.hpp index 759cf65b16..e606019a42 100644 --- a/modules/dnn/src/cuda4dnn/primitives/max_unpooling.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/max_unpooling.hpp @@ -111,6 +111,22 @@ namespace cv { namespace dnn { namespace cuda4dnn { ); } + void forward( + const std::vector& inputs, + const std::vector& outputs, + csl::Workspace& workspace) override + { + CV_Assert(inputs.size() == 1 && outputs.size() == 2); + + auto input_data = csl::viewOf(inputs[0]); + auto output_data = csl::spanOf(outputs[0]); + auto output_indices = csl::spanOf(outputs[1]); + + kernels::max_pooling_with_indices( + stream, output_data, output_indices, input_data, window_size, strides, padding_left + ); + } + private: csl::Stream stream; diff --git a/modules/dnn/src/layer.cpp b/modules/dnn/src/layer.cpp index 662fa99325..a30d1c5a8b 100644 --- a/modules/dnn/src/layer.cpp +++ b/modules/dnn/src/layer.cpp @@ -67,9 +67,9 @@ Ptr Layer::initCUDA( // new graph engine only need to override initCUDA(context, inputs, outputs) with GpuMatND. std::vector inGpu(inputs.size()), outGpu(outputs.size()); for (size_t i = 0; i < inputs.size(); i++) - inGpu[i] = inputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/false); + inGpu[i] = inputs[i].dynamicCast()->getDeviceMatND(); for (size_t i = 0; i < outputs.size(); i++) - outGpu[i] = outputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/true); + outGpu[i] = outputs[i].dynamicCast()->getDeviceMatND(); return initCUDA(context, inGpu, outGpu); #else CV_UNUSED(context); CV_UNUSED(inputs); CV_UNUSED(outputs); diff --git a/modules/dnn/src/layers/batch_norm2_layer.cpp b/modules/dnn/src/layers/batch_norm2_layer.cpp index bfd194fa8a..92595632af 100644 --- a/modules/dnn/src/layers/batch_norm2_layer.cpp +++ b/modules/dnn/src/layers/batch_norm2_layer.cpp @@ -369,8 +369,8 @@ public: #ifdef HAVE_CUDA // Channel-wise scale+shift on CUDA; reused by the new graph engine via CUDALegacyExec. Ptr initCUDA(void* context_, - const std::vector >&, - const std::vector >&) CV_OVERRIDE + InputArrayOfArrays, + InputArrayOfArrays) CV_OVERRIDE { auto context = reinterpret_cast(context_); Mat scale, bias; diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 9760ba34b8..5219c599f4 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -356,8 +356,8 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - InputArrayOfArrays /*inputs*/, - InputArrayOfArrays /*outputs*/ + const std::vector>& inputs, + const std::vector>& outputs ) override { auto context = reinterpret_cast(context_); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 077ee80afe..a28e508850 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -1361,6 +1361,16 @@ static inline cuda::Stream wrapCudaStream(cuda4dnn::csl::Stream& s) return cuda::StreamAccessor::wrapStream(s.get()); } +// GpuMatND upload/download require at least one dimension (rank-0 shapes trip GpuMatND::setFields). +// View a rank-0 scalar as a 1-element 1D header over the same data; higher-rank Mats pass through. +static inline Mat asAtLeast1D(const Mat& m) +{ + if (m.shape().dims != 0) + return m; + int one = 1; + return Mat(1, &one, m.type(), const_cast(m.data)); +} + // Device element type for a host tensor: float tensors are stored as half under the FP16 target, // everything else mirrors the host type. fit() reuses the existing allocation when large enough, // so buffers persist across forwards. @@ -1379,7 +1389,12 @@ cuda::GpuMatND& Net::Impl::getCudaArgBuffer(Arg arg, const Mat& hostMat) cudaArgHostDirty.assign(args.size(), 1); cudaArgDeviceDirty.assign(args.size(), 0); } - cudaArgBuffers[idx].fit(hostMat.shape(), cudaDeviceType(hostMat)); + // GpuMatND (and cuda4dnn tensors) require at least one dimension; represent a rank-0 scalar + // tensor as a 1-element 1D tensor so element-wise ops still see the single value. + MatShape shape = hostMat.shape(); + if (shape.dims == 0) + shape = MatShape({1}); + cudaArgBuffers[idx].fit(shape, cudaDeviceType(hostMat)); return cudaArgBuffers[idx]; } @@ -1395,15 +1410,16 @@ void Net::Impl::cudaSetHostDirty(Arg arg) void Net::Impl::cudaUploadArg(Arg arg, const Mat& hostMat) { int idx = arg.idx; - cuda::GpuMatND& g = getCudaArgBuffer(arg, hostMat); + Mat src = asAtLeast1D(hostMat); + cuda::GpuMatND& g = getCudaArgBuffer(arg, src); if (cudaArgHostDirty[idx]) { cuda::Stream s = wrapCudaStream(cudaInfo->context.stream); - if (g.type() == hostMat.type()) { - g.upload(hostMat, s); + if (g.type() == src.type()) { + g.upload(src, s); } else { // FP32 -> FP16 (device stores half): convert on host, then copy up. Mat tmp; - hostMat.convertTo(tmp, CV_MAT_DEPTH(g.type())); + src.convertTo(tmp, CV_MAT_DEPTH(g.type())); g.upload(tmp, s); } cudaArgHostDirty[idx] = 0; @@ -1419,15 +1435,16 @@ void Net::Impl::cudaDownloadArg(Arg arg, Mat& hostMat) if (cudaArgDeviceDirty[idx]) { cuda::GpuMatND& g = cudaArgBuffers[idx]; cuda::Stream s = wrapCudaStream(cudaInfo->context.stream); - if (g.type() == hostMat.type()) { - g.download(hostMat, s); + Mat dst = asAtLeast1D(hostMat); // fill the scalar's storage through a 1D header + if (g.type() == dst.type()) { + g.download(dst, s); cudaInfo->context.stream.synchronize(); // host read follows immediately } else { // device stores half: copy down, then convert up to the host FP32 tensor. Mat tmp; g.download(tmp, s); cudaInfo->context.stream.synchronize(); - tmp.convertTo(hostMat, hostMat.type()); + tmp.convertTo(dst, dst.type()); } cudaArgDeviceDirty[idx] = 0; cudaArgHostDirty[idx] = 0; @@ -1523,6 +1540,13 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, for (i = 0; i < ninputs; i++) { Arg inp = inputs[i]; +#ifdef HAVE_CUDA + // CPU op: bring any device-resident input back to host before reading its + // shape/data. This must happen before allocateLayerOutputs(), which may fit() + // an in-place op's output onto this very buffer and rewrite its header. + if (opBackend != DNN_BACKEND_CUDA) + cudaDownloadArg(inp, argTensor(inp)); +#endif const Mat& m = argTensor(inp); inpMats[i] = m; inpTypes[i] = m.type(); @@ -1562,14 +1586,7 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, } else #endif { -#ifdef HAVE_CUDA - // CPU op: bring any device-resident inputs back to host before reading them. - for (size_t k = 0; k < ninputs; k++) { - Mat& t = argTensor(inputs[k]); - cudaDownloadArg(inputs[k], t); - inpMats[k] = t; - } -#endif + // Device-resident inputs were already synced to host in the capture loop above. if (finalizeLayers) layer->finalize(inpMats, outMats); layer->forward(inpMats, outMats, tempMats); diff --git a/modules/dnn/src/op_cuda.hpp b/modules/dnn/src/op_cuda.hpp index 7e79055142..2914d40e85 100644 --- a/modules/dnn/src/op_cuda.hpp +++ b/modules/dnn/src/op_cuda.hpp @@ -335,14 +335,14 @@ namespace cv { namespace dnn { virtual void update(const MatShape& shape, std::size_t offset) = 0; - /** @brief returns a GpuMatND header over the device memory (no copy) + /** @brief returns a GpuMatND header over the device memory (no copy, no synchronization) * - * @p forWrite selects the same synchronization semantics as getSpan()/getView(): - * for reads the host->device transfer is performed if needed; for writes the device - * memory is marked dirty. The returned header borrows the device memory and is only - * valid while this wrapper (and its device tensor) is alive. + * The header borrows the device memory and is only valid while this wrapper (and its + * device tensor) is alive. Host<->device synchronization is the caller's responsibility + * (see copyToDevice()/setDeviceDirty()); this accessor must stay side-effect free so it + * can be used at init time before any stream is attached. */ - virtual cuda::GpuMatND getDeviceMatND(bool forWrite) = 0; + virtual cuda::GpuMatND getDeviceMatND() = 0; }; namespace cuda4dnn { namespace detail { @@ -691,11 +691,7 @@ namespace cv { namespace dnn { return tensor_view_type(shared_block->device.get() + offset, std::begin(shape), std::end(shape)); } - cuda::GpuMatND getDeviceMatND(bool forWrite) override { - if (forWrite) - setDeviceDirty(); - else - copyToDevice(); + cuda::GpuMatND getDeviceMatND() override { DEVICE_T* ptr = shared_block->device.get().get() + offset; // Report the host element type (e.g. CV_32F even for an FP16 device buffer): the type is // only consumed by initCUDA(), while the compute reinterprets the pointer per its own T. @@ -759,10 +755,16 @@ namespace cv { namespace dnn { cuda4dnn::csl::Workspace& workspace) { std::vector inGpu(inputs.size()), outGpu(outputs.size()); - for (size_t i = 0; i < inputs.size(); i++) - inGpu[i] = inputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/false); - for (size_t i = 0; i < outputs.size(); i++) - outGpu[i] = outputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/true); + for (size_t i = 0; i < inputs.size(); i++) { + auto w = inputs[i].dynamicCast(); + w->copyToDevice(); // host->device if needed (mirrors getView()) + inGpu[i] = w->getDeviceMatND(); + } + for (size_t i = 0; i < outputs.size(); i++) { + auto w = outputs[i].dynamicCast(); + w->setDeviceDirty(); // op writes the device buffer (mirrors getSpan()) + outGpu[i] = w->getDeviceMatND(); + } forward(inGpu, outGpu, workspace); }