diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 4964eff494..779f66979d 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -437,6 +437,20 @@ CV__DNN_INLINE_NS_BEGIN const std::vector>& outputs ); + /** + * @brief Returns a CUDA backend node for the new graph engine (wrapper-free). + * + * Inputs and outputs are device tensors (arrays of cuda::GpuMatND) carrying shape and type; + * only that metadata is needed to build the node, the buffers are filled later by + * forwardCUDA(). The default adapts the wrapper-based initCUDA() so classic-engine ops keep + * working. @p context is a void pointer to a CSLContext object. + */ + virtual Ptr initCUDA( + void *context, + InputArrayOfArrays inputs, + InputArrayOfArrays outputs + ); + /** * @brief Returns a TimVX backend node * @@ -479,12 +493,13 @@ CV__DNN_INLINE_NS_BEGIN /** * @brief Executes the operation on the CUDA backend (new graph engine). * - * Called by the engine for nodes assigned to DNN_BACKEND_CUDA. The default - * implementation raises an error. @p workspace is an opaque pointer to a - * cuda4dnn::csl::Workspace (kept void* to avoid leaking internal CUDA types). + * Called by the engine for nodes assigned to DNN_BACKEND_CUDA. Inputs and outputs are + * device-resident tensors passed as arrays of cuda::GpuMatND; no backend wrappers are + * involved. The default implementation raises an error. @p workspace is an opaque pointer + * to a cuda4dnn::csl::Workspace (kept void* to avoid leaking internal CUDA types). */ - virtual void forwardCUDA(const std::vector >& inputs, - const std::vector >& outputs, + virtual void forwardCUDA(InputArrayOfArrays inputs, + OutputArrayOfArrays outputs, void* workspace); /** diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index c10f9014a5..ecd306e584 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -25,17 +25,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { using wrapper_type = GetCUDABackendWrapperType; void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { for (int i = 0; i < inputs.size(); i++) { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto input = csl::viewOf(inputs[i]); + auto output = csl::spanOf(outputs[i]); static_cast*>(this)->calculate(output, input); } diff --git a/modules/dnn/src/cuda4dnn/primitives/batch_norm.hpp b/modules/dnn/src/cuda4dnn/primitives/batch_norm.hpp index 293811f73d..6180ff0453 100644 --- a/modules/dnn/src/cuda4dnn/primitives/batch_norm.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/batch_norm.hpp @@ -32,17 +32,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { CV_Assert(inputs.size() == 1 && outputs.size() == 1); - auto input_wrapper = inputs[0].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[0].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto input = csl::viewOf(inputs[0]); + auto output = csl::spanOf(outputs[0]); std::size_t inner_size = input.size_range(2, input.rank()); kernels::scaleN_with_biasN(stream, output, input, inner_size, weightsTensor, biasTensor); diff --git a/modules/dnn/src/cuda4dnn/primitives/convolution.hpp b/modules/dnn/src/cuda4dnn/primitives/convolution.hpp index 12cf97404e..a4534a19f5 100644 --- a/modules/dnn/src/cuda4dnn/primitives/convolution.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/convolution.hpp @@ -286,8 +286,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { /* input[0] = conv input, input[1] = bias (from fused eltwise layer) */ @@ -296,8 +296,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { csl::WorkspaceAllocator allocator(workspace); - auto input_wrapper = inputs[0].dynamicCast(); - auto input = input_wrapper->getView(); + auto input = csl::viewOf(inputs[0]); if (!transformed_shape.empty()) { @@ -309,8 +308,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto conv_scratchpad = allocator.get_instance(); - auto output_wrapper = outputs[0].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto output = csl::spanOf(outputs[0]); if (fusion_location == InternalFusionLocation::CUDNN) { @@ -320,8 +318,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { convoluter.convolve_with_bias_activation(output, input, filtersTensor, biasTensor, conv_scratchpad); else if (fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION) { - auto eltwise_wrapper = inputs[1].dynamicCast(); - auto eltwise = eltwise_wrapper->getView(); + auto eltwise = csl::viewOf(inputs[1]); CV_Assert(is_shape_same(eltwise, output)); convoluter.convolve_with_bias_eltwise_activation(output, input, filtersTensor, biasTensor, eltwise, conv_scratchpad); @@ -357,8 +354,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION || fusion_mode == ConvolutionConfiguration::FusionMode::ACTIVATION_THEN_ELTWISE_SUM); - auto eltwise_wrapper = inputs[1].dynamicCast(); - auto eltwise = eltwise_wrapper->getView(); + auto eltwise = csl::viewOf(inputs[1]); CV_Assert(is_shape_same(eltwise, output)); std::size_t inner_size = output.size_range(2, output.rank()); @@ -472,8 +468,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION || fusion_mode == ConvolutionConfiguration::FusionMode::ACTIVATION_THEN_ELTWISE_SUM); - auto eltwise_wrapper = inputs[1].dynamicCast(); - auto eltwise = eltwise_wrapper->getView(); + auto eltwise = csl::viewOf(inputs[1]); CV_Assert(is_shape_same(eltwise, output)); /* we pass `eltwise` as `bias` (with `inner_size` as one) to bias-activation kernels */ diff --git a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp index 634ea2d806..b283401eb0 100644 --- a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp @@ -59,8 +59,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { CV_Assert(outputs.size() == 1); @@ -68,16 +68,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { CV_Assert(coeffs.size() == 0 || op == EltwiseOpType::SUM); CV_Assert(coeffs.size() == 0 || inputs.size() == coeffs.size()); - auto output_wrapper = outputs[0].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto output = csl::spanOf(outputs[0]); if (inputs.size() == 2) { - auto input_wrapper_x = inputs[0].dynamicCast(); - auto input_x = input_wrapper_x->getView(); - - auto input_wrapper_y = inputs[1].dynamicCast(); - auto input_y = input_wrapper_y->getView(); + auto input_x = csl::viewOf(inputs[0]); + auto input_y = csl::viewOf(inputs[1]); switch (op) { @@ -97,20 +93,17 @@ namespace cv { namespace dnn { namespace cuda4dnn { case EltwiseOpType::POW: kernels::eltwise_pow_2(stream, output, input_x, input_y); break; } } else if (inputs.size() == 1) { - auto input_wrapper_0 = inputs[0].dynamicCast(); - auto input_0 = input_wrapper_0->getView(); + auto input_0 = csl::viewOf(inputs[0]); csl::tensor_ops::copy(stream, output, input_0); } else { - auto input_wrapper_0 = inputs[0].dynamicCast(); - auto input_0 = input_wrapper_0->getView(); + auto input_0 = csl::viewOf(inputs[0]); /* we first make a copy and then apply EltwiseOp cumulatively */ csl::tensor_ops::copy(stream, output, input_0); for (int i = 1; i < inputs.size(); i++) { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); + auto input = csl::viewOf(inputs[i]); switch (op) { diff --git a/modules/dnn/src/cuda4dnn/primitives/inner_product.hpp b/modules/dnn/src/cuda4dnn/primitives/inner_product.hpp index a0a00ce125..70f1f79c52 100644 --- a/modules/dnn/src/cuda4dnn/primitives/inner_product.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/inner_product.hpp @@ -43,17 +43,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { for (int i = 0; i < inputs.size(); i++) { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto input = csl::viewOf(inputs[i]); + auto output = csl::spanOf(outputs[i]); std::size_t batch_size = input.size_range(0, axis); diff --git a/modules/dnn/src/cuda4dnn/primitives/pooling.hpp b/modules/dnn/src/cuda4dnn/primitives/pooling.hpp index bd8a73c2f2..7ab328f1a4 100644 --- a/modules/dnn/src/cuda4dnn/primitives/pooling.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/pooling.hpp @@ -224,14 +224,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { CV_Assert(inputs.size() == 1 && outputs.size() == 1); - auto input_wrapper = inputs[0].dynamicCast(); - auto input = input_wrapper->getView(); + auto input = csl::viewOf(inputs[0]); if (!transformedInput.empty()) { @@ -239,8 +238,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { input = csl::TensorView(transformedInput); } - auto output_wrapper = outputs[0].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto output = csl::spanOf(outputs[0]); pooler.pool(input, output); } diff --git a/modules/dnn/src/cuda4dnn/primitives/reshape.hpp b/modules/dnn/src/cuda4dnn/primitives/reshape.hpp index 2cf1d676ea..676a30c76b 100644 --- a/modules/dnn/src/cuda4dnn/primitives/reshape.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/reshape.hpp @@ -23,8 +23,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { ReshapeOp(csl::Stream stream_) : stream(std::move(stream_)) { } void forward( - const std::vector>& inputs, - const std::vector>& outputs, + const std::vector& inputs, + const std::vector& outputs, csl::Workspace& workspace) override { /* sometimes the output shape is passed as extra inputs; hence, >= instead of == */ @@ -32,11 +32,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { for (int i = 0; i < outputs.size(); i++) { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + auto input = csl::viewOf(inputs[i]); + auto output = csl::spanOf(outputs[i]); if (input.get() != output.get()) { diff --git a/modules/dnn/src/layer.cpp b/modules/dnn/src/layer.cpp index aa44e2fc39..662fa99325 100644 --- a/modules/dnn/src/layer.cpp +++ b/modules/dnn/src/layer.cpp @@ -57,10 +57,31 @@ bool Layer::supportBackend(int backendId) return backendId == DNN_BACKEND_OPENCV; } +Ptr Layer::initCUDA( + void* context, + const std::vector>& inputs, + const std::vector>& outputs) +{ +#ifdef HAVE_CUDA + // Adapt the classic wrapper-based entry point to the array-based one, so ops ported to the + // new graph engine only need to override initCUDA(context, inputs, outputs) with GpuMatND. + std::vector inGpu(inputs.size()), outGpu(outputs.size()); + for (size_t i = 0; i < inputs.size(); i++) + inGpu[i] = inputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/false); + for (size_t i = 0; i < outputs.size(); i++) + outGpu[i] = outputs[i].dynamicCast()->getDeviceMatND(/*forWrite=*/true); + return initCUDA(context, inGpu, outGpu); +#else + CV_UNUSED(context); CV_UNUSED(inputs); CV_UNUSED(outputs); + CV_Error(Error::StsNotImplemented, "CUDA pipeline of " + type + " layers is not defined."); + return Ptr(); +#endif +} + Ptr Layer::initCUDA( void*, - const std::vector>&, - const std::vector>&) + InputArrayOfArrays, + InputArrayOfArrays) { CV_Error(Error::StsNotImplemented, "CUDA pipeline of " + type + " layers is not defined."); return Ptr(); @@ -106,9 +127,7 @@ Ptr Layer::initCann(const std::vector > &inputs bool Layer::setActivation(const Ptr&) { return false; } bool Layer::tryFuse(Ptr&) { return false; } -void Layer::forwardCUDA(const std::vector >&, - const std::vector >&, - void*) +void Layer::forwardCUDA(InputArrayOfArrays, OutputArrayOfArrays, void*) { CV_Error(Error::StsNotImplemented, "CUDA forward of " + type + " layers is not defined."); } diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 5219c599f4..9760ba34b8 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -356,8 +356,8 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - const std::vector>& inputs, - const std::vector>& outputs + InputArrayOfArrays /*inputs*/, + InputArrayOfArrays /*outputs*/ ) override { auto context = reinterpret_cast(context_); diff --git a/modules/dnn/src/layers/conv2_layer.cpp b/modules/dnn/src/layers/conv2_layer.cpp index 8f2991a409..562b35bd22 100644 --- a/modules/dnn/src/layers/conv2_layer.cpp +++ b/modules/dnn/src/layers/conv2_layer.cpp @@ -802,16 +802,18 @@ public: return layer; } - void forwardCUDA(const std::vector >& inputs, - const std::vector >& outputs, + void forwardCUDA(InputArrayOfArrays inputs_, + OutputArrayOfArrays outputs_, void* workspace) CV_OVERRIDE { + std::vector inputs, outputs; + inputs_.getGpuMatNDVector(inputs); + outputs_.getGpuMatNDVector(outputs); CV_Assert(!inputs.empty() && !outputs.empty()); + auto& ws = *reinterpret_cast(workspace); if (!node) { - auto inW = inputs[0].dynamicCast(); - auto outW = outputs[0].dynamicCast(); - node = conv->initCudaConvNode(ctx, inW->getShape(), outW->getShape(), preferableTarget); + node = conv->initCudaConvNode(ctx, inputs[0].size, outputs[0].size, preferableTarget); cudaNode = node.dynamicCast(); CV_Assert(cudaNode); ws.require(cudaNode->get_workspace_memory_in_bytes()); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 269b3fe96a..174b4d02f2 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -310,8 +310,8 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - const std::vector>& inputs, - const std::vector>& outputs + InputArrayOfArrays /*inputs*/, + InputArrayOfArrays /*outputs*/ ) override { auto context = reinterpret_cast(context_); diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index e628bc3b42..7d29408f0c 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -281,15 +281,18 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - const std::vector>& inputs, - const std::vector>& outputs + InputArrayOfArrays inputs_, + InputArrayOfArrays outputs ) override { auto context = reinterpret_cast(context_); - if (inputs[0]->getHostMatDepth() == CV_Bool) + std::vector inputs; + inputs_.getGpuMatNDVector(inputs); + int depth = CV_MAT_DEPTH(inputs[0].type()); + if (depth == CV_Bool) return make_cuda_node_bool(std::move(context->stream)); else - return make_cuda_node_with_type(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream)); + return make_cuda_node_with_type(preferableTarget, depth, std::move(context->stream)); } #endif diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp index 8e4bd9465a..bac4998455 100644 --- a/modules/dnn/src/layers/gemm_layer.cpp +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -476,18 +476,19 @@ public: #ifdef HAVE_CUDA // Y = A * B + C. B should be guaranteed as two dimensional. Ptr initCUDA(void *context_, - const std::vector>& inputs, - const std::vector>& outputs) CV_OVERRIDE { + InputArrayOfArrays inputs_, + InputArrayOfArrays outputs) CV_OVERRIDE { CV_CheckFalse(trans_a, "DNN/Gemm/Cuda: does not support transA"); CV_CheckTrue(const_B, "DNN/Gemm/Cuda: input B (weight) is required to be constant"); auto context = reinterpret_cast(context_); - auto wrapper_A = inputs[0].dynamicCast(); + std::vector inputs; + inputs_.getGpuMatNDVector(inputs); auto B = blobs[0]; auto C = have_bias && const_C ? blobs[1] : Mat(); // in most cases C is constant if (!trans_b) cv::transpose(B, B); - auto flatten_start_axis = normalize_axis(1, wrapper_A->getRank()); + auto flatten_start_axis = normalize_axis(1, (int)inputs[0].size.size()); return make_cuda_node(preferableTarget, std::move(context->stream), std::move(context->cublas_handle), flatten_start_axis, B, C); } #endif // HAVE_CUDA diff --git a/modules/dnn/src/layers/maxpool_layer.cpp b/modules/dnn/src/layers/maxpool_layer.cpp index 46016187de..d4f000ca08 100644 --- a/modules/dnn/src/layers/maxpool_layer.cpp +++ b/modules/dnn/src/layers/maxpool_layer.cpp @@ -493,12 +493,13 @@ public: #ifdef HAVE_CUDA Ptr initCUDA(void* context_, - const std::vector >& inputs, - const std::vector >&) CV_OVERRIDE + InputArrayOfArrays inputs_, + InputArrayOfArrays) CV_OVERRIDE { auto context = reinterpret_cast(context_); - auto inW = inputs[0].dynamicCast(); - MatShape inShape = inW->getShape(); + std::vector inputs; + inputs_.getGpuMatNDVector(inputs); + MatShape inShape = inputs[0].size; const int nspatial = (int)kernel_shape.size(); cuda4dnn::PoolingConfiguration config; diff --git a/modules/dnn/src/layers/nary_eltwise_layers.cpp b/modules/dnn/src/layers/nary_eltwise_layers.cpp index 650036d0c3..b48b11787f 100644 --- a/modules/dnn/src/layers/nary_eltwise_layers.cpp +++ b/modules/dnn/src/layers/nary_eltwise_layers.cpp @@ -1320,11 +1320,13 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - const std::vector>& inputs, - const std::vector>& outputs + InputArrayOfArrays inputs_, + InputArrayOfArrays outputs ) override { auto context = reinterpret_cast(context_); + std::vector inputs; + inputs_.getGpuMatNDVector(inputs); cuda4dnn::EltwiseOpType op_ = cuda4dnn::EltwiseOpType::SUM; switch (op) { @@ -1361,7 +1363,7 @@ public: default: return Ptr(); // return empty cuda_node if the EltwiseOpType is unsupported type. }; - return make_cuda_node_with_type(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), op_, std::vector()); + return make_cuda_node_with_type(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), op_, std::vector()); } #endif diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 1f3932dc7b..a912eeb791 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -373,16 +373,18 @@ public: #ifdef HAVE_CUDA Ptr initCUDA( void *context_, - const std::vector>& inputs, - const std::vector>& outputs + InputArrayOfArrays inputs_, + InputArrayOfArrays outputs_ ) override { auto context = reinterpret_cast(context_); if (type == ROI) return make_cuda_node(preferableTarget, std::move(context->stream), spatialScale); - auto input_wrapper = inputs[0].dynamicCast(); - auto input_shape = input_wrapper->getShape(); + std::vector inputs, outputs; + inputs_.getGpuMatNDVector(inputs); + outputs_.getGpuMatNDVector(outputs); + MatShape input_shape = inputs[0].size; /* storing max indices is a special case and we deal with it separately */ if (computeMaxIdx) { @@ -412,13 +414,13 @@ public: config.input_shape.assign(std::begin(input_shape), std::end(input_shape)); - int indicesType = outputs[1]->getHostMatDepth(); + int indicesType = CV_MAT_DEPTH(outputs[1].type()); CV_CheckType(indicesType, indicesType == CV_32S || indicesType == CV_64S, "Unsupported indices type"); if (indicesType == CV_32S) - return make_cuda_node_with_indices(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), config); + return make_cuda_node_with_indices(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), config); else if (indicesType == CV_64S) - return make_cuda_node_with_indices(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), config); + return make_cuda_node_with_indices(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), config); CV_Error(Error::BadDepth, "Unsupported indices type"); return Ptr(); diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index 6afdaadb71..c59e5cb290 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -156,8 +156,6 @@ struct Net::Impl : public detail::NetImplBase }; bool fusedSnapshotValid = false; std::vector fusedSnapshot; - std::vector > argWrappers; - std::vector argWrapperData; TracingMode tracingMode; ProfilingMode profilingMode; std::vector dimvalues; @@ -295,6 +293,22 @@ struct Net::Impl : public detail::NetImplBase std::unique_ptr cudaInfo; void initCUDABackend(const std::vector& blobsToKeep_); + + // New graph engine: per-Arg device-resident tensors owned directly by the net (no backend + // wrappers). Sized lazily via GpuMatND::fit() and reused across forwards. Dirty flags track + // which copy (host cv::Mat vs device GpuMatND) is authoritative so transfers happen only at + // CPU<->CUDA boundaries; intermediates stay device-resident across consecutive CUDA ops. + std::vector cudaArgBuffers; + std::vector cudaArgHostDirty; // 1: host copy is authoritative -> needs H2D before device read + std::vector cudaArgDeviceDirty; // 1: device copy is authoritative -> needs D2H before host read + + // Device element type for a host tensor (half for float tensors under the FP16 target). + int cudaDeviceType(const Mat& hostMat) const; + // Returns the device buffer for @p arg, fit() to the host Mat's shape and device type. + cuda::GpuMatND& getCudaArgBuffer(Arg arg, const Mat& hostMat); + void cudaSetHostDirty(Arg arg); // mark host authoritative (e.g. after a CPU op wrote it) + void cudaUploadArg(Arg arg, const Mat& hostMat); // H2D if host dirty + void cudaDownloadArg(Arg arg, Mat& hostMat); // D2H if device dirty #endif #ifdef HAVE_ONNXRUNTIME @@ -441,9 +455,6 @@ struct Net::Impl : public detail::NetImplBase // Save/restore the fused graph so finalize() is re-entrant across backend changes. void saveFusedSnapshot(); void restoreFusedSnapshot(); -#ifdef HAVE_CUDA - Ptr getCudaArgWrapper(Arg arg, Mat& hostMat); -#endif // pre-allocates memory for output tensors. // if useBufferPool==true, the method uses 'buffers' diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 4f88f1bbf4..077ee80afe 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -8,6 +8,10 @@ #include +#ifdef HAVE_CUDA +#include +#endif + #ifdef HAVE_ONNXRUNTIME #include #endif @@ -706,8 +710,9 @@ void Net::Impl::finalize() bool useCUDA = false; #ifdef HAVE_CUDA - argWrappers.clear(); - argWrapperData.clear(); + cudaArgBuffers.clear(); + cudaArgHostDirty.clear(); + cudaArgDeviceDirty.clear(); if (preferableBackend == DNN_BACKEND_CUDA && haveCUDA()) { useCUDA = true; if (!cudaInfo) { @@ -1349,22 +1354,84 @@ static Mat stackScanAxis(const std::vector& perIter, int axis, bool reverse return stacked; } #ifdef HAVE_CUDA -Ptr Net::Impl::getCudaArgWrapper(Arg arg, Mat& hostMat) +// cv::cuda::Stream view over the (non-owning) cuda4dnn inference stream, so GpuMatND transfers +// are ordered against the op compute that runs on the same cudaStream_t. +static inline cuda::Stream wrapCudaStream(cuda4dnn::csl::Stream& s) +{ + return cuda::StreamAccessor::wrapStream(s.get()); +} + +// Device element type for a host tensor: float tensors are stored as half under the FP16 target, +// everything else mirrors the host type. fit() reuses the existing allocation when large enough, +// so buffers persist across forwards. +int Net::Impl::cudaDeviceType(const Mat& hostMat) const +{ + if (preferableTarget == DNN_TARGET_CUDA_FP16 && CV_MAT_DEPTH(hostMat.type()) == CV_32F) + return CV_MAKETYPE(CV_16F, CV_MAT_CN(hostMat.type())); + return hostMat.type(); +} + +cuda::GpuMatND& Net::Impl::getCudaArgBuffer(Arg arg, const Mat& hostMat) { int idx = arg.idx; - if ((int)argWrappers.size() != (int)args.size()) { - argWrappers.assign(args.size(), Ptr()); - argWrapperData.assign(args.size(), nullptr); + if ((int)cudaArgBuffers.size() != (int)args.size()) { + cudaArgBuffers.assign(args.size(), cuda::GpuMatND()); + cudaArgHostDirty.assign(args.size(), 1); + cudaArgDeviceDirty.assign(args.size(), 0); } - Ptr cw = argWrappers[idx].dynamicCast(); - if (!cw || argWrapperData[idx] != (const void*)hostMat.data) { - Ptr w = wrapMat(DNN_BACKEND_CUDA, preferableTarget, hostMat); - cw = w.dynamicCast(); - cw->setStream(cudaInfo->context.stream, cudaInfo->d2h_stream); - argWrappers[idx] = w; - argWrapperData[idx] = (const void*)hostMat.data; + cudaArgBuffers[idx].fit(hostMat.shape(), cudaDeviceType(hostMat)); + return cudaArgBuffers[idx]; +} + +void Net::Impl::cudaSetHostDirty(Arg arg) +{ + int idx = arg.idx; + if (idx >= 0 && idx < (int)cudaArgHostDirty.size()) { + cudaArgHostDirty[idx] = 1; + cudaArgDeviceDirty[idx] = 0; + } +} + +void Net::Impl::cudaUploadArg(Arg arg, const Mat& hostMat) +{ + int idx = arg.idx; + cuda::GpuMatND& g = getCudaArgBuffer(arg, hostMat); + if (cudaArgHostDirty[idx]) { + cuda::Stream s = wrapCudaStream(cudaInfo->context.stream); + if (g.type() == hostMat.type()) { + g.upload(hostMat, s); + } else { + // FP32 -> FP16 (device stores half): convert on host, then copy up. + Mat tmp; + hostMat.convertTo(tmp, CV_MAT_DEPTH(g.type())); + g.upload(tmp, s); + } + cudaArgHostDirty[idx] = 0; + cudaArgDeviceDirty[idx] = 0; + } +} + +void Net::Impl::cudaDownloadArg(Arg arg, Mat& hostMat) +{ + int idx = arg.idx; + if (idx < 0 || idx >= (int)cudaArgDeviceDirty.size()) + return; + if (cudaArgDeviceDirty[idx]) { + cuda::GpuMatND& g = cudaArgBuffers[idx]; + cuda::Stream s = wrapCudaStream(cudaInfo->context.stream); + if (g.type() == hostMat.type()) { + g.download(hostMat, s); + cudaInfo->context.stream.synchronize(); // host read follows immediately + } else { + // device stores half: copy down, then convert up to the host FP32 tensor. + Mat tmp; + g.download(tmp, s); + cudaInfo->context.stream.synchronize(); + tmp.convertTo(hostMat, hostMat.type()); + } + cudaArgDeviceDirty[idx] = 0; + cudaArgHostDirty[idx] = 0; } - return argWrappers[idx]; } static void forwardOpCUDA(Net::Impl* netimpl, GraphImpl* gimpl, size_t opidx, @@ -1373,12 +1440,18 @@ static void forwardOpCUDA(Net::Impl* netimpl, GraphImpl* gimpl, size_t opidx, { Ptr exec = gimpl->exec_[opidx]; CV_Assert(exec && netimpl->cudaInfo); - std::vector > inpWrappers(inputs.size()), outWrappers(outputs.size()); - for (size_t i = 0; i < inputs.size(); i++) - inpWrappers[i] = netimpl->getCudaArgWrapper(inputs[i], inpMats[i]); - for (size_t i = 0; i < outputs.size(); i++) - outWrappers[i] = netimpl->getCudaArgWrapper(outputs[i], outMats[i]); - exec->forwardCUDA(inpWrappers, outWrappers, &netimpl->cudaInfo->workspace); + std::vector inpG(inputs.size()), outG(outputs.size()); + for (size_t i = 0; i < inputs.size(); i++) { + netimpl->cudaUploadArg(inputs[i], inpMats[i]); // H2D only if host-authoritative + inpG[i] = netimpl->cudaArgBuffers[inputs[i].idx]; + } + for (size_t i = 0; i < outputs.size(); i++) { + outG[i] = netimpl->getCudaArgBuffer(outputs[i], outMats[i]); + int oidx = outputs[i].idx; // op writes the device buffer + netimpl->cudaArgDeviceDirty[oidx] = 1; + netimpl->cudaArgHostDirty[oidx] = 0; + } + exec->forwardCUDA(inpG, outG, &netimpl->cudaInfo->workspace); } #endif @@ -1422,13 +1495,8 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, // forward so the current input is re-uploaded; otherwise a second forward with a // changed input would read the previous forward's stale device data. if (cudaInfo) { - for (i = 0; i < n_gr_inputs; i++) { - Arg ginp = gr_inputs[i]; - if (ginp.idx >= 0 && ginp.idx < (int)argWrappers.size()) { - Ptr cw = argWrappers[ginp.idx].dynamicCast(); - if (cw) cw->setHostDirty(); - } - } + for (i = 0; i < n_gr_inputs; i++) + cudaSetHostDirty(gr_inputs[i]); } #endif @@ -1497,10 +1565,9 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, #ifdef HAVE_CUDA // CPU op: bring any device-resident inputs back to host before reading them. for (size_t k = 0; k < ninputs; k++) { - if (inputs[k].idx < (int)argWrappers.size()) { - Ptr cw = argWrappers[inputs[k].idx].dynamicCast(); - if (cw) { cw->copyToHost(); inpMats[k] = argTensor(inputs[k]); } - } + Mat& t = argTensor(inputs[k]); + cudaDownloadArg(inputs[k], t); + inpMats[k] = t; } #endif if (finalizeLayers) @@ -1508,12 +1575,8 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, layer->forward(inpMats, outMats, tempMats); #ifdef HAVE_CUDA // CPU produced fresh host data; invalidate any stale device copy of its outputs. - for (size_t k = 0; k < noutputs; k++) { - if (outputs[k].idx < (int)argWrappers.size()) { - Ptr cw = argWrappers[outputs[k].idx].dynamicCast(); - if (cw) cw->setHostDirty(); - } - } + for (size_t k = 0; k < noutputs; k++) + cudaSetHostDirty(outputs[k]); #endif } } @@ -1767,9 +1830,9 @@ void Net::Impl::forwardGraph(Ptr& graph, InputArrayOfArrays inputs_, Arg out = gr_outputs[i]; #ifdef HAVE_CUDA // A graph output produced on the device must be brought back to host before it is read. - if (out.idx < (int)argWrappers.size()) { - Ptr cw = argWrappers[out.idx].dynamicCast(); - if (cw) cw->copyToHost(); + if (out.idx >= 0) { + Mat& t = argTensor(out); + cudaDownloadArg(out, t); } #endif const Mat& outm = argTensor(out); diff --git a/modules/dnn/src/op_cuda.cpp b/modules/dnn/src/op_cuda.cpp index 12901c00b5..b1e267c10f 100644 --- a/modules/dnn/src/op_cuda.cpp +++ b/modules/dnn/src/op_cuda.cpp @@ -37,15 +37,19 @@ public: impl->finalize(inputs, outputs); } - void forwardCUDA(const std::vector >& inputs, - const std::vector >& outputs, + void forwardCUDA(InputArrayOfArrays inputs_, + OutputArrayOfArrays outputs_, void* workspace) CV_OVERRIDE { + std::vector inputs, outputs; + inputs_.getGpuMatNDVector(inputs); + outputs_.getGpuMatNDVector(outputs); + cuda4dnn::csl::Workspace& ws = *reinterpret_cast(workspace); if (!node) { impl->preferableTarget = preferableTarget; // initCUDA may pick FP16/FP32 by target cuda4dnn::csl::CSLContext context = *reinterpret_cast(ctx); - node = impl->initCUDA(&context, inputs, outputs); + node = impl->initCUDA(&context, inputs_, outputs_); CV_Assert(node); cudaNode = node.dynamicCast(); CV_Assert(cudaNode); diff --git a/modules/dnn/src/op_cuda.hpp b/modules/dnn/src/op_cuda.hpp index ecc8b3c856..7e79055142 100644 --- a/modules/dnn/src/op_cuda.hpp +++ b/modules/dnn/src/op_cuda.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -177,6 +178,22 @@ namespace cv { namespace dnn { if (temp.data != destMat.data) temp.copyTo(destMat); } + + /** @brief builds a read-only TensorView over the device memory of a GpuMatND (no copy) */ + template + TensorView viewOf(const cuda::GpuMatND& g) { + using const_ptr = typename TensorView::const_pointer; + return TensorView(const_ptr(reinterpret_cast(g.getDevicePtr())), + std::begin(g.size), std::end(g.size)); + } + + /** @brief builds a writable TensorSpan over the device memory of a GpuMatND (no copy) */ + template + TensorSpan spanOf(const cuda::GpuMatND& g) { + using ptr = typename TensorSpan::pointer; + return TensorSpan(ptr(reinterpret_cast(g.getDevicePtr())), + std::begin(g.size), std::end(g.size)); + } }} /* namespace cuda4dnn::csl */ /** base class for CUDA operation nodes (for all supported targets) */ @@ -185,10 +202,25 @@ namespace cv { namespace dnn { CUDABackendNode() : BackendNode(DNN_BACKEND_CUDA) { } virtual ~CUDABackendNode() { } + /** classic-engine entry point (wrapper-based). + * + * The default adapts the wrappers to GpuMatND headers and dispatches to the GpuMatND + * overload, so ops ported to the new graph engine only implement the GpuMatND forward. + * Ops not yet ported keep overriding this method directly. + */ virtual void forward( const std::vector>& inputs, const std::vector>& outputs, - cuda4dnn::csl::Workspace& workspace) = 0; + cuda4dnn::csl::Workspace& workspace); + + /** new graph-engine entry point (wrapper-free): operates directly on GpuMatND device tensors */ + virtual void forward( + const std::vector& inputs, + const std::vector& outputs, + cuda4dnn::csl::Workspace& workspace) + { + CV_Error(Error::StsNotImplemented, "GpuMatND CUDA forward is not implemented for this operation"); + } virtual std::size_t get_workspace_memory_in_bytes() const noexcept { return 0; } }; @@ -229,7 +261,7 @@ namespace cv { namespace dnn { template