potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-31 00:03:03 +04:00
Code Packages Releases Wiki Activity

using gpuMat instead of backend wrappers

Browse Source
This commit is contained in:
Abhishek Gola
2026-07-08 17:17:48 +05:30
parent 7f63fede4c
commit 12fb9a6c24
21 changed files with 319 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/include/opencv2/dnn/dnn.hpp
+20 -5
View File
@@ -437,6 +437,20 @@ CV__DNN_INLINE_NS_BEGIN
const std::vector<Ptr<BackendWrapper>>& outputs
);
/**
* @brief Returns a CUDA backend node for the new graph engine (wrapper-free).
*
* Inputs and outputs are device tensors (arrays of cuda::GpuMatND) carrying shape and type;
* only that metadata is needed to build the node, the buffers are filled later by
* forwardCUDA(). The default adapts the wrapper-based initCUDA() so classic-engine ops keep
* working. @p context is a void pointer to a CSLContext object.
*/
virtual Ptr<BackendNode> initCUDA(
void *context,
InputArrayOfArrays inputs,
InputArrayOfArrays outputs
);
/**
* @brief Returns a TimVX backend node
*
@@ -479,12 +493,13 @@ CV__DNN_INLINE_NS_BEGIN
/**
* @brief Executes the operation on the CUDA backend (new graph engine).
*
* Called by the engine for nodes assigned to DNN_BACKEND_CUDA. The default
* implementation raises an error. @p workspace is an opaque pointer to a
* cuda4dnn::csl::Workspace (kept void* to avoid leaking internal CUDA types).
* Called by the engine for nodes assigned to DNN_BACKEND_CUDA. Inputs and outputs are
* device-resident tensors passed as arrays of cuda::GpuMatND; no backend wrappers are
* involved. The default implementation raises an error. @p workspace is an opaque pointer
* to a cuda4dnn::csl::Workspace (kept void* to avoid leaking internal CUDA types).
*/
virtual void forwardCUDA(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendWrapper> >& outputs,
virtual void forwardCUDA(InputArrayOfArrays inputs,
OutputArrayOfArrays outputs,
void* workspace);
/**
modules/dnn/src/cuda4dnn/primitives/activation.hpp
+4 -7
View File
@@ -25,17 +25,14 @@ namespace cv { namespace dnn { namespace cuda4dnn {
using wrapper_type = GetCUDABackendWrapperType<T>;
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
for (int i = 0; i < inputs.size(); i++)
{
auto input_wrapper = inputs[i].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto output_wrapper = outputs[i].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto input = csl::viewOf<T>(inputs[i]);
auto output = csl::spanOf<T>(outputs[i]);
static_cast<const Op<T>*>(this)->calculate(output, input);
}
modules/dnn/src/cuda4dnn/primitives/batch_norm.hpp
+4 -7
View File
@@ -32,17 +32,14 @@ namespace cv { namespace dnn { namespace cuda4dnn {
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
CV_Assert(inputs.size() == 1 && outputs.size() == 1);
auto input_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto input = csl::viewOf<T>(inputs[0]);
auto output = csl::spanOf<T>(outputs[0]);
std::size_t inner_size = input.size_range(2, input.rank());
kernels::scaleN_with_biasN<T>(stream, output, input, inner_size, weightsTensor, biasTensor);
modules/dnn/src/cuda4dnn/primitives/convolution.hpp
+7 -12
View File
@@ -286,8 +286,8 @@ namespace cv { namespace dnn { namespace cuda4dnn {
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
/* input[0] = conv input, input[1] = bias (from fused eltwise layer) */
@@ -296,8 +296,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
csl::WorkspaceAllocator allocator(workspace);
auto input_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto input = csl::viewOf<T>(inputs[0]);
if (!transformed_shape.empty())
{
@@ -309,8 +308,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
auto conv_scratchpad = allocator.get_instance();
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto output = csl::spanOf<T>(outputs[0]);
if (fusion_location == InternalFusionLocation::CUDNN)
{
@@ -320,8 +318,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
convoluter.convolve_with_bias_activation(output, input, filtersTensor, biasTensor, conv_scratchpad);
else if (fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION)
{
auto eltwise_wrapper = inputs[1].dynamicCast<wrapper_type>();
auto eltwise = eltwise_wrapper->getView();
auto eltwise = csl::viewOf<T>(inputs[1]);
CV_Assert(is_shape_same(eltwise, output));
convoluter.convolve_with_bias_eltwise_activation(output, input, filtersTensor, biasTensor, eltwise, conv_scratchpad);
@@ -357,8 +354,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION ||
fusion_mode == ConvolutionConfiguration::FusionMode::ACTIVATION_THEN_ELTWISE_SUM);
auto eltwise_wrapper = inputs[1].dynamicCast<wrapper_type>();
auto eltwise = eltwise_wrapper->getView();
auto eltwise = csl::viewOf<T>(inputs[1]);
CV_Assert(is_shape_same(eltwise, output));
std::size_t inner_size = output.size_range(2, output.rank());
@@ -472,8 +468,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
fusion_mode == ConvolutionConfiguration::FusionMode::ELTWISE_SUM_THEN_ACTIVATION ||
fusion_mode == ConvolutionConfiguration::FusionMode::ACTIVATION_THEN_ELTWISE_SUM);
auto eltwise_wrapper = inputs[1].dynamicCast<wrapper_type>();
auto eltwise = eltwise_wrapper->getView();
auto eltwise = csl::viewOf<T>(inputs[1]);
CV_Assert(is_shape_same(eltwise, output));
/* we pass `eltwise` as `bias` (with `inner_size` as one) to bias-activation kernels */
modules/dnn/src/cuda4dnn/primitives/eltwise.hpp
+8 -15
View File
@@ -59,8 +59,8 @@ namespace cv { namespace dnn { namespace cuda4dnn {
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
CV_Assert(outputs.size() == 1);
@@ -68,16 +68,12 @@ namespace cv { namespace dnn { namespace cuda4dnn {
CV_Assert(coeffs.size() == 0 || op == EltwiseOpType::SUM);
CV_Assert(coeffs.size() == 0 || inputs.size() == coeffs.size());
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto output = csl::spanOf<T>(outputs[0]);
if (inputs.size() == 2)
{
auto input_wrapper_x = inputs[0].dynamicCast<wrapper_type>();
auto input_x = input_wrapper_x->getView();
auto input_wrapper_y = inputs[1].dynamicCast<wrapper_type>();
auto input_y = input_wrapper_y->getView();
auto input_x = csl::viewOf<T>(inputs[0]);
auto input_y = csl::viewOf<T>(inputs[1]);
switch (op)
{
@@ -97,20 +93,17 @@ namespace cv { namespace dnn { namespace cuda4dnn {
case EltwiseOpType::POW: kernels::eltwise_pow_2<T>(stream, output, input_x, input_y); break;
}
} else if (inputs.size() == 1) {
auto input_wrapper_0 = inputs[0].dynamicCast<wrapper_type>();
auto input_0 = input_wrapper_0->getView();
auto input_0 = csl::viewOf<T>(inputs[0]);
csl::tensor_ops::copy(stream, output, input_0);
} else {
auto input_wrapper_0 = inputs[0].dynamicCast<wrapper_type>();
auto input_0 = input_wrapper_0->getView();
auto input_0 = csl::viewOf<T>(inputs[0]);
/* we first make a copy and then apply EltwiseOp cumulatively */
csl::tensor_ops::copy(stream, output, input_0);
for (int i = 1; i < inputs.size(); i++)
{
auto input_wrapper = inputs[i].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto input = csl::viewOf<T>(inputs[i]);
switch (op)
{
modules/dnn/src/cuda4dnn/primitives/inner_product.hpp
+4 -7
View File
@@ -43,17 +43,14 @@ namespace cv { namespace dnn { namespace cuda4dnn {
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
for (int i = 0; i < inputs.size(); i++)
{
auto input_wrapper = inputs[i].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto output_wrapper = outputs[i].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto input = csl::viewOf<T>(inputs[i]);
auto output = csl::spanOf<T>(outputs[i]);
std::size_t batch_size = input.size_range(0, axis);
modules/dnn/src/cuda4dnn/primitives/pooling.hpp
+4 -6
View File
@@ -224,14 +224,13 @@ namespace cv { namespace dnn { namespace cuda4dnn {
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
CV_Assert(inputs.size() == 1 && outputs.size() == 1);
auto input_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto input = csl::viewOf<T>(inputs[0]);
if (!transformedInput.empty())
{
@@ -239,8 +238,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
input = csl::TensorView<T>(transformedInput);
}
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto output = csl::spanOf<T>(outputs[0]);
pooler.pool(input, output);
}
modules/dnn/src/cuda4dnn/primitives/reshape.hpp
+4 -7
View File
@@ -23,8 +23,8 @@ namespace cv { namespace dnn { namespace cuda4dnn {
ReshapeOp(csl::Stream stream_) : stream(std::move(stream_)) { }
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
csl::Workspace& workspace) override
{
/* sometimes the output shape is passed as extra inputs; hence, >= instead of == */
@@ -32,11 +32,8 @@ namespace cv { namespace dnn { namespace cuda4dnn {
for (int i = 0; i < outputs.size(); i++)
{
auto input_wrapper = inputs[i].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto output_wrapper = outputs[i].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto input = csl::viewOf<T>(inputs[i]);
auto output = csl::spanOf<T>(outputs[i]);
if (input.get() != output.get())
{
modules/dnn/src/layer.cpp
+24 -5
View File
@@ -57,10 +57,31 @@ bool Layer::supportBackend(int backendId)
return backendId == DNN_BACKEND_OPENCV;
}
Ptr<BackendNode> Layer::initCUDA(
void* context,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs)
{
#ifdef HAVE_CUDA
// Adapt the classic wrapper-based entry point to the array-based one, so ops ported to the
// new graph engine only need to override initCUDA(context, inputs, outputs) with GpuMatND.
std::vector<cuda::GpuMatND> inGpu(inputs.size()), outGpu(outputs.size());
for (size_t i = 0; i < inputs.size(); i++)
inGpu[i] = inputs[i].dynamicCast<CUDABackendWrapper>()->getDeviceMatND(/*forWrite=*/false);
for (size_t i = 0; i < outputs.size(); i++)
outGpu[i] = outputs[i].dynamicCast<CUDABackendWrapper>()->getDeviceMatND(/*forWrite=*/true);
return initCUDA(context, inGpu, outGpu);
#else
CV_UNUSED(context); CV_UNUSED(inputs); CV_UNUSED(outputs);
CV_Error(Error::StsNotImplemented, "CUDA pipeline of " + type + " layers is not defined.");
return Ptr<BackendNode>();
#endif
}
Ptr<BackendNode> Layer::initCUDA(
void*,
const std::vector<Ptr<BackendWrapper>>&,
const std::vector<Ptr<BackendWrapper>>&)
InputArrayOfArrays,
InputArrayOfArrays)
{
CV_Error(Error::StsNotImplemented, "CUDA pipeline of " + type + " layers is not defined.");
return Ptr<BackendNode>();
@@ -106,9 +127,7 @@ Ptr<BackendNode> Layer::initCann(const std::vector<Ptr<BackendWrapper> > &inputs
bool Layer::setActivation(const Ptr<ActivationLayer>&) { return false; }
bool Layer::tryFuse(Ptr<Layer>&) { return false; }
void Layer::forwardCUDA(const std::vector<Ptr<BackendWrapper> >&,
const std::vector<Ptr<BackendWrapper> >&,
void*)
void Layer::forwardCUDA(InputArrayOfArrays, OutputArrayOfArrays, void*)
{
CV_Error(Error::StsNotImplemented, "CUDA forward of " + type + " layers is not defined.");
}
modules/dnn/src/layers/batch_norm_layer.cpp
+2 -2
View File
@@ -356,8 +356,8 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(
void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
InputArrayOfArrays /*inputs*/,
InputArrayOfArrays /*outputs*/
) override
{
auto context = reinterpret_cast<csl::CSLContext*>(context_);
modules/dnn/src/layers/conv2_layer.cpp
+7 -5
View File
@@ -802,16 +802,18 @@ public:
return layer;
}
void forwardCUDA(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendWrapper> >& outputs,
void forwardCUDA(InputArrayOfArrays inputs_,
OutputArrayOfArrays outputs_,
void* workspace) CV_OVERRIDE
{
std::vector<cuda::GpuMatND> inputs, outputs;
inputs_.getGpuMatNDVector(inputs);
outputs_.getGpuMatNDVector(outputs);
CV_Assert(!inputs.empty() && !outputs.empty());
auto& ws = *reinterpret_cast<cuda4dnn::csl::Workspace*>(workspace);
if (!node) {
auto inW = inputs[0].dynamicCast<CUDABackendWrapper>();
auto outW = outputs[0].dynamicCast<CUDABackendWrapper>();
node = conv->initCudaConvNode(ctx, inW->getShape(), outW->getShape(), preferableTarget);
node = conv->initCudaConvNode(ctx, inputs[0].size, outputs[0].size, preferableTarget);
cudaNode = node.dynamicCast<CUDABackendNode>();
CV_Assert(cudaNode);
ws.require(cudaNode->get_workspace_memory_in_bytes());
modules/dnn/src/layers/elementwise_layers.cpp
+2 -2
View File
@@ -310,8 +310,8 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(
void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
InputArrayOfArrays /*inputs*/,
InputArrayOfArrays /*outputs*/
) override
{
auto context = reinterpret_cast<csl::CSLContext*>(context_);
modules/dnn/src/layers/flatten_layer.cpp
+7 -4
View File
@@ -281,15 +281,18 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(
void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
InputArrayOfArrays inputs_,
InputArrayOfArrays outputs
) override
{
auto context = reinterpret_cast<csl::CSLContext*>(context_);
if (inputs[0]->getHostMatDepth() == CV_Bool)
std::vector<cuda::GpuMatND> inputs;
inputs_.getGpuMatNDVector(inputs);
int depth = CV_MAT_DEPTH(inputs[0].type());
if (depth == CV_Bool)
return make_cuda_node_bool<cuda4dnn::ReshapeOp>(std::move(context->stream));
else
return make_cuda_node_with_type<cuda4dnn::ReshapeOp>(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream));
return make_cuda_node_with_type<cuda4dnn::ReshapeOp>(preferableTarget, depth, std::move(context->stream));
}
#endif
modules/dnn/src/layers/gemm_layer.cpp
+5 -4
View File
@@ -476,18 +476,19 @@ public:
#ifdef HAVE_CUDA
// Y = A * B + C. B should be guaranteed as two dimensional.
Ptr<BackendNode> initCUDA(void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs) CV_OVERRIDE {
InputArrayOfArrays inputs_,
InputArrayOfArrays outputs) CV_OVERRIDE {
CV_CheckFalse(trans_a, "DNN/Gemm/Cuda: does not support transA");
CV_CheckTrue(const_B, "DNN/Gemm/Cuda: input B (weight) is required to be constant");
auto context = reinterpret_cast<csl::CSLContext*>(context_);
auto wrapper_A = inputs[0].dynamicCast<CUDABackendWrapper>();
std::vector<cuda::GpuMatND> inputs;
inputs_.getGpuMatNDVector(inputs);
auto B = blobs[0];
auto C = have_bias && const_C ? blobs[1] : Mat(); // in most cases C is constant
if (!trans_b)
cv::transpose(B, B);
auto flatten_start_axis = normalize_axis(1, wrapper_A->getRank());
auto flatten_start_axis = normalize_axis(1, (int)inputs[0].size.size());
return make_cuda_node<cuda4dnn::InnerProductOp>(preferableTarget, std::move(context->stream), std::move(context->cublas_handle), flatten_start_axis, B, C);
}
#endif // HAVE_CUDA
modules/dnn/src/layers/maxpool_layer.cpp
+5 -4
View File
@@ -493,12 +493,13 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(void* context_,
const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendWrapper> >&) CV_OVERRIDE
InputArrayOfArrays inputs_,
InputArrayOfArrays) CV_OVERRIDE
{
auto context = reinterpret_cast<cuda4dnn::csl::CSLContext*>(context_);
auto inW = inputs[0].dynamicCast<CUDABackendWrapper>();
MatShape inShape = inW->getShape();
std::vector<cuda::GpuMatND> inputs;
inputs_.getGpuMatNDVector(inputs);
MatShape inShape = inputs[0].size;
const int nspatial = (int)kernel_shape.size();
cuda4dnn::PoolingConfiguration config;
modules/dnn/src/layers/nary_eltwise_layers.cpp
+5 -3
View File
@@ -1320,11 +1320,13 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(
void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
InputArrayOfArrays inputs_,
InputArrayOfArrays outputs
) override
{
auto context = reinterpret_cast<csl::CSLContext*>(context_);
std::vector<cuda::GpuMatND> inputs;
inputs_.getGpuMatNDVector(inputs);
cuda4dnn::EltwiseOpType op_ = cuda4dnn::EltwiseOpType::SUM;
switch (op) {
@@ -1361,7 +1363,7 @@ public:
default: return Ptr<BackendNode>(); // return empty cuda_node if the EltwiseOpType is unsupported type.
};
return make_cuda_node_with_type<cuda4dnn::EltwiseOp>(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), op_, std::vector<float>());
return make_cuda_node_with_type<cuda4dnn::EltwiseOp>(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), op_, std::vector<float>());
}
#endif
modules/dnn/src/layers/pooling_layer.cpp
+9 -7
View File
@@ -373,16 +373,18 @@ public:
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(
void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
InputArrayOfArrays inputs_,
InputArrayOfArrays outputs_
) override
{
auto context = reinterpret_cast<csl::CSLContext*>(context_);
if (type == ROI)
return make_cuda_node<cuda4dnn::ROIPoolingOp>(preferableTarget, std::move(context->stream), spatialScale);
auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
auto input_shape = input_wrapper->getShape();
std::vector<cuda::GpuMatND> inputs, outputs;
inputs_.getGpuMatNDVector(inputs);
outputs_.getGpuMatNDVector(outputs);
MatShape input_shape = inputs[0].size;
/* storing max indices is a special case and we deal with it separately */
if (computeMaxIdx) {
@@ -412,13 +414,13 @@ public:
config.input_shape.assign(std::begin(input_shape), std::end(input_shape));
int indicesType = outputs[1]->getHostMatDepth();
int indicesType = CV_MAT_DEPTH(outputs[1].type());
CV_CheckType(indicesType, indicesType == CV_32S || indicesType == CV_64S, "Unsupported indices type");
if (indicesType == CV_32S)
return make_cuda_node_with_indices<cuda4dnn::MaxPoolingOp, int32_t>(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), config);
return make_cuda_node_with_indices<cuda4dnn::MaxPoolingOp, int32_t>(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), config);
else if (indicesType == CV_64S)
return make_cuda_node_with_indices<cuda4dnn::MaxPoolingOp, int64_t>(preferableTarget, inputs[0]->getHostMatDepth(), std::move(context->stream), config);
return make_cuda_node_with_indices<cuda4dnn::MaxPoolingOp, int64_t>(preferableTarget, CV_MAT_DEPTH(inputs[0].type()), std::move(context->stream), config);
CV_Error(Error::BadDepth, "Unsupported indices type");
return Ptr<BackendNode>();
modules/dnn/src/net_impl.hpp
+16 -5
View File
@@ -156,8 +156,6 @@ struct Net::Impl : public detail::NetImplBase
};
bool fusedSnapshotValid = false;
std::vector<FusedGraphSnapshot> fusedSnapshot;
std::vector<Ptr<BackendWrapper> > argWrappers;
std::vector<const void*> argWrapperData;
TracingMode tracingMode;
ProfilingMode profilingMode;
std::vector<int64_t> dimvalues;
@@ -295,6 +293,22 @@ struct Net::Impl : public detail::NetImplBase
std::unique_ptr<CudaInfo_t> cudaInfo;
void initCUDABackend(const std::vector<LayerPin>& blobsToKeep_);
// New graph engine: per-Arg device-resident tensors owned directly by the net (no backend
// wrappers). Sized lazily via GpuMatND::fit() and reused across forwards. Dirty flags track
// which copy (host cv::Mat vs device GpuMatND) is authoritative so transfers happen only at
// CPU<->CUDA boundaries; intermediates stay device-resident across consecutive CUDA ops.
std::vector<cuda::GpuMatND> cudaArgBuffers;
std::vector<uchar> cudaArgHostDirty; // 1: host copy is authoritative -> needs H2D before device read
std::vector<uchar> cudaArgDeviceDirty; // 1: device copy is authoritative -> needs D2H before host read
// Device element type for a host tensor (half for float tensors under the FP16 target).
int cudaDeviceType(const Mat& hostMat) const;
// Returns the device buffer for @p arg, fit() to the host Mat's shape and device type.
cuda::GpuMatND& getCudaArgBuffer(Arg arg, const Mat& hostMat);
void cudaSetHostDirty(Arg arg); // mark host authoritative (e.g. after a CPU op wrote it)
void cudaUploadArg(Arg arg, const Mat& hostMat); // H2D if host dirty
void cudaDownloadArg(Arg arg, Mat& hostMat); // D2H if device dirty
#endif
#ifdef HAVE_ONNXRUNTIME
@@ -441,9 +455,6 @@ struct Net::Impl : public detail::NetImplBase
// Save/restore the fused graph so finalize() is re-entrant across backend changes.
void saveFusedSnapshot();
void restoreFusedSnapshot();
#ifdef HAVE_CUDA
Ptr<BackendWrapper> getCudaArgWrapper(Arg arg, Mat& hostMat);
#endif
// pre-allocates memory for output tensors.
// if useBufferPool==true, the method uses 'buffers'
modules/dnn/src/net_impl2.cpp
+103 -40
View File
@@ -8,6 +8,10 @@
#include <limits>
#ifdef HAVE_CUDA
#include <opencv2/core/cuda_stream_accessor.hpp>
#endif
#ifdef HAVE_ONNXRUNTIME
#include <onnxruntime_cxx_api.h>
#endif
@@ -706,8 +710,9 @@ void Net::Impl::finalize()
bool useCUDA = false;
#ifdef HAVE_CUDA
argWrappers.clear();
argWrapperData.clear();
cudaArgBuffers.clear();
cudaArgHostDirty.clear();
cudaArgDeviceDirty.clear();
if (preferableBackend == DNN_BACKEND_CUDA && haveCUDA()) {
useCUDA = true;
if (!cudaInfo) {
@@ -1349,22 +1354,84 @@ static Mat stackScanAxis(const std::vector<Mat>& perIter, int axis, bool reverse
return stacked;
}
#ifdef HAVE_CUDA
Ptr<BackendWrapper> Net::Impl::getCudaArgWrapper(Arg arg, Mat& hostMat)
// cv::cuda::Stream view over the (non-owning) cuda4dnn inference stream, so GpuMatND transfers
// are ordered against the op compute that runs on the same cudaStream_t.
static inline cuda::Stream wrapCudaStream(cuda4dnn::csl::Stream& s)
{
return cuda::StreamAccessor::wrapStream(s.get());
}
// Device element type for a host tensor: float tensors are stored as half under the FP16 target,
// everything else mirrors the host type. fit() reuses the existing allocation when large enough,
// so buffers persist across forwards.
int Net::Impl::cudaDeviceType(const Mat& hostMat) const
{
if (preferableTarget == DNN_TARGET_CUDA_FP16 && CV_MAT_DEPTH(hostMat.type()) == CV_32F)
return CV_MAKETYPE(CV_16F, CV_MAT_CN(hostMat.type()));
return hostMat.type();
}
cuda::GpuMatND& Net::Impl::getCudaArgBuffer(Arg arg, const Mat& hostMat)
{
int idx = arg.idx;
if ((int)argWrappers.size() != (int)args.size()) {
argWrappers.assign(args.size(), Ptr<BackendWrapper>());
argWrapperData.assign(args.size(), nullptr);
if ((int)cudaArgBuffers.size() != (int)args.size()) {
cudaArgBuffers.assign(args.size(), cuda::GpuMatND());
cudaArgHostDirty.assign(args.size(), 1);
cudaArgDeviceDirty.assign(args.size(), 0);
}
Ptr<CUDABackendWrapper> cw = argWrappers[idx].dynamicCast<CUDABackendWrapper>();
if (!cw || argWrapperData[idx] != (const void*)hostMat.data) {
Ptr<BackendWrapper> w = wrapMat(DNN_BACKEND_CUDA, preferableTarget, hostMat);
cw = w.dynamicCast<CUDABackendWrapper>();
cw->setStream(cudaInfo->context.stream, cudaInfo->d2h_stream);
argWrappers[idx] = w;
argWrapperData[idx] = (const void*)hostMat.data;
cudaArgBuffers[idx].fit(hostMat.shape(), cudaDeviceType(hostMat));
return cudaArgBuffers[idx];
}
void Net::Impl::cudaSetHostDirty(Arg arg)
{
int idx = arg.idx;
if (idx >= 0 && idx < (int)cudaArgHostDirty.size()) {
cudaArgHostDirty[idx] = 1;
cudaArgDeviceDirty[idx] = 0;
}
}
void Net::Impl::cudaUploadArg(Arg arg, const Mat& hostMat)
{
int idx = arg.idx;
cuda::GpuMatND& g = getCudaArgBuffer(arg, hostMat);
if (cudaArgHostDirty[idx]) {
cuda::Stream s = wrapCudaStream(cudaInfo->context.stream);
if (g.type() == hostMat.type()) {
g.upload(hostMat, s);
} else {
// FP32 -> FP16 (device stores half): convert on host, then copy up.
Mat tmp;
hostMat.convertTo(tmp, CV_MAT_DEPTH(g.type()));
g.upload(tmp, s);
}
cudaArgHostDirty[idx] = 0;
cudaArgDeviceDirty[idx] = 0;
}
}
void Net::Impl::cudaDownloadArg(Arg arg, Mat& hostMat)
{
int idx = arg.idx;
if (idx < 0 || idx >= (int)cudaArgDeviceDirty.size())
return;
if (cudaArgDeviceDirty[idx]) {
cuda::GpuMatND& g = cudaArgBuffers[idx];
cuda::Stream s = wrapCudaStream(cudaInfo->context.stream);
if (g.type() == hostMat.type()) {
g.download(hostMat, s);
cudaInfo->context.stream.synchronize(); // host read follows immediately
} else {
// device stores half: copy down, then convert up to the host FP32 tensor.
Mat tmp;
g.download(tmp, s);
cudaInfo->context.stream.synchronize();
tmp.convertTo(hostMat, hostMat.type());
}
cudaArgDeviceDirty[idx] = 0;
cudaArgHostDirty[idx] = 0;
}
return argWrappers[idx];
}
static void forwardOpCUDA(Net::Impl* netimpl, GraphImpl* gimpl, size_t opidx,
@@ -1373,12 +1440,18 @@ static void forwardOpCUDA(Net::Impl* netimpl, GraphImpl* gimpl, size_t opidx,
{
Ptr<Layer> exec = gimpl->exec_[opidx];
CV_Assert(exec && netimpl->cudaInfo);
std::vector<Ptr<BackendWrapper> > inpWrappers(inputs.size()), outWrappers(outputs.size());
for (size_t i = 0; i < inputs.size(); i++)
inpWrappers[i] = netimpl->getCudaArgWrapper(inputs[i], inpMats[i]);
for (size_t i = 0; i < outputs.size(); i++)
outWrappers[i] = netimpl->getCudaArgWrapper(outputs[i], outMats[i]);
exec->forwardCUDA(inpWrappers, outWrappers, &netimpl->cudaInfo->workspace);
std::vector<cuda::GpuMatND> inpG(inputs.size()), outG(outputs.size());
for (size_t i = 0; i < inputs.size(); i++) {
netimpl->cudaUploadArg(inputs[i], inpMats[i]); // H2D only if host-authoritative
inpG[i] = netimpl->cudaArgBuffers[inputs[i].idx];
}
for (size_t i = 0; i < outputs.size(); i++) {
outG[i] = netimpl->getCudaArgBuffer(outputs[i], outMats[i]);
int oidx = outputs[i].idx; // op writes the device buffer
netimpl->cudaArgDeviceDirty[oidx] = 1;
netimpl->cudaArgHostDirty[oidx] = 0;
}
exec->forwardCUDA(inpG, outG, &netimpl->cudaInfo->workspace);
}
#endif
@@ -1422,13 +1495,8 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
// forward so the current input is re-uploaded; otherwise a second forward with a
// changed input would read the previous forward's stale device data.
if (cudaInfo) {
for (i = 0; i < n_gr_inputs; i++) {
Arg ginp = gr_inputs[i];
if (ginp.idx >= 0 && ginp.idx < (int)argWrappers.size()) {
Ptr<CUDABackendWrapper> cw = argWrappers[ginp.idx].dynamicCast<CUDABackendWrapper>();
if (cw) cw->setHostDirty();
}
}
for (i = 0; i < n_gr_inputs; i++)
cudaSetHostDirty(gr_inputs[i]);
}
#endif
@@ -1497,10 +1565,9 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
#ifdef HAVE_CUDA
// CPU op: bring any device-resident inputs back to host before reading them.
for (size_t k = 0; k < ninputs; k++) {
if (inputs[k].idx < (int)argWrappers.size()) {
Ptr<CUDABackendWrapper> cw = argWrappers[inputs[k].idx].dynamicCast<CUDABackendWrapper>();
if (cw) { cw->copyToHost(); inpMats[k] = argTensor(inputs[k]); }
}
Mat& t = argTensor(inputs[k]);
cudaDownloadArg(inputs[k], t);
inpMats[k] = t;
}
#endif
if (finalizeLayers)
@@ -1508,12 +1575,8 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
layer->forward(inpMats, outMats, tempMats);
#ifdef HAVE_CUDA
// CPU produced fresh host data; invalidate any stale device copy of its outputs.
for (size_t k = 0; k < noutputs; k++) {
if (outputs[k].idx < (int)argWrappers.size()) {
Ptr<CUDABackendWrapper> cw = argWrappers[outputs[k].idx].dynamicCast<CUDABackendWrapper>();
if (cw) cw->setHostDirty();
}
}
for (size_t k = 0; k < noutputs; k++)
cudaSetHostDirty(outputs[k]);
#endif
}
}
@@ -1767,9 +1830,9 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
Arg out = gr_outputs[i];
#ifdef HAVE_CUDA
// A graph output produced on the device must be brought back to host before it is read.
if (out.idx < (int)argWrappers.size()) {
Ptr<CUDABackendWrapper> cw = argWrappers[out.idx].dynamicCast<CUDABackendWrapper>();
if (cw) cw->copyToHost();
if (out.idx >= 0) {
Mat& t = argTensor(out);
cudaDownloadArg(out, t);
}
#endif
const Mat& outm = argTensor(out);
modules/dnn/src/op_cuda.cpp
+7 -3
View File
@@ -37,15 +37,19 @@ public:
impl->finalize(inputs, outputs);
}
void forwardCUDA(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendWrapper> >& outputs,
void forwardCUDA(InputArrayOfArrays inputs_,
OutputArrayOfArrays outputs_,
void* workspace) CV_OVERRIDE
{
std::vector<cuda::GpuMatND> inputs, outputs;
inputs_.getGpuMatNDVector(inputs);
outputs_.getGpuMatNDVector(outputs);
cuda4dnn::csl::Workspace& ws = *reinterpret_cast<cuda4dnn::csl::Workspace*>(workspace);
if (!node) {
impl->preferableTarget = preferableTarget; // initCUDA may pick FP16/FP32 by target
cuda4dnn::csl::CSLContext context = *reinterpret_cast<cuda4dnn::csl::CSLContext*>(ctx);
node = impl->initCUDA(&context, inputs, outputs);
node = impl->initCUDA(&context, inputs_, outputs_);
CV_Assert(node);
cudaNode = node.dynamicCast<CUDABackendNode>();
CV_Assert(cudaNode);
modules/dnn/src/op_cuda.hpp
+72 -3
View File
@@ -18,6 +18,7 @@
#include <opencv2/dnn/shape_utils.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/cuda.hpp>
#include <cstddef>
#include <memory>
@@ -177,6 +178,22 @@ namespace cv { namespace dnn {
if (temp.data != destMat.data)
temp.copyTo(destMat);
}
/** @brief builds a read-only TensorView<T> over the device memory of a GpuMatND (no copy) */
template <class T>
TensorView<T> viewOf(const cuda::GpuMatND& g) {
using const_ptr = typename TensorView<T>::const_pointer;
return TensorView<T>(const_ptr(reinterpret_cast<const T*>(g.getDevicePtr())),
std::begin(g.size), std::end(g.size));
}
/** @brief builds a writable TensorSpan<T> over the device memory of a GpuMatND (no copy) */
template <class T>
TensorSpan<T> spanOf(const cuda::GpuMatND& g) {
using ptr = typename TensorSpan<T>::pointer;
return TensorSpan<T>(ptr(reinterpret_cast<T*>(g.getDevicePtr())),
std::begin(g.size), std::end(g.size));
}
}} /* namespace cuda4dnn::csl */
/** base class for CUDA operation nodes (for all supported targets) */
@@ -185,10 +202,25 @@ namespace cv { namespace dnn {
CUDABackendNode() : BackendNode(DNN_BACKEND_CUDA) { }
virtual ~CUDABackendNode() { }
/** classic-engine entry point (wrapper-based).
*
* The default adapts the wrappers to GpuMatND headers and dispatches to the GpuMatND
* overload, so ops ported to the new graph engine only implement the GpuMatND forward.
* Ops not yet ported keep overriding this method directly.
*/
virtual void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
cuda4dnn::csl::Workspace& workspace) = 0;
cuda4dnn::csl::Workspace& workspace);
/** new graph-engine entry point (wrapper-free): operates directly on GpuMatND device tensors */
virtual void forward(
const std::vector<cuda::GpuMatND>& inputs,
const std::vector<cuda::GpuMatND>& outputs,
cuda4dnn::csl::Workspace& workspace)
{
CV_Error(Error::StsNotImplemented, "GpuMatND CUDA forward is not implemented for this operation");
}
virtual std::size_t get_workspace_memory_in_bytes() const noexcept { return 0; }
};
@@ -229,7 +261,7 @@ namespace cv { namespace dnn {
template <template <class> class NodeType, class ...Args>
cv::Ptr<BackendNode> make_cuda_node_with_type(int targetId, int hostMatType, Args&& ...args) {
CV_CheckType(hostMatType, hostMatType == CV_32F || hostMatType == CV_8S || hostMatType == CV_8U || hostMatType == CV_32S || hostMatType == CV_64S, "");
CV_CheckType(hostMatType, hostMatType == CV_32F || hostMatType == CV_16F || hostMatType == CV_8S || hostMatType == CV_8U || hostMatType == CV_32S || hostMatType == CV_64S, "");
if (hostMatType == CV_8S)
return Ptr<BackendNode>(new NodeType<int8_t>(std::forward<Args>(args)...));
@@ -239,6 +271,8 @@ namespace cv { namespace dnn {
return Ptr<BackendNode>(new NodeType<int32_t>(std::forward<Args>(args)...));
else if (hostMatType == CV_64S)
return Ptr<BackendNode>(new NodeType<int64_t>(std::forward<Args>(args)...));
else if (hostMatType == CV_16F) // device tensor already stored as half (FP16 target)
return Ptr<BackendNode>(new NodeType<half>(std::forward<Args>(args)...));
else if (hostMatType == CV_32F)
{
if (targetId == DNN_TARGET_CUDA_FP16)
@@ -252,7 +286,7 @@ namespace cv { namespace dnn {
template <template <class, class> class NodeType, class T_INDEX, class ...Args>
cv::Ptr<BackendNode> make_cuda_node_with_indices(int targetId, int hostMatType, Args&& ...args) {
CV_CheckType(hostMatType, hostMatType == CV_32F || hostMatType == CV_8S || hostMatType == CV_8U || hostMatType == CV_32S || hostMatType == CV_64S, "");
CV_CheckType(hostMatType, hostMatType == CV_32F || hostMatType == CV_16F || hostMatType == CV_8S || hostMatType == CV_8U || hostMatType == CV_32S || hostMatType == CV_64S, "");
if (hostMatType == CV_8S)
return Ptr<BackendNode>(new NodeType<int8_t, T_INDEX>(std::forward<Args>(args)...));
@@ -262,6 +296,8 @@ namespace cv { namespace dnn {
return Ptr<BackendNode>(new NodeType<int32_t, T_INDEX>(std::forward<Args>(args)...));
else if (hostMatType == CV_64S)
return Ptr<BackendNode>(new NodeType<int64_t, T_INDEX>(std::forward<Args>(args)...));
else if (hostMatType == CV_16F) // device tensor already stored as half (FP16 target)
return Ptr<BackendNode>(new NodeType<half, T_INDEX>(std::forward<Args>(args)...));
else if (hostMatType == CV_32F)
{
if (targetId == DNN_TARGET_CUDA_FP16)
@@ -298,6 +334,15 @@ namespace cv { namespace dnn {
virtual void setStream(cuda4dnn::csl::Stream stream, cuda4dnn::csl::Stream h2d_stream) noexcept = 0;
virtual void update(const MatShape& shape, std::size_t offset) = 0;
/** @brief returns a GpuMatND header over the device memory (no copy)
*
* @p forWrite selects the same synchronization semantics as getSpan()/getView():
* for reads the host->device transfer is performed if needed; for writes the device
* memory is marked dirty. The returned header borrows the device memory and is only
* valid while this wrapper (and its device tensor) is alive.
*/
virtual cuda::GpuMatND getDeviceMatND(bool forWrite) = 0;
};
namespace cuda4dnn { namespace detail {
@@ -646,6 +691,17 @@ namespace cv { namespace dnn {
return tensor_view_type(shared_block->device.get() + offset, std::begin(shape), std::end(shape));
}
cuda::GpuMatND getDeviceMatND(bool forWrite) override {
if (forWrite)
setDeviceDirty();
else
copyToDevice();
DEVICE_T* ptr = shared_block->device.get().get() + offset;
// Report the host element type (e.g. CV_32F even for an FP16 device buffer): the type is
// only consumed by initCUDA(), while the compute reinterprets the pointer per its own T.
return cuda::GpuMatND(shape, hostMatDepth, static_cast<void*>(ptr));
}
private:
/* The same tensor memory can be reused by different layers whenever possible.
* Hence, it is possible for different backend wrappers to point to the same memory.
@@ -697,6 +753,19 @@ namespace cv { namespace dnn {
template <class T>
using GetCUDABackendWrapperType = typename GetCUDABackendWrapperType_<T>::type;
inline void CUDABackendNode::forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
cuda4dnn::csl::Workspace& workspace)
{
std::vector<cuda::GpuMatND> inGpu(inputs.size()), outGpu(outputs.size());
for (size_t i = 0; i < inputs.size(); i++)
inGpu[i] = inputs[i].dynamicCast<CUDABackendWrapper>()->getDeviceMatND(/*forWrite=*/false);
for (size_t i = 0; i < outputs.size(); i++)
outGpu[i] = outputs[i].dynamicCast<CUDABackendWrapper>()->getDeviceMatND(/*forWrite=*/true);
forward(inGpu, outGpu, workspace);
}
#endif
}} /* namespace cv::dnn */