diff --git a/modules/dnn/src/layers/avgpool_layer.cpp b/modules/dnn/src/layers/avgpool_layer.cpp index aa7a4d6bf0..987789eb94 100644 --- a/modules/dnn/src/layers/avgpool_layer.cpp +++ b/modules/dnn/src/layers/avgpool_layer.cpp @@ -52,7 +52,7 @@ static void avgPool32f(const void* inp_, void* out_, float* out = (float*)out_ + nc0*planesize; float iksize = 1.f/ksize; -#if CV_SIMD +#if CV_SIMD || CV_SIMD_SCALABLE int nlanes = VTraits::vlanes(); CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0); v_float32 z = vx_setzero_f32(); @@ -68,12 +68,12 @@ static void avgPool32f(const void* inp_, void* out_, y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W; int yi_ = y0*SY - padY0; - #if !(CV_SIMD) + #if !(CV_SIMD || CV_SIMD_SCALABLE) memset(out, 0, W*C0*sizeof(out[0])); #endif for(;;) { - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; @@ -152,7 +152,7 @@ static void avgPool32f(const void* inp_, void* out_, break; x1 = inner_x1; - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; diff --git a/modules/dnn/src/layers/batch_norm2_layer.cpp b/modules/dnn/src/layers/batch_norm2_layer.cpp index 43d8dda5c5..7d686567cb 100644 --- a/modules/dnn/src/layers/batch_norm2_layer.cpp +++ b/modules/dnn/src/layers/batch_norm2_layer.cpp @@ -19,7 +19,7 @@ namespace dnn { */ #undef CV_SIMD_ONLY -#if CV_SIMD +#if CV_SIMD || CV_SIMD_SCALABLE #define CV_SIMD_ONLY(expr) expr #else #define CV_SIMD_ONLY(expr) @@ -135,7 +135,7 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale, } } } - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE /* [TODO] support C0 == vlanes/2, maybe C0 == vlanes/4. in this case, load everything into vsc0 and vb0, process @@ -153,7 +153,7 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale, scalebuf[c] = biasbuf[c] = 0.f; } v_float32 vsc0, vsc1, vsc2, vsc3; - v_float32 vb0, vb1, vb2, vb3, vb4; + v_float32 vb0, vb1, vb2, vb3; vsc0 = vx_load(scalebuf); vsc1 = vx_load(scalebuf + vlanes); vsc2 = vx_load(scalebuf + vlanes*2); diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp b/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp index 1aada84c28..25b0f55d56 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp @@ -63,7 +63,7 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/, const float* wdata = (const float*)weights__; const float* bias = bias__; -#if (CV_SIMD) +#if (CV_SIMD || CV_SIMD_SCALABLE) // SIMD path is safe when ngroups==1 or Kg%C0==0; repackDeconvWeights() // zero-fills padded lanes. const bool simd_ok = @@ -83,7 +83,7 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/, float* out_k1 = (float*)out__ + ((int64_t)n * K1 + k1) * ospatial * C0; const float* inp_n = (const float*)inp__ + (int64_t)n * C1 * ispatial * C0; -#if (CV_SIMD) +#if (CV_SIMD || CV_SIMD_SCALABLE) // Precompute the shared (g, kblk, c1_abs_base) for the SIMD path. int simd_g = 0, simd_kblk = 0, simd_c1_abs_base = 0; if (simd_ok) { @@ -115,7 +115,7 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/, } } -#if (CV_SIMD) +#if (CV_SIMD || CV_SIMD_SCALABLE) if (simd_ok) { const int VLANES = VTraits::vlanes(); const int v_per_block = C0 / VLANES; diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp index d6feb2d443..642695dfbe 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp @@ -38,7 +38,14 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, parallel_for_(Range(0, NC1), [&](const Range& range) { constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; +#if CV_SIMD_SCALABLE + // RVV: universal intrinsics use LMUL=2, so the float vector width tracks + // the hardware VLEN at runtime. The block size C0 == defaultC0 == vlanes() + // (e.g. 16 at VLEN=256), so read it at runtime instead of fixing it to 8 (#28852). + const int C0 = (int)cs.inpshape.back(); +#else constexpr int C0 = 8; +#endif CV_Assert(cs.nspatialdims <= MAX_CONV_DIMS && MAX_CONV_DIMS == 3); CV_Assert(C0 == cs.inpshape.back()); @@ -73,7 +80,14 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, const float* activParams = cs.activParams.data(); ActivationFunc activation = cs.activation; float maxval = FLT_MAX, defaultAlpha = 0.f; +#if CV_SIMD_SCALABLE + // C0 is runtime on RVV; size scratch by the compile-time upper bound. + float scalebuf[VTraits::max_nlanes]; + float biasbuf[VTraits::max_nlanes]; + float alphabuf[VTraits::max_nlanes]; +#else float scalebuf[C0], biasbuf[C0], alphabuf[C0]; +#endif if (fastActivation == FAST_ACTIV_CLIP) { CV_Assert(cs.activParams.size() == 2u); maxval = activParams[1]; @@ -89,7 +103,7 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, defaultAlpha = 1.f; } - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE v_float32 v_maxval = vx_setall_f32(maxval); v_float32 z = vx_setzero_f32(); const int nlanes = VTraits::vlanes(); @@ -126,12 +140,12 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W; int yi_ = y0*SY - padY0; - #if !(CV_SIMD) + #if !(CV_SIMD || CV_SIMD_SCALABLE) memset(out, 0, W*C0*sizeof(out[0])); #endif for(;;) { - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf); v_float32 alpha0 = vx_load(alphabuf); @@ -218,7 +232,7 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, break; x1 = inner_x1; - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf), alpha0 = vx_load(alphabuf); for (; x0 < x1; x0++) { @@ -335,7 +349,7 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, x1 = W; } - #if !(CV_SIMD) + #if !(CV_SIMD || CV_SIMD_SCALABLE) if (residual) { for (int x = 0; x < W*C0; x += C0) { for (int c = 0; c < C0; c++) { diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp index 11ac5189ae..65adfa8547 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp @@ -432,14 +432,14 @@ static void setupActivation(const ConvState& cs, int K, static void fillCoeffBufs(FastActivation fastActivation, const float* activParams, float defaultAlpha, int k_count, int k_base, const float* scaleptr, const float* biasptr, - float* scalebuf, float* biasbuf, float* alphabuf) { + float* scalebuf, float* biasbuf, float* alphabuf, int K0) { int kk = 0; for (; kk < k_count; kk++) { scalebuf[kk] = scaleptr ? scaleptr[k_base + kk] : 1.f; biasbuf[kk] = biasptr ? biasptr[k_base + kk] : 0.f; alphabuf[kk] = fastActivation == FAST_ACTIV_PRELU ? activParams[k_base + kk] : defaultAlpha; } - for (; kk < 8; kk++) { + for (; kk < K0; kk++) { scalebuf[kk] = 0.f; biasbuf[kk] = 0.f; alphabuf[kk] = 0.f; @@ -570,6 +570,10 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif } #endif +// RVV uses the generic runtime-C0 path below; the C0=8-specialized kernels are +// unused on scalable backends, so do not compile them there (avoids -Wunused-function). +#if !CV_SIMD_SCALABLE + // Specialized 1x1 convolution kernel with stride=1. static void conv32fC8_1x1(const void* inp__, const void* residual__, void* out__, const ConvState& cs, const void* weights__, @@ -655,7 +659,7 @@ static void conv32fC8_1x1(const void* inp__, const void* residual__, void* out__ const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(1*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf, K0); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; @@ -824,8 +828,8 @@ static void conv32fC8_1x1_kpair(const void* inp__, const void* residual__, void* const float* wbaseptrA = (float*)weights__ + (g*Kblk + kblkA)*(1*C1Max*C0*K0); const float* wbaseptrB = (float*)weights__ + (g*Kblk + kblkB)*(1*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseA, scaleptr, biasptr, scalebufA, biasbufA, alphabufA); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseB, scaleptr, biasptr, scalebufB, biasbufB, alphabufB); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseA, scaleptr, biasptr, scalebufA, biasbufA, alphabufA, K0); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseB, scaleptr, biasptr, scalebufB, biasbufB, alphabufB, K0); float* outbaseA = (float*)out__ + n*(K1*planesize) + k_baseA*planeblocks; float* outbaseB = (float*)out__ + n*(K1*planesize) + k_baseB*planeblocks; @@ -1050,7 +1054,7 @@ static void conv32fC8_3x3s1(const void* inp__, const void* residual__, void* out const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(9*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf, K0); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; @@ -1329,8 +1333,8 @@ static void conv32fC8_3x3s1_kpair(const void* inp__, const void* residual__, voi const float* wbaseptrA = (float*)weights__ + (g*Kblk + kblkA)*(9*C1Max*C0*K0); const float* wbaseptrB = (float*)weights__ + (g*Kblk + kblkB)*(9*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseA, scaleptr, biasptr, scalebufA, biasbufA, alphabufA); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseB, scaleptr, biasptr, scalebufB, biasbufB, alphabufB); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseA, scaleptr, biasptr, scalebufA, biasbufA, alphabufA, K0); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, K0, k_baseB, scaleptr, biasptr, scalebufB, biasbufB, alphabufB, K0); float* outbaseA = (float*)out__ + n*(K1*planesize) + k_baseA*planeblocks; float* outbaseB = (float*)out__ + n*(K1*planesize) + k_baseB*planeblocks; @@ -1635,7 +1639,7 @@ static void conv32fC8_1x1_strided(const void* inp__, const void* residual__, voi const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(1*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf, K0); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; @@ -1831,7 +1835,7 @@ static void conv32fC8_3x3_strided(const void* inp__, const void* residual__, voi const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(9*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf, K0); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; @@ -2019,10 +2023,13 @@ static void conv32fC8_3x3_strided(const void* inp__, const void* residual__, voi }); } +#endif // !CV_SIMD_SCALABLE + static void conv32fC8(const void* inp__, const void* residual__, void* out__, const ConvState& cs, const void* weights__, const float* scale__, const float* bias__) { +#if !CV_SIMD_SCALABLE // RVV: skip the C0=8-specialized kernels; use the generic runtime-C0 path int ksize = cs.wshape[2]; if (ksize == 1 && cs.strides[0]*cs.strides[1]*cs.strides[2] == 1) { #if CV_SIMD256 && defined(__AVX2__) @@ -2063,6 +2070,7 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, cs.outshape.dims <= 5) { return conv32fC8_3x3_strided(inp__, residual__, out__, cs, weights__, scale__, bias__); } +#endif // !CV_SIMD_SCALABLE const MatShape& inpshape = cs.inpshape; const MatShape& outshape = cs.outshape; @@ -2091,8 +2099,17 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, parallel_for_(Range(0, total_tasks_gen), [&](const Range& range) { constexpr int SPAT_BLOCK_SIZE = 10; +#if CV_SIMD_SCALABLE + // RVV: block size follows the runtime vector width (defaultC0 = vlanes(), LMUL=2). + const int C0 = (int)inpshape.back(); + int C0shift = 0; while ((1 << C0shift) < C0) C0shift++; + const int K0 = C0, K0shift = C0shift; + constexpr int C0BUF = VTraits::max_nlanes; // compile-time scratch bound +#else constexpr int C0shift = 3, K0shift = C0shift; constexpr int C0 = 1 << C0shift, K0 = C0; + constexpr int C0BUF = K0; +#endif CV_Assert_N(inpshape.back() == C0, outshape.back() == K0); @@ -2122,14 +2139,14 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS]; int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; - float zbuf[C0] = {}; + float zbuf[C0BUF] = {}; #endif FastActivation fastActivation; const float* activParams; ActivationFunc activation; float maxval, defaultAlpha; - float scalebuf[K0], biasbuf[K0], alphabuf[K0]; + float scalebuf[C0BUF], biasbuf[C0BUF], alphabuf[C0BUF]; setupActivation(cs, K, fastActivation, activParams, activation, maxval, defaultAlpha); // 1x1x1 convolution with (1,1,1) strides: @@ -2167,11 +2184,11 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(ksize*C1Max*C0*K0); - fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf, K0); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; - float tmpbuf[SPAT_BLOCK_SIZE*K0] = {}; + float tmpbuf[SPAT_BLOCK_SIZE*C0BUF] = {}; int p = p0; #ifdef CONV_ENABLE_SIMD @@ -2343,7 +2360,7 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, } #endif - float resbuf[K0] = {}; + float resbuf[C0BUF] = {}; for (; p < p1; p++, outptr += K0, resptr += (resptr ? K0 : 0)) { @@ -2425,9 +2442,16 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, cv::dnn::ConvFunc getConvFunc_(int depth, int C0) { ConvFunc func = nullptr; +#if CV_SIMD_SCALABLE + // RVV: block size follows the runtime vector width; accept the supported pow2 widths. + if (depth == CV_32F && (C0 == 8 || C0 == 16 || C0 == 32 || C0 == 64)) { + func = conv32fC8; + } +#else if (depth == CV_32F && C0 == 8) { func = conv32fC8; } +#endif return func; } diff --git a/modules/dnn/src/layers/maxpool_layer.cpp b/modules/dnn/src/layers/maxpool_layer.cpp index ce64e7ceb0..0cc6c5ac4c 100644 --- a/modules/dnn/src/layers/maxpool_layer.cpp +++ b/modules/dnn/src/layers/maxpool_layer.cpp @@ -50,7 +50,7 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) float* out = (float*)out_ + nc0*planesize; const float INITVAL = -FLT_MAX; - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE int nlanes = VTraits::vlanes(); v_float32 s_min = vx_setall_f32(INITVAL); CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0); @@ -65,13 +65,13 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W; int yi_ = y0*SY - padY0; - #if !(CV_SIMD) + #if !(CV_SIMD || CV_SIMD_SCALABLE) for (int c = 0; c < C0*W; c++) out[c] = INITVAL; #endif for(;;) { - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; @@ -136,7 +136,7 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) break; x1 = inner_x1; - #if CV_SIMD + #if CV_SIMD || CV_SIMD_SCALABLE if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; diff --git a/modules/dnn/src/net_impl.cpp b/modules/dnn/src/net_impl.cpp index a12109eeca..92445d657e 100644 --- a/modules/dnn/src/net_impl.cpp +++ b/modules/dnn/src/net_impl.cpp @@ -5,6 +5,7 @@ #include "precomp.hpp" #include "net_impl.hpp" +#include "opencv2/core/hal/intrin.hpp" #ifdef HAVE_ONNXRUNTIME #include @@ -74,6 +75,13 @@ Net::Impl::Impl() accuracy = CV_32F; defaultC0 = DEFAULT_C0; +#if CV_SIMD_SCALABLE + // RVV: the universal intrinsics use LMUL=2, so the float vector width is + // (VLEN/32)*2 (16 at VLEN=256). The blocked-layout kernels fill a full + // vector per channel block, so the net-wide block size must match that + // width rather than the fixed default of 8 (#28852). + defaultC0 = std::max((int)DEFAULT_C0, VTraits::vlanes()); +#endif enableFP16 = haveFP16 = false; // FP16 is not ready yet in the new DNN engine // Ticket: https://github.com/opencv/opencv/issues/26196 diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index d497352dd1..f7fb5d45f0 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -5,6 +5,7 @@ #include "precomp.hpp" #include "net_impl.hpp" +#include "opencv2/core/hal/intrin.hpp" #include @@ -545,6 +546,16 @@ void Net::Impl::prepareForInference() #endif if (!prepared) { +#if CV_SIMD_SCALABLE + // RVV (#28852): keep quantized graphs at C0=8. The int8 kernels are hardwired to + // C0=8 (VNNI/NEON weight packing + per-channel quantization) and run scalar on RVV, + // so a wider block gives no benefit and breaks them. Only fp32 graphs use the wider + // vlanes()-based defaultC0. Signed-int8 (CV_8S) args are the quantization signature + // (uint8 image inputs are CV_8U, so they don't trigger this). + for (const ArgData& a : args) { + if (a.type == CV_8S) { defaultC0 = 8; break; } + } +#endif fuseQDQ(); constFold(); fuseBN();