diff --git a/modules/dnn/src/layers/avgpool_layer.cpp b/modules/dnn/src/layers/avgpool_layer.cpp index 861baec27b..5c371c5f5b 100644 --- a/modules/dnn/src/layers/avgpool_layer.cpp +++ b/modules/dnn/src/layers/avgpool_layer.cpp @@ -53,7 +53,18 @@ static void avgPool32f(const void* inp_, void* out_, float* out = (float*)out_ + nc0*planesize; float iksize = 1.f/ksize; -#if CV_SIMD || CV_SIMD_SCALABLE +#if CV_SIMD_SCALABLE + // RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so + // the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide + // and there is no way to make it span exactly C0 elements. Drop to native RVV and + // pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) = + // VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one + // vector spans precisely one channel block -- no VLEN-dependent branches, and + // nothing can overrun or under-fill the block. + const size_t _vl = __riscv_vsetvl_e32m2(C0); + vfloat32m2_t z = __riscv_vfmv_v_f_f32m2(0.f, _vl); + vfloat32m2_t vscale0 = __riscv_vfmv_v_f_f32m2(iksize, _vl); +#elif CV_SIMD int nlanes = VTraits::vlanes(); CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0); v_float32 z = vx_setzero_f32(); @@ -74,7 +85,30 @@ static void avgPool32f(const void* inp_, void* out_, #endif for(;;) { - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + vfloat32m2_t s0 = z; + int nitems = 0, npadded = 0; + for (int k = 0; k < ksize; k++) { + int zi = zi_ + zyxtab[k*MAX_POOL_DIMS]; + int yi = yi_ + zyxtab[k*MAX_POOL_DIMS+1]; + int xi = xi_ + zyxtab[k*MAX_POOL_DIMS+2]; + npadded += (zi >= -padZ0 && zi < Di + padZ1 && + yi >= -padY0 && yi < Hi + padY1 && + xi >= -padX0 && xi < Wi + padX1); + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) + continue; + vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl); + s0 = __riscv_vfadd_vv_f32m2(s0, v0, _vl); + nitems++; + } + s0 = __riscv_vfmul_vf_f32m2(s0, 1.f/(count_include_pad ? npadded : nitems), _vl); + __riscv_vse32_v_f32m2(out + x0*C0, s0, _vl); + } + #elif CV_SIMD if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; @@ -162,7 +196,17 @@ static void avgPool32f(const void* inp_, void* out_, break; x1 = inner_x1; - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0; + + vfloat32m2_t s0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[0], _vl); + for (int k = 1; k < ksize; k++) + s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl), _vl); + __riscv_vse32_v_f32m2(out + x0*C0, __riscv_vfmul_vv_f32m2(s0, vscale0, _vl), _vl); + } + #elif CV_SIMD if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; diff --git a/modules/dnn/src/layers/batch_norm2_layer.cpp b/modules/dnn/src/layers/batch_norm2_layer.cpp index 7d686567cb..d29022399a 100644 --- a/modules/dnn/src/layers/batch_norm2_layer.cpp +++ b/modules/dnn/src/layers/batch_norm2_layer.cpp @@ -136,12 +136,6 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale, } } #if CV_SIMD || CV_SIMD_SCALABLE - /* - [TODO] support C0 == vlanes/2, maybe C0 == vlanes/4. - in this case, load everything into vsc0 and vb0, process - most part of the plane using vector code as if C0 == vlanes and - then process the tail using scalar code - */ else if (C0 == vlanes*4 || C0 == vlanes*2 || C0 == vlanes) { // accelerated block layout case int c = 0; @@ -268,6 +262,63 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale, } } } + else if (C0 < vlanes && vlanes % C0 == 0) { + /* + Vector wider than the channel block (e.g. C0 == 8 on a VLEN >= 512 + machine). The plane is contiguous [pixel][C0] and the per-channel + coefficients repeat every C0 elements, so replicate the C0-wide + scale/bias pattern vlanes/C0 times. One vector then spans vlanes/C0 + consecutive pixels with correctly aligned coefficients, and the plane + can be swept in plain vlanes-sized steps at full lane utilisation. + */ + const int reps = vlanes / C0; + int c = 0; + for (; c < c_delta; c++) { + scalebuf[c] = scaleptr[c]; + biasbuf[c] = biasptr[c]; + } + for (; c < C0; c++) // padded channels of the last block -> zero output + scalebuf[c] = biasbuf[c] = 0.f; + for (int r = 1; r < reps; r++) { + for (int c2 = 0; c2 < C0; c2++) { + scalebuf[r*C0 + c2] = scalebuf[c2]; + biasbuf[r*C0 + c2] = biasbuf[c2]; + } + } + v_float32 vsc = vx_load(scalebuf); + v_float32 vb = vx_load(biasbuf); + if (type == CV_32F) { + const float* inptr = (const float*)inptr_; + float* outptr = (float*)outptr_; + for (; i <= planesize_C0 - vlanes; i += vlanes) { + v_float32 x0 = vx_load(inptr + i); + x0 = v_fma(x0, vsc, vb); + v_store(outptr + i, x0); + } + for (; i < planesize_C0; i++) + outptr[i] = inptr[i]*scalebuf[i % C0] + biasbuf[i % C0]; + } else if (type == CV_16F) { + const hfloat* inptr = (const hfloat*)inptr_; + hfloat* outptr = (hfloat*)outptr_; + for (; i <= planesize_C0 - vlanes; i += vlanes) { + v_float32 x0 = vx_load_expand(inptr + i); + x0 = v_fma(x0, vsc, vb); + v_pack_store(outptr + i, x0); + } + for (; i < planesize_C0; i++) + outptr[i] = hfloat(float(inptr[i])*scalebuf[i % C0] + biasbuf[i % C0]); + } else if (type == CV_16BF) { + const bfloat* inptr = (const bfloat*)inptr_; + bfloat* outptr = (bfloat*)outptr_; + for (; i <= planesize_C0 - vlanes; i += vlanes) { + v_float32 x0 = vx_load_expand(inptr + i); + x0 = v_fma(x0, vsc, vb); + v_pack_store(outptr + i, x0); + } + for (; i < planesize_C0; i++) + outptr[i] = bfloat(float(inptr[i])*scalebuf[i % C0] + biasbuf[i % C0]); + } + } #endif else { // general block layout or NHWC case diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp b/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp index 25b0f55d56..34f56ffce2 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp @@ -63,9 +63,14 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/, const float* wdata = (const float*)weights__; const float* bias = bias__; -#if (CV_SIMD || CV_SIMD_SCALABLE) +#if CV_SIMD_SCALABLE + // RVV: the vector length is pinned to K0 (see below), so one vector always spans + // exactly one output block whatever the hardware VLEN. Only the weight-packing + // constraint remains; repackDeconvWeights() zero-fills padded lanes. + const bool simd_ok = ((ngroups == 1) || (Kg % C0 == 0)); +#elif CV_SIMD // SIMD path is safe when ngroups==1 or Kg%C0==0; repackDeconvWeights() - // zero-fills padded lanes. + // zero-fills padded lanes. The block must also be a whole number of vectors. const bool simd_ok = ((ngroups == 1) || (Kg % C0 == 0)) && (C0 % VTraits::vlanes() == 0); #endif @@ -115,7 +120,55 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/, } } -#if (CV_SIMD || CV_SIMD_SCALABLE) +#if CV_SIMD_SCALABLE + if (simd_ok) { + // Set the vector length to the block size instead of taking whatever + // the hardware offers: vlmax(e32,m2) = VLEN/32*2 >= K0 for every + // VLEN >= 128, so __riscv_vsetvl_e32m2(K0) returns exactly K0. One + // accumulator therefore spans the whole output block on any VLEN, and + // the block never has to be a whole number of vectors (#29454). + const size_t _vl = __riscv_vsetvl_e32m2(K0); + vfloat32m2_t acc = __riscv_vle32_v_f32m2(out_ptr, _vl); + + for (int ks = 0; ks < ksize; ks++) { + bool valid = true; + int ipos_flat = 0; + for (int i = 0; i < sdims; i++) { + int di = MAX_DIMS - sdims + i; + int raw = ocoords[di] + cs.pads[di] + - kcoords_tab[ks][di] * cs.dilations[di]; + if (raw < 0 || raw % cs.strides[di] != 0) { + valid = false; break; + } + int ic = raw / cs.strides[di]; + if (ic >= iDims[di]) { valid = false; break; } + ipos_flat = ipos_flat * iDims[di] + ic; + } + if (!valid) continue; + + const float* w_base = wdata + + ((int64_t)(simd_g * Kblk + simd_kblk) * ksize + ks) + * C1Max * C0 * K0; + + for (int c1p = 0; c1p < C1Max; c1p++) { + const int c1_abs = simd_c1_abs_base + c1p; + if (c1_abs >= C1) break; + + const float* inp_ptr = inp_n + + (int64_t)(c1_abs * ispatial + ipos_flat) * C0; + const float* w_c1p = w_base + (int64_t)c1p * C0 * K0; + + for (int c0 = 0; c0 < C0; c0++) { + vfloat32m2_t w0 = __riscv_vle32_v_f32m2(w_c1p + c0 * K0, _vl); + acc = __riscv_vfmacc_vf_f32m2(acc, inp_ptr[c0], w0, _vl); + } + } + } + + __riscv_vse32_v_f32m2(out_ptr, acc, _vl); + continue; + } +#elif CV_SIMD if (simd_ok) { const int VLANES = VTraits::vlanes(); const int v_per_block = C0 / VLANES; diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp index e6933584f5..9929a5b9fa 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp @@ -38,6 +38,8 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, parallel_for_(Range(0, NC1), [&](const Range& range) { constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + // The block size is a fixed property of the blocked layout on every platform; on RVV + // the kernel pins the vector length to C0 instead of sizing C0 to the vector (#29454). constexpr int C0 = 8; CV_Assert(cs.nspatialdims <= MAX_CONV_DIMS && MAX_CONV_DIMS == 3); @@ -89,7 +91,18 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, defaultAlpha = 1.f; } - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + // RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so + // the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide + // and there is no way to make it span exactly C0 elements. Drop to native RVV and + // pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) = + // VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one + // vector spans precisely one channel block -- no VLEN-dependent branches, and + // nothing can overrun or under-fill the block. + const size_t _vl = __riscv_vsetvl_e32m2(C0); + vfloat32m2_t v_maxval = __riscv_vfmv_v_f_f32m2(maxval, _vl); + vfloat32m2_t z = __riscv_vfmv_v_f_f32m2(0.f, _vl); + #elif CV_SIMD v_float32 v_maxval = vx_setall_f32(maxval); v_float32 z = vx_setzero_f32(); const int nlanes = VTraits::vlanes(); @@ -131,7 +144,36 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, #endif for(;;) { - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + { + vfloat32m2_t sc0 = __riscv_vle32_v_f32m2(scalebuf, _vl); + vfloat32m2_t b0 = __riscv_vle32_v_f32m2(biasbuf, _vl); + vfloat32m2_t alpha0 = __riscv_vle32_v_f32m2(alphabuf, _vl); + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + vfloat32m2_t s0 = z; + for (int k = 0; k < ksize; k++) { + int zi = zi_ + zyxtab[k*MAX_CONV_DIMS]; + int yi = yi_ + zyxtab[k*MAX_CONV_DIMS + 1]; + int xi = xi_ + zyxtab[k*MAX_CONV_DIMS + 2]; + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) + continue; + vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl); + vfloat32m2_t w0 = __riscv_vle32_v_f32m2(weights + k*C0, _vl); + s0 = __riscv_vfmacc_vv_f32m2(s0, v0, w0, _vl); + } + s0 = __riscv_vfmadd_vv_f32m2(s0, sc0, b0, _vl); + if (residual) + s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(residual + x0*C0, _vl), _vl); + s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, alpha0, _vl), s0, + __riscv_vmfge_vv_f32m2_b16(s0, z, _vl), _vl); + s0 = __riscv_vfmin_vv_f32m2(s0, v_maxval, _vl); + __riscv_vse32_v_f32m2(out + x0*C0, s0, _vl); + } + } + #elif CV_SIMD if (nlanes == C0) { v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf); v_float32 alpha0 = vx_load(alphabuf); @@ -218,7 +260,31 @@ static void depthwiseConv32f(const void* inp__, const void* residual__, break; x1 = inner_x1; - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + { + vfloat32m2_t sc0 = __riscv_vle32_v_f32m2(scalebuf, _vl); + vfloat32m2_t b0 = __riscv_vle32_v_f32m2(biasbuf, _vl); + vfloat32m2_t alpha0 = __riscv_vle32_v_f32m2(alphabuf, _vl); + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0; + + vfloat32m2_t s0 = z; + for (int k = 0; k < ksize; k++) { + vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl); + vfloat32m2_t w0 = __riscv_vle32_v_f32m2(weights + k*C0, _vl); + s0 = __riscv_vfmacc_vv_f32m2(s0, v0, w0, _vl); + } + s0 = __riscv_vfmadd_vv_f32m2(s0, sc0, b0, _vl); + if (residual) + s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(residual + x0*C0, _vl), _vl); + s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, alpha0, _vl), s0, + __riscv_vmfge_vv_f32m2_b16(s0, z, _vl), _vl); + s0 = __riscv_vfmin_vv_f32m2(s0, v_maxval, _vl); + __riscv_vse32_v_f32m2(out + x0*C0, s0, _vl); + } + } + #elif CV_SIMD if (nlanes == C0) { v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf), alpha0 = vx_load(alphabuf); for (; x0 < x1; x0++) { diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp index e16af3e22a..6f1e1d3522 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp @@ -389,32 +389,42 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN #elif CV_SIMD_SCALABLE /////////////////////////// scalable (RVV) implementation ///////////////////////////// -// K0 == vlanes(), so each of the 10 spatial positions needs exactly one vector -// accumulator (structurally identical to the AVX2 path, which uses one v_float32x8 per -// position for its K0=8). One K0-wide weight load is reused across all 10 positions, so -// the per-output-point load bottleneck of the scalar tail is amortized away. C0 (= vlanes()) -// is a runtime value, so the c0 sweep is a loop rather than a fixed unroll. +// The blocked layout fixes K0 == C0 == 8 regardless of the hardware vector width, so the +// universal intrinsics cannot be used here: v_float32 is always vlanes() wide and there is +// no way to make it span exactly K0 elements. Instead we drop to native RVV and set the +// vector length explicitly to K0 (#29454). +// +// LMUL=2 is chosen so that vlmax(e32,m2) = VLEN/32*2 >= 8 for every VLEN >= 128 (the +// smallest RVV configuration). __riscv_vsetvl_e32m2(K0) therefore returns exactly K0 = 8 +// on every machine -- 8 of 8 lanes at VLEN=128, 8 of 64 at VLEN=1024 -- so one accumulator +// per spatial position always spans precisely one output block. No strip-mining, no +// VLEN-dependent branches, and nothing can overrun or under-fill the block. +// +// Note this LMUL is a property of *this kernel only*; it is independent of whatever LMUL +// the universal intrinsics are built with. #undef CONV_INIT_SUMS #define CONV_INIT_SUMS() \ - v_float32 zz = vx_setzero_f32(); \ - v_float32 s0 = zz, s1 = zz, s2 = zz, s3 = zz, s4 = zz; \ - v_float32 s5 = zz, s6 = zz, s7 = zz, s8 = zz, s9 = zz + const size_t _vl = __riscv_vsetvl_e32m2(K0); \ + vfloat32m2_t zz = __riscv_vfmv_v_f_f32m2(0.f, _vl); \ + vfloat32m2_t s0 = zz, s1 = zz, s2 = zz, s3 = zz, s4 = zz; \ + vfloat32m2_t s5 = zz, s6 = zz, s7 = zz, s8 = zz, s9 = zz +// s_j += inptr[j][_c0] * w (vfmacc_vf: vd += rs1 * vs2) #undef CONV_UPDATE_LOOP_BODY #define CONV_UPDATE_LOOP_BODY() \ for (int _c0 = 0; _c0 < C0; _c0++) { \ - v_float32 _w = vx_load(wptr + _c0*K0); \ - s0 = v_fma(vx_setall_f32(inptr[0][_c0]), _w, s0); \ - s1 = v_fma(vx_setall_f32(inptr[1][_c0]), _w, s1); \ - s2 = v_fma(vx_setall_f32(inptr[2][_c0]), _w, s2); \ - s3 = v_fma(vx_setall_f32(inptr[3][_c0]), _w, s3); \ - s4 = v_fma(vx_setall_f32(inptr[4][_c0]), _w, s4); \ - s5 = v_fma(vx_setall_f32(inptr[5][_c0]), _w, s5); \ - s6 = v_fma(vx_setall_f32(inptr[6][_c0]), _w, s6); \ - s7 = v_fma(vx_setall_f32(inptr[7][_c0]), _w, s7); \ - s8 = v_fma(vx_setall_f32(inptr[8][_c0]), _w, s8); \ - s9 = v_fma(vx_setall_f32(inptr[9][_c0]), _w, s9); \ + vfloat32m2_t _w = __riscv_vle32_v_f32m2(wptr + _c0*K0, _vl); \ + s0 = __riscv_vfmacc_vf_f32m2(s0, inptr[0][_c0], _w, _vl); \ + s1 = __riscv_vfmacc_vf_f32m2(s1, inptr[1][_c0], _w, _vl); \ + s2 = __riscv_vfmacc_vf_f32m2(s2, inptr[2][_c0], _w, _vl); \ + s3 = __riscv_vfmacc_vf_f32m2(s3, inptr[3][_c0], _w, _vl); \ + s4 = __riscv_vfmacc_vf_f32m2(s4, inptr[4][_c0], _w, _vl); \ + s5 = __riscv_vfmacc_vf_f32m2(s5, inptr[5][_c0], _w, _vl); \ + s6 = __riscv_vfmacc_vf_f32m2(s6, inptr[6][_c0], _w, _vl); \ + s7 = __riscv_vfmacc_vf_f32m2(s7, inptr[7][_c0], _w, _vl); \ + s8 = __riscv_vfmacc_vf_f32m2(s8, inptr[8][_c0], _w, _vl); \ + s9 = __riscv_vfmacc_vf_f32m2(s9, inptr[9][_c0], _w, _vl); \ } \ inptr[0] += inpstep[0]; inptr[1] += inpstep[1]; \ inptr[2] += inpstep[2]; inptr[3] += inpstep[3]; \ @@ -424,26 +434,30 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN #undef CONV_START_FINALIZE_OUT #define CONV_START_FINALIZE_OUT() \ - v_float32 vscale = vx_load(scalebuf); \ - v_float32 vbias = vx_load(biasbuf); \ - v_float32 valpha = vx_load(alphabuf); \ - v_float32 vmaxval = vx_setall_f32(maxval) + vfloat32m2_t vscale = __riscv_vle32_v_f32m2(scalebuf, _vl); \ + vfloat32m2_t vbias = __riscv_vle32_v_f32m2(biasbuf, _vl); \ + vfloat32m2_t valpha = __riscv_vle32_v_f32m2(alphabuf, _vl); \ + vfloat32m2_t vmaxval = __riscv_vfmv_v_f_f32m2(maxval, _vl) #define CONV_ADD_RESIDUAL2(idx0, idx1) \ - s##idx0 = v_add(s##idx0, vx_load(tmpbuf + idx0*K0)); \ - s##idx1 = v_add(s##idx1, vx_load(tmpbuf + idx1*K0)) + s##idx0 = __riscv_vfadd_vv_f32m2(s##idx0, __riscv_vle32_v_f32m2(tmpbuf + idx0*K0, _vl), _vl); \ + s##idx1 = __riscv_vfadd_vv_f32m2(s##idx1, __riscv_vle32_v_f32m2(tmpbuf + idx1*K0, _vl), _vl) +// s = s*scale + bias; s = (s >= 0 ? s : s*alpha); s = min(s, maxval) +// (vfmadd_vv: vd = vd*vs1 + vs2; vmerge_vvm picks vs1 where the mask is set, else vs2) #undef CONV_FINALIZE_OUT2 #define CONV_FINALIZE_OUT2(idx0, idx1, add_residual2) \ - s##idx0 = v_fma(s##idx0, vscale, vbias); \ - s##idx1 = v_fma(s##idx1, vscale, vbias); \ + s##idx0 = __riscv_vfmadd_vv_f32m2(s##idx0, vscale, vbias, _vl); \ + s##idx1 = __riscv_vfmadd_vv_f32m2(s##idx1, vscale, vbias, _vl); \ add_residual2(idx0, idx1); \ - s##idx0 = v_select(v_ge(s##idx0, zz), s##idx0, v_mul(s##idx0, valpha)); \ - s##idx1 = v_select(v_ge(s##idx1, zz), s##idx1, v_mul(s##idx1, valpha)); \ - s##idx0 = v_min(s##idx0, vmaxval); \ - s##idx1 = v_min(s##idx1, vmaxval); \ - v_store(outbuf + idx0*K0, s##idx0); \ - v_store(outbuf + idx1*K0, s##idx1) + s##idx0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s##idx0, valpha, _vl), s##idx0, \ + __riscv_vmfge_vv_f32m2_b16(s##idx0, zz, _vl), _vl); \ + s##idx1 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s##idx1, valpha, _vl), s##idx1, \ + __riscv_vmfge_vv_f32m2_b16(s##idx1, zz, _vl), _vl); \ + s##idx0 = __riscv_vfmin_vv_f32m2(s##idx0, vmaxval, _vl); \ + s##idx1 = __riscv_vfmin_vv_f32m2(s##idx1, vmaxval, _vl); \ + __riscv_vse32_v_f32m2(outbuf + idx0*K0, s##idx0, _vl); \ + __riscv_vse32_v_f32m2(outbuf + idx1*K0, s##idx1, _vl) #undef CONV_FINALIZE_OUT_ALL #define CONV_FINALIZE_OUT_ALL() \ @@ -604,10 +618,12 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif v_float32x4 zz = v_setzero_f32(); \ v_float32x4 s0 = zz, s1 = zz #elif CV_SIMD_SCALABLE -// RVV: K0 == vlanes(), so a single scalable vector spans the whole output block. +// RVV: native LMUL=2 with vl fixed to K0, so one vector spans exactly one output block +// on any VLEN >= 128. See the CONV_INIT_SUMS block above. #define CONV_INIT_SCALAR_SUMS() \ - v_float32 zz = vx_setzero_f32(); \ - v_float32 s0 = zz + const size_t _vl = __riscv_vsetvl_e32m2(K0); \ + vfloat32m2_t zz = __riscv_vfmv_v_f_f32m2(0.f, _vl); \ + vfloat32m2_t s0 = zz #else #define CONV_INIT_SCALAR_SUMS() \ for (int _ks = 0; _ks < K0; _ks++) tmpbuf[_ks] = 0.f @@ -645,15 +661,16 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif #elif CV_SIMD_SCALABLE #define CONV_FINALIZE_SCALAR_OUT(outbuf) \ { \ - v_float32 _vsc = vx_load(scalebuf); \ - v_float32 _vbi = vx_load(biasbuf); \ - v_float32 _val = vx_load(alphabuf); \ - v_float32 _vmx = vx_setall_f32(maxval); \ - s0 = v_fma(s0, _vsc, _vbi); \ - s0 = v_add(s0, vx_load(resbuf)); \ - s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, _val)); \ - s0 = v_min(s0, _vmx); \ - v_store((outbuf), s0); \ + vfloat32m2_t _vsc = __riscv_vle32_v_f32m2(scalebuf, _vl); \ + vfloat32m2_t _vbi = __riscv_vle32_v_f32m2(biasbuf, _vl); \ + vfloat32m2_t _val = __riscv_vle32_v_f32m2(alphabuf, _vl); \ + vfloat32m2_t _vmx = __riscv_vfmv_v_f_f32m2(maxval, _vl); \ + s0 = __riscv_vfmadd_vv_f32m2(s0, _vsc, _vbi, _vl); \ + s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(resbuf, _vl), _vl); \ + s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, _val, _vl), s0, \ + __riscv_vmfge_vv_f32m2_b16(s0, zz, _vl), _vl); \ + s0 = __riscv_vfmin_vv_f32m2(s0, _vmx, _vl); \ + __riscv_vse32_v_f32m2((outbuf), s0, _vl); \ } #else #define CONV_FINALIZE_SCALAR_OUT(outbuf) \ @@ -664,8 +681,9 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif } #endif -// RVV uses the generic runtime-C0 path below; the C0=8-specialized kernels are -// unused on scalable backends, so do not compile them there (avoids -Wunused-function). +// The C0=8-specialized kernels below are written in fixed-width intrinsics (v_float32x4/x8), +// which do not exist on scalable backends; RVV uses the generic kernel instead, so do not +// compile them there (avoids -Wunused-function). #if !CV_SIMD_SCALABLE // Specialized 1x1 convolution kernel with stride=1. @@ -2132,7 +2150,7 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, const ConvState& cs, const void* weights__, const float* scale__, const float* bias__) { -#if !CV_SIMD_SCALABLE // RVV: skip the C0=8-specialized kernels; use the generic runtime-C0 path +#if !CV_SIMD_SCALABLE // RVV: skip the fixed-width C0=8 kernels; use the generic kernel int ksize = cs.wshape[2]; if (ksize == 1 && cs.strides[0]*cs.strides[1]*cs.strides[2] == 1) { #if CV_SIMD256 && defined(__AVX2__) @@ -2203,7 +2221,8 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, parallel_for_(Range(0, total_tasks_gen), [&](const Range& range) { constexpr int SPAT_BLOCK_SIZE = 10; // The block size is a fixed property of the blocked layout on every platform, - // so C0/K0 are compile-time constants here (#29454). + // so C0/K0 are compile-time constants here. On RVV the kernels pin the vector + // length to K0 rather than sizing the block to the vector (#29454). constexpr int C0shift = 3, K0shift = C0shift; constexpr int C0 = 1 << C0shift, K0 = C0; constexpr int C0BUF = K0; @@ -2518,13 +2537,12 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, CONV_UPDATE_BLOCK1(6); CONV_UPDATE_BLOCK1(7); #elif CV_SIMD_SCALABLE - // RVV: K0 == vlanes(); weights are contiguous over kk for a fixed c0, - // so one vx_load gives the whole K0-wide weight vector. Broadcast the - // input lane and accumulate into the persistent block accumulator. + // RVV: weights are contiguous over kk for a fixed c0, and _vl == K0 + // (see CONV_INIT_SCALAR_SUMS), so one load gives the whole K0-wide + // weight vector. Broadcast the input lane into the accumulator. for (int c0 = 0; c0 < C0; ++c0) { - v_float32 w = vx_load(wptr + c0*K0); - v_float32 x = vx_setall_f32(inptr[c0]); - s0 = v_fma(x, w, s0); + vfloat32m2_t w = __riscv_vle32_v_f32m2(wptr + c0*K0, _vl); + s0 = __riscv_vfmacc_vf_f32m2(s0, inptr[c0], w, _vl); } #else for (int c0 = 0; c0 < C0; ++c0) { diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.cpp b/modules/dnn/src/layers/cpu_kernels/convolution.cpp index fb5f3b493d..97dc2ec74a 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.cpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.cpp @@ -2214,9 +2214,9 @@ void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bo convBlockNoSIMD(np, a, b, c, ldc, init_c, outLen, convMR, convNR); #elif CV_RVV { - // LMUL=1 kernel: vlanes = VLEN/32 (8 at VLEN=256). - // OpenCV's v_float32 is vfloat32m2_t (LMUL=2, vlanes=2*VLEN/32=16 at VLEN=256); - // 24 % 16 != 0, so the 24-wide tile requires LMUL=1 and native intrinsics. + // LMUL=1 kernel: vlanes = VLEN/32 (8 at VLEN=256). Native intrinsics are used so the + // tile width is fixed by the kernel rather than by whatever LMUL the universal + // intrinsics happen to be built with (#29454). // vfmacc.vf (scalar-into-vector FMA) avoids a broadcast register. // Only the exact full-tile case (outLen==convNR) is handled; tails fall to scalar. const size_t vl = __riscv_vsetvlmax_e32m1(); diff --git a/modules/dnn/src/layers/maxpool_layer.cpp b/modules/dnn/src/layers/maxpool_layer.cpp index 0cc6c5ac4c..998c964873 100644 --- a/modules/dnn/src/layers/maxpool_layer.cpp +++ b/modules/dnn/src/layers/maxpool_layer.cpp @@ -50,7 +50,17 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) float* out = (float*)out_ + nc0*planesize; const float INITVAL = -FLT_MAX; - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + // RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so + // the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide + // and there is no way to make it span exactly C0 elements. Drop to native RVV and + // pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) = + // VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one + // vector spans precisely one channel block -- no VLEN-dependent branches, and + // nothing can overrun or under-fill the block. + const size_t _vl = __riscv_vsetvl_e32m2(C0); + vfloat32m2_t s_min = __riscv_vfmv_v_f_f32m2(INITVAL, _vl); + #elif CV_SIMD int nlanes = VTraits::vlanes(); v_float32 s_min = vx_setall_f32(INITVAL); CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0); @@ -71,7 +81,24 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) #endif for(;;) { - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + vfloat32m2_t s0 = s_min; + for (int k = 0; k < ksize; k++) { + int zi = zi_ + zyxtab[k*MAX_POOL_DIMS]; + int yi = yi_ + zyxtab[k*MAX_POOL_DIMS+1]; + int xi = xi_ + zyxtab[k*MAX_POOL_DIMS+2]; + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) + continue; + vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl); + s0 = __riscv_vfmax_vv_f32m2(s0, v0, _vl); + } + __riscv_vse32_v_f32m2(out + x0*C0, s0, _vl); + } + #elif CV_SIMD if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0; @@ -136,7 +163,17 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs) break; x1 = inner_x1; - #if CV_SIMD || CV_SIMD_SCALABLE + #if CV_SIMD_SCALABLE + for (; x0 < x1; x0++) { + int xi_ = x0*SX - padX0; + const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0; + + vfloat32m2_t s0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[0], _vl); + for (int k = 1; k < ksize; k++) + s0 = __riscv_vfmax_vv_f32m2(s0, __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl), _vl); + __riscv_vse32_v_f32m2(out + x0*C0, s0, _vl); + } + #elif CV_SIMD if (nlanes == C0) { for (; x0 < x1; x0++) { int xi_ = x0*SX - padX0;