potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 15:23:05 +04:00
Code Packages Releases Wiki Activity

dnn:rvv: pin the vector length to the channel block

Browse Source 
The blocked kernels assumed one channel block is exactly one vector
(C0 == vlanes()). That is not a property of the layout, it is a property of the
hardware, so the kernels only worked where the two happened to coincide: with
C0 fixed at 8 they break wherever vlanes() != 8 -- under-filling the block when
the vector is narrower (silently leaving output channels unwritten) and
overrunning it when the vector is wider (CV_Assert).

The universal intrinsics cannot express this: v_float32 is always vlanes() wide
and there is no way to make it span exactly C0 elements. So the RVV paths drop to
native intrinsics and set the vector length explicitly:

    const size_t _vl = __riscv_vsetvl_e32m2(C0);   // == 8 on every VLEN >= 128

LMUL=2 is required, not incidental: vlmax(e32,m2) = VLEN/32*2 >= 8 even at
VLEN=128, the narrowest RVV configuration, so _vl == C0 on every RVV 1.0 core.
One vector then spans precisely one channel block regardless of VLEN, and the
C0-vs-vlanes asserts and VLEN-dependent branches are removed rather than extended.
The existing int8 kernels already choose LMUL=2 for the same reason
(conv2_int8_kernels.simd.hpp).

This is a native per-kernel choice and is independent of the LMUL the universal
intrinsics are built with; dnn no longer reads the vector width of
intrin_rvv_scalable.hpp at all.

conv, deconv, maxpool, avgpool and depthwise use native intrinsics. batch_norm
stays portable and instead replicates the C0-wide coefficient pattern across the
vector, which keeps full lane utilisation where a fixed vl would idle most of it.
The x86/ARM paths are restored to their pre-#29304 form and are unchanged.

Tested on SpaceMIT K3 (X100 VLEN=256, A100 VLEN=1024), opencv_test_dnn: no
regressions against upstream on either core.
This commit is contained in:
velonica0
2026-07-13 00:12:15 -07:00
parent 11e0a41bf6
commit 40587c6dd9
7 changed files with 345 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/src/layers/avgpool_layer.cpp
+47 -3
View File
@@ -53,7 +53,18 @@ static void avgPool32f(const void* inp_, void* out_,
float* out = (float*)out_ + nc0*planesize;
float iksize = 1.f/ksize;
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
// RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so
// the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide
// and there is no way to make it span exactly C0 elements. Drop to native RVV and
// pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) =
// VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one
// vector spans precisely one channel block -- no VLEN-dependent branches, and
// nothing can overrun or under-fill the block.
const size_t _vl = __riscv_vsetvl_e32m2(C0);
vfloat32m2_t z = __riscv_vfmv_v_f_f32m2(0.f, _vl);
vfloat32m2_t vscale0 = __riscv_vfmv_v_f_f32m2(iksize, _vl);
#elif CV_SIMD
int nlanes = VTraits<v_float32>::vlanes();
CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0);
v_float32 z = vx_setzero_f32();
@@ -74,7 +85,30 @@ static void avgPool32f(const void* inp_, void* out_,
#endif
for(;;) {
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
vfloat32m2_t s0 = z;
int nitems = 0, npadded = 0;
for (int k = 0; k < ksize; k++) {
int zi = zi_ + zyxtab[k*MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k*MAX_POOL_DIMS+1];
int xi = xi_ + zyxtab[k*MAX_POOL_DIMS+2];
npadded += (zi >= -padZ0 && zi < Di + padZ1 &&
yi >= -padY0 && yi < Hi + padY1 &&
xi >= -padX0 && xi < Wi + padX1);
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi)
continue;
vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl);
s0 = __riscv_vfadd_vv_f32m2(s0, v0, _vl);
nitems++;
}
s0 = __riscv_vfmul_vf_f32m2(s0, 1.f/(count_include_pad ? npadded : nitems), _vl);
__riscv_vse32_v_f32m2(out + x0*C0, s0, _vl);
}
#elif CV_SIMD
if (nlanes == C0) {
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
@@ -162,7 +196,17 @@ static void avgPool32f(const void* inp_, void* out_,
break;
x1 = inner_x1;
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0;
vfloat32m2_t s0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[0], _vl);
for (int k = 1; k < ksize; k++)
s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl), _vl);
__riscv_vse32_v_f32m2(out + x0*C0, __riscv_vfmul_vv_f32m2(s0, vscale0, _vl), _vl);
}
#elif CV_SIMD
if (nlanes == C0) {
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
modules/dnn/src/layers/batch_norm2_layer.cpp
+57 -6
View File
@@ -136,12 +136,6 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale,
}
}
#if CV_SIMD || CV_SIMD_SCALABLE
/*
[TODO] support C0 == vlanes/2, maybe C0 == vlanes/4.
in this case, load everything into vsc0 and vb0, process
most part of the plane using vector code as if C0 == vlanes and
then process the tail using scalar code
*/
else if (C0 == vlanes*4 || C0 == vlanes*2 || C0 == vlanes) {
// accelerated block layout case
int c = 0;
@@ -268,6 +262,63 @@ static void batchnorm(const Mat& inp, Mat& out, const Mat& scale,
}
}
}
else if (C0 < vlanes && vlanes % C0 == 0) {
/*
Vector wider than the channel block (e.g. C0 == 8 on a VLEN >= 512
machine). The plane is contiguous [pixel][C0] and the per-channel
coefficients repeat every C0 elements, so replicate the C0-wide
scale/bias pattern vlanes/C0 times. One vector then spans vlanes/C0
consecutive pixels with correctly aligned coefficients, and the plane
can be swept in plain vlanes-sized steps at full lane utilisation.
*/
const int reps = vlanes / C0;
int c = 0;
for (; c < c_delta; c++) {
scalebuf[c] = scaleptr[c];
biasbuf[c] = biasptr[c];
}
for (; c < C0; c++) // padded channels of the last block -> zero output
scalebuf[c] = biasbuf[c] = 0.f;
for (int r = 1; r < reps; r++) {
for (int c2 = 0; c2 < C0; c2++) {
scalebuf[r*C0 + c2] = scalebuf[c2];
biasbuf[r*C0 + c2] = biasbuf[c2];
}
}
v_float32 vsc = vx_load(scalebuf);
v_float32 vb = vx_load(biasbuf);
if (type == CV_32F) {
const float* inptr = (const float*)inptr_;
float* outptr = (float*)outptr_;
for (; i <= planesize_C0 - vlanes; i += vlanes) {
v_float32 x0 = vx_load(inptr + i);
x0 = v_fma(x0, vsc, vb);
v_store(outptr + i, x0);
}
for (; i < planesize_C0; i++)
outptr[i] = inptr[i]*scalebuf[i % C0] + biasbuf[i % C0];
} else if (type == CV_16F) {
const hfloat* inptr = (const hfloat*)inptr_;
hfloat* outptr = (hfloat*)outptr_;
for (; i <= planesize_C0 - vlanes; i += vlanes) {
v_float32 x0 = vx_load_expand(inptr + i);
x0 = v_fma(x0, vsc, vb);
v_pack_store(outptr + i, x0);
}
for (; i < planesize_C0; i++)
outptr[i] = hfloat(float(inptr[i])*scalebuf[i % C0] + biasbuf[i % C0]);
} else if (type == CV_16BF) {
const bfloat* inptr = (const bfloat*)inptr_;
bfloat* outptr = (bfloat*)outptr_;
for (; i <= planesize_C0 - vlanes; i += vlanes) {
v_float32 x0 = vx_load_expand(inptr + i);
x0 = v_fma(x0, vsc, vb);
v_pack_store(outptr + i, x0);
}
for (; i < planesize_C0; i++)
outptr[i] = bfloat(float(inptr[i])*scalebuf[i % C0] + biasbuf[i % C0]);
}
}
#endif
else {
// general block layout or NHWC case
modules/dnn/src/layers/cpu_kernels/conv2_deconv.cpp
+56 -3
View File
@@ -63,9 +63,14 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/,
const float* wdata = (const float*)weights__;
const float* bias = bias__;
#if (CV_SIMD || CV_SIMD_SCALABLE)
#if CV_SIMD_SCALABLE
// RVV: the vector length is pinned to K0 (see below), so one vector always spans
// exactly one output block whatever the hardware VLEN. Only the weight-packing
// constraint remains; repackDeconvWeights() zero-fills padded lanes.
const bool simd_ok = ((ngroups == 1) || (Kg % C0 == 0));
#elif CV_SIMD
// SIMD path is safe when ngroups==1 or Kg%C0==0; repackDeconvWeights()
// zero-fills padded lanes.
// zero-fills padded lanes. The block must also be a whole number of vectors.
const bool simd_ok =
((ngroups == 1) || (Kg % C0 == 0)) && (C0 % VTraits<v_float32>::vlanes() == 0);
#endif
@@ -115,7 +120,55 @@ static void deconvBlock32f(const void* inp__, const void* /*residual*/,
}
}
#if (CV_SIMD || CV_SIMD_SCALABLE)
#if CV_SIMD_SCALABLE
if (simd_ok) {
// Set the vector length to the block size instead of taking whatever
// the hardware offers: vlmax(e32,m2) = VLEN/32*2 >= K0 for every
// VLEN >= 128, so __riscv_vsetvl_e32m2(K0) returns exactly K0. One
// accumulator therefore spans the whole output block on any VLEN, and
// the block never has to be a whole number of vectors (#29454).
const size_t _vl = __riscv_vsetvl_e32m2(K0);
vfloat32m2_t acc = __riscv_vle32_v_f32m2(out_ptr, _vl);
for (int ks = 0; ks < ksize; ks++) {
bool valid = true;
int ipos_flat = 0;
for (int i = 0; i < sdims; i++) {
int di = MAX_DIMS - sdims + i;
int raw = ocoords[di] + cs.pads[di]
- kcoords_tab[ks][di] * cs.dilations[di];
if (raw < 0 || raw % cs.strides[di] != 0) {
valid = false; break;
}
int ic = raw / cs.strides[di];
if (ic >= iDims[di]) { valid = false; break; }
ipos_flat = ipos_flat * iDims[di] + ic;
}
if (!valid) continue;
const float* w_base = wdata +
((int64_t)(simd_g * Kblk + simd_kblk) * ksize + ks)
* C1Max * C0 * K0;
for (int c1p = 0; c1p < C1Max; c1p++) {
const int c1_abs = simd_c1_abs_base + c1p;
if (c1_abs >= C1) break;
const float* inp_ptr = inp_n +
(int64_t)(c1_abs * ispatial + ipos_flat) * C0;
const float* w_c1p = w_base + (int64_t)c1p * C0 * K0;
for (int c0 = 0; c0 < C0; c0++) {
vfloat32m2_t w0 = __riscv_vle32_v_f32m2(w_c1p + c0 * K0, _vl);
acc = __riscv_vfmacc_vf_f32m2(acc, inp_ptr[c0], w0, _vl);
}
}
}
__riscv_vse32_v_f32m2(out_ptr, acc, _vl);
continue;
}
#elif CV_SIMD
if (simd_ok) {
const int VLANES = VTraits<v_float32>::vlanes();
const int v_per_block = C0 / VLANES;
modules/dnn/src/layers/cpu_kernels/conv2_depthwise.simd.hpp
+69 -3
View File
@@ -38,6 +38,8 @@ static void depthwiseConv32f(const void* inp__, const void* residual__,
parallel_for_(Range(0, NC1), [&](const Range& range)
{
constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS;
// The block size is a fixed property of the blocked layout on every platform; on RVV
// the kernel pins the vector length to C0 instead of sizing C0 to the vector (#29454).
constexpr int C0 = 8;
CV_Assert(cs.nspatialdims <= MAX_CONV_DIMS && MAX_CONV_DIMS == 3);
@@ -89,7 +91,18 @@ static void depthwiseConv32f(const void* inp__, const void* residual__,
defaultAlpha = 1.f;
}
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
// RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so
// the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide
// and there is no way to make it span exactly C0 elements. Drop to native RVV and
// pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) =
// VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one
// vector spans precisely one channel block -- no VLEN-dependent branches, and
// nothing can overrun or under-fill the block.
const size_t _vl = __riscv_vsetvl_e32m2(C0);
vfloat32m2_t v_maxval = __riscv_vfmv_v_f_f32m2(maxval, _vl);
vfloat32m2_t z = __riscv_vfmv_v_f_f32m2(0.f, _vl);
#elif CV_SIMD
v_float32 v_maxval = vx_setall_f32(maxval);
v_float32 z = vx_setzero_f32();
const int nlanes = VTraits<v_float32>::vlanes();
@@ -131,7 +144,36 @@ static void depthwiseConv32f(const void* inp__, const void* residual__,
#endif
for(;;) {
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
{
vfloat32m2_t sc0 = __riscv_vle32_v_f32m2(scalebuf, _vl);
vfloat32m2_t b0 = __riscv_vle32_v_f32m2(biasbuf, _vl);
vfloat32m2_t alpha0 = __riscv_vle32_v_f32m2(alphabuf, _vl);
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
vfloat32m2_t s0 = z;
for (int k = 0; k < ksize; k++) {
int zi = zi_ + zyxtab[k*MAX_CONV_DIMS];
int yi = yi_ + zyxtab[k*MAX_CONV_DIMS + 1];
int xi = xi_ + zyxtab[k*MAX_CONV_DIMS + 2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi)
continue;
vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl);
vfloat32m2_t w0 = __riscv_vle32_v_f32m2(weights + k*C0, _vl);
s0 = __riscv_vfmacc_vv_f32m2(s0, v0, w0, _vl);
}
s0 = __riscv_vfmadd_vv_f32m2(s0, sc0, b0, _vl);
if (residual)
s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(residual + x0*C0, _vl), _vl);
s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, alpha0, _vl), s0,
__riscv_vmfge_vv_f32m2_b16(s0, z, _vl), _vl);
s0 = __riscv_vfmin_vv_f32m2(s0, v_maxval, _vl);
__riscv_vse32_v_f32m2(out + x0*C0, s0, _vl);
}
}
#elif CV_SIMD
if (nlanes == C0) {
v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf);
v_float32 alpha0 = vx_load(alphabuf);
@@ -218,7 +260,31 @@ static void depthwiseConv32f(const void* inp__, const void* residual__,
break;
x1 = inner_x1;
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
{
vfloat32m2_t sc0 = __riscv_vle32_v_f32m2(scalebuf, _vl);
vfloat32m2_t b0 = __riscv_vle32_v_f32m2(biasbuf, _vl);
vfloat32m2_t alpha0 = __riscv_vle32_v_f32m2(alphabuf, _vl);
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0;
vfloat32m2_t s0 = z;
for (int k = 0; k < ksize; k++) {
vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl);
vfloat32m2_t w0 = __riscv_vle32_v_f32m2(weights + k*C0, _vl);
s0 = __riscv_vfmacc_vv_f32m2(s0, v0, w0, _vl);
}
s0 = __riscv_vfmadd_vv_f32m2(s0, sc0, b0, _vl);
if (residual)
s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(residual + x0*C0, _vl), _vl);
s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, alpha0, _vl), s0,
__riscv_vmfge_vv_f32m2_b16(s0, z, _vl), _vl);
s0 = __riscv_vfmin_vv_f32m2(s0, v_maxval, _vl);
__riscv_vse32_v_f32m2(out + x0*C0, s0, _vl);
}
}
#elif CV_SIMD
if (nlanes == C0) {
v_float32 sc0 = vx_load(scalebuf), b0 = vx_load(biasbuf), alpha0 = vx_load(alphabuf);
for (; x0 < x1; x0++) {
modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp
+73 -55
View File
@@ -389,32 +389,42 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
#elif CV_SIMD_SCALABLE
/////////////////////////// scalable (RVV) implementation /////////////////////////////
// K0 == vlanes(), so each of the 10 spatial positions needs exactly one vector
// accumulator (structurally identical to the AVX2 path, which uses one v_float32x8 per
// position for its K0=8). One K0-wide weight load is reused across all 10 positions, so
// the per-output-point load bottleneck of the scalar tail is amortized away. C0 (= vlanes())
// is a runtime value, so the c0 sweep is a loop rather than a fixed unroll.
// The blocked layout fixes K0 == C0 == 8 regardless of the hardware vector width, so the
// universal intrinsics cannot be used here: v_float32 is always vlanes() wide and there is
// no way to make it span exactly K0 elements. Instead we drop to native RVV and set the
// vector length explicitly to K0 (#29454).
//
// LMUL=2 is chosen so that vlmax(e32,m2) = VLEN/32*2 >= 8 for every VLEN >= 128 (the
// smallest RVV configuration). __riscv_vsetvl_e32m2(K0) therefore returns exactly K0 = 8
// on every machine -- 8 of 8 lanes at VLEN=128, 8 of 64 at VLEN=1024 -- so one accumulator
// per spatial position always spans precisely one output block. No strip-mining, no
// VLEN-dependent branches, and nothing can overrun or under-fill the block.
//
// Note this LMUL is a property of *this kernel only*; it is independent of whatever LMUL
// the universal intrinsics are built with.
#undef CONV_INIT_SUMS
#define CONV_INIT_SUMS() \
v_float32 zz = vx_setzero_f32(); \
v_float32 s0 = zz, s1 = zz, s2 = zz, s3 = zz, s4 = zz; \
v_float32 s5 = zz, s6 = zz, s7 = zz, s8 = zz, s9 = zz
const size_t _vl = __riscv_vsetvl_e32m2(K0); \
vfloat32m2_t zz = __riscv_vfmv_v_f_f32m2(0.f, _vl); \
vfloat32m2_t s0 = zz, s1 = zz, s2 = zz, s3 = zz, s4 = zz; \
vfloat32m2_t s5 = zz, s6 = zz, s7 = zz, s8 = zz, s9 = zz
// s_j += inptr[j][_c0] * w (vfmacc_vf: vd += rs1 * vs2)
#undef CONV_UPDATE_LOOP_BODY
#define CONV_UPDATE_LOOP_BODY() \
for (int _c0 = 0; _c0 < C0; _c0++) { \
v_float32 _w = vx_load(wptr + _c0*K0); \
s0 = v_fma(vx_setall_f32(inptr[0][_c0]), _w, s0); \
s1 = v_fma(vx_setall_f32(inptr[1][_c0]), _w, s1); \
s2 = v_fma(vx_setall_f32(inptr[2][_c0]), _w, s2); \
s3 = v_fma(vx_setall_f32(inptr[3][_c0]), _w, s3); \
s4 = v_fma(vx_setall_f32(inptr[4][_c0]), _w, s4); \
s5 = v_fma(vx_setall_f32(inptr[5][_c0]), _w, s5); \
s6 = v_fma(vx_setall_f32(inptr[6][_c0]), _w, s6); \
s7 = v_fma(vx_setall_f32(inptr[7][_c0]), _w, s7); \
s8 = v_fma(vx_setall_f32(inptr[8][_c0]), _w, s8); \
s9 = v_fma(vx_setall_f32(inptr[9][_c0]), _w, s9); \
vfloat32m2_t _w = __riscv_vle32_v_f32m2(wptr + _c0*K0, _vl); \
s0 = __riscv_vfmacc_vf_f32m2(s0, inptr[0][_c0], _w, _vl); \
s1 = __riscv_vfmacc_vf_f32m2(s1, inptr[1][_c0], _w, _vl); \
s2 = __riscv_vfmacc_vf_f32m2(s2, inptr[2][_c0], _w, _vl); \
s3 = __riscv_vfmacc_vf_f32m2(s3, inptr[3][_c0], _w, _vl); \
s4 = __riscv_vfmacc_vf_f32m2(s4, inptr[4][_c0], _w, _vl); \
s5 = __riscv_vfmacc_vf_f32m2(s5, inptr[5][_c0], _w, _vl); \
s6 = __riscv_vfmacc_vf_f32m2(s6, inptr[6][_c0], _w, _vl); \
s7 = __riscv_vfmacc_vf_f32m2(s7, inptr[7][_c0], _w, _vl); \
s8 = __riscv_vfmacc_vf_f32m2(s8, inptr[8][_c0], _w, _vl); \
s9 = __riscv_vfmacc_vf_f32m2(s9, inptr[9][_c0], _w, _vl); \
} \
inptr[0] += inpstep[0]; inptr[1] += inpstep[1]; \
inptr[2] += inpstep[2]; inptr[3] += inpstep[3]; \
@@ -424,26 +434,30 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
#undef CONV_START_FINALIZE_OUT
#define CONV_START_FINALIZE_OUT() \
v_float32 vscale = vx_load(scalebuf); \
v_float32 vbias = vx_load(biasbuf); \
v_float32 valpha = vx_load(alphabuf); \
v_float32 vmaxval = vx_setall_f32(maxval)
vfloat32m2_t vscale = __riscv_vle32_v_f32m2(scalebuf, _vl); \
vfloat32m2_t vbias = __riscv_vle32_v_f32m2(biasbuf, _vl); \
vfloat32m2_t valpha = __riscv_vle32_v_f32m2(alphabuf, _vl); \
vfloat32m2_t vmaxval = __riscv_vfmv_v_f_f32m2(maxval, _vl)
#define CONV_ADD_RESIDUAL2(idx0, idx1) \
s##idx0 = v_add(s##idx0, vx_load(tmpbuf + idx0*K0)); \
s##idx1 = v_add(s##idx1, vx_load(tmpbuf + idx1*K0))
s##idx0 = __riscv_vfadd_vv_f32m2(s##idx0, __riscv_vle32_v_f32m2(tmpbuf + idx0*K0, _vl), _vl); \
s##idx1 = __riscv_vfadd_vv_f32m2(s##idx1, __riscv_vle32_v_f32m2(tmpbuf + idx1*K0, _vl), _vl)
// s = s*scale + bias; s = (s >= 0 ? s : s*alpha); s = min(s, maxval)
// (vfmadd_vv: vd = vd*vs1 + vs2; vmerge_vvm picks vs1 where the mask is set, else vs2)
#undef CONV_FINALIZE_OUT2
#define CONV_FINALIZE_OUT2(idx0, idx1, add_residual2) \
s##idx0 = v_fma(s##idx0, vscale, vbias); \
s##idx1 = v_fma(s##idx1, vscale, vbias); \
s##idx0 = __riscv_vfmadd_vv_f32m2(s##idx0, vscale, vbias, _vl); \
s##idx1 = __riscv_vfmadd_vv_f32m2(s##idx1, vscale, vbias, _vl); \
add_residual2(idx0, idx1); \
s##idx0 = v_select(v_ge(s##idx0, zz), s##idx0, v_mul(s##idx0, valpha)); \
s##idx1 = v_select(v_ge(s##idx1, zz), s##idx1, v_mul(s##idx1, valpha)); \
s##idx0 = v_min(s##idx0, vmaxval); \
s##idx1 = v_min(s##idx1, vmaxval); \
v_store(outbuf + idx0*K0, s##idx0); \
v_store(outbuf + idx1*K0, s##idx1)
s##idx0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s##idx0, valpha, _vl), s##idx0, \
__riscv_vmfge_vv_f32m2_b16(s##idx0, zz, _vl), _vl); \
s##idx1 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s##idx1, valpha, _vl), s##idx1, \
__riscv_vmfge_vv_f32m2_b16(s##idx1, zz, _vl), _vl); \
s##idx0 = __riscv_vfmin_vv_f32m2(s##idx0, vmaxval, _vl); \
s##idx1 = __riscv_vfmin_vv_f32m2(s##idx1, vmaxval, _vl); \
__riscv_vse32_v_f32m2(outbuf + idx0*K0, s##idx0, _vl); \
__riscv_vse32_v_f32m2(outbuf + idx1*K0, s##idx1, _vl)
#undef CONV_FINALIZE_OUT_ALL
#define CONV_FINALIZE_OUT_ALL() \
@@ -604,10 +618,12 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif
v_float32x4 zz = v_setzero_f32(); \
v_float32x4 s0 = zz, s1 = zz
#elif CV_SIMD_SCALABLE
// RVV: K0 == vlanes(), so a single scalable vector spans the whole output block.
// RVV: native LMUL=2 with vl fixed to K0, so one vector spans exactly one output block
// on any VLEN >= 128. See the CONV_INIT_SUMS block above.
#define CONV_INIT_SCALAR_SUMS() \
v_float32 zz = vx_setzero_f32(); \
v_float32 s0 = zz
const size_t _vl = __riscv_vsetvl_e32m2(K0); \
vfloat32m2_t zz = __riscv_vfmv_v_f_f32m2(0.f, _vl); \
vfloat32m2_t s0 = zz
#else
#define CONV_INIT_SCALAR_SUMS() \
for (int _ks = 0; _ks < K0; _ks++) tmpbuf[_ks] = 0.f
@@ -645,15 +661,16 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif
#elif CV_SIMD_SCALABLE
#define CONV_FINALIZE_SCALAR_OUT(outbuf) \
{ \
v_float32 _vsc = vx_load(scalebuf); \
v_float32 _vbi = vx_load(biasbuf); \
v_float32 _val = vx_load(alphabuf); \
v_float32 _vmx = vx_setall_f32(maxval); \
s0 = v_fma(s0, _vsc, _vbi); \
s0 = v_add(s0, vx_load(resbuf)); \
s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, _val)); \
s0 = v_min(s0, _vmx); \
v_store((outbuf), s0); \
vfloat32m2_t _vsc = __riscv_vle32_v_f32m2(scalebuf, _vl); \
vfloat32m2_t _vbi = __riscv_vle32_v_f32m2(biasbuf, _vl); \
vfloat32m2_t _val = __riscv_vle32_v_f32m2(alphabuf, _vl); \
vfloat32m2_t _vmx = __riscv_vfmv_v_f_f32m2(maxval, _vl); \
s0 = __riscv_vfmadd_vv_f32m2(s0, _vsc, _vbi, _vl); \
s0 = __riscv_vfadd_vv_f32m2(s0, __riscv_vle32_v_f32m2(resbuf, _vl), _vl); \
s0 = __riscv_vmerge_vvm_f32m2(__riscv_vfmul_vv_f32m2(s0, _val, _vl), s0, \
__riscv_vmfge_vv_f32m2_b16(s0, zz, _vl), _vl); \
s0 = __riscv_vfmin_vv_f32m2(s0, _vmx, _vl); \
__riscv_vse32_v_f32m2((outbuf), s0, _vl); \
}
#else
#define CONV_FINALIZE_SCALAR_OUT(outbuf) \
@@ -664,8 +681,9 @@ static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shif
}
#endif
// RVV uses the generic runtime-C0 path below; the C0=8-specialized kernels are
// unused on scalable backends, so do not compile them there (avoids -Wunused-function).
// The C0=8-specialized kernels below are written in fixed-width intrinsics (v_float32x4/x8),
// which do not exist on scalable backends; RVV uses the generic kernel instead, so do not
// compile them there (avoids -Wunused-function).
#if !CV_SIMD_SCALABLE
// Specialized 1x1 convolution kernel with stride=1.
@@ -2132,7 +2150,7 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__,
const ConvState& cs, const void* weights__,
const float* scale__, const float* bias__)
{
#if !CV_SIMD_SCALABLE // RVV: skip the C0=8-specialized kernels; use the generic runtime-C0 path
#if !CV_SIMD_SCALABLE // RVV: skip the fixed-width C0=8 kernels; use the generic kernel
int ksize = cs.wshape[2];
if (ksize == 1 && cs.strides[0]*cs.strides[1]*cs.strides[2] == 1) {
#if CV_SIMD256 && defined(__AVX2__)
@@ -2203,7 +2221,8 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__,
parallel_for_(Range(0, total_tasks_gen), [&](const Range& range) {
constexpr int SPAT_BLOCK_SIZE = 10;
// The block size is a fixed property of the blocked layout on every platform,
// so C0/K0 are compile-time constants here (#29454).
// so C0/K0 are compile-time constants here. On RVV the kernels pin the vector
// length to K0 rather than sizing the block to the vector (#29454).
constexpr int C0shift = 3, K0shift = C0shift;
constexpr int C0 = 1 << C0shift, K0 = C0;
constexpr int C0BUF = K0;
@@ -2518,13 +2537,12 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__,
CONV_UPDATE_BLOCK1(6);
CONV_UPDATE_BLOCK1(7);
#elif CV_SIMD_SCALABLE
// RVV: K0 == vlanes(); weights are contiguous over kk for a fixed c0,
// so one vx_load gives the whole K0-wide weight vector. Broadcast the
// input lane and accumulate into the persistent block accumulator.
// RVV: weights are contiguous over kk for a fixed c0, and _vl == K0
// (see CONV_INIT_SCALAR_SUMS), so one load gives the whole K0-wide
// weight vector. Broadcast the input lane into the accumulator.
for (int c0 = 0; c0 < C0; ++c0) {
v_float32 w = vx_load(wptr + c0*K0);
v_float32 x = vx_setall_f32(inptr[c0]);
s0 = v_fma(x, w, s0);
vfloat32m2_t w = __riscv_vle32_v_f32m2(wptr + c0*K0, _vl);
s0 = __riscv_vfmacc_vf_f32m2(s0, inptr[c0], w, _vl);
}
#else
for (int c0 = 0; c0 < C0; ++c0) {
modules/dnn/src/layers/cpu_kernels/convolution.cpp
+3 -3
View File
@@ -2214,9 +2214,9 @@ void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bo
convBlockNoSIMD(np, a, b, c, ldc, init_c, outLen, convMR, convNR);
#elif CV_RVV
{
// LMUL=1 kernel: vlanes = VLEN/32 (8 at VLEN=256).
// OpenCV's v_float32 is vfloat32m2_t (LMUL=2, vlanes=2*VLEN/32=16 at VLEN=256);
// 24 % 16 != 0, so the 24-wide tile requires LMUL=1 and native intrinsics.
// LMUL=1 kernel: vlanes = VLEN/32 (8 at VLEN=256). Native intrinsics are used so the
// tile width is fixed by the kernel rather than by whatever LMUL the universal
// intrinsics happen to be built with (#29454).
// vfmacc.vf (scalar-into-vector FMA) avoids a broadcast register.
// Only the exact full-tile case (outLen==convNR) is handled; tails fall to scalar.
const size_t vl = __riscv_vsetvlmax_e32m1();
modules/dnn/src/layers/maxpool_layer.cpp
+40 -3
View File
@@ -50,7 +50,17 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs)
float* out = (float*)out_ + nc0*planesize;
const float INITVAL = -FLT_MAX;
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
// RVV: the blocked layout fixes C0 == 8 regardless of the hardware vector width, so
// the universal intrinsics cannot be used here -- v_float32 is always vlanes() wide
// and there is no way to make it span exactly C0 elements. Drop to native RVV and
// pin the vector length to C0 (#29454). LMUL=2 guarantees vlmax(e32,m2) =
// VLEN/32*2 >= 8 for every VLEN >= 128, so _vl == C0 == 8 on every machine and one
// vector spans precisely one channel block -- no VLEN-dependent branches, and
// nothing can overrun or under-fill the block.
const size_t _vl = __riscv_vsetvl_e32m2(C0);
vfloat32m2_t s_min = __riscv_vfmv_v_f_f32m2(INITVAL, _vl);
#elif CV_SIMD
int nlanes = VTraits<v_float32>::vlanes();
v_float32 s_min = vx_setall_f32(INITVAL);
CV_Assert(C0 == nlanes || C0 == nlanes*2 || C0 % (nlanes*4) == 0);
@@ -71,7 +81,24 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs)
#endif
for(;;) {
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
vfloat32m2_t s0 = s_min;
for (int k = 0; k < ksize; k++) {
int zi = zi_ + zyxtab[k*MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k*MAX_POOL_DIMS+1];
int xi = xi_ + zyxtab[k*MAX_POOL_DIMS+2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi)
continue;
vfloat32m2_t v0 = __riscv_vle32_v_f32m2(inp + ((zi*Hi + yi)*Wi + xi)*C0, _vl);
s0 = __riscv_vfmax_vv_f32m2(s0, v0, _vl);
}
__riscv_vse32_v_f32m2(out + x0*C0, s0, _vl);
}
#elif CV_SIMD
if (nlanes == C0) {
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
@@ -136,7 +163,17 @@ static void maxPool32f(const void* inp_, void* out_, const ConvState& cs)
break;
x1 = inner_x1;
#if CV_SIMD || CV_SIMD_SCALABLE
#if CV_SIMD_SCALABLE
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;
const float* inp_xi = inp + ((Hi*zi_ + yi_)*Wi + xi_)*C0;
vfloat32m2_t s0 = __riscv_vle32_v_f32m2(inp_xi + ofstab[0], _vl);
for (int k = 1; k < ksize; k++)
s0 = __riscv_vfmax_vv_f32m2(s0, __riscv_vle32_v_f32m2(inp_xi + ofstab[k], _vl), _vl);
__riscv_vse32_v_f32m2(out + x0*C0, s0, _vl);
}
#elif CV_SIMD
if (nlanes == C0) {
for (; x0 < x1; x0++) {
int xi_ = x0*SX - padX0;