potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-21 19:33:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #29421 from Teddy-Yangjiale:rvv-k1-01-02-fastgemm-packb

Browse Source 
dnn: add RISC-V RVV FP32 fastGemm micro-kernel and Pack-B support
This commit is contained in:
Alexander Smorkalov
2026-07-02 13:06:31 +03:00
committed by GitHub
parent 1439136ef8 dcae1f1dc1
commit d82fbe3530
7 changed files with 354 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/CMakeLists.txt
+1 -1
View File
@@ -9,7 +9,7 @@ ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX RVV
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEON_FP16 RVV)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX RVV)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2)
modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp
+89 -2
View File
@@ -22,6 +22,11 @@
namespace cv { namespace dnn {
int fastGemmMC(const FastGemmOpt &opt) {
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
return opt_RVV::fastGemmMC();
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
return opt_NEON::fastGemmMC();
@@ -48,6 +53,11 @@ int fastGemmMC(const FastGemmOpt &opt) {
}
int fastGemmNC(const FastGemmOpt &opt) {
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
return opt_RVV::fastGemmNC();
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
return opt_NEON::fastGemmNC();
@@ -74,6 +84,11 @@ int fastGemmNC(const FastGemmOpt &opt) {
}
int fastGemmKC(const FastGemmOpt &opt) {
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
return opt_RVV::fastGemmKC();
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
return opt_NEON::fastGemmKC();
@@ -100,6 +115,11 @@ int fastGemmKC(const FastGemmOpt &opt) {
}
int fastGemmNR(const FastGemmOpt &opt) {
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
return opt_RVV::fastGemmNR();
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
return opt_NEON::fastGemmNR();
@@ -128,6 +148,11 @@ int fastGemmNR(const FastGemmOpt &opt) {
size_t fastGemmPackBSize(size_t N, size_t K, const FastGemmOpt &opt) {
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
return static_cast<size_t>(opt_RVV::fastGemmPackBSize(N, K));
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
return static_cast<size_t>(opt_NEON::fastGemmPackBSize(N, K));
@@ -167,6 +192,18 @@ void fastGemmPackB(const Mat &B, std::vector<float> &packed_B, bool trans, FastG
const auto *b = B.ptr<const char>();
int esz = B.elemSize();
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
size_t size_packed_B = opt_RVV::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
opt_RVV::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += (size_t)N * (size_t)K * (size_t)esz;
packed_b += size_packed_B * (size_t)esz;
}
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
size_t size_packed_B = opt_NEON::fastGemmPackBSize(N, K);
@@ -237,6 +274,11 @@ void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, f
const auto &b = (const char *)B;
auto *packed_b = (char *)packed_B;
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
@@ -482,6 +524,11 @@ void fastGemm(bool trans_a, int M, int N, int K,
std::swap(lda0, lda1);
}
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
@@ -532,7 +579,7 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
#ifdef HAVE_MLAS
const bool a_row_major = (lda0 == 1 || lda1 == 1);
const bool b_row_major = (ldb0 == 1 || ldb1 == 1);
if (a_row_major && b_row_major) {
if (!opt.use_rvv && a_row_major && b_row_major) {
const int phys_lda = std::max(lda0, lda1);
const int phys_ldb = std::max(ldb0, ldb1);
if (mlasSgemm(trans_a, trans_b, M, N, K,
@@ -543,6 +590,13 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
}
#endif
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
@@ -645,7 +699,7 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset
else if (lda0 == 1) { a_ok = true; mlas_trans_a = true; mlas_lda = lda1; }
if (ldb1 == 1) { b_ok = true; mlas_trans_b = false; mlas_ldb = ldb0; }
else if (ldb0 == 1) { b_ok = true; mlas_trans_b = true; mlas_ldb = ldb1; }
if (a_ok && b_ok) {
if (!opt.use_rvv && a_ok && b_ok) {
if (mlasSgemmBatch(batch, A_offsets, B_offsets, C_offsets,
mlas_trans_a, mlas_trans_b, M, N, K,
alpha, A, mlas_lda, B, mlas_ldb,
@@ -655,6 +709,11 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset
}
#endif
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
@@ -687,6 +746,11 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *packed_B
const char *b = (const char *)packed_B;
char *c = (char *)C;
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
@@ -768,6 +832,11 @@ void fastGemmBatch(size_t batch,
return;
}
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv) {
opt_RVV::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
@@ -839,6 +908,15 @@ void pagedAttnQKGemm(
char*a = A.ptr<char>();
bool isQ3D = shape_q.size() == 3;
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv)
opt_RVV::pagedAttnQKGemmKernel(
Q.ptr<const char>(), packed_K, a,
B, T_q, Nq, N_k, T_s, D, T_k,
sm_scale, esz, isQ3D
);
else
#endif
#if CV_TRY_NEON
if (opt.use_neon)
opt_NEON::pagedAttnQKGemmKernel(
@@ -928,6 +1006,15 @@ void pagedAttnAVGemm(
}
bool canonical_layout = shape(Out).size() == 3;
#if CV_TRY_RVV && CV_RVV
if (opt.use_rvv)
opt_RVV::pagedAttnAVGemmKernel(
A.ptr<const char>(), packed_V, Out.ptr<char>(),
B, T_q, Nq, N_k, T_s, D, T_v,
esz, canonical_layout, fastGemmPackBSize(D, T_s, opt)
);
else
#endif
#if CV_TRY_NEON
if (opt.use_neon)
opt_NEON::pagedAttnAVGemmKernel(
modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp
+4 -1
View File
@@ -22,6 +22,7 @@ struct FastGemmOpt {
bool use_avx2;
bool use_neon;
bool use_lasx;
bool use_rvv;
bool multi_thread;
FastGemmOpt() {
@@ -29,6 +30,7 @@ struct FastGemmOpt {
use_avx2 = false;
use_neon = false;
use_lasx = false;
use_rvv = false;
multi_thread = false;
}
@@ -37,11 +39,12 @@ struct FastGemmOpt {
use_avx2 = checkHardwareSupport(CPU_AVX2);
use_neon = checkHardwareSupport(CPU_NEON);
use_lasx = checkHardwareSupport(CPU_LASX);
use_rvv = checkHardwareSupport(CPU_RVV);
multi_thread = true;
}
bool all() {
return use_avx || use_avx2 || use_neon || use_lasx;
return use_avx || use_avx2 || use_neon || use_lasx || use_rvv;
}
};
modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp
+118 -14
View File
@@ -15,7 +15,10 @@
#define FAST_GEMM_STORAGE (1<<20) // 2^20
#define FAST_GEMM_MAX_STACKBUF (1 << 13)
#if CV_AVX
#if CV_RVV
#define FAST_GEMM_F32_MC 64
#define FAST_GEMM_F32_NC 128
#elif CV_AVX
#define FAST_GEMM_F32_MC 60
#define FAST_GEMM_F32_NC 320
#elif CV_LASX
@@ -26,7 +29,10 @@
#define FAST_GEMM_F32_NC 72
#endif
#if CV_AVX
#if CV_RVV
#define FAST_GEMM_F32_MR 8
#define FAST_GEMM_F32_NR 16
#elif CV_AVX
#define FAST_GEMM_F32_MR 12
#define FAST_GEMM_F32_NR 8
#elif CV_LASX
@@ -37,7 +43,9 @@
#define FAST_GEMM_F32_NR 12
#endif
#if CV_AVX
#if CV_RVV
#define FAST_GEMM_F32_PACKED_STRIDE_K 128
#elif CV_AVX
#define FAST_GEMM_F32_PACKED_STRIDE_K 128
#else // CV_LASX, CV_NEON_AARCH64, CV_SIMD128
#define FAST_GEMM_F32_PACKED_STRIDE_K 64
@@ -156,7 +164,83 @@ void pagedAttnAVGemmKernel(
/*
Compute kernels that optimized for different platforms
*/
#if CV_NEON && CV_NEON_AARCH64 // AARCH64: 32 x 128-bit registers
#if CV_RVV // RVV (32 x VLEN-bit registers, LMUL=m2 for NR=16)
FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) // a packer
FAST_GEMM_IMPLEMENT_PACK(16, _f32, float, float) // b packer
static inline void fast_gemm8x16_f32(int k, const char *a_, const char *b_,
char *c_, int ldc, float alpha) {
const float* a = (const float*)a_;
const float* b = (const float*)b_;
float* c = (float*)c_;
if (__riscv_vsetvlmax_e32m1() >= 8) {
const size_t vl = __riscv_vsetvl_e32m2(FAST_GEMM_F32_NR);
vfloat32m2_t s0 = __riscv_vfmv_v_f_f32m2(0.f, vl);
vfloat32m2_t s1 = s0, s2 = s0, s3 = s0, s4 = s0, s5 = s0, s6 = s0, s7 = s0;
for (int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) {
vfloat32m2_t bp = __riscv_vle32_v_f32m2(b, vl);
s0 = __riscv_vfmacc_vf_f32m2(s0, a[0], bp, vl);
s1 = __riscv_vfmacc_vf_f32m2(s1, a[1], bp, vl);
s2 = __riscv_vfmacc_vf_f32m2(s2, a[2], bp, vl);
s3 = __riscv_vfmacc_vf_f32m2(s3, a[3], bp, vl);
s4 = __riscv_vfmacc_vf_f32m2(s4, a[4], bp, vl);
s5 = __riscv_vfmacc_vf_f32m2(s5, a[5], bp, vl);
s6 = __riscv_vfmacc_vf_f32m2(s6, a[6], bp, vl);
s7 = __riscv_vfmacc_vf_f32m2(s7, a[7], bp, vl);
}
#define FAST_GEMM_RVV_STORE(row) \
__riscv_vse32_v_f32m2(c + (row) * ldc, \
__riscv_vfmacc_vf_f32m2(__riscv_vle32_v_f32m2(c + (row) * ldc, vl), alpha, s##row, vl), vl)
FAST_GEMM_RVV_STORE(0);
FAST_GEMM_RVV_STORE(1);
FAST_GEMM_RVV_STORE(2);
FAST_GEMM_RVV_STORE(3);
FAST_GEMM_RVV_STORE(4);
FAST_GEMM_RVV_STORE(5);
FAST_GEMM_RVV_STORE(6);
FAST_GEMM_RVV_STORE(7);
#undef FAST_GEMM_RVV_STORE
return;
}
// fallback for VLEN < 256: strip-mine NR columns with m1
for (int j = 0; j < FAST_GEMM_F32_NR; ) {
const size_t vl = __riscv_vsetvl_e32m1(FAST_GEMM_F32_NR - j);
vfloat32m1_t s0 = __riscv_vfmv_v_f_f32m1(0.f, vl);
vfloat32m1_t s1 = s0, s2 = s0, s3 = s0, s4 = s0, s5 = s0, s6 = s0, s7 = s0;
for (int p = 0; p < k; p++) {
vfloat32m1_t bp = __riscv_vle32_v_f32m1(b + p * FAST_GEMM_F32_NR + j, vl);
const float* ap = a + p * FAST_GEMM_F32_MR;
s0 = __riscv_vfmacc_vf_f32m1(s0, ap[0], bp, vl);
s1 = __riscv_vfmacc_vf_f32m1(s1, ap[1], bp, vl);
s2 = __riscv_vfmacc_vf_f32m1(s2, ap[2], bp, vl);
s3 = __riscv_vfmacc_vf_f32m1(s3, ap[3], bp, vl);
s4 = __riscv_vfmacc_vf_f32m1(s4, ap[4], bp, vl);
s5 = __riscv_vfmacc_vf_f32m1(s5, ap[5], bp, vl);
s6 = __riscv_vfmacc_vf_f32m1(s6, ap[6], bp, vl);
s7 = __riscv_vfmacc_vf_f32m1(s7, ap[7], bp, vl);
}
#define FAST_GEMM_RVV_STORE_TAIL(row) \
__riscv_vse32_v_f32m1(c + (row) * ldc + j, \
__riscv_vfmacc_vf_f32m1(__riscv_vle32_v_f32m1(c + (row) * ldc + j, vl), alpha, s##row, vl), vl)
FAST_GEMM_RVV_STORE_TAIL(0);
FAST_GEMM_RVV_STORE_TAIL(1);
FAST_GEMM_RVV_STORE_TAIL(2);
FAST_GEMM_RVV_STORE_TAIL(3);
FAST_GEMM_RVV_STORE_TAIL(4);
FAST_GEMM_RVV_STORE_TAIL(5);
FAST_GEMM_RVV_STORE_TAIL(6);
FAST_GEMM_RVV_STORE_TAIL(7);
#undef FAST_GEMM_RVV_STORE_TAIL
j += (int)vl;
}
}
#elif CV_NEON && CV_NEON_AARCH64 // AARCH64: 32 x 128-bit registers
FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) // a packer
FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float) // b packer
@@ -531,7 +615,9 @@ static inline void fast_gemm_macro_kernel(int m, int n, int k,
memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz);
}
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm8x16_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm8x12_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha);
#elif CV_AVX
fast_gemm12x8_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha);
@@ -574,7 +660,9 @@ void fastGemmPackBKernel(const char *B, char *packed_B, size_t N, size_t K, size
for (size_t k = 0; k < K; k += KC) {
size_t kc = K - k < KC ? K - k : KC;
size_t step = (k * ldb0 + j0 * ldb1) * esz;
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack16_f32(nc, kc, B + step, ldb1, ldb0, packed_B);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack12_f32(nc, kc, B + step, ldb1, ldb0, packed_B);
#elif CV_AVX
fast_gemm_pack8_f32(nc, kc, B + step, ldb1, ldb0, packed_B);
@@ -638,7 +726,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K,
{
size_t kc = K - k0 < KC ? K - k0 : KC;
// pack a
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
@@ -648,7 +738,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K,
fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#endif
// pack b
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack16_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack12_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#elif CV_AVX
fast_gemm_pack8_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
@@ -730,7 +822,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K,
size_t kc = K - k0 < KC ? K - k0 : KC;
size_t step_a = (i0 * lda0 + k0 * lda1) * esz;
// pack a
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, A + step_a, lda0, lda1, packed_a);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, A + step_a, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, A + step_a, lda0, lda1, packed_a);
@@ -815,7 +909,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_
size_t step_a = (i0 * lda0 + k0 * lda1) * esz;
// pack a
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a);
@@ -826,7 +922,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_
#endif
size_t step_b = (k0 * ldb0 + j0 * ldb1) * esz;
// pack b
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack16_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack12_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b);
#elif CV_AVX
fast_gemm_pack8_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b);
@@ -914,7 +1012,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_
// size_t step = ((size_t)k0 * (size_t)lda1 + (size_t)i0 * (size_t)lda0) * (size_t)esz;
size_t step = (k0 * lda1 + i0 * lda0) * esz;
// pack a
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, a_block + step, lda0, lda1, packed_a);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, a_block + step, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, a_block + step, lda0, lda1, packed_a);
@@ -1026,7 +1126,9 @@ void pagedAttnQKGemmKernel(
int kc = D - k0 < KC ? D - k0 : KC;
// pack q
size_t step_q = (i0 * ldq0 + k0) * esz;
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q);
@@ -1142,7 +1244,9 @@ void pagedAttnAVGemmKernel(
const char *v_block = V[sk] + v_offset;
// pack
#if CV_NEON && CV_NEON_AARCH64
#if CV_RVV
fast_gemm_pack8_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a);
#elif CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a);
modules/dnn/src/layers/gemm_layer.cpp
+2 -2
View File
@@ -265,7 +265,7 @@ public:
#ifdef HAVE_MLAS
packed_B_mlas.release();
packed_B_mlas_M = packed_B_mlas_N = packed_B_mlas_K = 0;
if (mlasAvailable()) {
if (mlasAvailable() && !opt.use_rvv) {
std::vector<Mat> outputs;
outputs_arr.getMatVector(outputs);
const auto shape_A = shape(inputs[0]);
@@ -448,7 +448,7 @@ public:
if (constB(mode)) {
#ifdef HAVE_MLAS
if (!packed_B_mlas.empty() &&
if (!opt.use_rvv && !packed_B_mlas.empty() &&
packed_B_mlas_N == N && packed_B_mlas_K == K)
{
if (mlasSgemmPacked(trans_a, trans_b, rows, N, K,
modules/dnn/src/layers/matmul_layer.cpp
+1 -1
View File
@@ -275,7 +275,7 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
const auto &B = inputs[1];
const auto *b = B.ptr<const float>();
bool done = false;
if (mlasAvailable() && helper.M > 0 && helper.N > 0 && helper.K > 0) {
if (mlasAvailable() && !opt.use_rvv && helper.M > 0 && helper.N > 0 && helper.K > 0) {
const auto A_shape = shape(A);
const auto B_shape = shape(B);
const int lda_mem = A_shape.back();
modules/dnn/test/test_layers.cpp
+139 -1
View File
@@ -2446,7 +2446,9 @@ public:
Net netWithoutKVCache = readNetFromONNX(findDataFile(model_path, true), cv::dnn::ENGINE_NEW);
int T = 523, Nq = 8, Nkv = 4, D = 256;
int T_pref = T;
// Keep the prefill larger than one cache page, then exercise generation
// across the partially filled last page.
int T_pref = T - 7;
std::vector<int> q_sz, k_sz, v_sz;
if (layout == "3d") {
@@ -2608,4 +2610,140 @@ TEST(Layer_Test_Softmax, NoNaN_AllNegInf)
}
}
TEST(Test_Gemm, FastGemmBlockedTails)
{
struct TestCase
{
int M, N, K;
bool transB;
};
const TestCase cases[] = {
{7, 15, 129, false}, // partial M/N and K tail
{8, 16, 128, false}, // one full RVV micro-tile
{9, 17, 65, false}, // full tile plus M/N/K tails
{31, 33, 129, true} // multiple tiles and transposed B
};
for (const TestCase& tc : cases)
{
Mat A(tc.M, tc.K, CV_32F);
Mat B(tc.transB ? tc.N : tc.K, tc.transB ? tc.K : tc.N, CV_32F);
randu(A, -1.f, 1.f);
randu(B, -1.f, 1.f);
LayerParams lp;
lp.type = "Gemm";
lp.name = "fast_gemm_blocked_tails";
lp.set("transA", false);
lp.set("transB", tc.transB);
lp.set("alpha", 0.75f);
lp.set("beta", 0.f);
lp.set("real_ndims_C", 0);
lp.set("constB", true);
lp.blobs.push_back(B);
Net net;
net.addLayerToPrev(lp.name, lp.type, lp);
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(DNN_TARGET_CPU);
net.setInput(A);
Mat actual = net.forward();
Mat expected;
gemm(A, B, 0.75, noArray(), 0., expected, tc.transB ? GEMM_2_T : 0);
normAssert(actual, expected, "fastGemm blocked/tail mismatch", 1e-4, 1e-4);
}
}
TEST(Test_Gemm, FastGemmDynamicTransposeAlphaBeta)
{
const int M = 11, N = 19, K = 67;
const float alpha = 0.75f, beta = -0.25f;
for (int flags = 0; flags < 4; flags++)
{
const bool transA = (flags & 1) != 0;
const bool transB = (flags & 2) != 0;
Mat A(transA ? K : M, transA ? M : K, CV_32F);
Mat B(transB ? N : K, transB ? K : N, CV_32F);
Mat C(M, N, CV_32F);
randu(A, -1.f, 1.f);
randu(B, -1.f, 1.f);
randu(C, -1.f, 1.f);
LayerParams lp;
lp.type = "Gemm";
lp.name = "fast_gemm_dynamic";
lp.set("transA", transA);
lp.set("transB", transB);
lp.set("alpha", alpha);
lp.set("beta", beta);
lp.set("have_bias", true);
lp.set("real_ndims_C", 2);
Ptr<Layer> layer = LayerFactory::createLayerInstance(lp.type, lp);
ASSERT_TRUE(layer);
std::vector<Mat> inputs = {A, B, C}, outputs;
runLayer(layer, inputs, outputs);
ASSERT_EQ(outputs.size(), (size_t)1);
Mat expected;
int gemmFlags = (transA ? GEMM_1_T : 0) | (transB ? GEMM_2_T : 0);
gemm(A, B, alpha, C, beta, expected, gemmFlags);
normAssert(outputs[0], expected, "fastGemm dynamic transpose/alpha/beta mismatch", 1e-4, 1e-4);
}
}
TEST(Test_MatMul, FastGemmBatchDynamicAndPackedBroadcast)
{
const int batch = 3, M = 11, N = 19, K = 67;
Mat A({batch, M, K}, CV_32F);
Mat dynamicB({batch, N, K}, CV_32F); // transposed B
Mat packedB(K, N, CV_32F); // shared constant B
randu(A, -1.f, 1.f);
randu(dynamicB, -1.f, 1.f);
randu(packedB, -1.f, 1.f);
auto reference = [&](const Mat& B, bool transB, bool broadcastB)
{
Mat expected({batch, M, N}, CV_32F);
for (int b = 0; b < batch; b++)
{
Mat a2d(M, K, CV_32F, A.ptr<float>(b));
Mat b2d(transB ? N : K, transB ? K : N, CV_32F,
broadcastB ? const_cast<float*>(B.ptr<float>()) : const_cast<float*>(B.ptr<float>(b)));
Mat out2d(M, N, CV_32F, expected.ptr<float>(b));
gemm(a2d, b2d, 1., noArray(), 0., out2d, transB ? GEMM_2_T : 0);
}
return expected;
};
LayerParams dynamicParams;
dynamicParams.type = "MatMul";
dynamicParams.name = "fast_gemm_batch_dynamic";
dynamicParams.set("transA", false);
dynamicParams.set("transB", true);
Ptr<Layer> dynamicLayer = LayerFactory::createLayerInstance(dynamicParams.type, dynamicParams);
ASSERT_TRUE(dynamicLayer);
std::vector<Mat> dynamicInputs = {A, dynamicB}, dynamicOutputs;
runLayer(dynamicLayer, dynamicInputs, dynamicOutputs);
ASSERT_EQ(dynamicOutputs.size(), (size_t)1);
Mat dynamicExpected = reference(dynamicB, true, false);
normAssert(dynamicOutputs[0], dynamicExpected, "fastGemm dynamic batch mismatch", 1e-4, 1e-4);
LayerParams packedParams;
packedParams.type = "MatMul";
packedParams.name = "fast_gemm_batch_packed";
packedParams.set("transA", false);
packedParams.set("transB", false);
packedParams.blobs.push_back(packedB);
Ptr<Layer> packedLayer = LayerFactory::createLayerInstance(packedParams.type, packedParams);
ASSERT_TRUE(packedLayer);
std::vector<Mat> packedInputs = {A}, packedOutputs;
runLayer(packedLayer, packedInputs, packedOutputs);
ASSERT_EQ(packedOutputs.size(), (size_t)1);
Mat packedExpected = reference(packedB, false, true);
normAssert(packedOutputs[0], packedExpected, "fastGemm packed broadcast batch mismatch", 1e-4, 1e-4);
}
}} // namespace