diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index f0065c68da..422852ee0f 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -9,7 +9,7 @@ ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX RVV ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEON_FP16 RVV) -ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX) +ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX RVV) ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2) diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp index 6516a69aac..5da3b8a034 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp @@ -22,6 +22,11 @@ namespace cv { namespace dnn { int fastGemmMC(const FastGemmOpt &opt) { +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + return opt_RVV::fastGemmMC(); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { return opt_NEON::fastGemmMC(); @@ -48,6 +53,11 @@ int fastGemmMC(const FastGemmOpt &opt) { } int fastGemmNC(const FastGemmOpt &opt) { +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + return opt_RVV::fastGemmNC(); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { return opt_NEON::fastGemmNC(); @@ -74,6 +84,11 @@ int fastGemmNC(const FastGemmOpt &opt) { } int fastGemmKC(const FastGemmOpt &opt) { +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + return opt_RVV::fastGemmKC(); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { return opt_NEON::fastGemmKC(); @@ -100,6 +115,11 @@ int fastGemmKC(const FastGemmOpt &opt) { } int fastGemmNR(const FastGemmOpt &opt) { +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + return opt_RVV::fastGemmNR(); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { return opt_NEON::fastGemmNR(); @@ -128,6 +148,11 @@ int fastGemmNR(const FastGemmOpt &opt) { size_t fastGemmPackBSize(size_t N, size_t K, const FastGemmOpt &opt) { +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + return static_cast(opt_RVV::fastGemmPackBSize(N, K)); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { return static_cast(opt_NEON::fastGemmPackBSize(N, K)); @@ -167,6 +192,18 @@ void fastGemmPackB(const Mat &B, std::vector &packed_B, bool trans, FastG const auto *b = B.ptr(); int esz = B.elemSize(); +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + size_t size_packed_B = opt_RVV::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B * batch); + auto *packed_b = (char*)packed_B.data(); + for (int i = 0; i < batch; i++) { + opt_RVV::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz); + b += (size_t)N * (size_t)K * (size_t)esz; + packed_b += size_packed_B * (size_t)esz; + } + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { size_t size_packed_B = opt_NEON::fastGemmPackBSize(N, K); @@ -237,6 +274,11 @@ void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, f const auto &b = (const char *)B; auto *packed_b = (char *)packed_B; +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float)); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float)); @@ -482,6 +524,11 @@ void fastGemm(bool trans_a, int M, int N, int K, std::swap(lda0, lda1); } +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread); @@ -532,7 +579,7 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb, #ifdef HAVE_MLAS const bool a_row_major = (lda0 == 1 || lda1 == 1); const bool b_row_major = (ldb0 == 1 || ldb1 == 1); - if (a_row_major && b_row_major) { + if (!opt.use_rvv && a_row_major && b_row_major) { const int phys_lda = std::max(lda0, lda1); const int phys_ldb = std::max(ldb0, ldb1); if (mlasSgemm(trans_a, trans_b, M, N, K, @@ -543,6 +590,13 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb, } #endif +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, + b, ldb0, ldb1, beta, + c, ldc, sizeof(float), opt.multi_thread); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, @@ -645,7 +699,7 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset else if (lda0 == 1) { a_ok = true; mlas_trans_a = true; mlas_lda = lda1; } if (ldb1 == 1) { b_ok = true; mlas_trans_b = false; mlas_ldb = ldb0; } else if (ldb0 == 1) { b_ok = true; mlas_trans_b = true; mlas_ldb = ldb1; } - if (a_ok && b_ok) { + if (!opt.use_rvv && a_ok && b_ok) { if (mlasSgemmBatch(batch, A_offsets, B_offsets, C_offsets, mlas_trans_a, mlas_trans_b, M, N, K, alpha, A, mlas_lda, B, mlas_ldb, @@ -655,6 +709,11 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset } #endif +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); @@ -687,6 +746,11 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *packed_B const char *b = (const char *)packed_B; char *c = (char *)C; +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float)); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float)); @@ -768,6 +832,11 @@ void fastGemmBatch(size_t batch, return; } +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) { + opt_RVV::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); + } else +#endif #if CV_TRY_NEON if (opt.use_neon) { opt_NEON::fastGemmBatchKernel(batch, A_offsets.data(), B_offsets.data(), C_offsets.data(), M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float)); @@ -839,6 +908,15 @@ void pagedAttnQKGemm( char*a = A.ptr(); bool isQ3D = shape_q.size() == 3; +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) + opt_RVV::pagedAttnQKGemmKernel( + Q.ptr(), packed_K, a, + B, T_q, Nq, N_k, T_s, D, T_k, + sm_scale, esz, isQ3D + ); + else +#endif #if CV_TRY_NEON if (opt.use_neon) opt_NEON::pagedAttnQKGemmKernel( @@ -928,6 +1006,15 @@ void pagedAttnAVGemm( } bool canonical_layout = shape(Out).size() == 3; +#if CV_TRY_RVV && CV_RVV + if (opt.use_rvv) + opt_RVV::pagedAttnAVGemmKernel( + A.ptr(), packed_V, Out.ptr(), + B, T_q, Nq, N_k, T_s, D, T_v, + esz, canonical_layout, fastGemmPackBSize(D, T_s, opt) + ); + else +#endif #if CV_TRY_NEON if (opt.use_neon) opt_NEON::pagedAttnAVGemmKernel( diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp index cbe8b819f0..a0c08c092b 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp @@ -22,6 +22,7 @@ struct FastGemmOpt { bool use_avx2; bool use_neon; bool use_lasx; + bool use_rvv; bool multi_thread; FastGemmOpt() { @@ -29,6 +30,7 @@ struct FastGemmOpt { use_avx2 = false; use_neon = false; use_lasx = false; + use_rvv = false; multi_thread = false; } @@ -37,11 +39,12 @@ struct FastGemmOpt { use_avx2 = checkHardwareSupport(CPU_AVX2); use_neon = checkHardwareSupport(CPU_NEON); use_lasx = checkHardwareSupport(CPU_LASX); + use_rvv = checkHardwareSupport(CPU_RVV); multi_thread = true; } bool all() { - return use_avx || use_avx2 || use_neon || use_lasx; + return use_avx || use_avx2 || use_neon || use_lasx || use_rvv; } }; diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp index 09a05068f9..64fb8a2eb1 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp @@ -15,7 +15,10 @@ #define FAST_GEMM_STORAGE (1<<20) // 2^20 #define FAST_GEMM_MAX_STACKBUF (1 << 13) -#if CV_AVX +#if CV_RVV +#define FAST_GEMM_F32_MC 64 +#define FAST_GEMM_F32_NC 128 +#elif CV_AVX #define FAST_GEMM_F32_MC 60 #define FAST_GEMM_F32_NC 320 #elif CV_LASX @@ -26,7 +29,10 @@ #define FAST_GEMM_F32_NC 72 #endif -#if CV_AVX +#if CV_RVV +#define FAST_GEMM_F32_MR 8 +#define FAST_GEMM_F32_NR 16 +#elif CV_AVX #define FAST_GEMM_F32_MR 12 #define FAST_GEMM_F32_NR 8 #elif CV_LASX @@ -37,7 +43,9 @@ #define FAST_GEMM_F32_NR 12 #endif -#if CV_AVX +#if CV_RVV +#define FAST_GEMM_F32_PACKED_STRIDE_K 128 +#elif CV_AVX #define FAST_GEMM_F32_PACKED_STRIDE_K 128 #else // CV_LASX, CV_NEON_AARCH64, CV_SIMD128 #define FAST_GEMM_F32_PACKED_STRIDE_K 64 @@ -156,7 +164,83 @@ void pagedAttnAVGemmKernel( /* Compute kernels that optimized for different platforms */ -#if CV_NEON && CV_NEON_AARCH64 // AARCH64: 32 x 128-bit registers +#if CV_RVV // RVV (32 x VLEN-bit registers, LMUL=m2 for NR=16) + +FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) // a packer +FAST_GEMM_IMPLEMENT_PACK(16, _f32, float, float) // b packer + +static inline void fast_gemm8x16_f32(int k, const char *a_, const char *b_, + char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + if (__riscv_vsetvlmax_e32m1() >= 8) { + const size_t vl = __riscv_vsetvl_e32m2(FAST_GEMM_F32_NR); + vfloat32m2_t s0 = __riscv_vfmv_v_f_f32m2(0.f, vl); + vfloat32m2_t s1 = s0, s2 = s0, s3 = s0, s4 = s0, s5 = s0, s6 = s0, s7 = s0; + + for (int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) { + vfloat32m2_t bp = __riscv_vle32_v_f32m2(b, vl); + s0 = __riscv_vfmacc_vf_f32m2(s0, a[0], bp, vl); + s1 = __riscv_vfmacc_vf_f32m2(s1, a[1], bp, vl); + s2 = __riscv_vfmacc_vf_f32m2(s2, a[2], bp, vl); + s3 = __riscv_vfmacc_vf_f32m2(s3, a[3], bp, vl); + s4 = __riscv_vfmacc_vf_f32m2(s4, a[4], bp, vl); + s5 = __riscv_vfmacc_vf_f32m2(s5, a[5], bp, vl); + s6 = __riscv_vfmacc_vf_f32m2(s6, a[6], bp, vl); + s7 = __riscv_vfmacc_vf_f32m2(s7, a[7], bp, vl); + } + +#define FAST_GEMM_RVV_STORE(row) \ + __riscv_vse32_v_f32m2(c + (row) * ldc, \ + __riscv_vfmacc_vf_f32m2(__riscv_vle32_v_f32m2(c + (row) * ldc, vl), alpha, s##row, vl), vl) + FAST_GEMM_RVV_STORE(0); + FAST_GEMM_RVV_STORE(1); + FAST_GEMM_RVV_STORE(2); + FAST_GEMM_RVV_STORE(3); + FAST_GEMM_RVV_STORE(4); + FAST_GEMM_RVV_STORE(5); + FAST_GEMM_RVV_STORE(6); + FAST_GEMM_RVV_STORE(7); +#undef FAST_GEMM_RVV_STORE + return; + } + + // fallback for VLEN < 256: strip-mine NR columns with m1 + for (int j = 0; j < FAST_GEMM_F32_NR; ) { + const size_t vl = __riscv_vsetvl_e32m1(FAST_GEMM_F32_NR - j); + vfloat32m1_t s0 = __riscv_vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t s1 = s0, s2 = s0, s3 = s0, s4 = s0, s5 = s0, s6 = s0, s7 = s0; + for (int p = 0; p < k; p++) { + vfloat32m1_t bp = __riscv_vle32_v_f32m1(b + p * FAST_GEMM_F32_NR + j, vl); + const float* ap = a + p * FAST_GEMM_F32_MR; + s0 = __riscv_vfmacc_vf_f32m1(s0, ap[0], bp, vl); + s1 = __riscv_vfmacc_vf_f32m1(s1, ap[1], bp, vl); + s2 = __riscv_vfmacc_vf_f32m1(s2, ap[2], bp, vl); + s3 = __riscv_vfmacc_vf_f32m1(s3, ap[3], bp, vl); + s4 = __riscv_vfmacc_vf_f32m1(s4, ap[4], bp, vl); + s5 = __riscv_vfmacc_vf_f32m1(s5, ap[5], bp, vl); + s6 = __riscv_vfmacc_vf_f32m1(s6, ap[6], bp, vl); + s7 = __riscv_vfmacc_vf_f32m1(s7, ap[7], bp, vl); + } +#define FAST_GEMM_RVV_STORE_TAIL(row) \ + __riscv_vse32_v_f32m1(c + (row) * ldc + j, \ + __riscv_vfmacc_vf_f32m1(__riscv_vle32_v_f32m1(c + (row) * ldc + j, vl), alpha, s##row, vl), vl) + FAST_GEMM_RVV_STORE_TAIL(0); + FAST_GEMM_RVV_STORE_TAIL(1); + FAST_GEMM_RVV_STORE_TAIL(2); + FAST_GEMM_RVV_STORE_TAIL(3); + FAST_GEMM_RVV_STORE_TAIL(4); + FAST_GEMM_RVV_STORE_TAIL(5); + FAST_GEMM_RVV_STORE_TAIL(6); + FAST_GEMM_RVV_STORE_TAIL(7); +#undef FAST_GEMM_RVV_STORE_TAIL + j += (int)vl; + } +} + +#elif CV_NEON && CV_NEON_AARCH64 // AARCH64: 32 x 128-bit registers FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) // a packer FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float) // b packer @@ -531,7 +615,9 @@ static inline void fast_gemm_macro_kernel(int m, int n, int k, memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz); } -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm8x16_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm8x12_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); #elif CV_AVX fast_gemm12x8_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); @@ -574,7 +660,9 @@ void fastGemmPackBKernel(const char *B, char *packed_B, size_t N, size_t K, size for (size_t k = 0; k < K; k += KC) { size_t kc = K - k < KC ? K - k : KC; size_t step = (k * ldb0 + j0 * ldb1) * esz; -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack16_f32(nc, kc, B + step, ldb1, ldb0, packed_B); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack12_f32(nc, kc, B + step, ldb1, ldb0, packed_B); #elif CV_AVX fast_gemm_pack8_f32(nc, kc, B + step, ldb1, ldb0, packed_B); @@ -638,7 +726,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K, { size_t kc = K - k0 < KC ? K - k0 : KC; // pack a -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); @@ -648,7 +738,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K, fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); #endif // pack b -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack16_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack12_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); #elif CV_AVX fast_gemm_pack8_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); @@ -730,7 +822,9 @@ void fastGemmKernel(size_t M, size_t N, size_t K, size_t kc = K - k0 < KC ? K - k0 : KC; size_t step_a = (i0 * lda0 + k0 * lda1) * esz; // pack a -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, A + step_a, lda0, lda1, packed_a); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, A + step_a, lda0, lda1, packed_a); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, A + step_a, lda0, lda1, packed_a); @@ -815,7 +909,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_ size_t step_a = (i0 * lda0 + k0 * lda1) * esz; // pack a -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, a_block + step_a, lda0, lda1, packed_a); @@ -826,7 +922,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_ #endif size_t step_b = (k0 * ldb0 + j0 * ldb1) * esz; // pack b -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack16_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack12_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b); #elif CV_AVX fast_gemm_pack8_f32(nc, kc, b_block + step_b, ldb1, ldb0, packed_b); @@ -914,7 +1012,9 @@ void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_ // size_t step = ((size_t)k0 * (size_t)lda1 + (size_t)i0 * (size_t)lda0) * (size_t)esz; size_t step = (k0 * lda1 + i0 * lda0) * esz; // pack a -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, a_block + step, lda0, lda1, packed_a); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, a_block + step, lda0, lda1, packed_a); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, a_block + step, lda0, lda1, packed_a); @@ -1026,7 +1126,9 @@ void pagedAttnQKGemmKernel( int kc = D - k0 < KC ? D - k0 : KC; // pack q size_t step_q = (i0 * ldq0 + k0) * esz; -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, q_block + step_q, ldq0, 1, packed_q); @@ -1142,7 +1244,9 @@ void pagedAttnAVGemmKernel( const char *v_block = V[sk] + v_offset; // pack -#if CV_NEON && CV_NEON_AARCH64 +#if CV_RVV + fast_gemm_pack8_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a); +#elif CV_NEON && CV_NEON_AARCH64 fast_gemm_pack8_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a); #elif CV_AVX fast_gemm_pack12_f32(mc, kc, a_block + k0 * esz, T_v, 1, packed_a); diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp index 2996074fe4..8e4bd9465a 100644 --- a/modules/dnn/src/layers/gemm_layer.cpp +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -265,7 +265,7 @@ public: #ifdef HAVE_MLAS packed_B_mlas.release(); packed_B_mlas_M = packed_B_mlas_N = packed_B_mlas_K = 0; - if (mlasAvailable()) { + if (mlasAvailable() && !opt.use_rvv) { std::vector outputs; outputs_arr.getMatVector(outputs); const auto shape_A = shape(inputs[0]); @@ -448,7 +448,7 @@ public: if (constB(mode)) { #ifdef HAVE_MLAS - if (!packed_B_mlas.empty() && + if (!opt.use_rvv && !packed_B_mlas.empty() && packed_B_mlas_N == N && packed_B_mlas_K == K) { if (mlasSgemmPacked(trans_a, trans_b, rows, N, K, diff --git a/modules/dnn/src/layers/matmul_layer.cpp b/modules/dnn/src/layers/matmul_layer.cpp index c75897ddf9..f3a1f01e41 100644 --- a/modules/dnn/src/layers/matmul_layer.cpp +++ b/modules/dnn/src/layers/matmul_layer.cpp @@ -275,7 +275,7 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { const auto &B = inputs[1]; const auto *b = B.ptr(); bool done = false; - if (mlasAvailable() && helper.M > 0 && helper.N > 0 && helper.K > 0) { + if (mlasAvailable() && !opt.use_rvv && helper.M > 0 && helper.N > 0 && helper.K > 0) { const auto A_shape = shape(A); const auto B_shape = shape(B); const int lda_mem = A_shape.back(); diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index c97e641b9d..c5e6a665e2 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -2446,7 +2446,9 @@ public: Net netWithoutKVCache = readNetFromONNX(findDataFile(model_path, true), cv::dnn::ENGINE_NEW); int T = 523, Nq = 8, Nkv = 4, D = 256; - int T_pref = T; + // Keep the prefill larger than one cache page, then exercise generation + // across the partially filled last page. + int T_pref = T - 7; std::vector q_sz, k_sz, v_sz; if (layout == "3d") { @@ -2608,4 +2610,140 @@ TEST(Layer_Test_Softmax, NoNaN_AllNegInf) } } +TEST(Test_Gemm, FastGemmBlockedTails) +{ + struct TestCase + { + int M, N, K; + bool transB; + }; + const TestCase cases[] = { + {7, 15, 129, false}, // partial M/N and K tail + {8, 16, 128, false}, // one full RVV micro-tile + {9, 17, 65, false}, // full tile plus M/N/K tails + {31, 33, 129, true} // multiple tiles and transposed B + }; + + for (const TestCase& tc : cases) + { + Mat A(tc.M, tc.K, CV_32F); + Mat B(tc.transB ? tc.N : tc.K, tc.transB ? tc.K : tc.N, CV_32F); + randu(A, -1.f, 1.f); + randu(B, -1.f, 1.f); + + LayerParams lp; + lp.type = "Gemm"; + lp.name = "fast_gemm_blocked_tails"; + lp.set("transA", false); + lp.set("transB", tc.transB); + lp.set("alpha", 0.75f); + lp.set("beta", 0.f); + lp.set("real_ndims_C", 0); + lp.set("constB", true); + lp.blobs.push_back(B); + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + net.setPreferableBackend(DNN_BACKEND_OPENCV); + net.setPreferableTarget(DNN_TARGET_CPU); + net.setInput(A); + Mat actual = net.forward(); + + Mat expected; + gemm(A, B, 0.75, noArray(), 0., expected, tc.transB ? GEMM_2_T : 0); + normAssert(actual, expected, "fastGemm blocked/tail mismatch", 1e-4, 1e-4); + } +} + +TEST(Test_Gemm, FastGemmDynamicTransposeAlphaBeta) +{ + const int M = 11, N = 19, K = 67; + const float alpha = 0.75f, beta = -0.25f; + + for (int flags = 0; flags < 4; flags++) + { + const bool transA = (flags & 1) != 0; + const bool transB = (flags & 2) != 0; + Mat A(transA ? K : M, transA ? M : K, CV_32F); + Mat B(transB ? N : K, transB ? K : N, CV_32F); + Mat C(M, N, CV_32F); + randu(A, -1.f, 1.f); + randu(B, -1.f, 1.f); + randu(C, -1.f, 1.f); + + LayerParams lp; + lp.type = "Gemm"; + lp.name = "fast_gemm_dynamic"; + lp.set("transA", transA); + lp.set("transB", transB); + lp.set("alpha", alpha); + lp.set("beta", beta); + lp.set("have_bias", true); + lp.set("real_ndims_C", 2); + + Ptr layer = LayerFactory::createLayerInstance(lp.type, lp); + ASSERT_TRUE(layer); + std::vector inputs = {A, B, C}, outputs; + runLayer(layer, inputs, outputs); + ASSERT_EQ(outputs.size(), (size_t)1); + + Mat expected; + int gemmFlags = (transA ? GEMM_1_T : 0) | (transB ? GEMM_2_T : 0); + gemm(A, B, alpha, C, beta, expected, gemmFlags); + normAssert(outputs[0], expected, "fastGemm dynamic transpose/alpha/beta mismatch", 1e-4, 1e-4); + } +} + +TEST(Test_MatMul, FastGemmBatchDynamicAndPackedBroadcast) +{ + const int batch = 3, M = 11, N = 19, K = 67; + Mat A({batch, M, K}, CV_32F); + Mat dynamicB({batch, N, K}, CV_32F); // transposed B + Mat packedB(K, N, CV_32F); // shared constant B + randu(A, -1.f, 1.f); + randu(dynamicB, -1.f, 1.f); + randu(packedB, -1.f, 1.f); + + auto reference = [&](const Mat& B, bool transB, bool broadcastB) + { + Mat expected({batch, M, N}, CV_32F); + for (int b = 0; b < batch; b++) + { + Mat a2d(M, K, CV_32F, A.ptr(b)); + Mat b2d(transB ? N : K, transB ? K : N, CV_32F, + broadcastB ? const_cast(B.ptr()) : const_cast(B.ptr(b))); + Mat out2d(M, N, CV_32F, expected.ptr(b)); + gemm(a2d, b2d, 1., noArray(), 0., out2d, transB ? GEMM_2_T : 0); + } + return expected; + }; + + LayerParams dynamicParams; + dynamicParams.type = "MatMul"; + dynamicParams.name = "fast_gemm_batch_dynamic"; + dynamicParams.set("transA", false); + dynamicParams.set("transB", true); + Ptr dynamicLayer = LayerFactory::createLayerInstance(dynamicParams.type, dynamicParams); + ASSERT_TRUE(dynamicLayer); + std::vector dynamicInputs = {A, dynamicB}, dynamicOutputs; + runLayer(dynamicLayer, dynamicInputs, dynamicOutputs); + ASSERT_EQ(dynamicOutputs.size(), (size_t)1); + Mat dynamicExpected = reference(dynamicB, true, false); + normAssert(dynamicOutputs[0], dynamicExpected, "fastGemm dynamic batch mismatch", 1e-4, 1e-4); + + LayerParams packedParams; + packedParams.type = "MatMul"; + packedParams.name = "fast_gemm_batch_packed"; + packedParams.set("transA", false); + packedParams.set("transB", false); + packedParams.blobs.push_back(packedB); + Ptr packedLayer = LayerFactory::createLayerInstance(packedParams.type, packedParams); + ASSERT_TRUE(packedLayer); + std::vector packedInputs = {A}, packedOutputs; + runLayer(packedLayer, packedInputs, packedOutputs); + ASSERT_EQ(packedOutputs.size(), (size_t)1); + Mat packedExpected = reference(packedB, false, true); + normAssert(packedOutputs[0], packedExpected, "fastGemm packed broadcast batch mismatch", 1e-4, 1e-4); +} + }} // namespace