diff --git a/modules/dnn/perf/perf_gemm.cpp b/modules/dnn/perf/perf_gemm.cpp index 40fd66865b..f2fd70dd1b 100644 --- a/modules/dnn/perf/perf_gemm.cpp +++ b/modules/dnn/perf/perf_gemm.cpp @@ -83,6 +83,13 @@ static const GemmParam_t test_matmul_configs[] = { { {16, 197, 64 }, {16, 64, 197} }, { {16, 50, 64}, {16, 64, 50} }, { {16, 50, 50}, {16, 50, 64} }, + + // transformer token generation cases (thin-M GEMV, fastGemmThin path) + { {1, 1, 768}, {1, 768, 768} }, + { {1, 1, 768}, {1, 768, 3072} }, + { {1, 1, 3072}, {1, 3072, 768} }, + { {1, 2, 768}, {1, 768, 768} }, + { {1, 4, 768}, {1, 768, 768} }, }; struct GemmParamId diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp index 5da3b8a034..3f3e3d9650 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp @@ -318,6 +318,45 @@ static inline int fast_gemm_thin_lanes() { #endif } +#if CV_SIMD_SCALABLE +// Apply the alpha/beta epilogue to one accumulator row. +static inline void fast_gemm_thin_store_row(float* c, v_float32 acc, + v_float32 v_alpha, float beta) { + if (beta == 0.f) + vx_store(c, v_mul(acc, v_alpha)); + else if (beta == 1.f) + vx_store(c, v_fma(acc, v_alpha, vx_load(c))); + else + vx_store(c, v_fma(acc, v_alpha, v_mul(vx_load(c), vx_setall_f32(beta)))); +} + +// Multiply MR (1..4) rows of A by one packed-B strip. Sizeless vector types +// cannot form the acc[] array the CV_SIMD path uses, so the accumulators are +// named variables selected by the compile-time MR (the `if (MR > n)` branches +// fold away) and stay in registers for the whole K loop. Accumulation remains +// in increasing-k order, bit-exact with the previous scratch-buffer code. +template +static inline void fast_gemm_thin_block(int K, const float* A, int lda0, int lda1, + const float* b_strip, int NR, + v_float32 v_alpha, float beta, + float* c_strip, int ldc) { + v_float32 c0 = vx_setzero_f32(), c1 = vx_setzero_f32(); + v_float32 c2 = vx_setzero_f32(), c3 = vx_setzero_f32(); + for (int k = 0; k < K; k++) { + v_float32 bv = vx_load(b_strip + k * NR); + const float* a = A + k * lda1; + c0 = v_fma(bv, vx_setall_f32(a[0]), c0); + if (MR > 1) c1 = v_fma(bv, vx_setall_f32(a[lda0]), c1); + if (MR > 2) c2 = v_fma(bv, vx_setall_f32(a[2 * lda0]), c2); + if (MR > 3) c3 = v_fma(bv, vx_setall_f32(a[3 * lda0]), c3); + } + fast_gemm_thin_store_row(c_strip, c0, v_alpha, beta); + if (MR > 1) fast_gemm_thin_store_row(c_strip + ldc, c1, v_alpha, beta); + if (MR > 2) fast_gemm_thin_store_row(c_strip + 2 * ldc, c2, v_alpha, beta); + if (MR > 3) fast_gemm_thin_store_row(c_strip + 3 * ldc, c3, v_alpha, beta); +} +#endif // CV_SIMD_SCALABLE + static inline void fast_gemm_thin_strip(int M, int K, float alpha, const float* A, int lda0, int lda1, const float* b_strip, @@ -353,37 +392,17 @@ static inline void fast_gemm_thin_strip(int M, int K, float alpha, } } #elif CV_SIMD_SCALABLE - // Scalable vector types (e.g. RVV) are sizeless and cannot form arrays; - // back the per-row accumulators with a scalar scratch buffer. + // Process the strip in register-resident row blocks; the common thin case + // M <= 4 (transformer token generation) is a single pass. const int NR = VTraits::vlanes(); - float acc_buf[FAST_GEMM_THIN_MAX_M * VTraits::max_nlanes]; - for (int m = 0; m < M; m++) vx_store(acc_buf + m * NR, vx_setzero_f32()); - - for (int k = 0; k < K; k++) { - v_float32 bv = vx_load(b_strip + k * NR); - for (int m = 0; m < M; m++) { - v_float32 am = vx_setall_f32(A[m * lda0 + k * lda1]); - v_float32 acc_m = vx_load(acc_buf + m * NR); - vx_store(acc_buf + m * NR, v_fma(bv, am, acc_m)); - } - } - const v_float32 v_alpha = vx_setall_f32(alpha); - if (beta == 0.f) { - for (int m = 0; m < M; m++) - vx_store(c_strip + m * ldc, v_mul(vx_load(acc_buf + m * NR), v_alpha)); - } else if (beta == 1.f) { - for (int m = 0; m < M; m++) { - v_float32 cv = vx_load(c_strip + m * ldc); - vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv)); - } - } else { - const v_float32 v_beta = vx_setall_f32(beta); - for (int m = 0; m < M; m++) { - v_float32 cv = vx_load(c_strip + m * ldc); - cv = v_mul(cv, v_beta); - vx_store(c_strip + m * ldc, v_fma(vx_load(acc_buf + m * NR), v_alpha, cv)); - } + int m = 0; + for (; m + 4 <= M; m += 4) + fast_gemm_thin_block<4>(K, A + m * lda0, lda0, lda1, b_strip, NR, v_alpha, beta, c_strip + m * ldc, ldc); + switch (M - m) { + case 1: fast_gemm_thin_block<1>(K, A + m * lda0, lda0, lda1, b_strip, NR, v_alpha, beta, c_strip + m * ldc, ldc); break; + case 2: fast_gemm_thin_block<2>(K, A + m * lda0, lda0, lda1, b_strip, NR, v_alpha, beta, c_strip + m * ldc, ldc); break; + case 3: fast_gemm_thin_block<3>(K, A + m * lda0, lda0, lda1, b_strip, NR, v_alpha, beta, c_strip + m * ldc, ldc); break; } #else const int NR = FAST_GEMM_THIN_SCALAR_NR; diff --git a/modules/dnn/test/test_fast_gemm.cpp b/modules/dnn/test/test_fast_gemm.cpp new file mode 100644 index 0000000000..e245389e57 --- /dev/null +++ b/modules/dnn/test/test_fast_gemm.cpp @@ -0,0 +1,77 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +static Mat referenceMatMul(const Mat& A, const Mat& B, bool trans_a, float alpha) +{ + const int M = trans_a ? A.cols : A.rows; + const int K = trans_a ? A.rows : A.cols; + const int N = B.cols; + Mat expected(M, N, CV_32F); + + for (int m = 0; m < M; m++) + { + float* dst = expected.ptr(m); + for (int n = 0; n < N; n++) + { + float acc = 0.f; + for (int k = 0; k < K; k++) + { + const float a = trans_a ? A.ptr(k)[m] : A.ptr(m)[k]; + acc += a * B.ptr(k)[n]; + } + dst[n] = alpha * acc; + } + } + return expected; +} + +// Exercises the fastGemmThin path (constant-B MatMul). M covers every register +// block width and the multi-block remainder, N covers full strips and partial +// column tails for any VLEN <= 512. +typedef testing::TestWithParam> DNN_FastGemmThin; + +TEST_P(DNN_FastGemmThin, MatMulAccuracy) +{ + int M = get<0>(GetParam()), N = get<1>(GetParam()), K = get<2>(GetParam()); + int trans_a = get<3>(GetParam()); + float alpha = get<4>(GetParam()); + + RNG rng(0x5EED); + Mat B(K, N, CV_32F); rng.fill(B, RNG::UNIFORM, -1.f, 1.f); + Mat A(trans_a ? K : M, trans_a ? M : K, CV_32F); + rng.fill(A, RNG::UNIFORM, -1.f, 1.f); + + LayerParams lp; + lp.type = "MatMul"; + lp.name = "thin_matmul"; + lp.set("transA", trans_a != 0); + lp.set("transB", false); + lp.set("alpha", alpha); + lp.blobs.push_back(B); + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + net.setInputsNames(std::vector{ "A" }); + net.setInput(A, "A"); + Mat actual = net.forward(); + Mat expected = referenceMatMul(A, B, trans_a != 0, alpha); + + EXPECT_LE(cv::norm(expected, actual, NORM_INF), 2e-5f * std::max(K, 1)) + << "M=" << M << ", N=" << N << ", K=" << K + << ", trans_a=" << trans_a << ", alpha=" << alpha; +} + +INSTANTIATE_TEST_CASE_P(/*nothing*/, DNN_FastGemmThin, Combine( + Values(1, 2, 3, 4, 5), + Values(16, 19, 64, 67, 80, 83), + Values(1, 3, 4, 7, 16, 64), + Values(0, 1), + Values(1.f, -0.5f) +)); + +}} // namespace