mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge pull request #28934 from abhishek-gola:mlas_gemm
Added MLAS third party module and integrated into GeMM path #28934 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -487,6 +487,30 @@ if(NOT EMSCRIPTEN)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Vendored MLAS (Microsoft Linear Algebra Subprograms) from ONNX Runtime.
|
||||
# Sources live in 3rdparty/mlas/. Builds to an OBJECT library whose objects
|
||||
# link directly into opencv_dnn. Skipped under Emscripten: MLAS is a native
|
||||
# CPU SGEMM accelerator (asm/intrinsic kernels) and the wasm scalar fallback
|
||||
# offers no benefit for the JS bindings build that produces opencv.js.
|
||||
set(HAVE_MLAS 0)
|
||||
# Reset status flags + arch booleans
|
||||
foreach(_v OPENCV_DNN_MLAS_ENABLED OPENCV_DNN_MLAS_SKIP_REASON
|
||||
MLAS_X86_64 MLAS_X86 MLAS_ARM MLAS_ARM64 MLAS_POWER
|
||||
MLAS_LOONGARCH64 MLAS_S390X MLAS_RISCV64 MLAS_WASM
|
||||
MLAS_HAS_ASM MLAS_HAS_POWER10 MLAS_HAS_RISCV64_RVV)
|
||||
unset(${_v} CACHE)
|
||||
endforeach()
|
||||
if(NOT EMSCRIPTEN)
|
||||
add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/mlas" "${CMAKE_BINARY_DIR}/3rdparty/mlas")
|
||||
endif()
|
||||
if(HAVE_MLAS)
|
||||
add_definitions(-DHAVE_MLAS=1)
|
||||
list(APPEND include_dirs ${MLAS_INCLUDE_DIRS})
|
||||
message(STATUS "DNN: MLAS (vendored) enabled.")
|
||||
else()
|
||||
message(STATUS "DNN: MLAS (vendored) disabled — host arch/OS not wired up.")
|
||||
endif()
|
||||
|
||||
ocv_module_include_directories(${include_dirs})
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
ocv_append_source_files_cxx_compiler_options(fw_srcs "-Wno-suggest-override") # GCC
|
||||
@@ -534,7 +558,7 @@ endif()
|
||||
|
||||
ocv_install_used_external_targets(${libs} ${dnn_runtime_libs})
|
||||
|
||||
ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs} ${webnn_srcs})
|
||||
ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs} ${webnn_srcs} ${MLAS_OBJECTS})
|
||||
ocv_create_module(${libs} ${dnn_runtime_libs})
|
||||
ocv_add_samples()
|
||||
ocv_add_accuracy_tests(${dnn_runtime_libs})
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "../precomp.hpp"
|
||||
#include "cpu_kernels/fast_gemm.hpp"
|
||||
#include "cpu_kernels/softmax.hpp"
|
||||
#include "cpu_kernels/mlas_gemm.hpp"
|
||||
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
|
||||
@@ -366,7 +367,39 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer {
|
||||
parallel_for_(Range(0, loops), fn, nstripes);
|
||||
}
|
||||
|
||||
// attention_prob = softmax(scale * Q @ K^T + mask)
|
||||
const int num_non_blob_inputs = (int)inputs.size();
|
||||
const bool has_mask = (!blobs.empty() && num_non_blob_inputs >= 2) ||
|
||||
( blobs.empty() && num_non_blob_inputs >= 4);
|
||||
|
||||
if (mlasAvailable() && !has_mask &&
|
||||
batch_size > 0 && num_heads > 0 && seq_len > 0 &&
|
||||
qkv_head_sizes[0] > 0 && qkv_head_sizes[2] > 0)
|
||||
{
|
||||
const int B = (int)batch_size;
|
||||
const int H = (int)num_heads;
|
||||
const int S = (int)seq_len;
|
||||
const int Dqk = (int)qkv_head_sizes[0];
|
||||
const int Dv = (int)qkv_head_sizes[2];
|
||||
// ORT-style default tiling, clamped to the actual sequence length.
|
||||
const int q_block = std::min(256, S);
|
||||
const int kv_block = std::min(256, S);
|
||||
const int threads = std::max(1, cv::getNumThreads());
|
||||
|
||||
const size_t per_thread =
|
||||
mlasFlashAttentionBufferBytesPerThread(q_block, kv_block, Dv);
|
||||
flash_scratch.resize((size_t)threads * per_thread);
|
||||
|
||||
if (mlasFlashAttention(Q, K, V,
|
||||
outputs[0].ptr<float>(),
|
||||
B, H, S, S, Dqk, Dv, scale,
|
||||
q_block, kv_block,
|
||||
flash_scratch.data(), threads))
|
||||
{
|
||||
return; // fast path done; outputs[0] is fully written
|
||||
}
|
||||
}
|
||||
|
||||
// Compute Softmax(scale * MatMul(Q, K))
|
||||
auto &attention_prob = internals[1];
|
||||
{
|
||||
auto *output = attention_prob.ptr<float>();
|
||||
@@ -526,6 +559,7 @@ class AttentionLayerImpl CV_FINAL : public AttentionLayer {
|
||||
std::vector<float> packed_weight_q;
|
||||
std::vector<float> packed_weight_k;
|
||||
std::vector<float> packed_weight_v;
|
||||
std::vector<unsigned char> flash_scratch;
|
||||
|
||||
FastGemmOpt opt;
|
||||
};
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
#include "../../precomp.hpp"
|
||||
#include "fast_gemm.hpp"
|
||||
#include "mlas_gemm.hpp"
|
||||
|
||||
#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
|
||||
#include "fast_gemm_kernels.simd.hpp"
|
||||
@@ -500,6 +501,20 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
|
||||
return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc, opt.multi_thread);
|
||||
}
|
||||
|
||||
#ifdef HAVE_MLAS
|
||||
const bool a_row_major = (lda0 == 1 || lda1 == 1);
|
||||
const bool b_row_major = (ldb0 == 1 || ldb1 == 1);
|
||||
if (a_row_major && b_row_major) {
|
||||
const int phys_lda = std::max(lda0, lda1);
|
||||
const int phys_ldb = std::max(ldb0, ldb1);
|
||||
if (mlasSgemm(trans_a, trans_b, M, N, K,
|
||||
alpha, A, phys_lda, B, phys_ldb,
|
||||
beta, C, ldc)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if CV_TRY_NEON
|
||||
if (opt.use_neon) {
|
||||
opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
|
||||
@@ -594,6 +609,24 @@ void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offset
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef HAVE_MLAS
|
||||
bool a_ok = false, b_ok = false;
|
||||
bool mlas_trans_a = false, mlas_trans_b = false;
|
||||
int mlas_lda = 0, mlas_ldb = 0;
|
||||
if (lda1 == 1) { a_ok = true; mlas_trans_a = false; mlas_lda = lda0; }
|
||||
else if (lda0 == 1) { a_ok = true; mlas_trans_a = true; mlas_lda = lda1; }
|
||||
if (ldb1 == 1) { b_ok = true; mlas_trans_b = false; mlas_ldb = ldb0; }
|
||||
else if (ldb0 == 1) { b_ok = true; mlas_trans_b = true; mlas_ldb = ldb1; }
|
||||
if (a_ok && b_ok) {
|
||||
if (mlasSgemmBatch(batch, A_offsets, B_offsets, C_offsets,
|
||||
mlas_trans_a, mlas_trans_b, M, N, K,
|
||||
alpha, A, mlas_lda, B, mlas_ldb,
|
||||
beta, C, ldc)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if CV_TRY_NEON
|
||||
if (opt.use_neon) {
|
||||
opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
#include "../../precomp.hpp"
|
||||
#include "mlas_gemm.hpp"
|
||||
|
||||
#ifdef HAVE_MLAS
|
||||
|
||||
#include "mlas.h"
|
||||
#include <vector>
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
bool mlasAvailable() {
|
||||
static const bool ok = []() {
|
||||
const size_t a = MlasGetPreferredBufferAlignment();
|
||||
return a > 0 && a <= 256;
|
||||
}();
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool mlasSgemm(bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A, int lda,
|
||||
const float* B, int ldb,
|
||||
float beta,
|
||||
float* C, int ldc)
|
||||
{
|
||||
if (!mlasAvailable()) return false;
|
||||
if (M <= 0 || N <= 0 || K <= 0) return false;
|
||||
|
||||
MLAS_SGEMM_DATA_PARAMS data;
|
||||
data.A = A;
|
||||
data.lda = static_cast<size_t>(lda);
|
||||
data.B = B;
|
||||
data.ldb = static_cast<size_t>(ldb);
|
||||
data.C = C;
|
||||
data.ldc = static_cast<size_t>(ldc);
|
||||
data.alpha = alpha;
|
||||
data.beta = beta;
|
||||
data.BIsPacked = false;
|
||||
|
||||
MlasGemm(trans_a ? CblasTrans : CblasNoTrans,
|
||||
trans_b ? CblasTrans : CblasNoTrans,
|
||||
static_cast<size_t>(M),
|
||||
static_cast<size_t>(N),
|
||||
static_cast<size_t>(K),
|
||||
data,
|
||||
/*ThreadPool=*/nullptr,
|
||||
/*BackendKernelSelectorConfig=*/nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mlasSgemmBatch(size_t batch,
|
||||
const size_t* A_offsets,
|
||||
const size_t* B_offsets,
|
||||
const size_t* C_offsets,
|
||||
bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A_base, int lda,
|
||||
const float* B_base, int ldb,
|
||||
float beta,
|
||||
float* C_base, int ldc)
|
||||
{
|
||||
if (!mlasAvailable()) return false;
|
||||
if (batch == 0 || M <= 0 || N <= 0 || K <= 0) return false;
|
||||
|
||||
std::vector<MLAS_SGEMM_DATA_PARAMS> data(batch);
|
||||
for (size_t i = 0; i < batch; i++) {
|
||||
data[i].A = A_base + A_offsets[i];
|
||||
data[i].lda = static_cast<size_t>(lda);
|
||||
data[i].B = B_base + B_offsets[i];
|
||||
data[i].ldb = static_cast<size_t>(ldb);
|
||||
data[i].C = C_base + C_offsets[i];
|
||||
data[i].ldc = static_cast<size_t>(ldc);
|
||||
data[i].alpha = alpha;
|
||||
data[i].beta = beta;
|
||||
data[i].BIsPacked = false;
|
||||
}
|
||||
|
||||
MlasGemmBatch(trans_a ? CblasTrans : CblasNoTrans,
|
||||
trans_b ? CblasTrans : CblasNoTrans,
|
||||
static_cast<size_t>(M),
|
||||
static_cast<size_t>(N),
|
||||
static_cast<size_t>(K),
|
||||
data.data(),
|
||||
batch,
|
||||
/*ThreadPool=*/nullptr,
|
||||
/*BackendKernelSelectorConfig=*/nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t mlasSgemmPackBSize(bool trans_a, bool trans_b, int N, int K)
|
||||
{
|
||||
if (!mlasAvailable()) return 0;
|
||||
if (N <= 0 || K <= 0) return 0;
|
||||
return MlasGemmPackBSize(trans_a ? CblasTrans : CblasNoTrans,
|
||||
trans_b ? CblasTrans : CblasNoTrans,
|
||||
static_cast<size_t>(N),
|
||||
static_cast<size_t>(K),
|
||||
/*BackendKernelSelectorConfig=*/nullptr);
|
||||
}
|
||||
|
||||
bool mlasSgemmPackB(bool trans_a, bool trans_b, int N, int K,
|
||||
const float* B, int ldb, void* packed_B)
|
||||
{
|
||||
if (!mlasAvailable()) return false;
|
||||
if (N <= 0 || K <= 0 || B == nullptr || packed_B == nullptr) return false;
|
||||
MlasGemmPackB(trans_a ? CblasTrans : CblasNoTrans,
|
||||
trans_b ? CblasTrans : CblasNoTrans,
|
||||
static_cast<size_t>(N),
|
||||
static_cast<size_t>(K),
|
||||
B, static_cast<size_t>(ldb),
|
||||
packed_B,
|
||||
/*BackendKernelSelectorConfig=*/nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mlasSgemmPacked(bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A, int lda,
|
||||
const void* packed_B,
|
||||
float beta,
|
||||
float* C, int ldc)
|
||||
{
|
||||
if (!mlasAvailable()) return false;
|
||||
if (M <= 0 || N <= 0 || K <= 0) return false;
|
||||
|
||||
MLAS_SGEMM_DATA_PARAMS data;
|
||||
data.A = A;
|
||||
data.lda = static_cast<size_t>(lda);
|
||||
data.B = static_cast<const float*>(packed_B);
|
||||
data.ldb = 0; // ignored when BIsPacked
|
||||
data.C = C;
|
||||
data.ldc = static_cast<size_t>(ldc);
|
||||
data.alpha = alpha;
|
||||
data.beta = beta;
|
||||
data.BIsPacked = true;
|
||||
|
||||
MlasGemm(trans_a ? CblasTrans : CblasNoTrans,
|
||||
trans_b ? CblasTrans : CblasNoTrans,
|
||||
static_cast<size_t>(M),
|
||||
static_cast<size_t>(N),
|
||||
static_cast<size_t>(K),
|
||||
data,
|
||||
/*ThreadPool=*/nullptr,
|
||||
/*BackendKernelSelectorConfig=*/nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t mlasFlashAttentionBufferBytesPerThread(int q_block_size,
|
||||
int kv_block_size,
|
||||
int v_head_size)
|
||||
{
|
||||
if (q_block_size <= 0 || kv_block_size <= 0 || v_head_size <= 0) return 0;
|
||||
// flashattn.cpp lays out the per-thread scratch as:
|
||||
// l[q_block_size] + m[q_block_size]
|
||||
// + intermediate[q_block_size * kv_block_size]
|
||||
// + temp_output[q_block_size * v_head_size]
|
||||
const size_t q = static_cast<size_t>(q_block_size);
|
||||
const size_t kv = static_cast<size_t>(kv_block_size);
|
||||
const size_t vd = static_cast<size_t>(v_head_size);
|
||||
return (q * (2 + kv + vd)) * sizeof(float);
|
||||
}
|
||||
|
||||
bool mlasFlashAttention(const float* query, const float* key, const float* value,
|
||||
float* output,
|
||||
int batch_size, int num_heads,
|
||||
int q_seq_len, int kv_seq_len,
|
||||
int qk_head_size, int v_head_size,
|
||||
float scale,
|
||||
int q_block_size, int kv_block_size,
|
||||
void* scratch, int thread_count)
|
||||
{
|
||||
if (!mlasAvailable()) return false;
|
||||
if (batch_size <= 0 || num_heads <= 0) return false;
|
||||
if (q_seq_len <= 0 || kv_seq_len <= 0) return false;
|
||||
if (qk_head_size <= 0 || v_head_size <= 0) return false;
|
||||
if (q_block_size <= 0 || kv_block_size <= 0) return false;
|
||||
if (thread_count <= 0 || scratch == nullptr) return false;
|
||||
if (query == nullptr || key == nullptr || value == nullptr || output == nullptr)
|
||||
return false;
|
||||
|
||||
MlasFlashAttentionThreadedArgs args;
|
||||
args.batch_size = batch_size;
|
||||
args.num_heads = num_heads;
|
||||
args.q_sequence_length = q_seq_len;
|
||||
args.kv_sequence_length = kv_seq_len;
|
||||
args.qk_head_size = qk_head_size;
|
||||
args.v_head_size = v_head_size;
|
||||
args.q_block_size = q_block_size;
|
||||
args.kv_block_size = kv_block_size;
|
||||
args.scale = scale;
|
||||
args.thread_count = thread_count;
|
||||
args.buffer = static_cast<float*>(scratch);
|
||||
args.buffer_size_per_thread = mlasFlashAttentionBufferBytesPerThread(
|
||||
q_block_size, kv_block_size, v_head_size);
|
||||
args.query = query;
|
||||
args.key = key;
|
||||
args.value = value;
|
||||
args.output = output;
|
||||
|
||||
MlasFlashAttention(&args, /*ThreadPool=*/nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
}} // cv::dnn
|
||||
|
||||
#endif // HAVE_MLAS
|
||||
@@ -0,0 +1,122 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
#ifndef OPENCV_DNN_MLAS_GEMM_HPP
|
||||
#define OPENCV_DNN_MLAS_GEMM_HPP
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
#ifdef HAVE_MLAS
|
||||
|
||||
// True if MLAS is usable on this host. False signals callers to fall back.
|
||||
bool mlasAvailable();
|
||||
|
||||
// Row-major SGEMM: C := alpha * op(A) * op(B) + beta * C, op(X) = X or X^T.
|
||||
// Returns false if MLAS is unavailable or M/N/K <= 0.
|
||||
bool mlasSgemm(bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A, int lda,
|
||||
const float* B, int ldb,
|
||||
float beta,
|
||||
float* C, int ldc);
|
||||
|
||||
// Batched SGEMM with per-batch element offsets into A_base/B_base/C_base.
|
||||
// M/N/K and leading dims are shared across the batch.
|
||||
bool mlasSgemmBatch(size_t batch,
|
||||
const size_t* A_offsets,
|
||||
const size_t* B_offsets,
|
||||
const size_t* C_offsets,
|
||||
bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A_base, int lda,
|
||||
const float* B_base, int ldb,
|
||||
float beta,
|
||||
float* C_base, int ldc);
|
||||
|
||||
// Pack B once, reuse across many mlasSgemmPacked() calls. Returns the
|
||||
// required buffer size in bytes; caller allocates and passes to mlasSgemmPackB.
|
||||
size_t mlasSgemmPackBSize(bool trans_a, bool trans_b, int N, int K);
|
||||
|
||||
bool mlasSgemmPackB(bool trans_a, bool trans_b, int N, int K,
|
||||
const float* B, int ldb, void* packed_B);
|
||||
|
||||
// mlasSgemm with a pre-packed B from mlasSgemmPackB.
|
||||
bool mlasSgemmPacked(bool trans_a, bool trans_b,
|
||||
int M, int N, int K,
|
||||
float alpha,
|
||||
const float* A, int lda,
|
||||
const void* packed_B,
|
||||
float beta,
|
||||
float* C, int ldc);
|
||||
|
||||
// Scratch-buffer size (in bytes) per worker thread for mlasFlashAttention.
|
||||
// The caller must allocate `thread_count * this` bytes for the scratch
|
||||
// pointer. Returns 0 if any argument is non-positive.
|
||||
size_t mlasFlashAttentionBufferBytesPerThread(int q_block_size,
|
||||
int kv_block_size,
|
||||
int v_head_size);
|
||||
|
||||
// Multi-head attention via MLAS flash-attention. Computes
|
||||
// output[b, i, h, :] = softmax(scale * Q[b,h,i,:] @ K[b,h,:,:]^T) @ V[b,h,:,:]
|
||||
// fused into one tiled kernel without materializing the q_seq x kv_seq
|
||||
// attention matrix.
|
||||
//
|
||||
// Layouts (row-major contiguous, FP32):
|
||||
// query : [batch, num_heads, q_seq_len, qk_head_size]
|
||||
// key : [batch, num_heads, kv_seq_len, qk_head_size]
|
||||
// value : [batch, num_heads, kv_seq_len, v_head_size]
|
||||
// output: [batch, q_seq_len, num_heads, v_head_size] (heads *after* seq)
|
||||
//
|
||||
// scale - usually 1 / sqrt(qk_head_size).
|
||||
// q_block_size - tile size along the q sequence (e.g. 256).
|
||||
// kv_block_size - tile size along the kv sequence (e.g. 256).
|
||||
// scratch - caller-owned buffer of at least
|
||||
// thread_count * mlasFlashAttentionBufferBytesPerThread(...).
|
||||
// thread_count - number of MLAS workers to fan out across (typically
|
||||
// cv::getNumThreads()).
|
||||
//
|
||||
// Returns false if MLAS is unavailable or arguments are invalid.
|
||||
bool mlasFlashAttention(const float* query, const float* key, const float* value,
|
||||
float* output,
|
||||
int batch_size, int num_heads,
|
||||
int q_seq_len, int kv_seq_len,
|
||||
int qk_head_size, int v_head_size,
|
||||
float scale,
|
||||
int q_block_size, int kv_block_size,
|
||||
void* scratch, int thread_count);
|
||||
|
||||
#else // HAVE_MLAS
|
||||
|
||||
inline bool mlasAvailable() { return false; }
|
||||
|
||||
inline bool mlasSgemm(bool, bool, int, int, int, float,
|
||||
const float*, int, const float*, int,
|
||||
float, float*, int) { return false; }
|
||||
|
||||
inline bool mlasSgemmBatch(size_t, const size_t*, const size_t*, const size_t*,
|
||||
bool, bool, int, int, int, float,
|
||||
const float*, int, const float*, int,
|
||||
float, float*, int) { return false; }
|
||||
|
||||
inline size_t mlasSgemmPackBSize(bool, bool, int, int) { return 0; }
|
||||
inline bool mlasSgemmPackB(bool, bool, int, int, const float*, int, void*) { return false; }
|
||||
inline bool mlasSgemmPacked(bool, bool, int, int, int, float,
|
||||
const float*, int, const void*, float, float*, int) { return false; }
|
||||
|
||||
inline size_t mlasFlashAttentionBufferBytesPerThread(int, int, int) { return 0; }
|
||||
inline bool mlasFlashAttention(const float*, const float*, const float*, float*,
|
||||
int, int, int, int, int, int, float,
|
||||
int, int, void*, int) { return false; }
|
||||
|
||||
#endif // HAVE_MLAS
|
||||
|
||||
}} // cv::dnn
|
||||
|
||||
#endif // OPENCV_DNN_MLAS_GEMM_HPP
|
||||
@@ -0,0 +1,98 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
// OpenCV-backed implementation of MLAS's threading primitives.
|
||||
//
|
||||
// Replaces lib/threading.cpp from upstream MLAS (which, when built with
|
||||
// BUILD_MLAS_NO_ONNXRUNTIME and a nullptr ThreadPool, runs everything in a
|
||||
// serial for-loop). MLAS internally calls MlasGetMaximumThreadCount() to
|
||||
// pick a partition count; for the standalone build that returns 1, so even
|
||||
// with a parallel `MlasTrySimpleParallel` MLAS would still emit a single
|
||||
// iteration. We patch both halves:
|
||||
//
|
||||
// 1. MlasGetMaximumThreadCount() in mlasi.h returns cv::getNumThreads()
|
||||
// when MLAS_OPENCV_THREADING is defined (see the small patch in
|
||||
// mlasi.h flagged with that define).
|
||||
// 2. The three threaded entry points below dispatch to cv::parallel_for_.
|
||||
//
|
||||
// Compiled into the opencv_dnn_mlas object library only — the
|
||||
// MLAS_OPENCV_THREADING guard makes the file a no-op when picked up by
|
||||
// opencv_dnn's recursive src glob, since MLAS internal headers and the
|
||||
// MLAS_GEMM_ONLY / BUILD_MLAS_NO_ONNXRUNTIME defines are not in scope there.
|
||||
|
||||
#if defined(MLAS_OPENCV_THREADING)
|
||||
|
||||
#include "mlasi.h"
|
||||
#include "qgemm.h" // for MLAS_GEMM_QUANT_DISPATCH definition (stub below)
|
||||
|
||||
#include <opencv2/core/utility.hpp>
|
||||
|
||||
// MLAS_GEMM_ONLY stub: mlasi.h's MLAS_PLATFORM struct uses
|
||||
// GemmS8S8Dispatch{&MlasGemmQuantDispatchDefault}
|
||||
// as in-class initializers. The real definition lives in qgemm_kernel_default.cpp
|
||||
// which we don't compile in the SGEMM-only build. Provide a zero-initialized
|
||||
// instance so mlasi.h links — it's never read because we never call MlasQgemm.
|
||||
extern "C++" const MLAS_GEMM_QUANT_DISPATCH MlasGemmQuantDispatchDefault{};
|
||||
|
||||
extern "C" int opencv_dnn_mlas_max_threads()
|
||||
{
|
||||
int n = cv::getNumThreads();
|
||||
return n > 0 ? n : 1;
|
||||
}
|
||||
|
||||
void
|
||||
MlasExecuteThreaded(
|
||||
MLAS_THREADED_ROUTINE* ThreadedRoutine,
|
||||
void* Context,
|
||||
ptrdiff_t Iterations,
|
||||
MLAS_THREADPOOL* /*ThreadPool*/)
|
||||
{
|
||||
if (Iterations <= 0) return;
|
||||
if (Iterations == 1) { ThreadedRoutine(Context, 0); return; }
|
||||
|
||||
cv::parallel_for_(cv::Range(0, static_cast<int>(Iterations)),
|
||||
[&](const cv::Range& r) {
|
||||
for (int tid = r.start; tid < r.end; tid++) {
|
||||
ThreadedRoutine(Context, static_cast<ptrdiff_t>(tid));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
MlasTrySimpleParallel(
|
||||
MLAS_THREADPOOL* /*ThreadPool*/,
|
||||
const std::ptrdiff_t Iterations,
|
||||
const std::function<void(std::ptrdiff_t tid)>& Work)
|
||||
{
|
||||
if (Iterations <= 0) return;
|
||||
if (Iterations == 1) { Work(0); return; }
|
||||
|
||||
cv::parallel_for_(cv::Range(0, static_cast<int>(Iterations)),
|
||||
[&](const cv::Range& r) {
|
||||
for (int tid = r.start; tid < r.end; tid++) {
|
||||
Work(static_cast<std::ptrdiff_t>(tid));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
MlasTryBatchParallel(
|
||||
MLAS_THREADPOOL* /*ThreadPool*/,
|
||||
const std::ptrdiff_t Iterations,
|
||||
const std::function<void(std::ptrdiff_t tid)>& Work)
|
||||
{
|
||||
// MLAS only calls this for "non-performance-critical" small batches, but
|
||||
// there is no reason to serialize it on a 24-core host either.
|
||||
if (Iterations <= 0) return;
|
||||
if (Iterations == 1) { Work(0); return; }
|
||||
|
||||
cv::parallel_for_(cv::Range(0, static_cast<int>(Iterations)),
|
||||
[&](const cv::Range& r) {
|
||||
for (int tid = r.start; tid < r.end; tid++) {
|
||||
Work(static_cast<std::ptrdiff_t>(tid));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#endif // MLAS_OPENCV_THREADING
|
||||
@@ -17,6 +17,7 @@ using namespace cv::dnn::cuda4dnn;
|
||||
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
#include "cpu_kernels/fast_gemm.hpp"
|
||||
#include "cpu_kernels/mlas_gemm.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
@@ -279,6 +280,30 @@ public:
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef HAVE_MLAS
|
||||
std::vector<Mat> outputs;
|
||||
outputs_arr.getMatVector(outputs);
|
||||
const auto shape_A = shape(inputs[0]);
|
||||
const auto shape_Y = shape(outputs[0]);
|
||||
const int na = shape_A[shape_A.size() - 1];
|
||||
const int ma = shape_A[shape_A.size() - 2];
|
||||
const int N = shape_Y[shape_Y.size() - 1];
|
||||
const int K = trans_a ? ma : na;
|
||||
const Mat& Bmat = blobs[0];
|
||||
const int ldb = Bmat.size[Bmat.dims - 1];
|
||||
const size_t packed_bytes = mlasSgemmPackBSize(trans_a, trans_b, N, K);
|
||||
if (packed_bytes > 0) {
|
||||
packed_B_mlas.create(1, static_cast<int>(packed_bytes), CV_8U);
|
||||
if (mlasSgemmPackB(trans_a, trans_b, N, K,
|
||||
Bmat.ptr<const float>(), ldb,
|
||||
packed_B_mlas.data)) {
|
||||
packed_B_mlas_N = N;
|
||||
packed_B_mlas_K = K;
|
||||
} else {
|
||||
packed_B_mlas.release();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (constC(mode) && flatten_a) {
|
||||
@@ -361,6 +386,20 @@ public:
|
||||
}
|
||||
|
||||
if (constB(mode)) {
|
||||
#ifdef HAVE_MLAS
|
||||
if (!packed_B_mlas.empty() &&
|
||||
packed_B_mlas_N == N && packed_B_mlas_K == K)
|
||||
{
|
||||
if (mlasSgemmPacked(trans_a, trans_b, rows, N, K,
|
||||
alpha,
|
||||
A.ptr<const float>(), na,
|
||||
packed_B_mlas.data,
|
||||
1.f,
|
||||
Y.ptr<float>(), N)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
CV_CheckGT(packed_B.size(), static_cast<size_t>(0), "DNN/Gemm: constant B is not pre-packed");
|
||||
if (!thin_packed_B.empty()) {
|
||||
fastGemmThin(rows, N, K, alpha, A.ptr<const float>(), na, 1,
|
||||
@@ -531,6 +570,11 @@ private:
|
||||
bool have_bias;
|
||||
std::vector<float> packed_B;
|
||||
std::vector<float> thin_packed_B;
|
||||
#ifdef HAVE_MLAS
|
||||
cv::Mat packed_B_mlas;
|
||||
int packed_B_mlas_N = 0;
|
||||
int packed_B_mlas_K = 0;
|
||||
#endif
|
||||
std::vector<float> broadcast_C;
|
||||
int real_ndims_C;
|
||||
FastGemmOpt opt;
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
#include "cpu_kernels/fast_gemm.hpp"
|
||||
#include "cpu_kernels/mlas_gemm.hpp"
|
||||
|
||||
// OpenVINO backend
|
||||
#include "../op_inf_engine.hpp"
|
||||
@@ -267,9 +268,22 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
if (blobs.empty()) {
|
||||
const auto &B = inputs[1];
|
||||
const auto *b = B.ptr<const float>();
|
||||
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
||||
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
||||
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
|
||||
bool done = false;
|
||||
if (mlasAvailable() && helper.M > 0 && helper.N > 0 && helper.K > 0) {
|
||||
const auto A_shape = shape(A);
|
||||
const auto B_shape = shape(B);
|
||||
const int lda_mem = A_shape.back();
|
||||
const int ldb_mem = B_shape.back();
|
||||
done = mlasSgemmBatch(helper.batch,
|
||||
helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
||||
trans_a, trans_b, helper.M, helper.N, helper.K,
|
||||
alpha, a, lda_mem, b, ldb_mem, beta, y, helper.ldc);
|
||||
}
|
||||
if (!done) {
|
||||
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
||||
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
||||
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
|
||||
}
|
||||
} else if (!thin_packed_B.empty()) {
|
||||
fastGemmThin(helper.M, helper.N, helper.K, alpha,
|
||||
a, helper.lda0, helper.lda1,
|
||||
|
||||
Reference in New Issue
Block a user