mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge pull request #27988 from nklskyoy:attention-2-layer
AttentionOnnxAiLayer #27988 Implements https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
nklskyoy
GitHub
@@ -1519,6 +1519,11 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
static Ptr<RotaryEmbeddingLayer> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
class CV_EXPORTS AttentionOnnxAiLayer : public Layer {
|
||||
public:
|
||||
static Ptr<AttentionOnnxAiLayer> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
class CV_EXPORTS GroupNormLayer : public Layer {
|
||||
public:
|
||||
static Ptr<GroupNormLayer> create(const LayerParams ¶ms);
|
||||
|
||||
@@ -203,6 +203,7 @@ void initializeLayerFactory()
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(AttentionOnnxAi, AttentionOnnxAiLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(RotaryEmbedding, RotaryEmbeddingLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer);
|
||||
CV_DNN_REGISTER_LAYER_CLASS(Cast, CastLayer);
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "../precomp.hpp"
|
||||
#include "cpu_kernels/fast_gemm.hpp"
|
||||
#include "cpu_kernels/fast_attn.hpp"
|
||||
|
||||
#include "layers_common.hpp"
|
||||
#include "../net_impl.hpp"
|
||||
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
// Operator spec: https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23
|
||||
class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
public:
|
||||
AttentionOnnxAiLayerImpl(const LayerParams ¶ms) {
|
||||
setParamsFrom(params);
|
||||
is_causal = params.get<bool>("is_causal", false);
|
||||
kv_num_heads = params.get<int>("kv_num_heads", 0);
|
||||
q_num_heads = params.get<int>("q_num_heads", 0);
|
||||
qk_matmul_output_mode = params.get<int>("qk_matmul_output_mode", 0);
|
||||
scale = params.get<float>("scale", 1.0f );
|
||||
is_scale_set = params.has("scale");
|
||||
softcap = params.get<float>("softcap", 0.f);
|
||||
softmax_precision = params.get<int>("softmax_precision", 0);
|
||||
}
|
||||
|
||||
virtual bool supportBackend(int backendId) CV_OVERRIDE {
|
||||
return backendId == DNN_BACKEND_OPENCV;
|
||||
}
|
||||
|
||||
virtual void getTypes(const std::vector<MatType>&inputs,
|
||||
const int requiredOutputs,
|
||||
const int requiredInternals,
|
||||
std::vector<MatType>&outputs,
|
||||
std::vector<MatType>&internals) const CV_OVERRIDE {
|
||||
// type checks
|
||||
CV_CheckTrue(inputs.size() >= 3, "At least three inputs (query, key, value) are required");
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
CV_CheckType(inputs[i], inputs[i] == CV_16F || inputs[i] == CV_32F, "");
|
||||
}
|
||||
|
||||
CV_CheckType(inputs[0], inputs[0] == inputs[1] && inputs[0] == inputs[2], "");
|
||||
|
||||
if (inputs.size() >= 4) {
|
||||
CV_CheckType(inputs[3], inputs[3] == CV_8U || inputs[3] == CV_8S ||
|
||||
inputs[3] == CV_16U || inputs[3] == CV_16S ||
|
||||
inputs[3] == CV_32S || inputs[3] == CV_64S ||
|
||||
inputs[3] == CV_64U || inputs[3] == CV_Bool ||
|
||||
inputs[3] == inputs[2], ""); // attention_mask
|
||||
}
|
||||
|
||||
outputs.assign(requiredOutputs, inputs[0]);
|
||||
|
||||
// internals:
|
||||
internals.clear();
|
||||
|
||||
// 1. attention_prob
|
||||
internals.push_back(inputs[0]);
|
||||
}
|
||||
|
||||
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<MatShape> &outputs,
|
||||
std::vector<MatShape> &internals) const CV_OVERRIDE {
|
||||
CV_CheckTrue(inputs.size() >= 3, "At least three inputs (query, key, value) are required");
|
||||
CV_CheckTrue(inputs.size() < 5, "past key and past value are not supported yet");
|
||||
CV_CheckTrue(inputs[0].dims == inputs[1].dims &&
|
||||
inputs[0].dims == inputs[2].dims,
|
||||
"Query, key and value must have the same number of dimensions");
|
||||
|
||||
const int input_dims = inputs[0].dims;
|
||||
CV_CheckTrue(
|
||||
input_dims == 4 || (q_num_heads > 0 && kv_num_heads > 0 && input_dims == 3),
|
||||
"Input dimensions must be 4D or 3D (in the latter case, q_num_heads and kv_num_heads must be set)"
|
||||
);
|
||||
|
||||
const int batch_size = inputs[0][0];
|
||||
const int seq_len_q = inputs[0][input_dims - 2];
|
||||
const int seq_len_kv = inputs[1][input_dims - 2];
|
||||
|
||||
const int q_hn = input_dims == 4 ?
|
||||
inputs[0][1] : q_num_heads;
|
||||
const int kv_hn =input_dims == 4 ?
|
||||
inputs[1][1] : kv_num_heads;
|
||||
|
||||
CV_CheckTrue(inputs[2][input_dims - 2] == seq_len_kv,
|
||||
"Key and query sequence lengths must be equal");
|
||||
const int nhq = input_dims == 4 ? inputs[0][1] : q_num_heads;
|
||||
|
||||
CV_CheckTrue(q_hn % kv_hn == 0,
|
||||
"q_num_heads must be divisible by kv_num_heads");
|
||||
|
||||
if (input_dims == 3)
|
||||
{
|
||||
CV_CheckTrue(kv_hn > 0,
|
||||
"For 3D input, kv_num_heads must be greater than 0 (this normally means that kv_num_heads is not set)");
|
||||
CV_CheckTrue(q_hn > 0,
|
||||
"For 3D input, q_num_heads must be greater than 0 (this normally means that q_num_heads is not set)");
|
||||
|
||||
int v_head_size = inputs[2][2] / kv_hn;
|
||||
MatShape output_shape{batch_size, seq_len_q, v_head_size * q_num_heads};
|
||||
outputs.push_back(output_shape);
|
||||
}
|
||||
else
|
||||
{
|
||||
int v_head_size = inputs[2][3];
|
||||
MatShape output_shape{batch_size, nhq, seq_len_q, v_head_size};
|
||||
outputs.push_back(output_shape);
|
||||
}
|
||||
|
||||
MatShape attention_prob_shape{batch_size , nhq, seq_len_q, seq_len_kv};
|
||||
internals.push_back(attention_prob_shape);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE {
|
||||
opt.init();
|
||||
|
||||
if (inputs_arr.depth() == CV_16F)
|
||||
{
|
||||
forward_fallback(inputs_arr, outputs_arr, internals_arr);
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<Mat> inputs, outputs, internals;
|
||||
inputs_arr.getMatVector(inputs);
|
||||
outputs_arr.getMatVector(outputs);
|
||||
internals_arr.getMatVector(internals);
|
||||
|
||||
const int input_dims = inputs[0].dims;
|
||||
const int batch_size = inputs[0].size[0];
|
||||
|
||||
const int nhq = input_dims == 3 ?
|
||||
q_num_heads :
|
||||
inputs[0].size[1];
|
||||
const int nhkv = input_dims == 3 ?
|
||||
kv_num_heads :
|
||||
inputs[1].size[1];
|
||||
|
||||
const int qk_head_size = input_dims == 3 ?
|
||||
inputs[0].size[2] / nhq :
|
||||
inputs[0].size[3];
|
||||
const int v_head_size = input_dims == 3 ?
|
||||
inputs[2].size[2] / nhkv :
|
||||
inputs[2].size[3];
|
||||
const int num_gq_groups = nhq / nhkv;
|
||||
const int seq_len_q = input_dims == 3 ?
|
||||
inputs[0].size[1]:
|
||||
inputs[0].size[2];
|
||||
const int seq_len_kv = input_dims == 3 ?
|
||||
inputs[1].size[1]:
|
||||
inputs[1].size[2];
|
||||
const auto seq_len_square = seq_len_q * seq_len_kv;
|
||||
|
||||
const auto* Q = inputs[0].ptr<const float>();
|
||||
const auto* K = inputs[1].ptr<const float>();
|
||||
const auto* V = inputs[2].ptr<const float>();
|
||||
|
||||
scale = is_scale_set ? scale : 1.0f / std::sqrt(static_cast<float>(qk_head_size));
|
||||
|
||||
std::vector<size_t> _q_offsets(nhq * batch_size),
|
||||
_k_offsets(nhq * batch_size),
|
||||
_a_offsets(nhq * batch_size),
|
||||
_v_offsets(nhq * batch_size),
|
||||
_o_offsets(nhq * batch_size);
|
||||
|
||||
for (int b = 0; b < batch_size; b++)
|
||||
for (int n = 0; n < nhq; n++){
|
||||
_q_offsets[b * nhq + n] =
|
||||
b * seq_len_q * qk_head_size * nhq +
|
||||
(input_dims == 3 ? n * qk_head_size : n * qk_head_size * seq_len_q);
|
||||
_k_offsets[b * nhq + n] =
|
||||
b * seq_len_kv * qk_head_size * nhkv +
|
||||
(n / num_gq_groups * qk_head_size) * (input_dims == 3 ? 1 : seq_len_kv);
|
||||
_v_offsets[b * nhq + n] =
|
||||
b * seq_len_kv * v_head_size * nhkv +
|
||||
(n / num_gq_groups * v_head_size) * (input_dims == 3 ? 1 : seq_len_kv);
|
||||
_a_offsets[b * nhq + n] =
|
||||
b * seq_len_square * nhq +
|
||||
n * seq_len_square;
|
||||
_o_offsets[b * nhq + n] =
|
||||
b * seq_len_q * v_head_size * nhq +
|
||||
(input_dims == 3 ? n * v_head_size : n * v_head_size * seq_len_q);
|
||||
}
|
||||
|
||||
const int ldq0 = input_dims == 3 ? qk_head_size * nhq : qk_head_size;
|
||||
const int ldk0 = input_dims == 3 ? qk_head_size * nhkv : qk_head_size;
|
||||
auto &attention_prob = internals[internals.size() - 1];
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_q_offsets.data(), _k_offsets.data(), _a_offsets.data(),
|
||||
seq_len_q, seq_len_kv, qk_head_size , scale,
|
||||
Q, ldq0, 1,
|
||||
K, 1, ldk0,
|
||||
0.f,
|
||||
attention_prob.ptr<float>(), seq_len_kv,
|
||||
opt
|
||||
);
|
||||
|
||||
|
||||
fused_softmax_softcap_mask(
|
||||
attention_prob,
|
||||
inputs.size() > 3 ? inputs[3] : Mat(),
|
||||
softcap, softcap > 0.f,
|
||||
9.f,
|
||||
-FLT_MAX,
|
||||
inputs.size() > 3,
|
||||
is_causal
|
||||
);
|
||||
|
||||
|
||||
const int ldv0 = input_dims == 3 ? v_head_size * nhkv : v_head_size;
|
||||
const int ldout = input_dims == 3 ? v_head_size * nhq : v_head_size;
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_a_offsets.data(), _v_offsets.data(), _o_offsets.data(),
|
||||
seq_len_q, v_head_size, seq_len_kv, 1.f,
|
||||
attention_prob.ptr<float>(), seq_len_kv, 1,
|
||||
V, ldv0, 1,
|
||||
0.f,
|
||||
outputs[0].ptr<float>(), ldout,
|
||||
opt
|
||||
);
|
||||
}
|
||||
|
||||
private:
|
||||
bool is_causal;
|
||||
int kv_num_heads;
|
||||
int q_num_heads;
|
||||
int qk_matmul_output_mode;
|
||||
float scale;
|
||||
bool is_scale_set = false;
|
||||
float softcap;
|
||||
int softmax_precision;
|
||||
FastGemmOpt opt;
|
||||
};
|
||||
|
||||
Ptr<AttentionOnnxAiLayer> AttentionOnnxAiLayer::create(const LayerParams ¶ms) {
|
||||
return makePtr<AttentionOnnxAiLayerImpl>(params);
|
||||
}
|
||||
|
||||
}} // cv::dnn
|
||||
@@ -0,0 +1,256 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "../../precomp.hpp"
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
static inline v_float32 load_int_mask_as_f32(const int32_t* ptr) {
|
||||
v_int32 v = vx_load(ptr);
|
||||
return v_reinterpret_as_f32(v_ne(v, vx_setall_s32(0)));
|
||||
}
|
||||
|
||||
static inline v_float32 load_int_mask_as_f32(const uint8_t* ptr) {
|
||||
v_uint32 v = vx_load_expand_q(ptr);
|
||||
return v_reinterpret_as_f32(v_ne(v, vx_setall_u32(0)));
|
||||
}
|
||||
|
||||
static inline v_float32 load_int_mask_as_f32(const int8_t* ptr) {
|
||||
v_int32 v = vx_load_expand_q(ptr);
|
||||
return v_reinterpret_as_f32(v_ne(v, vx_setall_s32(0)));
|
||||
}
|
||||
|
||||
static inline v_float32 load_int_mask_as_f32(const uint16_t* ptr) {
|
||||
v_uint32 v = vx_load_expand(ptr);
|
||||
return v_reinterpret_as_f32(v_ne(v, vx_setall_u32(0)));
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
inline v_float32 load_int_mask_as_f32(const T* ptr) {
|
||||
return vx_setall_f32(0.f);
|
||||
}
|
||||
|
||||
|
||||
template <typename MaskT>
|
||||
struct MaskPolicyFloat {
|
||||
static inline v_float32 load_mask(const MaskT* ptr) { return vx_load((const float*)ptr); }
|
||||
static inline v_float32 load_mask_scalar(const MaskT* ptr) { return vx_setall_f32((float)*ptr); }
|
||||
static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return v_add(val, mask); }
|
||||
static inline float apply_scalar(float val, MaskT mask, float min_val) { return val + mask; }
|
||||
};
|
||||
|
||||
template <typename MaskT>
|
||||
struct MaskPolicyInt {
|
||||
static inline v_float32 load_mask(const MaskT* ptr) { return load_int_mask_as_f32(ptr); }
|
||||
static inline v_float32 load_mask_scalar(const MaskT* ptr) {
|
||||
return (*ptr != 0) ? v_reinterpret_as_f32(vx_setall_u32(0xFFFFFFFF)) : vx_setall_f32(0.f);
|
||||
}
|
||||
static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return v_select(mask, val, min_val); }
|
||||
static inline float apply_scalar(float val, MaskT mask, float min_val) { return (mask != 0) ? val : min_val; }
|
||||
};
|
||||
|
||||
struct MaskPolicyNone {
|
||||
static inline v_float32 load_mask(const void* ptr) { return vx_setzero_f32(); }
|
||||
static inline v_float32 load_mask_scalar(const void* ptr) { return vx_setzero_f32(); }
|
||||
static inline v_float32 apply(v_float32 val, v_float32 mask, v_float32 min_val) { return val; }
|
||||
static inline float apply_scalar(float val, int mask, float min_val) { return val; }
|
||||
};
|
||||
|
||||
template <typename MaskT, typename Policy>
|
||||
void run_fused_softmax(
|
||||
Mat &att_weights, const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool is_causal)
|
||||
{
|
||||
const int batch_size = att_weights.size[0];
|
||||
const int n_heads = att_weights.size[1];
|
||||
const int seq_len_q = att_weights.size[2];
|
||||
const int seq_len_kv = att_weights.size[3];
|
||||
|
||||
float* data = att_weights.ptr<float>();
|
||||
const MaskT* mask_data = att_mask.empty() ? nullptr : att_mask.ptr<MaskT>();
|
||||
|
||||
size_t mask_step_b = 0, mask_step_h = 0, mask_step_q = 0, mask_step_k = 0;
|
||||
if (mask_data) {
|
||||
int dims_m = att_mask.dims;
|
||||
int offset_dim = 4 - dims_m;
|
||||
|
||||
auto get_size = [&](int i) { return (i < offset_dim) ? 1 : att_mask.size[i - offset_dim]; };
|
||||
auto get_step = [&](int i) { return (i < offset_dim) ? 0 : att_mask.step[i - offset_dim] / sizeof(MaskT); };
|
||||
|
||||
if (get_size(0) > 1) mask_step_b = get_step(0);
|
||||
if (get_size(1) > 1) mask_step_h = get_step(1);
|
||||
if (get_size(2) > 1) mask_step_q = get_step(2);
|
||||
if (get_size(3) > 1) mask_step_k = 1;
|
||||
}
|
||||
|
||||
size_t total_tasks = (size_t)batch_size * n_heads;
|
||||
|
||||
parallel_for_(Range(0, (int)total_tasks), [&](const Range &range) {
|
||||
for (int i = range.start; i < range.end; i++) {
|
||||
int b = i / n_heads;
|
||||
int h = i % n_heads;
|
||||
|
||||
size_t offset = (size_t)i * seq_len_q * seq_len_kv;
|
||||
size_t mask_offset_q = b * mask_step_b + h * mask_step_h;
|
||||
|
||||
for (int tq = 0; tq < seq_len_q; tq++){
|
||||
const int tmax = is_causal ? std::min(tq + 1, seq_len_kv) : seq_len_kv;
|
||||
float maxVal = -FLT_MAX;
|
||||
int tk = 0;
|
||||
#if CV_SIMD
|
||||
const int w = VTraits<v_float32>::vlanes();
|
||||
v_float32 v_max_val = vx_setall_f32(maxVal);
|
||||
v_float32 v_softcap = vx_setall_f32(softcap);
|
||||
v_float32 v_inv_softcap = vx_setall_f32(1.f / softcap);
|
||||
v_float32 v_threshold = vx_setall_f32(threshold);
|
||||
v_float32 v_minus_threshold = vx_setall_f32(-threshold);
|
||||
v_float32 v_one = vx_setall_f32(1.f);
|
||||
v_float32 v_minus_one = vx_setall_f32(-1.f);
|
||||
v_float32 v_minus_two = vx_setall_f32(-2.f);
|
||||
v_float32 v_min_val = vx_setall_f32(min_val);
|
||||
|
||||
for (; tk <= tmax - w; tk += w) {
|
||||
v_float32 v_val = vx_load(&data[offset + tk]);
|
||||
|
||||
if (mask_data) {
|
||||
v_float32 v_mask_val;
|
||||
if (mask_step_k)
|
||||
v_mask_val = Policy::load_mask(&mask_data[mask_offset_q + tk]);
|
||||
else
|
||||
v_mask_val = Policy::load_mask_scalar(&mask_data[mask_offset_q]);
|
||||
v_val = Policy::apply(v_val, v_mask_val, v_min_val);
|
||||
}
|
||||
|
||||
if (do_softcap) {
|
||||
v_float32 v_scaled = v_mul(v_val, v_inv_softcap);
|
||||
|
||||
v_float32 v_mask_pos = v_gt(v_scaled, v_threshold);
|
||||
v_float32 v_mask_neg = v_lt(v_scaled, v_minus_threshold);
|
||||
|
||||
v_float32 v_scaled_safe = v_select(v_mask_neg, vx_setall_f32(0.f), v_scaled);
|
||||
|
||||
v_float32 v_exp_part = v_exp(v_mul(v_minus_two, v_scaled_safe));
|
||||
v_float32 v_tanh = v_div(v_sub(v_one, v_exp_part), v_add(v_one, v_exp_part));
|
||||
|
||||
v_float32 v_res = v_select(v_mask_pos, v_one,
|
||||
v_select(v_mask_neg, v_minus_one, v_tanh));
|
||||
|
||||
v_val = v_mul(v_res, v_softcap);
|
||||
}
|
||||
vx_store(&data[offset + tk], v_val);
|
||||
v_max_val = v_max(v_max_val, v_val);
|
||||
}
|
||||
maxVal = v_reduce_max(v_max_val);
|
||||
#endif
|
||||
for (; tk < tmax; tk++){
|
||||
if (mask_data) {
|
||||
size_t mask_idx = mask_offset_q + (mask_step_k ? tk : 0);
|
||||
data[offset + tk] = Policy::apply_scalar(data[offset + tk], mask_data[mask_idx], min_val);
|
||||
}
|
||||
|
||||
if (do_softcap) {
|
||||
float softcap_val = data[offset + tk] / softcap;
|
||||
if (softcap_val > threshold)
|
||||
data[offset + tk ] = 1.f;
|
||||
else if(softcap_val < -threshold)
|
||||
data[offset + tk ] = -1.f;
|
||||
else
|
||||
data[offset + tk] = (1.f - expf(-2 *softcap_val)) / (1.f + expf(-2 *softcap_val));
|
||||
data[offset + tk] *= softcap;
|
||||
}
|
||||
maxVal = std::max(maxVal, data[offset + tk]);
|
||||
}
|
||||
|
||||
for (; tk < seq_len_kv; tk++) {
|
||||
data[offset + tk] = min_val;
|
||||
}
|
||||
|
||||
float sum = 0.f;
|
||||
tk = 0;
|
||||
#if CV_SIMD
|
||||
v_float32 v_sum = vx_setzero_f32();
|
||||
v_float32 v_max_val_shift = vx_setall_f32(maxVal);
|
||||
for (; tk <= seq_len_kv - w; tk += w) {
|
||||
v_float32 v_val = vx_load(&data[offset + tk]);
|
||||
v_val = v_sub(v_val, v_max_val_shift);
|
||||
v_val = v_exp(v_val);
|
||||
v_sum = v_add(v_sum, v_val);
|
||||
vx_store(&data[offset + tk], v_val);
|
||||
}
|
||||
sum = v_reduce_sum(v_sum);
|
||||
#endif
|
||||
for (; tk < seq_len_kv; tk++){
|
||||
data[offset + tk] = expf(data[offset + tk] - maxVal);
|
||||
sum += data[offset + tk];
|
||||
}
|
||||
|
||||
float inv_sum = 1.f / sum;
|
||||
tk = 0;
|
||||
#if CV_SIMD
|
||||
v_float32 v_inv_sum = vx_setall_f32(inv_sum);
|
||||
for (; tk <= seq_len_kv - w; tk += w) {
|
||||
v_float32 v_val = vx_load(&data[offset + tk]);
|
||||
v_val = v_mul(v_val, v_inv_sum);
|
||||
vx_store(&data[offset + tk], v_val);
|
||||
}
|
||||
#endif
|
||||
for (; tk < seq_len_kv; tk++){
|
||||
data[offset + tk] *= inv_sum;
|
||||
}
|
||||
|
||||
offset += seq_len_kv;
|
||||
mask_offset_q += mask_step_q;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void fused_softmax_softcap_mask(
|
||||
Mat &att_weights,const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool has_mask, const bool is_causal
|
||||
){
|
||||
if (has_mask) {
|
||||
switch(att_mask.depth())
|
||||
{
|
||||
case CV_32F:
|
||||
run_fused_softmax<float, MaskPolicyFloat<float>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_Bool:
|
||||
case CV_8U:
|
||||
run_fused_softmax<uint8_t, MaskPolicyInt<uint8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_8S:
|
||||
run_fused_softmax<int8_t, MaskPolicyInt<int8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_16U:
|
||||
run_fused_softmax<uint16_t, MaskPolicyInt<uint16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_16S:
|
||||
run_fused_softmax<int16_t, MaskPolicyInt<int16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_32U:
|
||||
run_fused_softmax<uint32_t, MaskPolicyInt<uint32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_32S:
|
||||
run_fused_softmax<int32_t, MaskPolicyInt<int32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_64U:
|
||||
run_fused_softmax<uint64_t, MaskPolicyInt<uint64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
case CV_64S:
|
||||
run_fused_softmax<int64_t, MaskPolicyInt<int64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
break;
|
||||
default:
|
||||
CV_Error(Error::StsUnsupportedFormat, "Unsupported mask data type in fused_softmax_softcap_mask");
|
||||
}
|
||||
} else {
|
||||
run_fused_softmax<float, MaskPolicyNone>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
}
|
||||
}
|
||||
}}
|
||||
@@ -0,0 +1,23 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
|
||||
#ifndef OPENCV_DNN_FAST_ATTN
|
||||
#define OPENCV_DNN_FAST_ATTN
|
||||
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
|
||||
void fused_softmax_softcap_mask(
|
||||
Mat &att_weights,const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool has_mask, const bool is_causal
|
||||
);
|
||||
|
||||
}}
|
||||
|
||||
#endif
|
||||
@@ -107,7 +107,6 @@ struct Net::Impl : public detail::NetImplBase
|
||||
// FIXIT use inheritance
|
||||
virtual Ptr<BackendWrapper> wrap(Mat& host);
|
||||
|
||||
|
||||
virtual void clear();
|
||||
|
||||
|
||||
|
||||
@@ -501,6 +501,7 @@ class AttentionSubGraph : public Subgraph {
|
||||
std::vector<Ptr<ImportNodeWrapper> >&) CV_OVERRIDE {
|
||||
// add attrs
|
||||
opencv_onnx::NodeProto* node = fusedNode.dynamicCast<ONNXNodeWrapper>()->node;
|
||||
node->set_domain("com.microsoft");
|
||||
opencv_onnx::AttributeProto* attr_num_heads = node->add_attribute();
|
||||
attr_num_heads->set_name("num_heads");
|
||||
attr_num_heads->set_i(num_heads);
|
||||
@@ -611,6 +612,7 @@ class AttentionSingleHeadSubGraph : public Subgraph {
|
||||
std::vector<Ptr<ImportNodeWrapper> >&) CV_OVERRIDE {
|
||||
// add attrs
|
||||
opencv_onnx::NodeProto* node = fusedNode.dynamicCast<ONNXNodeWrapper>()->node;
|
||||
node->set_domain("com.microsoft");
|
||||
opencv_onnx::AttributeProto* attr_num_heads = node->add_attribute();
|
||||
attr_num_heads->set_name("num_heads");
|
||||
attr_num_heads->set_i(num_heads);
|
||||
|
||||
@@ -257,6 +257,7 @@ protected:
|
||||
// Domain: com.microsoft
|
||||
// URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md
|
||||
void parseAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseAttentionOnnxAi (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
|
||||
@@ -2660,6 +2661,26 @@ void ONNXImporter2::parseAttention(LayerParams& params, const opencv_onnx::NodeP
|
||||
addLayer(params, node_proto, n_inputs);
|
||||
}
|
||||
|
||||
void ONNXImporter2::parseAttentionOnnxAi(LayerParams& params, const opencv_onnx::NodeProto& node_proto) {
|
||||
int i, n_inputs = node_proto.input_size();
|
||||
|
||||
for (i = 1; i < n_inputs; i++) {
|
||||
if (!net.isConstArg(node_inputs[i]))
|
||||
break;
|
||||
}
|
||||
|
||||
if (i == n_inputs) {
|
||||
for (i = 1; i < n_inputs; i++) {
|
||||
Mat blob = net.argTensor(node_inputs[i]);
|
||||
params.blobs.push_back(blob);
|
||||
}
|
||||
n_inputs = 1;
|
||||
}
|
||||
params.type = "AttentionOnnxAi";
|
||||
|
||||
addLayer(params, node_proto, n_inputs);
|
||||
}
|
||||
|
||||
// Domain: ai.onnx (default)
|
||||
// URL: https://github.com/onnx/onnx/blob/master/docs/Operators.md
|
||||
void ONNXImporter2::buildDispatchMap_ONNX_AI()
|
||||
@@ -2743,6 +2764,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
|
||||
dispatch["GroupNormalization"] = &ONNXImporter2::parseInstanceNormalization;
|
||||
dispatch["NegativeLogLikelihoodLoss"] = &ONNXImporter2::parseNegativeLogLikelihoodLoss;
|
||||
dispatch["SoftmaxCrossEntropyLoss"] = &ONNXImporter2::parseSoftmaxCrossEntropyLoss;
|
||||
// @TODO@ONNX: Add support for SDPA
|
||||
|
||||
dispatch["Equal"] = dispatch["Greater"] = dispatch["Less"] = dispatch["Pow"] = dispatch["Add"] =
|
||||
dispatch["Sub"] = dispatch["Mul"] = dispatch["Div"] = dispatch["GreaterOrEqual"] =
|
||||
@@ -2777,7 +2799,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
|
||||
// com.microsft: This operator is added for compatibility via onnx graph simplifier.
|
||||
// Opset domain cannot be modified from onnx_graph_simplifier.cpp so this
|
||||
// operator cannot be parsed if only added in buildDispatchMap_COM_MICROSOFT
|
||||
dispatch["Attention"] = &ONNXImporter2::parseAttention;
|
||||
dispatch["Attention"] = &ONNXImporter2::parseAttentionOnnxAi;
|
||||
|
||||
domain_dispatch_map[str_domain_ai_onnx] = dispatch;
|
||||
}
|
||||
|
||||
@@ -102,3 +102,41 @@
|
||||
"test_quantizelinear", // Issue https://github.com/opencv/opencv/issues/25999
|
||||
"test_quantizelinear_axis", // Issue https://github.com/opencv/opencv/issues/25999
|
||||
"test_quantizelinear_blocked", // Issue https://github.com/opencv/opencv/issues/25999
|
||||
"test_attention_3d_attn_mask",
|
||||
"test_attention_3d_causal",
|
||||
"test_attention_3d_diff_heads_sizes",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_3d_diff_heads_sizes_causal",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_scaled",
|
||||
"test_attention_3d_gqa",
|
||||
"test_attention_3d_gqa_attn_mask",
|
||||
"test_attention_3d_gqa_causal",
|
||||
"test_attention_3d_gqa_scaled",
|
||||
"test_attention_3d_gqa_softcap",
|
||||
"test_attention_3d_scaled",
|
||||
"test_attention_3d_softcap",
|
||||
"test_attention_3d_transpose_verification",
|
||||
"test_attention_4d",
|
||||
"test_attention_4d_attn_mask",
|
||||
"test_attention_4d_attn_mask_3d",
|
||||
"test_attention_4d_attn_mask_3d_causal",
|
||||
"test_attention_4d_attn_mask_4d",
|
||||
"test_attention_4d_attn_mask_4d_causal",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
"test_attention_4d_causal",
|
||||
"test_attention_4d_diff_heads_sizes",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_4d_diff_heads_sizes_causal",
|
||||
"test_attention_4d_diff_heads_sizes_scaled",
|
||||
"test_attention_4d_diff_heads_sizes_softcap",
|
||||
"test_attention_4d_gqa",
|
||||
"test_attention_4d_gqa_attn_mask",
|
||||
"test_attention_4d_gqa_causal",
|
||||
"test_attention_4d_gqa_scaled",
|
||||
"test_attention_4d_gqa_softcap",
|
||||
"test_attention_4d_scaled",
|
||||
"test_attention_4d_softcap",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
@@ -38,3 +38,41 @@
|
||||
"test_max_float64",
|
||||
"test_min_float64",
|
||||
"test_mod_mixed_sign_float64",
|
||||
"test_attention_3d_attn_mask",
|
||||
"test_attention_3d_causal",
|
||||
"test_attention_3d_diff_heads_sizes",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_3d_diff_heads_sizes_causal",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_scaled",
|
||||
"test_attention_3d_gqa",
|
||||
"test_attention_3d_gqa_attn_mask",
|
||||
"test_attention_3d_gqa_causal",
|
||||
"test_attention_3d_gqa_scaled",
|
||||
"test_attention_3d_gqa_softcap",
|
||||
"test_attention_3d_scaled",
|
||||
"test_attention_3d_softcap",
|
||||
"test_attention_3d_transpose_verification",
|
||||
"test_attention_4d",
|
||||
"test_attention_4d_attn_mask",
|
||||
"test_attention_4d_attn_mask_3d",
|
||||
"test_attention_4d_attn_mask_3d_causal",
|
||||
"test_attention_4d_attn_mask_4d",
|
||||
"test_attention_4d_attn_mask_4d_causal",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
"test_attention_4d_causal",
|
||||
"test_attention_4d_diff_heads_sizes",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_4d_diff_heads_sizes_causal",
|
||||
"test_attention_4d_diff_heads_sizes_scaled",
|
||||
"test_attention_4d_diff_heads_sizes_softcap",
|
||||
"test_attention_4d_gqa",
|
||||
"test_attention_4d_gqa_attn_mask",
|
||||
"test_attention_4d_gqa_causal",
|
||||
"test_attention_4d_gqa_scaled",
|
||||
"test_attention_4d_gqa_softcap",
|
||||
"test_attention_4d_scaled",
|
||||
"test_attention_4d_softcap",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
@@ -2837,6 +2837,85 @@ CASE(test_xor_bcast4v3d)
|
||||
// no filter
|
||||
CASE(test_xor_bcast4v4d)
|
||||
// no filter
|
||||
CASE()
|
||||
CASE(test_attention_3d)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_transpose_verification)
|
||||
SKIP;
|
||||
CASE(test_attention_4d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_3d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_3d_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_4d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_4d_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool_4d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_attn_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_scaled)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_softcap)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool_4d)
|
||||
SKIP;
|
||||
END_SWITCH()
|
||||
#undef EOF_LABEL
|
||||
#undef BEGIN_SWITCH
|
||||
|
||||
@@ -95,3 +95,41 @@
|
||||
"test_scatternd_max",
|
||||
"test_scatternd_min",
|
||||
"test_scatternd_multiply",
|
||||
"test_attention_3d_attn_mask",
|
||||
"test_attention_3d_causal",
|
||||
"test_attention_3d_diff_heads_sizes",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_3d_diff_heads_sizes_causal",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_scaled",
|
||||
"test_attention_3d_gqa",
|
||||
"test_attention_3d_gqa_attn_mask",
|
||||
"test_attention_3d_gqa_causal",
|
||||
"test_attention_3d_gqa_scaled",
|
||||
"test_attention_3d_gqa_softcap",
|
||||
"test_attention_3d_scaled",
|
||||
"test_attention_3d_softcap",
|
||||
"test_attention_3d_transpose_verification",
|
||||
"test_attention_4d",
|
||||
"test_attention_4d_attn_mask",
|
||||
"test_attention_4d_attn_mask_3d",
|
||||
"test_attention_4d_attn_mask_3d_causal",
|
||||
"test_attention_4d_attn_mask_4d",
|
||||
"test_attention_4d_attn_mask_4d_causal",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
"test_attention_4d_causal",
|
||||
"test_attention_4d_diff_heads_sizes",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_4d_diff_heads_sizes_causal",
|
||||
"test_attention_4d_diff_heads_sizes_scaled",
|
||||
"test_attention_4d_diff_heads_sizes_softcap",
|
||||
"test_attention_4d_gqa",
|
||||
"test_attention_4d_gqa_attn_mask",
|
||||
"test_attention_4d_gqa_causal",
|
||||
"test_attention_4d_gqa_scaled",
|
||||
"test_attention_4d_gqa_softcap",
|
||||
"test_attention_4d_scaled",
|
||||
"test_attention_4d_softcap"
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
@@ -600,6 +600,45 @@
|
||||
"test_rotary_embedding_with_interleaved_rotary_dim",
|
||||
"test_rotary_embedding_with_interleaved_rotary_dim_expanded",
|
||||
"test_rotary_embedding_with_rotary_dim",
|
||||
"test_rotary_embedding_with_rotary_dim_expanded","test_attention_3d",
|
||||
"test_attention_3d_attn_mask",
|
||||
"test_attention_3d_causal",
|
||||
"test_attention_3d_diff_heads_sizes",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_3d_diff_heads_sizes_causal",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_scaled",
|
||||
"test_attention_3d_gqa",
|
||||
"test_attention_3d_gqa_attn_mask",
|
||||
"test_attention_3d_gqa_causal",
|
||||
"test_attention_3d_gqa_scaled",
|
||||
"test_attention_3d_gqa_softcap",
|
||||
"test_attention_3d_scaled",
|
||||
"test_attention_3d_softcap",
|
||||
"test_attention_3d_transpose_verification",
|
||||
"test_attention_4d",
|
||||
"test_attention_4d_attn_mask",
|
||||
"test_attention_4d_attn_mask_3d",
|
||||
"test_attention_4d_attn_mask_3d_causal",
|
||||
"test_attention_4d_attn_mask_4d",
|
||||
"test_attention_4d_attn_mask_4d_causal",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
"test_attention_4d_causal",
|
||||
"test_attention_4d_diff_heads_sizes",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_4d_diff_heads_sizes_causal",
|
||||
"test_attention_4d_diff_heads_sizes_scaled",
|
||||
"test_attention_4d_diff_heads_sizes_softcap",
|
||||
"test_attention_4d_gqa",
|
||||
"test_attention_4d_gqa_attn_mask",
|
||||
"test_attention_4d_gqa_causal",
|
||||
"test_attention_4d_gqa_scaled",
|
||||
"test_attention_4d_gqa_softcap",
|
||||
"test_attention_4d_scaled",
|
||||
"test_attention_4d_softcap",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
"test_rotary_embedding_with_rotary_dim_expanded",
|
||||
"test_hammingwindow",
|
||||
"test_hammingwindow_expanded",
|
||||
|
||||
@@ -16,107 +16,81 @@
|
||||
"test_ai_onnx_ml_label_encoder_tensor_value_only_mapping",
|
||||
"test_ai_onnx_ml_tree_ensemble_set_membership",
|
||||
"test_ai_onnx_ml_tree_ensemble_single_tree",
|
||||
"test_attention_3d",
|
||||
"test_attention_3d_attn_mask",
|
||||
|
||||
// autoSize <= INT_MAX && autoSize*outTotal == inpTotal in function 'getOutShape' @ C++ exception with description
|
||||
//"OpenCV(5.0.0-pre) opencv/modules/dnn/src/layers/reshape2_layer.cpp:110
|
||||
// expanded graphs are not imported correctly
|
||||
"test_attention_3d_expanded",
|
||||
"test_attention_3d_attn_mask_expanded",
|
||||
"test_attention_3d_causal",
|
||||
"test_attention_3d_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_causal",
|
||||
"test_attention_3d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_3d_scaled_expanded",
|
||||
"test_attention_3d_gqa_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_scaled",
|
||||
"test_attention_3d_transpose_verification_expanded",
|
||||
"test_attention_3d_softcap_expanded",
|
||||
"test_attention_3d_gqa_scaled_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_3d_with_past_and_present_expanded",
|
||||
"test_attention_3d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_3d_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_3d_diff_heads_with_past_and_present",
|
||||
"test_attention_3d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_3d_expanded",
|
||||
"test_attention_3d_gqa",
|
||||
"test_attention_3d_gqa_attn_mask",
|
||||
"test_attention_3d_gqa_attn_mask_expanded",
|
||||
"test_attention_3d_gqa_causal",
|
||||
"test_attention_3d_gqa_causal_expanded",
|
||||
"test_attention_3d_gqa_expanded",
|
||||
"test_attention_3d_gqa_scaled",
|
||||
"test_attention_3d_gqa_scaled_expanded",
|
||||
"test_attention_3d_gqa_softcap",
|
||||
"test_attention_3d_gqa_softcap_expanded",
|
||||
"test_attention_3d_gqa_with_past_and_present",
|
||||
"test_attention_3d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_3d_scaled",
|
||||
"test_attention_3d_scaled_expanded",
|
||||
"test_attention_3d_softcap",
|
||||
"test_attention_3d_softcap_expanded",
|
||||
"test_attention_3d_transpose_verification",
|
||||
"test_attention_3d_transpose_verification_expanded",
|
||||
"test_attention_3d_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded",
|
||||
"test_attention_4d",
|
||||
"test_attention_4d_attn_mask",
|
||||
"test_attention_4d_attn_mask_3d",
|
||||
"test_attention_4d_attn_mask_3d_causal",
|
||||
"test_attention_4d_attn_mask_3d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_3d_expanded",
|
||||
"test_attention_4d_attn_mask_4d",
|
||||
"test_attention_4d_attn_mask_4d_causal",
|
||||
"test_attention_4d_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv_expanded",
|
||||
"test_attention_4d_attn_mask_4d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_4d_expanded",
|
||||
"test_attention_4d_attn_mask_bool",
|
||||
"test_attention_4d_attn_mask_bool_4d",
|
||||
|
||||
"test_attention_4d_attn_mask_bool_4d_expanded",
|
||||
"test_attention_4d_attn_mask_bool_expanded",
|
||||
"test_attention_4d_attn_mask_expanded",
|
||||
"test_attention_4d_causal",
|
||||
"test_attention_4d_causal_expanded",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv_expanded",
|
||||
"test_attention_4d_diff_heads_sizes",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_causal",
|
||||
"test_attention_4d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_scaled",
|
||||
"test_attention_4d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_softcap",
|
||||
"test_attention_4d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_4d_expanded",
|
||||
|
||||
"test_attention_3d_gqa_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv",
|
||||
"test_attention_4d_diff_heads_with_past_and_present",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d_expanded",
|
||||
"test_attention_4d_expanded",
|
||||
"test_attention_4d_fp16",
|
||||
"test_attention_4d_fp16_expanded",
|
||||
"test_attention_4d_gqa",
|
||||
"test_attention_4d_gqa_attn_mask",
|
||||
"test_attention_4d_gqa_attn_mask_expanded",
|
||||
"test_attention_4d_gqa_causal",
|
||||
"test_attention_4d_gqa_causal_expanded",
|
||||
"test_attention_4d_gqa_expanded",
|
||||
"test_attention_4d_gqa_scaled",
|
||||
"test_attention_4d_gqa_scaled_expanded",
|
||||
"test_attention_4d_gqa_softcap",
|
||||
"test_attention_4d_gqa_softcap_expanded",
|
||||
|
||||
// fixme
|
||||
"test_attention_4d_fp16",
|
||||
|
||||
"test_attention_4d_gqa_with_past_and_present",
|
||||
"test_attention_4d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16_expanded",
|
||||
"test_attention_4d_scaled",
|
||||
|
||||
"test_attention_4d_scaled_expanded",
|
||||
"test_attention_4d_softcap",
|
||||
"test_attention_4d_softcap_expanded",
|
||||
"test_attention_4d_with_past_and_present",
|
||||
"test_attention_4d_with_past_and_present_expanded",
|
||||
|
||||
Reference in New Issue
Block a user