mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #29333 from abhishek-gola:attention_layer_extension
Extended Attention layer support #29333 Implemented present/past KV support. Merge with: https://github.com/opencv/opencv_extra/pull/1381 Co-authored by: @Akansha-977 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -32,6 +32,14 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
return backendId == DNN_BACKEND_OPENCV;
|
||||
}
|
||||
|
||||
// ONNX Attention optional-input layout (Q, K, V are always inputs 0..2):
|
||||
// inputs.size() == 4 : Q, K, V, attn_mask
|
||||
// inputs.size() == 5 : Q, K, V, past_key, past_value (no mask)
|
||||
// inputs.size() == 6 : Q, K, V, attn_mask, past_key, past_value
|
||||
static bool hasMaskInput(size_t n_inputs) { return n_inputs == 4 || n_inputs == 6; }
|
||||
static int pastKeyIdx(size_t n_inputs) { return n_inputs == 5 ? 3 : n_inputs == 6 ? 4 : -1; }
|
||||
static int pastValueIdx(size_t n_inputs) { return n_inputs == 5 ? 4 : n_inputs == 6 ? 5 : -1; }
|
||||
|
||||
virtual void getTypes(const std::vector<MatType>&inputs,
|
||||
const int requiredOutputs,
|
||||
const int requiredInternals,
|
||||
@@ -46,7 +54,7 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
|
||||
CV_CheckType(inputs[0], inputs[0] == inputs[1] && inputs[0] == inputs[2], "");
|
||||
|
||||
if (inputs.size() >= 4) {
|
||||
if (hasMaskInput(inputs.size())) {
|
||||
CV_CheckType(inputs[3], inputs[3] == CV_8U || inputs[3] == CV_8S ||
|
||||
inputs[3] == CV_16U || inputs[3] == CV_16S ||
|
||||
inputs[3] == CV_32S || inputs[3] == CV_64S ||
|
||||
@@ -54,6 +62,12 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
inputs[3] == inputs[2], ""); // attention_mask
|
||||
}
|
||||
|
||||
const int past_k_idx = pastKeyIdx(inputs.size());
|
||||
if (past_k_idx >= 0) {
|
||||
CV_CheckType(inputs[past_k_idx], inputs[past_k_idx] == inputs[0], ""); // past_key
|
||||
CV_CheckType(inputs[past_k_idx+1], inputs[past_k_idx+1] == inputs[0], ""); // past_value
|
||||
}
|
||||
|
||||
outputs.assign(requiredOutputs, inputs[0]);
|
||||
|
||||
// internals:
|
||||
@@ -68,7 +82,6 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
std::vector<MatShape> &outputs,
|
||||
std::vector<MatShape> &internals) const CV_OVERRIDE {
|
||||
CV_CheckTrue(inputs.size() >= 3, "At least three inputs (query, key, value) are required");
|
||||
CV_CheckTrue(inputs.size() < 5, "past key and past value are not supported yet");
|
||||
CV_CheckTrue(inputs[0].dims == inputs[1].dims &&
|
||||
inputs[0].dims == inputs[2].dims,
|
||||
"Query, key and value must have the same number of dimensions");
|
||||
@@ -79,11 +92,13 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
"Input dimensions must be 4D or 3D (in the latter case, q_num_heads and kv_num_heads must be set)"
|
||||
);
|
||||
|
||||
const int seq_dim = input_dims - 2;
|
||||
const int batch_size = inputs[0][0];
|
||||
const int seq_len_q = inputs[0][input_dims - 2];
|
||||
int seq_len_k = inputs[1][input_dims - 2];
|
||||
int seq_len_v = inputs[2][input_dims - 2];
|
||||
const int seq_len_q = inputs[0][seq_dim];
|
||||
int seq_len_k = inputs[1][seq_dim];
|
||||
int seq_len_v = inputs[2][seq_dim];
|
||||
|
||||
int past_seq_kv = 0;
|
||||
Net::Impl* netimpl = getNetImpl(const_cast<AttentionOnnxAiLayerImpl*>(this));
|
||||
if (netimpl && netimpl->useKVCache) {
|
||||
KVCacheManager& kvCacheManager = netimpl->kvCacheManager;
|
||||
@@ -98,6 +113,25 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
if (it_v != kvCacheManager.vData.end())
|
||||
seq_len_v += it_v->second.getNumTokens();
|
||||
}
|
||||
} else {
|
||||
const int past_k_idx = pastKeyIdx(inputs.size());
|
||||
if (past_k_idx >= 0) {
|
||||
const int past_v_idx = pastValueIdx(inputs.size());
|
||||
CV_CheckTrue(past_v_idx >= 0 && past_v_idx < (int)inputs.size(),
|
||||
"past_key and past_value must be provided as a pair");
|
||||
|
||||
// past_key/past_value are always 4D [batch, nhkv, past_seq, head] (even for 3D Q/K/V).
|
||||
const MatShape& pk = inputs[past_k_idx];
|
||||
const MatShape& pv = inputs[past_v_idx];
|
||||
CV_CheckEQ(pk.dims, 4, "past_key must be 4D [batch, nhkv, past_seq, head]");
|
||||
CV_CheckEQ(pv.dims, 4, "past_value must be 4D [batch, nhkv, past_seq, head]");
|
||||
CV_CheckEQ(pk[0], batch_size, "past_key batch dimension must match query batch");
|
||||
CV_CheckEQ(pv[0], batch_size, "past_value batch dimension must match query batch");
|
||||
|
||||
past_seq_kv = pk[pk.dims - 2];
|
||||
seq_len_k += past_seq_kv;
|
||||
seq_len_v += past_seq_kv;
|
||||
}
|
||||
}
|
||||
|
||||
const int q_hn = input_dims == 4 ?
|
||||
@@ -108,6 +142,7 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
CV_CheckTrue(seq_len_v == seq_len_k,
|
||||
"Key and value sequence lengths must be equal");
|
||||
const int nhq = input_dims == 4 ? inputs[0][1] : q_num_heads;
|
||||
const int total_seq_kv = seq_len_k;
|
||||
|
||||
CV_CheckTrue(q_hn % kv_hn == 0,
|
||||
"q_num_heads must be divisible by kv_num_heads");
|
||||
@@ -130,7 +165,16 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
outputs.push_back(output_shape);
|
||||
}
|
||||
|
||||
MatShape attention_prob_shape{batch_size , nhq, seq_len_q, seq_len_k};
|
||||
const int qk_head_size = input_dims == 4 ? inputs[1][3] : inputs[1][2] / kv_hn;
|
||||
const int v_head_size = input_dims == 4 ? inputs[2][3] : inputs[2][2] / kv_hn;
|
||||
if (requiredOutputs > 1)
|
||||
outputs.push_back(MatShape{batch_size, kv_hn, total_seq_kv, qk_head_size}); // present_key
|
||||
if (requiredOutputs > 2)
|
||||
outputs.push_back(MatShape{batch_size, kv_hn, total_seq_kv, v_head_size}); // present_value
|
||||
if (requiredOutputs > 3)
|
||||
outputs.push_back(MatShape{batch_size, nhq, seq_len_q, total_seq_kv}); // qk_matmul_output
|
||||
|
||||
MatShape attention_prob_shape{batch_size , nhq, seq_len_q, total_seq_kv};
|
||||
internals.push_back(attention_prob_shape);
|
||||
|
||||
return false;
|
||||
@@ -180,16 +224,12 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
Mat &attention_prob = internals[internals.size() - 1];
|
||||
|
||||
const int input_dims = inputs[0].dims;
|
||||
const int seq_dim = input_dims - 2;
|
||||
|
||||
const int batch_size = inputs[0].size[0];
|
||||
|
||||
const int seq_len_q = input_dims == 3 ?
|
||||
inputs[0].size[1]:
|
||||
inputs[0].size[2];
|
||||
|
||||
int seq_len_kv = input_dims == 3 ?
|
||||
inputs[1].size[1]:
|
||||
inputs[1].size[2];
|
||||
const int seq_len_q = inputs[0].size[seq_dim];
|
||||
int seq_len_kv = inputs[1].size[seq_dim];
|
||||
|
||||
const int nhq = input_dims == 3 ?
|
||||
q_num_heads :
|
||||
@@ -207,9 +247,16 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
inputs[2].size[2] / nhkv :
|
||||
inputs[2].size[3];
|
||||
|
||||
const bool has_mask_input = hasMaskInput(inputs.size());
|
||||
const Mat mask_mat = has_mask_input ? inputs[3] : Mat();
|
||||
|
||||
scale = is_scale_set ? scale : 1.0f / std::sqrt(static_cast<float>(qk_head_size));
|
||||
|
||||
std::vector<size_t> _q_offsets, _k_offsets, _v_offsets,
|
||||
_a_offsets, _o_offsets;
|
||||
|
||||
int past_seq_kv = 0;
|
||||
|
||||
if (with_kv_cache){
|
||||
KVCacheManager& kvCacheManager = netimpl->kvCacheManager;
|
||||
|
||||
@@ -222,77 +269,21 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
const std::vector<Mat>& kCachePages = kData.getPages();
|
||||
seq_len_kv = kData.getNumTokens();
|
||||
|
||||
scale = is_scale_set ? scale : 1.0f / std::sqrt(static_cast<float>(qk_head_size));
|
||||
|
||||
pagedAttnQKGemm(
|
||||
inputs[0], kCachePages, attention_prob,
|
||||
seq_len_q, nhq, nhkv, kData.getPageSize(),
|
||||
qk_head_size, seq_len_kv,
|
||||
scale, opt
|
||||
);
|
||||
} else {
|
||||
const auto* Q = inputs[0].ptr<const float>();
|
||||
const auto* K = inputs[1].ptr<const float>();
|
||||
|
||||
const int num_gq_groups = nhq / nhkv;
|
||||
past_seq_kv = seq_len_kv - seq_len_q;
|
||||
|
||||
const auto seq_len_square = seq_len_q * seq_len_kv;
|
||||
|
||||
scale = is_scale_set ? scale : 1.0f / std::sqrt(static_cast<float>(qk_head_size));
|
||||
|
||||
_q_offsets.resize(nhq * batch_size);
|
||||
_k_offsets.resize(nhq * batch_size);
|
||||
_a_offsets.resize(nhq * batch_size);
|
||||
_v_offsets.resize(nhq * batch_size);
|
||||
_o_offsets.resize(nhq * batch_size);
|
||||
|
||||
for (int b = 0; b < batch_size; b++)
|
||||
for (int n = 0; n < nhq; n++){
|
||||
_q_offsets[b * nhq + n] =
|
||||
b * seq_len_q * qk_head_size * nhq +
|
||||
(input_dims == 3 ? n * qk_head_size : n * qk_head_size * seq_len_q);
|
||||
_k_offsets[b * nhq + n] =
|
||||
b * seq_len_kv * qk_head_size * nhkv +
|
||||
(n / num_gq_groups * qk_head_size) * (input_dims == 3 ? 1 : seq_len_kv);
|
||||
_v_offsets[b * nhq + n] =
|
||||
b * seq_len_kv * v_head_size * nhkv +
|
||||
(n / num_gq_groups * v_head_size) * (input_dims == 3 ? 1 : seq_len_kv);
|
||||
_a_offsets[b * nhq + n] =
|
||||
b * seq_len_square * nhq +
|
||||
n * seq_len_square;
|
||||
_o_offsets[b * nhq + n] =
|
||||
b * seq_len_q * v_head_size * nhq +
|
||||
(input_dims == 3 ? n * v_head_size : n * v_head_size * seq_len_q);
|
||||
}
|
||||
|
||||
const int ldq0 = input_dims == 3 ? qk_head_size * nhq : qk_head_size;
|
||||
const int ldk0 = input_dims == 3 ? qk_head_size * nhkv : qk_head_size;
|
||||
auto &attention_prob = internals[internals.size() - 1];
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_q_offsets.data(), _k_offsets.data(), _a_offsets.data(),
|
||||
seq_len_q, seq_len_kv, qk_head_size , scale,
|
||||
Q, ldq0, 1,
|
||||
K, 1, ldk0,
|
||||
0.f,
|
||||
attention_prob.ptr<float>(), seq_len_kv,
|
||||
opt
|
||||
fused_softmax_softcap_mask(
|
||||
attention_prob, mask_mat,
|
||||
softcap, softcap > 0.f, 9.f, -FLT_MAX,
|
||||
has_mask_input, is_causal, past_seq_kv
|
||||
);
|
||||
}
|
||||
|
||||
fused_softmax_softcap_mask(
|
||||
attention_prob,
|
||||
inputs.size() > 3 ? inputs[3] : Mat(),
|
||||
softcap, softcap > 0.f,
|
||||
9.f,
|
||||
-FLT_MAX,
|
||||
inputs.size() > 3,
|
||||
is_causal
|
||||
);
|
||||
|
||||
if (with_kv_cache){
|
||||
KVCacheManager& kvCacheManager = netimpl->kvCacheManager;
|
||||
auto it_v = kvCacheManager.vData.find(name);
|
||||
CV_Assert(it_v != kvCacheManager.vData.end());
|
||||
VCache& vData = it_v->second;
|
||||
@@ -305,23 +296,146 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
|
||||
seq_len_q, nhq, nhkv, vData.getPageSize() , v_head_size, seq_len_kv,
|
||||
opt
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Standard (non-paged) path, with optional past_key/past_value graph inputs
|
||||
const int past_k_idx = pastKeyIdx(inputs.size());
|
||||
const int past_v_idx = pastValueIdx(inputs.size());
|
||||
const bool use_past = past_k_idx >= 0;
|
||||
// past_key/past_value are always 4D [batch, nhkv, past_seq, head]; use their own rank.
|
||||
past_seq_kv = use_past ? inputs[past_k_idx].size[inputs[past_k_idx].dims - 2] : 0;
|
||||
const int total_seq_kv = past_seq_kv + seq_len_kv;
|
||||
|
||||
Mat K_eff, V_eff;
|
||||
if (use_past && past_seq_kv > 0) {
|
||||
if (input_dims == 4) {
|
||||
// 4D [batch, nhkv, seq, head]: concat on axis 2.
|
||||
int dims_k[4] = {batch_size, nhkv, total_seq_kv, qk_head_size};
|
||||
int dims_v[4] = {batch_size, nhkv, total_seq_kv, v_head_size};
|
||||
K_eff.create(4, dims_k, CV_32F);
|
||||
V_eff.create(4, dims_v, CV_32F);
|
||||
const std::vector<Range> past_r{Range::all(), Range::all(), Range(0, past_seq_kv), Range::all()};
|
||||
const std::vector<Range> cur_r {Range::all(), Range::all(), Range(past_seq_kv, total_seq_kv), Range::all()};
|
||||
inputs[past_k_idx].copyTo(K_eff(past_r)); inputs[1].copyTo(K_eff(cur_r));
|
||||
inputs[past_v_idx].copyTo(V_eff(past_r)); inputs[2].copyTo(V_eff(cur_r));
|
||||
} else {
|
||||
const int k_elem = nhkv * qk_head_size;
|
||||
const int v_elem = nhkv * v_head_size;
|
||||
int dims_k[3] = {batch_size, total_seq_kv, k_elem};
|
||||
int dims_v[3] = {batch_size, total_seq_kv, v_elem};
|
||||
int pk_sz[3] = {batch_size, past_seq_kv, k_elem};
|
||||
int pv_sz[3] = {batch_size, past_seq_kv, v_elem};
|
||||
K_eff.create(3, dims_k, CV_32F);
|
||||
V_eff.create(3, dims_v, CV_32F);
|
||||
const std::vector<Range> past_r{Range::all(), Range(0, past_seq_kv), Range::all()};
|
||||
const std::vector<Range> cur_r {Range::all(), Range(past_seq_kv, total_seq_kv), Range::all()};
|
||||
Mat pk, pv;
|
||||
cv::transposeND(inputs[past_k_idx], {0, 2, 1, 3}, pk);
|
||||
cv::transposeND(inputs[past_v_idx], {0, 2, 1, 3}, pv);
|
||||
pk.reshape(1, 3, pk_sz).copyTo(K_eff(past_r)); inputs[1].copyTo(K_eff(cur_r));
|
||||
pv.reshape(1, 3, pv_sz).copyTo(V_eff(past_r)); inputs[2].copyTo(V_eff(cur_r));
|
||||
}
|
||||
} else {
|
||||
const auto* V = inputs[2].ptr<const float>();
|
||||
K_eff = inputs[1];
|
||||
V_eff = inputs[2];
|
||||
}
|
||||
|
||||
const int ldv0 = input_dims == 3 ? v_head_size * nhkv : v_head_size;
|
||||
const int ldout = input_dims == 3 ? v_head_size * nhq : v_head_size;
|
||||
const auto* Q = inputs[0].ptr<const float>();
|
||||
const auto* K = K_eff.ptr<const float>();
|
||||
const auto* V = V_eff.ptr<const float>();
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_a_offsets.data(), _v_offsets.data(), _o_offsets.data(),
|
||||
seq_len_q, v_head_size, seq_len_kv, 1.f,
|
||||
attention_prob.ptr<float>(), seq_len_kv, 1,
|
||||
V, ldv0, 1,
|
||||
0.f,
|
||||
outputs[0].ptr<float>(), ldout,
|
||||
opt
|
||||
const int num_gq_groups = nhq / nhkv;
|
||||
const auto seq_len_square = seq_len_q * total_seq_kv;
|
||||
|
||||
_q_offsets.resize(nhq * batch_size);
|
||||
_k_offsets.resize(nhq * batch_size);
|
||||
_a_offsets.resize(nhq * batch_size);
|
||||
_v_offsets.resize(nhq * batch_size);
|
||||
_o_offsets.resize(nhq * batch_size);
|
||||
|
||||
for (int b = 0; b < batch_size; b++)
|
||||
for (int n = 0; n < nhq; n++){
|
||||
_q_offsets[b * nhq + n] =
|
||||
b * seq_len_q * qk_head_size * nhq +
|
||||
(input_dims == 3 ? n * qk_head_size : n * qk_head_size * seq_len_q);
|
||||
_k_offsets[b * nhq + n] =
|
||||
b * total_seq_kv * qk_head_size * nhkv +
|
||||
(n / num_gq_groups * qk_head_size) * (input_dims == 3 ? 1 : total_seq_kv);
|
||||
_v_offsets[b * nhq + n] =
|
||||
b * total_seq_kv * v_head_size * nhkv +
|
||||
(n / num_gq_groups * v_head_size) * (input_dims == 3 ? 1 : total_seq_kv);
|
||||
_a_offsets[b * nhq + n] =
|
||||
b * seq_len_square * nhq +
|
||||
n * seq_len_square;
|
||||
_o_offsets[b * nhq + n] =
|
||||
b * seq_len_q * v_head_size * nhq +
|
||||
(input_dims == 3 ? n * v_head_size : n * v_head_size * seq_len_q);
|
||||
}
|
||||
|
||||
const int ldq0 = input_dims == 3 ? qk_head_size * nhq : qk_head_size;
|
||||
const int ldk0 = input_dims == 3 ? qk_head_size * nhkv : qk_head_size;
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_q_offsets.data(), _k_offsets.data(), _a_offsets.data(),
|
||||
seq_len_q, total_seq_kv, qk_head_size , scale,
|
||||
Q, ldq0, 1,
|
||||
K, 1, ldk0,
|
||||
0.f,
|
||||
attention_prob.ptr<float>(), total_seq_kv,
|
||||
opt
|
||||
);
|
||||
|
||||
// qk_matmul_output (optional 4th output), per qk_matmul_output_mode:
|
||||
// 0 = raw scaled QK^T, 1 = + attention bias, 2 = + softcap, 3 = post-softmax.
|
||||
const bool want_qk = outputs.size() > 3 && !outputs[3].empty();
|
||||
if (want_qk && qk_matmul_output_mode == 0) {
|
||||
attention_prob.copyTo(outputs[3]);
|
||||
} else if (want_qk && (qk_matmul_output_mode == 1 || qk_matmul_output_mode == 2)) {
|
||||
attention_prob.copyTo(outputs[3]);
|
||||
fused_softmax_softcap_mask(
|
||||
outputs[3], mask_mat,
|
||||
softcap, (qk_matmul_output_mode == 2) && (softcap > 0.f), 9.f,
|
||||
-std::numeric_limits<float>::infinity(),
|
||||
has_mask_input, is_causal, past_seq_kv, /*do_softmax=*/false
|
||||
);
|
||||
}
|
||||
|
||||
fused_softmax_softcap_mask(
|
||||
attention_prob, mask_mat,
|
||||
softcap, softcap > 0.f, 9.f, -FLT_MAX,
|
||||
has_mask_input, is_causal, past_seq_kv
|
||||
);
|
||||
|
||||
if (want_qk && qk_matmul_output_mode == 3)
|
||||
attention_prob.copyTo(outputs[3]);
|
||||
|
||||
const int ldv0 = input_dims == 3 ? v_head_size * nhkv : v_head_size;
|
||||
const int ldout = input_dims == 3 ? v_head_size * nhq : v_head_size;
|
||||
|
||||
fastGemmBatch(
|
||||
batch_size * nhq,
|
||||
_a_offsets.data(), _v_offsets.data(), _o_offsets.data(),
|
||||
seq_len_q, v_head_size, total_seq_kv, 1.f,
|
||||
attention_prob.ptr<float>(), total_seq_kv, 1,
|
||||
V, ldv0, 1,
|
||||
0.f,
|
||||
outputs[0].ptr<float>(), ldout,
|
||||
opt
|
||||
);
|
||||
|
||||
auto writePresent = [&](Mat& out, const Mat& eff, int head_size) {
|
||||
if (out.empty()) return;
|
||||
if (input_dims == 4) {
|
||||
eff.reshape(1, out.dims, out.size.p).copyTo(out);
|
||||
} else {
|
||||
int sz[4] = {batch_size, total_seq_kv, nhkv, head_size};
|
||||
cv::transposeND(eff.reshape(1, 4, sz), {0, 2, 1, 3}, out);
|
||||
}
|
||||
};
|
||||
if (outputs.size() > 1) writePresent(outputs[1], K_eff, qk_head_size);
|
||||
if (outputs.size() > 2) writePresent(outputs[2], V_eff, v_head_size);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -63,7 +63,8 @@ void run_fused_softmax(
|
||||
Mat &att_weights, const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool is_causal)
|
||||
const float min_val, const bool is_causal, const int past_seq_len,
|
||||
const bool do_softmax)
|
||||
{
|
||||
const int batch_size = att_weights.size[0];
|
||||
const int n_heads = att_weights.size[1];
|
||||
@@ -98,7 +99,7 @@ void run_fused_softmax(
|
||||
size_t mask_offset_q = b * mask_step_b + h * mask_step_h;
|
||||
|
||||
for (int tq = 0; tq < seq_len_q; tq++){
|
||||
const int tmax = is_causal ? std::min(tq + 1, seq_len_kv) : seq_len_kv;
|
||||
const int tmax = is_causal ? std::min(past_seq_len + tq + 1, seq_len_kv) : seq_len_kv;
|
||||
float maxVal = -FLT_MAX;
|
||||
int tk = 0;
|
||||
#if CV_SIMD
|
||||
@@ -169,6 +170,7 @@ void run_fused_softmax(
|
||||
data[offset + tk] = min_val;
|
||||
}
|
||||
|
||||
if (do_softmax) {
|
||||
float sum = 0.f;
|
||||
tk = 0;
|
||||
#if CV_SIMD
|
||||
@@ -201,6 +203,7 @@ void run_fused_softmax(
|
||||
for (; tk < seq_len_kv; tk++){
|
||||
data[offset + tk] *= inv_sum;
|
||||
}
|
||||
} // do_softmax
|
||||
|
||||
offset += seq_len_kv;
|
||||
mask_offset_q += mask_step_q;
|
||||
@@ -213,44 +216,45 @@ void fused_softmax_softcap_mask(
|
||||
Mat &att_weights,const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool has_mask, const bool is_causal
|
||||
const float min_val, const bool has_mask, const bool is_causal,
|
||||
int past_seq_len, bool do_softmax
|
||||
){
|
||||
if (has_mask) {
|
||||
switch(att_mask.depth())
|
||||
{
|
||||
case CV_32F:
|
||||
run_fused_softmax<float, MaskPolicyFloat<float>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<float, MaskPolicyFloat<float>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_Bool:
|
||||
case CV_8U:
|
||||
run_fused_softmax<uint8_t, MaskPolicyInt<uint8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<uint8_t, MaskPolicyInt<uint8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_8S:
|
||||
run_fused_softmax<int8_t, MaskPolicyInt<int8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<int8_t, MaskPolicyInt<int8_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_16U:
|
||||
run_fused_softmax<uint16_t, MaskPolicyInt<uint16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<uint16_t, MaskPolicyInt<uint16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_16S:
|
||||
run_fused_softmax<int16_t, MaskPolicyInt<int16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<int16_t, MaskPolicyInt<int16_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_32U:
|
||||
run_fused_softmax<uint32_t, MaskPolicyInt<uint32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<uint32_t, MaskPolicyInt<uint32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_32S:
|
||||
run_fused_softmax<int32_t, MaskPolicyInt<int32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<int32_t, MaskPolicyInt<int32_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_64U:
|
||||
run_fused_softmax<uint64_t, MaskPolicyInt<uint64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<uint64_t, MaskPolicyInt<uint64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
case CV_64S:
|
||||
run_fused_softmax<int64_t, MaskPolicyInt<int64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<int64_t, MaskPolicyInt<int64_t>>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
break;
|
||||
default:
|
||||
CV_Error(Error::StsUnsupportedFormat, "Unsupported mask data type in fused_softmax_softcap_mask");
|
||||
}
|
||||
} else {
|
||||
run_fused_softmax<float, MaskPolicyNone>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal);
|
||||
run_fused_softmax<float, MaskPolicyNone>(att_weights, att_mask, softcap, do_softcap, threshold, min_val, is_causal, past_seq_len, do_softmax);
|
||||
}
|
||||
}
|
||||
}}
|
||||
|
||||
@@ -15,7 +15,8 @@ void fused_softmax_softcap_mask(
|
||||
Mat &att_weights,const Mat &att_mask,
|
||||
const float softcap, const bool do_softcap,
|
||||
const float threshold,
|
||||
const float min_val, const bool has_mask, const bool is_causal
|
||||
const float min_val, const bool has_mask, const bool is_causal,
|
||||
int past_seq_len = 0, bool do_softmax = true
|
||||
);
|
||||
|
||||
}}
|
||||
|
||||
@@ -25,7 +25,7 @@ CV_CPU_OPTIMIZATION_NAMESPACE_END
|
||||
namespace cv { namespace dnn {
|
||||
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
#if (CV_SIMD)
|
||||
// AVX2 8x8 f32 in-register transpose. v_transpose4x4 transposes the two
|
||||
// 128-bit halves independently; v_combine_low/high then exchange halves so
|
||||
// element k of each target row comes from the right source row.
|
||||
@@ -88,7 +88,7 @@ void transpose2D_f32_(const float* inp, float* out,
|
||||
const float* inB = inp + b * batchStride;
|
||||
float* outB = out + b * batchStride;
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
#if (CV_SIMD)
|
||||
if (VTraits<v_float32>::vlanes() == 8) {
|
||||
int64_t r = r0;
|
||||
for (; r + 8 <= r1; r += 8) {
|
||||
|
||||
@@ -1665,6 +1665,11 @@ void ONNXImporter2::parseCastLike(LayerParams& layerParams, const opencv_onnx::N
|
||||
void ONNXImporter2::parseConstantOfShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
|
||||
{
|
||||
layerParams.type = "ConstantOfShape";
|
||||
// ONNX spec: the 'value' attribute is optional; when it is absent the fill value
|
||||
// defaults to a one-element float32 tensor containing 0. The generic attribute
|
||||
// parser only populates blobs when 'value' is present, so supply the default here.
|
||||
if (layerParams.blobs.empty())
|
||||
layerParams.blobs.push_back(Mat(1, 1, CV_32F, Scalar(0)));
|
||||
addLayer(layerParams, node_proto);
|
||||
}
|
||||
|
||||
|
||||
@@ -1903,6 +1903,15 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
|
||||
if (name == "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded") {
|
||||
default_l1 = 2e-5; // Expected: (normL1) <= (l1), actual: 1.06394e-05 vs 1e-05
|
||||
}
|
||||
// fp16 Attention models retain fp16 accumulation precision (~9e-5 L1, ~2.4e-4 Inf)
|
||||
// even when executed on an fp32 target.
|
||||
if (name == "test_attention_4d_fp16" ||
|
||||
name == "test_attention_4d_fp16_expanded" ||
|
||||
name == "test_attention_4d_gqa_with_past_and_present_fp16" ||
|
||||
name == "test_attention_4d_gqa_with_past_and_present_fp16_expanded") {
|
||||
default_l1 = std::max(default_l1, 2e-4);
|
||||
default_lInf = std::max(default_lInf, 1e-3);
|
||||
}
|
||||
}
|
||||
#ifdef HAVE_HALIDE
|
||||
else if (backend == DNN_BACKEND_HALIDE)
|
||||
@@ -1965,6 +1974,15 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
|
||||
if (name == "test_roialign_aligned_false" || name == "test_roialign_aligned_true") {
|
||||
default_l1 = 3e-5;
|
||||
}
|
||||
// fp16 Attention models retain fp16 accumulation precision (~9e-5 L1, ~2.4e-4 Inf)
|
||||
// even when executed on an fp32 target (the layer falls back to the CPU path).
|
||||
if (name == "test_attention_4d_fp16" ||
|
||||
name == "test_attention_4d_fp16_expanded" ||
|
||||
name == "test_attention_4d_gqa_with_past_and_present_fp16" ||
|
||||
name == "test_attention_4d_gqa_with_past_and_present_fp16_expanded") {
|
||||
default_l1 = std::max(default_l1, 2e-4);
|
||||
default_lInf = std::max(default_lInf, 1e-3);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
else
|
||||
|
||||
@@ -3162,6 +3162,170 @@ CASE(test_attention_4d_attn_mask_bool)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool_4d)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_transpose_verification_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul_bias_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_sizes_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_diff_heads_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_3d_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_3d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_mask4d_padded_kv_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_4d_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_4d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool_4d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_bool_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_sizes_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_gqa_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul_bias)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present_mask3d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present_mask3d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present_mask4d)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_diff_heads_with_past_and_present_mask4d_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_fp16_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_attn_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_3d_with_past_and_present_qk_matmul_softmax)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_fp16)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_with_past_and_present_fp16)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_gqa_with_past_and_present_fp16_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_scaled_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_softcap_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_bias_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_past_and_present_qk_matmul_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul_bias)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul_bias_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul_expanded)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul_softmax)
|
||||
SKIP;
|
||||
CASE(test_attention_4d_with_qk_matmul_softmax_expanded)
|
||||
SKIP;
|
||||
END_SWITCH()
|
||||
#undef EOF_LABEL
|
||||
#undef BEGIN_SWITCH
|
||||
|
||||
+84
-1
@@ -832,4 +832,87 @@
|
||||
"test_dynamicquantizelinear_max_adjusted",
|
||||
"test_dynamicquantizelinear_max_adjusted_expanded",
|
||||
"test_dynamicquantizelinear_min_adjusted",
|
||||
"test_dynamicquantizelinear_min_adjusted_expanded"
|
||||
"test_dynamicquantizelinear_min_adjusted_expanded",
|
||||
"test_attention_3d_expanded",
|
||||
"test_attention_3d_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_3d_scaled_expanded",
|
||||
"test_attention_3d_gqa_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_expanded",
|
||||
"test_attention_3d_transpose_verification_expanded",
|
||||
"test_attention_3d_softcap_expanded",
|
||||
"test_attention_3d_gqa_scaled_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_3d_with_past_and_present_expanded",
|
||||
"test_attention_3d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_3d_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_3d_diff_heads_with_past_and_present",
|
||||
"test_attention_3d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_3d_gqa_causal_expanded",
|
||||
"test_attention_3d_gqa_expanded",
|
||||
"test_attention_3d_gqa_softcap_expanded",
|
||||
"test_attention_4d_attn_mask_3d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_3d_expanded",
|
||||
"test_attention_4d_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv_expanded",
|
||||
"test_attention_4d_attn_mask_4d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_4d_expanded",
|
||||
"test_attention_4d_attn_mask_bool_4d_expanded",
|
||||
"test_attention_4d_attn_mask_bool_expanded",
|
||||
"test_attention_4d_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_4d_expanded",
|
||||
"test_attention_3d_gqa_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_4d_diff_heads_with_past_and_present",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d_expanded",
|
||||
"test_attention_4d_fp16_expanded",
|
||||
"test_attention_4d_gqa_attn_mask_expanded",
|
||||
"test_attention_4d_gqa_causal_expanded",
|
||||
"test_attention_4d_gqa_expanded",
|
||||
"test_attention_4d_gqa_scaled_expanded",
|
||||
"test_attention_4d_gqa_softcap_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax",
|
||||
"test_attention_4d_fp16",
|
||||
"test_attention_4d_gqa_with_past_and_present",
|
||||
"test_attention_4d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16_expanded",
|
||||
"test_attention_4d_scaled_expanded",
|
||||
"test_attention_4d_softcap_expanded",
|
||||
"test_attention_4d_with_past_and_present",
|
||||
"test_attention_4d_with_past_and_present_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_4d_with_qk_matmul",
|
||||
"test_attention_4d_with_qk_matmul_bias",
|
||||
"test_attention_4d_with_qk_matmul_bias_expanded",
|
||||
"test_attention_4d_with_qk_matmul_expanded",
|
||||
"test_attention_4d_with_qk_matmul_softmax",
|
||||
"test_attention_4d_with_qk_matmul_softmax_expanded",
|
||||
|
||||
@@ -16,104 +16,18 @@
|
||||
"test_ai_onnx_ml_label_encoder_tensor_value_only_mapping",
|
||||
"test_ai_onnx_ml_tree_ensemble_set_membership",
|
||||
"test_ai_onnx_ml_tree_ensemble_single_tree",
|
||||
|
||||
// autoSize <= INT_MAX && autoSize*outTotal == inpTotal in function 'getOutShape' @ C++ exception with description
|
||||
//"OpenCV(5.0.0-pre) opencv/modules/dnn/src/layers/reshape2_layer.cpp:110
|
||||
// expanded graphs are not imported correctly
|
||||
"test_attention_3d_expanded",
|
||||
"test_attention_3d_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_3d_scaled_expanded",
|
||||
"test_attention_3d_gqa_attn_mask_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_expanded",
|
||||
"test_attention_3d_transpose_verification_expanded",
|
||||
"test_attention_3d_softcap_expanded",
|
||||
"test_attention_3d_gqa_scaled_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_3d_with_past_and_present_expanded",
|
||||
"test_attention_3d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_3d_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_3d_diff_heads_sizes_softcap",
|
||||
"test_attention_3d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_3d_diff_heads_with_past_and_present",
|
||||
"test_attention_3d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_3d_gqa_causal_expanded",
|
||||
"test_attention_3d_gqa_expanded",
|
||||
"test_attention_3d_gqa_softcap_expanded",
|
||||
"test_attention_4d_attn_mask_3d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_3d_expanded",
|
||||
"test_attention_4d_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv_expanded",
|
||||
"test_attention_4d_attn_mask_4d_causal_expanded",
|
||||
"test_attention_4d_attn_mask_4d_expanded",
|
||||
|
||||
"test_attention_4d_attn_mask_bool_4d_expanded",
|
||||
"test_attention_4d_attn_mask_bool_expanded",
|
||||
"test_attention_4d_attn_mask_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_causal_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_scaled_expanded",
|
||||
"test_attention_4d_diff_heads_sizes_softcap_expanded",
|
||||
"test_attention_4d_expanded",
|
||||
|
||||
"test_attention_3d_gqa_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softmax",
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv",
|
||||
"test_attention_4d_diff_heads_with_past_and_present",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask3d_expanded",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d",
|
||||
"test_attention_4d_diff_heads_with_past_and_present_mask4d_expanded",
|
||||
"test_attention_4d_fp16_expanded",
|
||||
"test_attention_4d_gqa_attn_mask_expanded",
|
||||
"test_attention_4d_gqa_causal_expanded",
|
||||
"test_attention_4d_gqa_expanded",
|
||||
"test_attention_4d_gqa_scaled_expanded",
|
||||
"test_attention_4d_gqa_softcap_expanded",
|
||||
|
||||
// fixme
|
||||
"test_attention_4d_fp16",
|
||||
|
||||
"test_attention_4d_gqa_with_past_and_present",
|
||||
"test_attention_4d_gqa_with_past_and_present_expanded",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16",
|
||||
"test_attention_4d_gqa_with_past_and_present_fp16_expanded",
|
||||
|
||||
"test_attention_4d_scaled_expanded",
|
||||
"test_attention_4d_softcap_expanded",
|
||||
"test_attention_4d_with_past_and_present",
|
||||
"test_attention_4d_with_past_and_present_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_bias_expanded",
|
||||
"test_attention_4d_with_past_and_present_qk_matmul_expanded",
|
||||
"test_attention_4d_with_qk_matmul",
|
||||
"test_attention_4d_with_qk_matmul_bias",
|
||||
"test_attention_4d_with_qk_matmul_bias_expanded",
|
||||
"test_attention_4d_with_qk_matmul_expanded",
|
||||
"test_attention_4d_with_qk_matmul_softcap",
|
||||
"test_attention_4d_with_qk_matmul_softcap_expanded",
|
||||
"test_attention_4d_with_qk_matmul_softmax",
|
||||
"test_attention_4d_with_qk_matmul_softmax_expanded",
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap_expanded", // softcap qk_matmul_output reference disagrees with onnxruntime; bad ref data
|
||||
"test_attention_3d_with_past_and_present_qk_matmul_softcap", // softcap qk_matmul_output reference disagrees with onnxruntime; bad ref data
|
||||
"test_attention_4d_diff_heads_mask4d_padded_kv", // needs the nonpad_kv_seqlen input (padded-KV variant) which is not supported yet
|
||||
"test_attention_4d_with_qk_matmul_softcap", // softcap qk_matmul_output reference disagrees with onnxruntime (~0.77); bad ref data
|
||||
"test_attention_4d_with_qk_matmul_softcap_expanded", // ditto (same reference data)
|
||||
"test_averagepool_2d_ceil_last_window_starts_on_pad",
|
||||
"test_averagepool_2d_dilations",
|
||||
"test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_False",
|
||||
"test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_True",
|
||||
"test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_False",
|
||||
"test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_True",
|
||||
"test_averagepool_3d_dilations_small",
|
||||
"test_basic_convinteger", // Issues::Layer::Can't create layer "onnx_node_output_0!y" of type "ConvInteger" in function 'getLayerInstance'
|
||||
"test_basic_deform_conv_with_padding",
|
||||
"test_basic_deform_conv_without_padding",
|
||||
|
||||
Reference in New Issue
Block a user