potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 15:23:05 +04:00
Code Packages Releases Wiki Activity

Merge pull request #28840 from nklskyoy:key-value-cache

Browse Source 
FP32 KV Cache #28840

OpenCV Extra: https://github.com/opencv/opencv_extra/pull/1348

## This PR introduces basic (Paged ) KV-Cache to use on CPU

### Summary:
1. To ensure proper gemm-prepacking,
1.1. The Page Size of Key Cache is currently hardcoded as `FAST_GEMM_F32_NR`(which is 8, 12 or 16 depending on CPU architecture)  
1.2. The Page Size of Values Cache is hardcoded as `FAST_GEMM_F32_PACKED_STRIDE_K`
2. there are two phases supported - prefill & generate. 
2.1. prefill grows cache by `N` tokens and is allowed **only** for empty cache
2.2. generate grows cache by 1 token. 
2.3. **Improtant**: it is currently not allowed to grow non-empty cache by more than one token at a time (thisbehaviour is sufficient for normal LLM querying, but should be extended if we want to implement speculative decoding)

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
nklskyoy
2026-05-25 12:39:02 +02:00
committed by GitHub
parent 07e0dd2fbd
commit c2594b41bf
12 changed files with 855 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/test/test_layers.cpp
+129 -1
View File
@@ -3005,6 +3005,134 @@ TEST(ConvolutionWinograd, Accuracy)
normAssert(outLarge, refLarge, "Large input after small", 0.0, 0.0);
}
class TESTKVCache : public testing::TestWithParam<std::string>
{
public:
void testKVCache(const std::string& layout)
{
auto engine_forced = static_cast<cv::dnn::EngineType>(
cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO));
if (engine_forced == cv::dnn::ENGINE_CLASSIC)
{
// Mark the test as skipped and exit early.
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER);
return;
}
std::string model_path = "dnn/onnx/models/test_attention_kv_cache_" + layout + ".onnx";
Net netWithKVCache = readNetFromONNX(findDataFile(model_path, true), cv::dnn::ENGINE_NEW);
netWithKVCache.enableKVCache();
Net netWithoutKVCache = readNetFromONNX(findDataFile(model_path, true), cv::dnn::ENGINE_NEW);
int T = 523, Nq = 8, Nkv = 4, D = 256;
int T_pref = T;
std::vector<int> q_sz, k_sz, v_sz;
if (layout == "3d") {
q_sz = {1, T, Nq * D};
k_sz = {1, T, Nkv * D};
v_sz = {1, T, Nkv * D};
} else {
q_sz = {1, Nq, T, D};
k_sz = {1, Nkv, T, D};
v_sz = {1, Nkv, T, D};
}
Mat Q_all(q_sz, CV_32F);
Mat K_all(k_sz, CV_32F);
Mat V_all(v_sz, CV_32F);
cv::randn(Q_all, 0.0, 1.0);
cv::randn(K_all, 0.0, 1.0);
cv::randn(V_all, 0.0, 1.0);
std::vector<int> mask_sz = {1, Nq, T, T};
Mat mask(mask_sz, CV_32S, cv::Scalar(0));
int* mask_ptr = (int*)mask.data;
for (int n = 0; n < Nq; n++) {
for (int i = 0; i < T; i++) {
for (int j = 0; j < T; j++) {
int idx = n * T * T +
i * T + j;
if (i < T_pref) {
if (j < T_pref) mask_ptr[idx] = 1;
} else {
if (j <= i) mask_ptr[idx] = 1;
}
}
}
}
Mat Y;
if (layout == "3d") {
std::vector<int> sz = {1, T, Nq * D};
Y = Mat(sz, CV_32F);
} else {
std::vector<int> sz = {1, Nq, T, D};
Y = Mat(sz, CV_32F);
}
Y.setTo(0);
std::vector<Range> ranges_pref;
if (layout == "3d") {
ranges_pref = {Range::all(), Range(0, T_pref), Range::all()};
} else {
ranges_pref = {Range::all(), Range::all(), Range(0, T_pref), Range::all()};
}
Mat Q_pref = Q_all(ranges_pref);
Mat K_pref = K_all(ranges_pref);
Mat V_pref = V_all(ranges_pref);
// 1. Prefill
netWithKVCache.setInput(Q_pref, "Q");
netWithKVCache.setInput(K_pref, "K");
netWithKVCache.setInput(V_pref, "V");
Mat prefillResult = netWithKVCache.forward(); // prefill
prefillResult.copyTo(Y(ranges_pref));
// 2. Generate
for(int t = T_pref; t < T; t++)
{
std::vector<Range> ranges_gen;
if (layout == "3d") {
ranges_gen = {Range::all(), Range(t, t + 1), Range::all()};
} else {
ranges_gen = {Range::all(), Range::all(), Range(t, t + 1), Range::all()};
}
netWithKVCache.setInput(Q_all(ranges_gen), "Q");
netWithKVCache.setInput(K_all(ranges_gen), "K");
netWithKVCache.setInput(V_all(ranges_gen), "V");
Mat nextToken = netWithKVCache.forward();
nextToken.copyTo(Y(ranges_gen));
}
// 3. Standard path
netWithoutKVCache.setInput(Q_all, "Q");
netWithoutKVCache.setInput(K_all, "K");
netWithoutKVCache.setInput(V_all, "V");
netWithoutKVCache.setInput(mask, "Mask");
Mat Yref = netWithoutKVCache.forward();
std::string msg = "Attention generate " + layout + ": KV vs standard";
normAssert(Y, Yref, msg.c_str(), 1e-3, 1e-3);
}
};
TEST_P(TESTKVCache, layouts)
{
testKVCache(GetParam());
}
INSTANTIATE_TEST_CASE_P(KV_Cache, TESTKVCache, testing::Values("3d", "4d"));
TEST(Layer_Test_GeluApprox, NoNaN_LargeInput)
{
LayerParams lp;
@@ -3060,4 +3188,4 @@ TEST(Layer_Test_Softmax, NoNaN_AllNegInf)
}
}
}} // namespace
}} // namespace