mirror of
https://github.com/opencv/opencv.git
synced 2026-07-31 00:03:03 +04:00
Merge pull request #28840 from nklskyoy:key-value-cache
FP32 KV Cache #28840 OpenCV Extra: https://github.com/opencv/opencv_extra/pull/1348 ## This PR introduces basic (Paged ) KV-Cache to use on CPU ### Summary: 1. To ensure proper gemm-prepacking, 1.1. The Page Size of Key Cache is currently hardcoded as `FAST_GEMM_F32_NR`(which is 8, 12 or 16 depending on CPU architecture) 1.2. The Page Size of Values Cache is hardcoded as `FAST_GEMM_F32_PACKED_STRIDE_K` 2. there are two phases supported - prefill & generate. 2.1. prefill grows cache by `N` tokens and is allowed **only** for empty cache 2.2. generate grows cache by 1 token. 2.3. **Improtant**: it is currently not allowed to grow non-empty cache by more than one token at a time (thisbehaviour is sufficient for normal LLM querying, but should be extended if we want to implement speculative decoding) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
@@ -1876,6 +1876,8 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
class CV_EXPORTS AttentionOnnxAiLayer : public Layer {
|
||||
public:
|
||||
int kv_num_heads;
|
||||
|
||||
static Ptr<AttentionOnnxAiLayer> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
|
||||
@@ -1027,6 +1027,14 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
*/
|
||||
CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
|
||||
|
||||
/** @brief Enables KV-Cache for all AttentionOnnxI layers */
|
||||
CV_WRAP void enableKVCache();
|
||||
|
||||
/** @brief Disables KV-Cache for all AttentionOnnxI layers */
|
||||
CV_WRAP void disableKVCache();
|
||||
|
||||
/** @brief Resets KV-Cache for all AttentionOnnxI layers */
|
||||
CV_WRAP void resetKVCache();
|
||||
/** @brief Returns profiling data captured during the last forward pass.
|
||||
*
|
||||
* Entries are sorted by time in descending order. Empty vectors are returned
|
||||
@@ -1065,6 +1073,7 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
bool comma=true, bool dump_details=false) const;
|
||||
std::ostream& dumpDim(std::ostream& strm, int value) const;
|
||||
|
||||
|
||||
struct Impl;
|
||||
inline Impl* getImpl() const { return impl.get(); }
|
||||
inline Impl& getImplRef() const { CV_DbgAssert(impl); return *impl.get(); }
|
||||
|
||||
Reference in New Issue
Block a user