mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 15:23:05 +04:00
Merge pull request #29221 from abhishek-gola:oom_issue_fixed
Fixed Out-of-Memory issue and added VLM sample #29221 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -145,7 +145,7 @@ public:
|
||||
const bool big_enough = (int64_t)Cout * (int64_t)Cin >= (int64_t)(256 * 256);
|
||||
if (ksize_all_one && strides_all_one && big_enough) {
|
||||
size_t pack_bytes = mlasSgemmPackBSize(false, true, Cout, Cin);
|
||||
if (pack_bytes > 0) {
|
||||
if (pack_bytes > 0 && pack_bytes <= (size_t)INT_MAX) {
|
||||
mlas_packed_B_.create(1, (int)pack_bytes, CV_8U);
|
||||
// Weight is (Cout, Cin, 1, ..., 1) contiguous; reshape to
|
||||
// (Cout, Cin) and PackB with trans_b=true.
|
||||
|
||||
@@ -256,56 +256,67 @@ public:
|
||||
|
||||
// pack B if it is const
|
||||
if (constB(mode) && blobs[0].data != last_packed_blob_data) {
|
||||
fastGemmPackB(blobs[0], packed_B, trans_b, opt);
|
||||
|
||||
// Pre-pack B in the "thin" layout when the gemm shape has a
|
||||
// small leading dim (M <= FAST_GEMM_THIN_MAX_M).
|
||||
packed_B.clear();
|
||||
packed_B.shrink_to_fit();
|
||||
thin_packed_B.clear();
|
||||
if (!trans_a && blobs[0].type() == CV_32F) {
|
||||
thin_packed_B.shrink_to_fit();
|
||||
|
||||
bool mlas_packed = false;
|
||||
#ifdef HAVE_MLAS
|
||||
packed_B_mlas.release();
|
||||
packed_B_mlas_M = packed_B_mlas_N = packed_B_mlas_K = 0;
|
||||
if (mlasAvailable()) {
|
||||
std::vector<Mat> outputs;
|
||||
outputs_arr.getMatVector(outputs);
|
||||
if (!outputs.empty()) {
|
||||
const auto &Y = outputs[0];
|
||||
const auto shape_Y = shape(Y);
|
||||
const int N = shape_Y.back();
|
||||
const int K = trans_b ? blobs[0].size[1] : blobs[0].size[0];
|
||||
const int rows_thin = flatten_a ? shape_Y[shape_Y.size() - 2]
|
||||
: (int)(Y.total() / (size_t)N);
|
||||
if (fastGemmThinEligible(rows_thin, N, K)) {
|
||||
thin_packed_B.resize(fastGemmThinPackBSize(N, K));
|
||||
const size_t ldb_K = trans_b ? 1 : N;
|
||||
const size_t ldb_N = trans_b ? K : 1;
|
||||
fastGemmThinPackB(N, K, blobs[0].ptr<const float>(),
|
||||
ldb_K, ldb_N, thin_packed_B.data());
|
||||
const auto shape_A = shape(inputs[0]);
|
||||
const auto shape_Y = shape(outputs[0]);
|
||||
const int na = shape_A[shape_A.size() - 1];
|
||||
const int ma = shape_A[shape_A.size() - 2];
|
||||
const int N = shape_Y[shape_Y.size() - 1];
|
||||
const int M = shape_Y[shape_Y.size() - 2];
|
||||
const int K = trans_a ? ma : na;
|
||||
const Mat& Bmat = blobs[0];
|
||||
const int ldb = Bmat.size[Bmat.dims - 1];
|
||||
const size_t packed_bytes = mlasSgemmPackBSize(trans_a, trans_b, N, K);
|
||||
if (packed_bytes > 0 && packed_bytes <= static_cast<size_t>(INT_MAX)) {
|
||||
packed_B_mlas.create(1, static_cast<int>(packed_bytes), CV_8U);
|
||||
if (mlasSgemmPackB(trans_a, trans_b, N, K,
|
||||
Bmat.ptr<const float>(), ldb,
|
||||
packed_B_mlas.data)) {
|
||||
packed_B_mlas_M = M;
|
||||
packed_B_mlas_N = N;
|
||||
packed_B_mlas_K = K;
|
||||
mlas_packed = true;
|
||||
} else {
|
||||
packed_B_mlas.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef HAVE_MLAS
|
||||
std::vector<Mat> outputs;
|
||||
outputs_arr.getMatVector(outputs);
|
||||
const auto shape_A = shape(inputs[0]);
|
||||
const auto shape_Y = shape(outputs[0]);
|
||||
const int na = shape_A[shape_A.size() - 1];
|
||||
const int ma = shape_A[shape_A.size() - 2];
|
||||
const int N = shape_Y[shape_Y.size() - 1];
|
||||
const int M = shape_Y[shape_Y.size() - 2];
|
||||
const int K = trans_a ? ma : na;
|
||||
const Mat& Bmat = blobs[0];
|
||||
const int ldb = Bmat.size[Bmat.dims - 1];
|
||||
const size_t packed_bytes = mlasSgemmPackBSize(trans_a, trans_b, N, K);
|
||||
if (packed_bytes > 0) {
|
||||
packed_B_mlas.create(1, static_cast<int>(packed_bytes), CV_8U);
|
||||
if (mlasSgemmPackB(trans_a, trans_b, N, K,
|
||||
Bmat.ptr<const float>(), ldb,
|
||||
packed_B_mlas.data)) {
|
||||
packed_B_mlas_M = M;
|
||||
packed_B_mlas_N = N;
|
||||
packed_B_mlas_K = K;
|
||||
} else {
|
||||
packed_B_mlas.release();
|
||||
#endif
|
||||
if (!mlas_packed) {
|
||||
fastGemmPackB(blobs[0], packed_B, trans_b, opt);
|
||||
|
||||
if (!trans_a && blobs[0].type() == CV_32F) {
|
||||
std::vector<Mat> outputs;
|
||||
outputs_arr.getMatVector(outputs);
|
||||
if (!outputs.empty()) {
|
||||
const auto &Y = outputs[0];
|
||||
const auto shape_Y = shape(Y);
|
||||
const int N = shape_Y.back();
|
||||
const int K = trans_b ? blobs[0].size[1] : blobs[0].size[0];
|
||||
const int rows_thin = flatten_a ? shape_Y[shape_Y.size() - 2]
|
||||
: (int)(Y.total() / (size_t)N);
|
||||
if (fastGemmThinEligible(rows_thin, N, K)) {
|
||||
thin_packed_B.resize(fastGemmThinPackBSize(N, K));
|
||||
const size_t ldb_K = trans_b ? 1 : N;
|
||||
const size_t ldb_N = trans_b ? K : 1;
|
||||
fastGemmThinPackB(N, K, blobs[0].ptr<const float>(),
|
||||
ldb_K, ldb_N, thin_packed_B.data());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
last_packed_blob_data = blobs[0].data;
|
||||
}
|
||||
|
||||
|
||||
@@ -158,8 +158,9 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
const Mat* B_mat = !blobs.empty() ? &blobs[0] :
|
||||
(inputs.size() >= 2 && inputs[1].dims == 2 ? &inputs[1] : nullptr);
|
||||
if (B_mat && B_mat->data != last_packed_input_B_data) {
|
||||
fastGemmPackB(*B_mat, packed_input_B, trans_b, opt);
|
||||
helper.updatePackedBOffsets(packed_input_B.size());
|
||||
packed_input_B.clear();
|
||||
packed_input_B.shrink_to_fit();
|
||||
thin_packed_B.clear();
|
||||
|
||||
if (helper.batch == 1 && B_mat->type() == CV_32F &&
|
||||
fastGemmThinEligible(helper.M, helper.N, helper.K)) {
|
||||
@@ -169,7 +170,8 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
||||
(size_t)helper.ldb0, (size_t)helper.ldb1,
|
||||
thin_packed_B.data());
|
||||
} else {
|
||||
thin_packed_B.clear();
|
||||
fastGemmPackB(*B_mat, packed_input_B, trans_b, opt);
|
||||
helper.updatePackedBOffsets(packed_input_B.size());
|
||||
}
|
||||
last_packed_input_B_data = B_mat->data;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# This file is part of OpenCV project.
|
||||
# It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
# of this distribution and at http://opencv.org/license.html.
|
||||
# Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
# Third party copyrights are property of their respective owners.
|
||||
|
||||
'''
|
||||
This is a sample script to run PaliGemma2 vision-language inference in OpenCV using
|
||||
ONNX models. Given an image and a text prompt, it generates a text response
|
||||
(e.g. a caption).
|
||||
|
||||
The model is split into three ONNX files:
|
||||
- SigLIP vision encoder : image -> 256 image-feature tokens
|
||||
- Embedding : prompt token ids -> text embeddings
|
||||
- Gemma2 language model : [image_features | text_embeds] -> logits
|
||||
|
||||
Model: https://huggingface.co/google/paligemma2-3b-pt-224
|
||||
ONNX: https://huggingface.co/nklskyoy/paligemma2-3b-pt-224-onnx
|
||||
|
||||
Run the script:
|
||||
1. Install the required dependencies:
|
||||
|
||||
pip install numpy
|
||||
|
||||
2. Run the script:
|
||||
|
||||
python vlm_inference.py --siglip=<path-to-vision_model.onnx> \
|
||||
--embedding=<path-to-embedding.onnx> \
|
||||
--gemma=<path-to-gemma2_3b.onnx> \
|
||||
--tokenizer_path=<path-to-opencv-tokenizer-config.json> \
|
||||
--input=<path-to-image> \
|
||||
--prompt="cap en\n"
|
||||
|
||||
The tokenizer_path should point to an OpenCV-format config.json, NOT the
|
||||
HuggingFace tokenizer_config.json.
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import argparse
|
||||
import cv2 as cv
|
||||
|
||||
EOS_ID = 1
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Use this script to run PaliGemma2 vision-language inference in OpenCV',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--siglip', type=str, required=True, help='Path to SigLIP vision encoder ONNX model file.')
|
||||
parser.add_argument('--embedding', type=str, required=True, help='Path to embedding ONNX model file.')
|
||||
parser.add_argument('--gemma', type=str, required=True, help='Path to Gemma2 language model ONNX model file.')
|
||||
parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to tokenizer config.json.')
|
||||
parser.add_argument('--input', '-i', type=str, required=True, help='Path to the input image.')
|
||||
parser.add_argument('--prompt', type=str, default='cap en\n', help='Task prompt (e.g. "cap en\\n" to caption in English).')
|
||||
parser.add_argument('--max_new_tokens', type=int, default=64, help='Maximum number of new tokens to generate.')
|
||||
parser.add_argument('--seed', type=int, default=0, help='Random seed.')
|
||||
return parser.parse_args()
|
||||
|
||||
def preprocess_image(image_path):
|
||||
'''Resize to 224x224 and normalize to [-1, 1] in CHW order (SigLIP: mean=0.5, std=0.5).'''
|
||||
img = cv.imread(image_path)
|
||||
if img is None:
|
||||
raise IOError("Could not read image: " + image_path)
|
||||
img = cv.resize(img, (224, 224))
|
||||
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
|
||||
img = img.astype(np.float32) / 255.0
|
||||
img = (img - 0.5) / 0.5
|
||||
img = img.transpose(2, 0, 1)[np.newaxis]
|
||||
return img
|
||||
|
||||
def vlm_inference(siglip_net, embed_net, gemma_net, pixel_values, prompt, max_new_tokens, tokenizer):
|
||||
|
||||
print("Inferencing PaliGemma2 model...")
|
||||
|
||||
tokens = list(tokenizer.encode(prompt))
|
||||
input_ids = np.array([tokens], dtype=np.int64)
|
||||
|
||||
# SigLIP vision encoder: image -> image-feature tokens
|
||||
siglip_net.setInput(pixel_values, 'pixel_values')
|
||||
image_features = siglip_net.forward() # (1, 256, 2304)
|
||||
|
||||
# Text embedding: token ids -> text embeddings
|
||||
embed_net.setInput(input_ids, 'input_ids')
|
||||
text_embeds = embed_net.forward() # (1, text_len, 2304)
|
||||
|
||||
# Combine [image_features | text_embeds]
|
||||
inputs_embeds = np.concatenate([image_features, text_embeds], axis=1)
|
||||
|
||||
generated = []
|
||||
|
||||
# Prefill
|
||||
gemma_net.setInput(inputs_embeds, 'inputs_embeds')
|
||||
logits = gemma_net.forward()
|
||||
new_id = int(np.argmax(logits[0, -1, :]))
|
||||
generated.append(new_id)
|
||||
|
||||
# Decode (no KV-cache: feed full growing sequence each step)
|
||||
for _ in range(max_new_tokens - 1):
|
||||
if new_id == EOS_ID:
|
||||
break
|
||||
embed_net.setInput(np.array([[new_id]], dtype=np.int64), 'input_ids')
|
||||
new_embed = embed_net.forward()
|
||||
inputs_embeds = np.concatenate([inputs_embeds, new_embed], axis=1)
|
||||
gemma_net.setInput(inputs_embeds, 'inputs_embeds')
|
||||
logits = gemma_net.forward()
|
||||
new_id = int(np.argmax(logits[0, -1, :]))
|
||||
generated.append(new_id)
|
||||
|
||||
if generated and generated[-1] == EOS_ID:
|
||||
generated.pop()
|
||||
|
||||
return generated
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
args = parse_args()
|
||||
np.random.seed(args.seed)
|
||||
|
||||
print("Preparing PaliGemma2 model...")
|
||||
tokenizer = cv.dnn.Tokenizer.load(args.tokenizer_path)
|
||||
|
||||
siglip_net = cv.dnn.readNetFromONNX(args.siglip, cv.dnn.ENGINE_NEW)
|
||||
embed_net = cv.dnn.readNetFromONNX(args.embedding, cv.dnn.ENGINE_NEW)
|
||||
gemma_net = cv.dnn.readNetFromONNX(args.gemma, cv.dnn.ENGINE_NEW)
|
||||
|
||||
print(f"Prompt:\n{args.prompt}")
|
||||
pixel_values = preprocess_image(args.input)
|
||||
|
||||
generated = vlm_inference(siglip_net, embed_net, gemma_net, pixel_values,
|
||||
args.prompt, args.max_new_tokens, tokenizer)
|
||||
response = tokenizer.decode(generated)
|
||||
print(f"Response:\n{response}")
|
||||
Reference in New Issue
Block a user