diff --git a/modules/dnn/src/layers/conv2_layer.cpp b/modules/dnn/src/layers/conv2_layer.cpp index f8366c87ae..7781f788f1 100644 --- a/modules/dnn/src/layers/conv2_layer.cpp +++ b/modules/dnn/src/layers/conv2_layer.cpp @@ -145,7 +145,7 @@ public: const bool big_enough = (int64_t)Cout * (int64_t)Cin >= (int64_t)(256 * 256); if (ksize_all_one && strides_all_one && big_enough) { size_t pack_bytes = mlasSgemmPackBSize(false, true, Cout, Cin); - if (pack_bytes > 0) { + if (pack_bytes > 0 && pack_bytes <= (size_t)INT_MAX) { mlas_packed_B_.create(1, (int)pack_bytes, CV_8U); // Weight is (Cout, Cin, 1, ..., 1) contiguous; reshape to // (Cout, Cin) and PackB with trans_b=true. diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp index cee520babf..2996074fe4 100644 --- a/modules/dnn/src/layers/gemm_layer.cpp +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -256,56 +256,67 @@ public: // pack B if it is const if (constB(mode) && blobs[0].data != last_packed_blob_data) { - fastGemmPackB(blobs[0], packed_B, trans_b, opt); - - // Pre-pack B in the "thin" layout when the gemm shape has a - // small leading dim (M <= FAST_GEMM_THIN_MAX_M). + packed_B.clear(); + packed_B.shrink_to_fit(); thin_packed_B.clear(); - if (!trans_a && blobs[0].type() == CV_32F) { + thin_packed_B.shrink_to_fit(); + + bool mlas_packed = false; +#ifdef HAVE_MLAS + packed_B_mlas.release(); + packed_B_mlas_M = packed_B_mlas_N = packed_B_mlas_K = 0; + if (mlasAvailable()) { std::vector outputs; outputs_arr.getMatVector(outputs); - if (!outputs.empty()) { - const auto &Y = outputs[0]; - const auto shape_Y = shape(Y); - const int N = shape_Y.back(); - const int K = trans_b ? blobs[0].size[1] : blobs[0].size[0]; - const int rows_thin = flatten_a ? shape_Y[shape_Y.size() - 2] - : (int)(Y.total() / (size_t)N); - if (fastGemmThinEligible(rows_thin, N, K)) { - thin_packed_B.resize(fastGemmThinPackBSize(N, K)); - const size_t ldb_K = trans_b ? 1 : N; - const size_t ldb_N = trans_b ? K : 1; - fastGemmThinPackB(N, K, blobs[0].ptr(), - ldb_K, ldb_N, thin_packed_B.data()); + const auto shape_A = shape(inputs[0]); + const auto shape_Y = shape(outputs[0]); + const int na = shape_A[shape_A.size() - 1]; + const int ma = shape_A[shape_A.size() - 2]; + const int N = shape_Y[shape_Y.size() - 1]; + const int M = shape_Y[shape_Y.size() - 2]; + const int K = trans_a ? ma : na; + const Mat& Bmat = blobs[0]; + const int ldb = Bmat.size[Bmat.dims - 1]; + const size_t packed_bytes = mlasSgemmPackBSize(trans_a, trans_b, N, K); + if (packed_bytes > 0 && packed_bytes <= static_cast(INT_MAX)) { + packed_B_mlas.create(1, static_cast(packed_bytes), CV_8U); + if (mlasSgemmPackB(trans_a, trans_b, N, K, + Bmat.ptr(), ldb, + packed_B_mlas.data)) { + packed_B_mlas_M = M; + packed_B_mlas_N = N; + packed_B_mlas_K = K; + mlas_packed = true; + } else { + packed_B_mlas.release(); } } } -#ifdef HAVE_MLAS - std::vector outputs; - outputs_arr.getMatVector(outputs); - const auto shape_A = shape(inputs[0]); - const auto shape_Y = shape(outputs[0]); - const int na = shape_A[shape_A.size() - 1]; - const int ma = shape_A[shape_A.size() - 2]; - const int N = shape_Y[shape_Y.size() - 1]; - const int M = shape_Y[shape_Y.size() - 2]; - const int K = trans_a ? ma : na; - const Mat& Bmat = blobs[0]; - const int ldb = Bmat.size[Bmat.dims - 1]; - const size_t packed_bytes = mlasSgemmPackBSize(trans_a, trans_b, N, K); - if (packed_bytes > 0) { - packed_B_mlas.create(1, static_cast(packed_bytes), CV_8U); - if (mlasSgemmPackB(trans_a, trans_b, N, K, - Bmat.ptr(), ldb, - packed_B_mlas.data)) { - packed_B_mlas_M = M; - packed_B_mlas_N = N; - packed_B_mlas_K = K; - } else { - packed_B_mlas.release(); +#endif + if (!mlas_packed) { + fastGemmPackB(blobs[0], packed_B, trans_b, opt); + + if (!trans_a && blobs[0].type() == CV_32F) { + std::vector outputs; + outputs_arr.getMatVector(outputs); + if (!outputs.empty()) { + const auto &Y = outputs[0]; + const auto shape_Y = shape(Y); + const int N = shape_Y.back(); + const int K = trans_b ? blobs[0].size[1] : blobs[0].size[0]; + const int rows_thin = flatten_a ? shape_Y[shape_Y.size() - 2] + : (int)(Y.total() / (size_t)N); + if (fastGemmThinEligible(rows_thin, N, K)) { + thin_packed_B.resize(fastGemmThinPackBSize(N, K)); + const size_t ldb_K = trans_b ? 1 : N; + const size_t ldb_N = trans_b ? K : 1; + fastGemmThinPackB(N, K, blobs[0].ptr(), + ldb_K, ldb_N, thin_packed_B.data()); + } + } } } -#endif + last_packed_blob_data = blobs[0].data; } diff --git a/modules/dnn/src/layers/matmul_layer.cpp b/modules/dnn/src/layers/matmul_layer.cpp index 12ceca96be..c75897ddf9 100644 --- a/modules/dnn/src/layers/matmul_layer.cpp +++ b/modules/dnn/src/layers/matmul_layer.cpp @@ -158,8 +158,9 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { const Mat* B_mat = !blobs.empty() ? &blobs[0] : (inputs.size() >= 2 && inputs[1].dims == 2 ? &inputs[1] : nullptr); if (B_mat && B_mat->data != last_packed_input_B_data) { - fastGemmPackB(*B_mat, packed_input_B, trans_b, opt); - helper.updatePackedBOffsets(packed_input_B.size()); + packed_input_B.clear(); + packed_input_B.shrink_to_fit(); + thin_packed_B.clear(); if (helper.batch == 1 && B_mat->type() == CV_32F && fastGemmThinEligible(helper.M, helper.N, helper.K)) { @@ -169,7 +170,8 @@ class MatMulLayerImpl CV_FINAL : public MatMulLayer { (size_t)helper.ldb0, (size_t)helper.ldb1, thin_packed_B.data()); } else { - thin_packed_B.clear(); + fastGemmPackB(*B_mat, packed_input_B, trans_b, opt); + helper.updatePackedBOffsets(packed_input_B.size()); } last_packed_input_B_data = B_mat->data; } diff --git a/samples/dnn/vlm_inference.py b/samples/dnn/vlm_inference.py new file mode 100644 index 0000000000..06d1198f14 --- /dev/null +++ b/samples/dnn/vlm_inference.py @@ -0,0 +1,130 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html. +# Copyright (C) 2026, BigVision LLC, all rights reserved. +# Third party copyrights are property of their respective owners. + +''' +This is a sample script to run PaliGemma2 vision-language inference in OpenCV using +ONNX models. Given an image and a text prompt, it generates a text response +(e.g. a caption). + +The model is split into three ONNX files: + - SigLIP vision encoder : image -> 256 image-feature tokens + - Embedding : prompt token ids -> text embeddings + - Gemma2 language model : [image_features | text_embeds] -> logits + +Model: https://huggingface.co/google/paligemma2-3b-pt-224 +ONNX: https://huggingface.co/nklskyoy/paligemma2-3b-pt-224-onnx + +Run the script: +1. Install the required dependencies: + + pip install numpy + +2. Run the script: + + python vlm_inference.py --siglip= \ + --embedding= \ + --gemma= \ + --tokenizer_path= \ + --input= \ + --prompt="cap en\n" + + The tokenizer_path should point to an OpenCV-format config.json, NOT the + HuggingFace tokenizer_config.json. +''' + +import numpy as np +import argparse +import cv2 as cv + +EOS_ID = 1 + +def parse_args(): + parser = argparse.ArgumentParser(description='Use this script to run PaliGemma2 vision-language inference in OpenCV', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--siglip', type=str, required=True, help='Path to SigLIP vision encoder ONNX model file.') + parser.add_argument('--embedding', type=str, required=True, help='Path to embedding ONNX model file.') + parser.add_argument('--gemma', type=str, required=True, help='Path to Gemma2 language model ONNX model file.') + parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to tokenizer config.json.') + parser.add_argument('--input', '-i', type=str, required=True, help='Path to the input image.') + parser.add_argument('--prompt', type=str, default='cap en\n', help='Task prompt (e.g. "cap en\\n" to caption in English).') + parser.add_argument('--max_new_tokens', type=int, default=64, help='Maximum number of new tokens to generate.') + parser.add_argument('--seed', type=int, default=0, help='Random seed.') + return parser.parse_args() + +def preprocess_image(image_path): + '''Resize to 224x224 and normalize to [-1, 1] in CHW order (SigLIP: mean=0.5, std=0.5).''' + img = cv.imread(image_path) + if img is None: + raise IOError("Could not read image: " + image_path) + img = cv.resize(img, (224, 224)) + img = cv.cvtColor(img, cv.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + img = (img - 0.5) / 0.5 + img = img.transpose(2, 0, 1)[np.newaxis] + return img + +def vlm_inference(siglip_net, embed_net, gemma_net, pixel_values, prompt, max_new_tokens, tokenizer): + + print("Inferencing PaliGemma2 model...") + + tokens = list(tokenizer.encode(prompt)) + input_ids = np.array([tokens], dtype=np.int64) + + # SigLIP vision encoder: image -> image-feature tokens + siglip_net.setInput(pixel_values, 'pixel_values') + image_features = siglip_net.forward() # (1, 256, 2304) + + # Text embedding: token ids -> text embeddings + embed_net.setInput(input_ids, 'input_ids') + text_embeds = embed_net.forward() # (1, text_len, 2304) + + # Combine [image_features | text_embeds] + inputs_embeds = np.concatenate([image_features, text_embeds], axis=1) + + generated = [] + + # Prefill + gemma_net.setInput(inputs_embeds, 'inputs_embeds') + logits = gemma_net.forward() + new_id = int(np.argmax(logits[0, -1, :])) + generated.append(new_id) + + # Decode (no KV-cache: feed full growing sequence each step) + for _ in range(max_new_tokens - 1): + if new_id == EOS_ID: + break + embed_net.setInput(np.array([[new_id]], dtype=np.int64), 'input_ids') + new_embed = embed_net.forward() + inputs_embeds = np.concatenate([inputs_embeds, new_embed], axis=1) + gemma_net.setInput(inputs_embeds, 'inputs_embeds') + logits = gemma_net.forward() + new_id = int(np.argmax(logits[0, -1, :])) + generated.append(new_id) + + if generated and generated[-1] == EOS_ID: + generated.pop() + + return generated + +if __name__ == '__main__': + + args = parse_args() + np.random.seed(args.seed) + + print("Preparing PaliGemma2 model...") + tokenizer = cv.dnn.Tokenizer.load(args.tokenizer_path) + + siglip_net = cv.dnn.readNetFromONNX(args.siglip, cv.dnn.ENGINE_NEW) + embed_net = cv.dnn.readNetFromONNX(args.embedding, cv.dnn.ENGINE_NEW) + gemma_net = cv.dnn.readNetFromONNX(args.gemma, cv.dnn.ENGINE_NEW) + + print(f"Prompt:\n{args.prompt}") + pixel_values = preprocess_image(args.input) + + generated = vlm_inference(siglip_net, embed_net, gemma_net, pixel_values, + args.prompt, args.max_new_tokens, tokenizer) + response = tokenizer.decode(generated) + print(f"Response:\n{response}")