Merge pull request #29221 from abhishek-gola:oom_issue_fixed

Fixed Out-of-Memory issue and added VLM sample #29221

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

samples/dnn/vlm_inference.py

@@ -0,0 +1,130 @@
+# This file is part of OpenCV project.
+# It is subject to the license terms in the LICENSE file found in the top-level directory
+# of this distribution and at http://opencv.org/license.html.
+# Copyright (C) 2026, BigVision LLC, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+'''
+This is a sample script to run PaliGemma2 vision-language inference in OpenCV using
+ONNX models. Given an image and a text prompt, it generates a text response
+(e.g. a caption).
+
+The model is split into three ONNX files:
+    - SigLIP vision encoder : image -> 256 image-feature tokens
+    - Embedding             : prompt token ids -> text embeddings
+    - Gemma2 language model : [image_features | text_embeds] -> logits
+
+Model: https://huggingface.co/google/paligemma2-3b-pt-224
+ONNX:  https://huggingface.co/nklskyoy/paligemma2-3b-pt-224-onnx
+
+Run the script:
+1. Install the required dependencies:
+
+    pip install numpy
+
+2. Run the script:
+
+    python vlm_inference.py --siglip=<path-to-vision_model.onnx> \
+                            --embedding=<path-to-embedding.onnx> \
+                            --gemma=<path-to-gemma2_3b.onnx> \
+                            --tokenizer_path=<path-to-opencv-tokenizer-config.json> \
+                            --input=<path-to-image> \
+                            --prompt="cap en\n"
+
+    The tokenizer_path should point to an OpenCV-format config.json, NOT the
+    HuggingFace tokenizer_config.json.
+'''
+
+import numpy as np
+import argparse
+import cv2 as cv
+
+EOS_ID = 1
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Use this script to run PaliGemma2 vision-language inference in OpenCV',
+                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--siglip', type=str, required=True, help='Path to SigLIP vision encoder ONNX model file.')
+    parser.add_argument('--embedding', type=str, required=True, help='Path to embedding ONNX model file.')
+    parser.add_argument('--gemma', type=str, required=True, help='Path to Gemma2 language model ONNX model file.')
+    parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to tokenizer config.json.')
+    parser.add_argument('--input', '-i', type=str, required=True, help='Path to the input image.')
+    parser.add_argument('--prompt', type=str, default='cap en\n', help='Task prompt (e.g. "cap en\\n" to caption in English).')
+    parser.add_argument('--max_new_tokens', type=int, default=64, help='Maximum number of new tokens to generate.')
+    parser.add_argument('--seed', type=int, default=0, help='Random seed.')
+    return parser.parse_args()
+
+def preprocess_image(image_path):
+    '''Resize to 224x224 and normalize to [-1, 1] in CHW order (SigLIP: mean=0.5, std=0.5).'''
+    img = cv.imread(image_path)
+    if img is None:
+        raise IOError("Could not read image: " + image_path)
+    img = cv.resize(img, (224, 224))
+    img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
+    img = img.astype(np.float32) / 255.0
+    img = (img - 0.5) / 0.5
+    img = img.transpose(2, 0, 1)[np.newaxis]
+    return img
+
+def vlm_inference(siglip_net, embed_net, gemma_net, pixel_values, prompt, max_new_tokens, tokenizer):
+
+    print("Inferencing PaliGemma2 model...")
+
+    tokens = list(tokenizer.encode(prompt))
+    input_ids = np.array([tokens], dtype=np.int64)
+
+    # SigLIP vision encoder: image -> image-feature tokens
+    siglip_net.setInput(pixel_values, 'pixel_values')
+    image_features = siglip_net.forward()        # (1, 256, 2304)
+
+    # Text embedding: token ids -> text embeddings
+    embed_net.setInput(input_ids, 'input_ids')
+    text_embeds = embed_net.forward()            # (1, text_len, 2304)
+
+    # Combine [image_features | text_embeds]
+    inputs_embeds = np.concatenate([image_features, text_embeds], axis=1)
+
+    generated = []
+
+    # Prefill
+    gemma_net.setInput(inputs_embeds, 'inputs_embeds')
+    logits = gemma_net.forward()
+    new_id = int(np.argmax(logits[0, -1, :]))
+    generated.append(new_id)
+
+    # Decode (no KV-cache: feed full growing sequence each step)
+    for _ in range(max_new_tokens - 1):
+        if new_id == EOS_ID:
+            break
+        embed_net.setInput(np.array([[new_id]], dtype=np.int64), 'input_ids')
+        new_embed     = embed_net.forward()
+        inputs_embeds = np.concatenate([inputs_embeds, new_embed], axis=1)
+        gemma_net.setInput(inputs_embeds, 'inputs_embeds')
+        logits        = gemma_net.forward()
+        new_id        = int(np.argmax(logits[0, -1, :]))
+        generated.append(new_id)
+
+    if generated and generated[-1] == EOS_ID:
+        generated.pop()
+
+    return generated
+
+if __name__ == '__main__':
+
+    args = parse_args()
+    np.random.seed(args.seed)
+
+    print("Preparing PaliGemma2 model...")
+    tokenizer = cv.dnn.Tokenizer.load(args.tokenizer_path)
+
+    siglip_net = cv.dnn.readNetFromONNX(args.siglip, cv.dnn.ENGINE_NEW)
+    embed_net  = cv.dnn.readNetFromONNX(args.embedding, cv.dnn.ENGINE_NEW)
+    gemma_net  = cv.dnn.readNetFromONNX(args.gemma, cv.dnn.ENGINE_NEW)
+
+    print(f"Prompt:\n{args.prompt}")
+    pixel_values = preprocess_image(args.input)
+
+    generated = vlm_inference(siglip_net, embed_net, gemma_net, pixel_values,
+                              args.prompt, args.max_new_tokens, tokenizer)
+    response = tokenizer.decode(generated)
+    print(f"Response:\n{response}")