Merge pull request #27534 from JorgeV92:gsoc2025-tokenizer

GSoC 2025: Add Tokenizer Support to DNN Module #27534

merge with https://github.com/opencv/opencv_extra/pull/1276

### Summary
This pull request introduces initial support for a tokenizer module under `modules/dnn/src/tokenizer` as part of Google Summer of Code 2025 (Project: Tokenization for OpenCV DNN).

### Status
- [x] Project structure in place
- [x] Initial BPE tokenizer loading
- [x] Regex splitting (in progress)
- [x] Encoding logic for GPT-2 tokenizer (in progress)
- [ ] Documentation (to be improved)

### Goals
The goal is to support Hugging Face-compatible tokenization (e.g., GPT-2) natively in C++ to be integrated with DNN inference pipelines. 

The core pipeline lives in `dnn/src/tokenizer/core_bpe.hpp` and `dnn/src/tokenizer/encoding.hpp`. For Unicode handling I’m using `dnn/src/tokenizer/unicode.hpp`, which is adapted from llama.cpp.


### Feedback
Please share early feedback on:
- General design structure
- Integration strategy with `dnn`
- Code organization or naming conventions

### Reference
Project: https://summerofcode.withgoogle.com/programs/2025/projects/79SW6eNK

samples/dnn/gpt2_inference.py

@@ -10,7 +10,7 @@ To export GPT-2 model to ONNX, you can use the following procedure:
 
 1. Clone fork of Andrej Karpathy's GPT-2 repository:
 
-    git clone -b ash/export-gpt2-onnx-dynamic https://github.com/Abdurrahheem/build-nanogpt.git
+    git clone -b fix-dynamic-axis-export  https://github.com/nklskyoy/build-nanogpt
 
 2. Install the required dependencies:
 
@@ -27,11 +27,10 @@ Run the script:
     pip install tiktoken==0.7.0 numpy tqdm
 
 2. Run the script:
-    python gpt2_inference.py --model=<path-to-onnx-model>  --prompt=<use-promt-of-the-same-length-used-while-exporting>
+    python gpt2_inference.py --model=<path-to-onnx-model> --tokenizer_path=<path-to-tokenizer-config> --prompt=<use-promt-of-the-same-length-used-while-exporting>
 '''
 
 import numpy as np
-import tiktoken
 import argparse
 import cv2 as cv
 
@@ -39,8 +38,9 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Use this script to run GPT-2 inference in OpenCV',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--model', type=str, required=True, help='Path to GPT-2 model ONNX model file.')
+    parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to GPT-2 tokenizer config file.')
     parser.add_argument("--prompt", type=str, default="Hello, I'm a language model,", help="Prompt to start with.")
-    parser.add_argument("--max_seq_len", type=int, default=1024, help="Number of tokens to continue.")
+    parser.add_argument("--max_seq_len", type=int, default=32, help="Number of tokens to continue.")
     parser.add_argument("--seed", type=int, default=0, help="Random seed")
     return parser.parse_args()
 
@@ -50,71 +50,40 @@ def stable_softmax(logits):
 
 
 
-def gpt2_inference(net, tokens, max_length, tokenizer):
+def gpt2_inference(net, prompt, max_length, tokenizer):
 
     print("Inferencing GPT-2 model...")
-    x = np.array(tokens)
-    x = np.tile(x, (1, 1)).astype(np.int32)
-    pos = np.arange(0, len(x), dtype=np.int32)
 
-    # warm up
-    net.setInputsNames(['input_ids', 'position_ids'])
-    net.setInput(x, 'input_ids')
-    net.setInput(pos, 'position_ids')
-    logits = net.forward()
+    tokens = tokenizer.encode(prompt).reshape(1,-1)
 
     stop_tokens = (50256, ) ## could be extended to include more stop tokens
-    print("\n", tokenizer.decode(tokens), sep="", end="")
-    while 0 < max_length and x[:, -1] not in stop_tokens:
+    while 0 < max_length and tokens[:, -1] not in stop_tokens:
 
-        net.setInputsNames(['input_ids', 'position_ids'])
-        net.setInput(x, 'input_ids')
-        net.setInput(pos, 'position_ids')
+        net.setInputsNames(['idx'])
+        net.setInput(tokens, 'idx')
         logits = net.forward()
-
-        # logits is assumed to be (B, seq_length, vocab_size) and needs to be the last token's logits
         logits = logits[:, -1, :]  # (B, vocab_size)
 
-        # Get the probabilities using softmax
-        probs = stable_softmax(logits)
+        # use hard sampling
+        new_idx = np.argmax(logits.reshape(-1)).reshape(1,1)
 
-        # Do top-k sampling of 50
-        topk_indices = np.argpartition(probs, -50, axis=-1)[:, -50:]
-        topk_probs = np.take_along_axis(probs, topk_indices, axis=-1)
-
-        # Normalize top-k probabilities
-        topk_probs /= np.sum(topk_probs, axis=-1, keepdims=True)
-
-        # Select a token from the top-k probabilities
-        sampled_indices = [np.random.choice(topk_indices[i], p=topk_probs[i]) for i in range(len(topk_probs))]
-        sampled_indices = np.array(sampled_indices).reshape(-1, 1)
-
-        # Decode and print the new token
-        new_word = tokenizer.decode([sampled_indices[0, 0]])
-
-        ## clean the prints from the previous line
-        print(new_word, end='', flush=True)
-
-        # Append to the sequence
-        x = np.concatenate((x, sampled_indices), axis=1)
-        pos = np.arange(0, x.shape[1], dtype=np.int32) # shape (T)
+        tokens = np.concatenate((tokens, new_idx), axis=1)
 
         max_length -= 1
+    return tokens
+
 
-    print('\n')
 
 if __name__ == '__main__':
 
     args = parse_args()
     print("Preparing GPT-2 model...")
-
-    np.random.seed(args.seed)
     max_length = args.max_seq_len
     prompt = args.prompt
+    tokenizer_path = args.tokenizer_path
 
-    net = cv.dnn.readNet(args.model)
+    net = cv.dnn.readNetFromONNX(args.model, cv.dnn.ENGINE_NEW)
+    tokenizer = cv.dnn.Tokenizer.load(tokenizer_path)
 
-    enc = tiktoken.get_encoding('gpt2')
-    tokens = enc.encode(prompt)
-
-    gpt2_inference(net, tokens, max_length, enc)
+    tokens = gpt2_inference(net, prompt, max_length, tokenizer)
+    print(tokenizer.decode(tokens[0]))