Merge pull request #27534 from JorgeV92:gsoc2025-tokenizer

GSoC 2025: Add Tokenizer Support to DNN Module #27534

merge with https://github.com/opencv/opencv_extra/pull/1276

### Summary
This pull request introduces initial support for a tokenizer module under `modules/dnn/src/tokenizer` as part of Google Summer of Code 2025 (Project: Tokenization for OpenCV DNN).

### Status
- [x] Project structure in place
- [x] Initial BPE tokenizer loading
- [x] Regex splitting (in progress)
- [x] Encoding logic for GPT-2 tokenizer (in progress)
- [ ] Documentation (to be improved)

### Goals
The goal is to support Hugging Face-compatible tokenization (e.g., GPT-2) natively in C++ to be integrated with DNN inference pipelines. 

The core pipeline lives in `dnn/src/tokenizer/core_bpe.hpp` and `dnn/src/tokenizer/encoding.hpp`. For Unicode handling I’m using `dnn/src/tokenizer/unicode.hpp`, which is adapted from llama.cpp.


### Feedback
Please share early feedback on:
- General design structure
- Integration strategy with `dnn`
- Code organization or naming conventions

### Reference
Project: https://summerofcode.withgoogle.com/programs/2025/projects/79SW6eNK

modules/python/test/test_tokenizer.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+'''
+Test for Tokenizer Python bindings
+'''
+
+from __future__ import print_function
+
+import cv2 as cv
+import os
+import json
+
+from tests_common import NewOpenCVTests
+
+def _tf(filename=""):
+    base = os.environ.get("OPENCV_TEST_DATA_PATH") or os.getcwd()
+    path = os.path.join(base, "dnn", "llm", filename)
+    if not os.path.exists(path):
+        raise FileNotFoundError(
+            f"Missing test data: {path}. "
+            "Set OPENCV_TEST_DATA_PATH to the testdata root contains dnn/llm."
+        )
+    return path
+
+class TokenizerBindingTest(NewOpenCVTests):
+    def test_tokenizer_binding(self):
+        try:
+            tokenizer = cv.dnn.Tokenizer
+            print("Tokenizer binding is available.", tokenizer)
+            gpt2_model = _tf("gpt2/config.json")
+            tokenizer = cv.dnn.Tokenizer.load(gpt2_model)
+            print("Tokenizer loaded from:", gpt2_model)
+        except AttributeError:
+            self.fail("Tokenizer binding is NOT available.")
+
+    def test_tokenizer_gpt2(self):
+        tok = cv.dnn.Tokenizer.load((_tf("gpt2/config.json")))
+        ids = tok.encode("hello world")
+        print(ids)
+        txt = tok.decode(ids)
+        self.assertEqual(txt, "hello world")
+
+    def test_tokenizer_gpt4(self):
+        tok = cv.dnn.Tokenizer.load(_tf("gpt4/config.json"))
+        tokens = tok.encode("hello world")
+        # expects {15339, 1917}
+        self.assertEqual(list(tokens), [15339, 1917])
+        sent = tok.decode([15339, 1917])
+        self.assertEqual(sent, "hello world")
+
+    def test_with_hf_tiktoken(self):
+        tok = cv.dnn.Tokenizer.load(_tf("gpt2/config.json"))
+        with open(_tf("gpt2/gpt2_hf_tik_testdata.json"), "r", encoding="utf-8") as f:
+            golden = json.load(f)
+
+        for s in golden["samples"]:
+            text = s["text"]
+            expected = s["ids"]
+            got = tok.encode(text).tolist()
+            self.assertEqual(
+                got, expected,
+                msg=f"Mismatch for sample '{s['name']}'"
+            )
+            self.assertEqual(tok.decode(expected), text)
+
+if __name__ == '__main__':
+    NewOpenCVTests.bootstrap()