mirror of
https://github.com/opencv/opencv.git
synced 2026-07-24 04:43:04 +04:00
Jorge Velez
a49a293d3c
GSoC 2025: Add Tokenizer Support to DNN Module #27534 merge with https://github.com/opencv/opencv_extra/pull/1276 ### Summary This pull request introduces initial support for a tokenizer module under `modules/dnn/src/tokenizer` as part of Google Summer of Code 2025 (Project: Tokenization for OpenCV DNN). ### Status - [x] Project structure in place - [x] Initial BPE tokenizer loading - [x] Regex splitting (in progress) - [x] Encoding logic for GPT-2 tokenizer (in progress) - [ ] Documentation (to be improved) ### Goals The goal is to support Hugging Face-compatible tokenization (e.g., GPT-2) natively in C++ to be integrated with DNN inference pipelines. The core pipeline lives in `dnn/src/tokenizer/core_bpe.hpp` and `dnn/src/tokenizer/encoding.hpp`. For Unicode handling I’m using `dnn/src/tokenizer/unicode.hpp`, which is adapted from llama.cpp. ### Feedback Please share early feedback on: - General design structure - Integration strategy with `dnn` - Code organization or naming conventions ### Reference Project: https://summerofcode.withgoogle.com/programs/2025/projects/79SW6eNK
68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
#!/usr/bin/env python
|
|
|
|
'''
|
|
Test for Tokenizer Python bindings
|
|
'''
|
|
|
|
from __future__ import print_function
|
|
|
|
import cv2 as cv
|
|
import os
|
|
import json
|
|
|
|
from tests_common import NewOpenCVTests
|
|
|
|
def _tf(filename=""):
|
|
base = os.environ.get("OPENCV_TEST_DATA_PATH") or os.getcwd()
|
|
path = os.path.join(base, "dnn", "llm", filename)
|
|
if not os.path.exists(path):
|
|
raise FileNotFoundError(
|
|
f"Missing test data: {path}. "
|
|
"Set OPENCV_TEST_DATA_PATH to the testdata root contains dnn/llm."
|
|
)
|
|
return path
|
|
|
|
class TokenizerBindingTest(NewOpenCVTests):
|
|
def test_tokenizer_binding(self):
|
|
try:
|
|
tokenizer = cv.dnn.Tokenizer
|
|
print("Tokenizer binding is available.", tokenizer)
|
|
gpt2_model = _tf("gpt2/config.json")
|
|
tokenizer = cv.dnn.Tokenizer.load(gpt2_model)
|
|
print("Tokenizer loaded from:", gpt2_model)
|
|
except AttributeError:
|
|
self.fail("Tokenizer binding is NOT available.")
|
|
|
|
def test_tokenizer_gpt2(self):
|
|
tok = cv.dnn.Tokenizer.load((_tf("gpt2/config.json")))
|
|
ids = tok.encode("hello world")
|
|
print(ids)
|
|
txt = tok.decode(ids)
|
|
self.assertEqual(txt, "hello world")
|
|
|
|
def test_tokenizer_gpt4(self):
|
|
tok = cv.dnn.Tokenizer.load(_tf("gpt4/config.json"))
|
|
tokens = tok.encode("hello world")
|
|
# expects {15339, 1917}
|
|
self.assertEqual(list(tokens), [15339, 1917])
|
|
sent = tok.decode([15339, 1917])
|
|
self.assertEqual(sent, "hello world")
|
|
|
|
def test_with_hf_tiktoken(self):
|
|
tok = cv.dnn.Tokenizer.load(_tf("gpt2/config.json"))
|
|
with open(_tf("gpt2/gpt2_hf_tik_testdata.json"), "r", encoding="utf-8") as f:
|
|
golden = json.load(f)
|
|
|
|
for s in golden["samples"]:
|
|
text = s["text"]
|
|
expected = s["ids"]
|
|
got = tok.encode(text).tolist()
|
|
self.assertEqual(
|
|
got, expected,
|
|
msg=f"Mismatch for sample '{s['name']}'"
|
|
)
|
|
self.assertEqual(tok.decode(expected), text)
|
|
|
|
if __name__ == '__main__':
|
|
NewOpenCVTests.bootstrap()
|