From c0ab6b69a8b2085f5a4b44016f152225d288278b Mon Sep 17 00:00:00 2001 From: Varun Jaiswal <96684656+varun-jaiswal17@users.noreply.github.com> Date: Wed, 15 Apr 2026 11:12:49 +0530 Subject: [PATCH] Merge pull request #28781 from varun-jaiswal17:work-next Add Qwen2.5 tokenizer support for dnn #28781 OpenCV extra: https://github.com/opencv/opencv_extra/pull/1334 Extended the dnn tokenizer support to qwen2.5 tokenization. - Add QWEN2_5 pre-tokenizer regex pattern to utils.hpp - Generalised buildTokenizerGPT to buildTokenizerFromJson to handle gpt2/gpt4/ and qwen2.5 - Add qwen2/qwen2.5 model type support with special token handling - Add Qwen2.5 tests - Add end-to-end qwen_inference script for Qwen2.5 ONNX model ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/tokenizer/tokenizer.cpp | 27 +++++-- modules/dnn/src/tokenizer/utils.hpp | 3 + modules/dnn/test/test_tokenizer.cpp | 55 ++++++++++++++ samples/dnn/qwen_inference.py | 99 +++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 samples/dnn/qwen_inference.py diff --git a/modules/dnn/src/tokenizer/tokenizer.cpp b/modules/dnn/src/tokenizer/tokenizer.cpp index 2ffdf96c27..8928970eec 100644 --- a/modules/dnn/src/tokenizer/tokenizer.cpp +++ b/modules/dnn/src/tokenizer/tokenizer.cpp @@ -20,7 +20,8 @@ static std::unordered_map& tokenizerRegistry() { return reg; } -CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path); +CoreBPE buildTokenizerFromJson(const std::string& model_type, const std::string& json_path, + std::unordered_set* outSpecial = nullptr); struct Tokenizer::Impl { virtual ~Impl() {} @@ -30,13 +31,16 @@ struct Tokenizer::Impl { struct BpeTokenizerImpl : public Tokenizer::Impl { Ptr coreBPE; + std::unordered_set allowedSpecial; - explicit BpeTokenizerImpl(CoreBPE core) - : coreBPE(makePtr(std::move(core))) {} + explicit BpeTokenizerImpl(CoreBPE core, + std::unordered_set special = {}) + : coreBPE(makePtr(std::move(core))) + , allowedSpecial(std::move(special)) {} std::vector encode(const std::string& text) override { CV_Assert(coreBPE); - std::vector tok = coreBPE->encode(text, {}).first; + std::vector tok = coreBPE->encode(text, allowedSpecial).first; return std::vector(tok.begin(), tok.end()); } @@ -60,12 +64,15 @@ static void registerDefaultTokenizers() { std::string tok_json = dir + "tokenizer.json"; CoreBPE core; + std::unordered_set special; if (model_type == "gpt2" || model_type == "gpt4") { - core = buildTokenizerGPT(model_type, tok_json); + core = buildTokenizerFromJson(model_type, tok_json); + } else if (model_type == "qwen2" || model_type == "qwen2.5") { + core = buildTokenizerFromJson(model_type, tok_json, &special); } else { CV_Error(cv::Error::StsError, "Unsupported model_type for BPE: " + model_type); } - return makePtr(std::move(core)); + return makePtr(std::move(core), std::move(special)); }; } } @@ -82,7 +89,8 @@ std::string Tokenizer::decode(const std::vector& tokens) { return impl_->decode(tokens); }; -CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path) { +CoreBPE buildTokenizerFromJson(const std::string& model_type, const std::string& json_path, + std::unordered_set* outSpecial) { cv::FileStorage fs(json_path, cv::FileStorage::READ | cv::FileStorage::FORMAT_JSON); if (!fs.isOpened()) CV_Error(cv::Error::StsError, "Failed to open tokenizer.json: " + json_path); @@ -99,9 +107,11 @@ CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json skip_tokens.insert("<|endoftext|>"); } else if (model_type == "gpt4" || model_type == "cl100k_base") { pattern = CL100K_BASE; + } else if (model_type == "qwen2" || model_type == "qwen2.5") { + pattern = QWEN2_5; } else { CV_Error(cv::Error::StsError, - "Unsupported model_type: " + model_type + " (expected gpt2/r50k_base or gpt4/cl100k_base)"); + "Unsupported model_type: " + model_type + " (expected gpt2/r50k_base, gpt4/cl100k_base, or qwen2/qwen2.5)"); } auto token_to_bytes = [&](const std::string& token_utf8) -> std::vector { @@ -139,6 +149,7 @@ CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json if (special && id >= 0 && !content.empty()) { specialTokens.emplace(content, (uint32_t)id); if (id > max_id) max_id = id; + if (outSpecial) outSpecial->insert(content); } } } diff --git a/modules/dnn/src/tokenizer/utils.hpp b/modules/dnn/src/tokenizer/utils.hpp index c03d23d5f6..56612d4dd4 100644 --- a/modules/dnn/src/tokenizer/utils.hpp +++ b/modules/dnn/src/tokenizer/utils.hpp @@ -28,5 +28,8 @@ static const std::string R50K_UTF8 = "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N} // The resulting C++ pattern is compatible with std::regex and the tokenizer's Unicode handling logic. static const std::string CL100K_BASE = R"CL100K('(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s)CL100K"; +// Qwen2.5 pre-tokenizer split pattern (from tokenizer.json) +static const std::string QWEN2_5 = R"QWEN('(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+)QWEN"; + }} #endif diff --git a/modules/dnn/test/test_tokenizer.cpp b/modules/dnn/test/test_tokenizer.cpp index 5ad2b9d169..3daee402f7 100644 --- a/modules/dnn/test/test_tokenizer.cpp +++ b/modules/dnn/test/test_tokenizer.cpp @@ -96,4 +96,59 @@ TEST(Tokenizer_BPE, CatastrophicallyRepetitive_GPT2) { EXPECT_EQ(with_newline, gpt2_tok.decode(gpt2_tok.encode(with_newline))); } } + +// ---- Qwen2.5 tests ---- +// Ground truth generated with: +// from transformers import AutoTokenizer +// tok = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") +// tok.encode(text) + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_English) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + EXPECT_EQ(tok.encode("Hello world"), (std::vector{9707, 1879})); +} + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Chinese) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + // 你好世界 + EXPECT_EQ(tok.encode("\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xb8\x96\xe7\x95\x8c"), + (std::vector{108386, 99489})); +} + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Code) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + EXPECT_EQ(tok.encode("def hello(): print('hello')"), + (std::vector{750, 23811, 4555, 1173, 492, 14990, 863})); +} + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Numbers) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + EXPECT_EQ(tok.encode("2024"), (std::vector{17, 15, 17, 19})); +} + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_SpecialTokens) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + // <|im_start|>user\nHello<|im_end|> + EXPECT_EQ(tok.encode("<|im_start|>user\nHello<|im_end|>"), + (std::vector{151644, 872, 198, 9707, 151645})); +} + +TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Roundtrip) { + std::string model = _tf("qwen2.5/config.json"); + Tokenizer tok = Tokenizer::load(model); + std::vector cases = { + "Hello world", + "def hello(): print('hello')", + "2024", + }; + for (const auto& text : cases) { + EXPECT_EQ(tok.decode(tok.encode(text)), text); + } +} + }} diff --git a/samples/dnn/qwen_inference.py b/samples/dnn/qwen_inference.py new file mode 100644 index 0000000000..6fc33ca9f3 --- /dev/null +++ b/samples/dnn/qwen_inference.py @@ -0,0 +1,99 @@ +''' +This is a sample script to run Qwen2.5 inference in OpenCV using ONNX model. +The script loads the Qwen2.5 model and runs inference on a given prompt using +the ChatML format (<|im_start|> / <|im_end|> special tokens). + +Model: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct + +Exporting Qwen2.5 model to ONNX: + +1. Install the required dependencies: + + pip install optimum[exporters] torch transformers + +2. Export the model to ONNX: + + optimum-cli export onnx --model Qwen/Qwen2.5-0.5B-Instruct --task causal-lm qwen2.5_instruct_onnx/ + + +Run the script: +1. Install the required dependencies: + + pip install numpy + +2. Run the script: + + python qwen_inference.py --model= \ + --tokenizer_path= \ + --prompt="What is OpenCV?" +''' + +import numpy as np +import argparse +import cv2 as cv + +def parse_args(): + parser = argparse.ArgumentParser(description='Use this script to run Qwen2.5 inference in OpenCV', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--model', type=str, required=True, help='Path to Qwen2.5 ONNX model file.') + parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to Qwen2.5 tokenizer config.json.') + parser.add_argument('--prompt', type=str, default='What is OpenCV?', help='User prompt.') + parser.add_argument('--max_new_tokens', type=int, default=64, help='Maximum number of new tokens to generate.') + parser.add_argument('--seed', type=int, default=0, help='Random seed.') + return parser.parse_args() + +def stable_softmax(logits): + exp_logits = np.exp(logits - np.max(logits, axis=-1, keepdims=True)) + return exp_logits / np.sum(exp_logits, axis=-1, keepdims=True) + +def build_chatml_prompt(user_prompt): + '''Wrap user prompt in Qwen2.5 ChatML format.''' + return '<|im_start|>user\n' + user_prompt + '<|im_end|>\n<|im_start|>assistant\n' + +def qwen_inference(net, prompt, max_new_tokens, tokenizer): + + print("Inferencing Qwen2.5 model...") + + tokens = tokenizer.encode(prompt) + tokens = np.array(tokens, dtype=np.int64).reshape(1, -1) + + # Qwen2.5 special token IDs + im_end_id = 151645 # <|im_end|> + eos_id = 151643 # <|endoftext|> + stop_ids = (im_end_id, eos_id) + + for _ in range(max_new_tokens): + seq_len = tokens.shape[1] + attention_mask = np.ones((1, seq_len), dtype=np.int64) + position_ids = np.arange(seq_len, dtype=np.int64).reshape(1, -1) + + net.setInput(tokens, 'input_ids') + net.setInput(attention_mask, 'attention_mask') + net.setInput(position_ids, 'position_ids') + logits = net.forward() # (1, seq_len, vocab_size) + logits = logits[:, -1, :] # take last token logits + + new_id = int(np.argmax(logits.reshape(-1))) + tokens = np.concatenate((tokens, np.array([[new_id]], dtype=np.int64)), axis=1) + + if new_id in stop_ids: + break + + return tokens + +if __name__ == '__main__': + + args = parse_args() + np.random.seed(args.seed) + + print("Preparing Qwen2.5 model...") + tokenizer = cv.dnn.Tokenizer.load(args.tokenizer_path) + + net = cv.dnn.readNetFromONNX(args.model, cv.dnn.ENGINE_NEW) + + chatml_prompt = build_chatml_prompt(args.prompt) + print(f"Prompt:\n{chatml_prompt}") + + tokens = qwen_inference(net, chatml_prompt, args.max_new_tokens, tokenizer) + response = tokenizer.decode(tokens[0].tolist()) + print(f"Response:\n{response}")