mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge pull request #28781 from varun-jaiswal17:work-next
Add Qwen2.5 tokenizer support for dnn #28781 OpenCV extra: https://github.com/opencv/opencv_extra/pull/1334 Extended the dnn tokenizer support to qwen2.5 tokenization. - Add QWEN2_5 pre-tokenizer regex pattern to utils.hpp - Generalised buildTokenizerGPT to buildTokenizerFromJson to handle gpt2/gpt4/ and qwen2.5 - Add qwen2/qwen2.5 model type support with special token handling - Add Qwen2.5 tests - Add end-to-end qwen_inference script for Qwen2.5 ONNX model ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Varun Jaiswal
GitHub
@@ -20,7 +20,8 @@ static std::unordered_map<std::string, ImplRegestry>& tokenizerRegistry() {
|
||||
return reg;
|
||||
}
|
||||
|
||||
CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path);
|
||||
CoreBPE buildTokenizerFromJson(const std::string& model_type, const std::string& json_path,
|
||||
std::unordered_set<std::string>* outSpecial = nullptr);
|
||||
|
||||
struct Tokenizer::Impl {
|
||||
virtual ~Impl() {}
|
||||
@@ -30,13 +31,16 @@ struct Tokenizer::Impl {
|
||||
|
||||
struct BpeTokenizerImpl : public Tokenizer::Impl {
|
||||
Ptr<CoreBPE> coreBPE;
|
||||
std::unordered_set<std::string> allowedSpecial;
|
||||
|
||||
explicit BpeTokenizerImpl(CoreBPE core)
|
||||
: coreBPE(makePtr<CoreBPE>(std::move(core))) {}
|
||||
explicit BpeTokenizerImpl(CoreBPE core,
|
||||
std::unordered_set<std::string> special = {})
|
||||
: coreBPE(makePtr<CoreBPE>(std::move(core)))
|
||||
, allowedSpecial(std::move(special)) {}
|
||||
|
||||
std::vector<int> encode(const std::string& text) override {
|
||||
CV_Assert(coreBPE);
|
||||
std::vector<uint32_t> tok = coreBPE->encode(text, {}).first;
|
||||
std::vector<uint32_t> tok = coreBPE->encode(text, allowedSpecial).first;
|
||||
return std::vector<int>(tok.begin(), tok.end());
|
||||
}
|
||||
|
||||
@@ -60,12 +64,15 @@ static void registerDefaultTokenizers() {
|
||||
std::string tok_json = dir + "tokenizer.json";
|
||||
|
||||
CoreBPE core;
|
||||
std::unordered_set<std::string> special;
|
||||
if (model_type == "gpt2" || model_type == "gpt4") {
|
||||
core = buildTokenizerGPT(model_type, tok_json);
|
||||
core = buildTokenizerFromJson(model_type, tok_json);
|
||||
} else if (model_type == "qwen2" || model_type == "qwen2.5") {
|
||||
core = buildTokenizerFromJson(model_type, tok_json, &special);
|
||||
} else {
|
||||
CV_Error(cv::Error::StsError, "Unsupported model_type for BPE: " + model_type);
|
||||
}
|
||||
return makePtr<BpeTokenizerImpl>(std::move(core));
|
||||
return makePtr<BpeTokenizerImpl>(std::move(core), std::move(special));
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -82,7 +89,8 @@ std::string Tokenizer::decode(const std::vector<int>& tokens) {
|
||||
return impl_->decode(tokens);
|
||||
};
|
||||
|
||||
CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path) {
|
||||
CoreBPE buildTokenizerFromJson(const std::string& model_type, const std::string& json_path,
|
||||
std::unordered_set<std::string>* outSpecial) {
|
||||
cv::FileStorage fs(json_path, cv::FileStorage::READ | cv::FileStorage::FORMAT_JSON);
|
||||
if (!fs.isOpened())
|
||||
CV_Error(cv::Error::StsError, "Failed to open tokenizer.json: " + json_path);
|
||||
@@ -99,9 +107,11 @@ CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json
|
||||
skip_tokens.insert("<|endoftext|>");
|
||||
} else if (model_type == "gpt4" || model_type == "cl100k_base") {
|
||||
pattern = CL100K_BASE;
|
||||
} else if (model_type == "qwen2" || model_type == "qwen2.5") {
|
||||
pattern = QWEN2_5;
|
||||
} else {
|
||||
CV_Error(cv::Error::StsError,
|
||||
"Unsupported model_type: " + model_type + " (expected gpt2/r50k_base or gpt4/cl100k_base)");
|
||||
"Unsupported model_type: " + model_type + " (expected gpt2/r50k_base, gpt4/cl100k_base, or qwen2/qwen2.5)");
|
||||
}
|
||||
|
||||
auto token_to_bytes = [&](const std::string& token_utf8) -> std::vector<uint8_t> {
|
||||
@@ -139,6 +149,7 @@ CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json
|
||||
if (special && id >= 0 && !content.empty()) {
|
||||
specialTokens.emplace(content, (uint32_t)id);
|
||||
if (id > max_id) max_id = id;
|
||||
if (outSpecial) outSpecial->insert(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,5 +28,8 @@ static const std::string R50K_UTF8 = "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}
|
||||
// The resulting C++ pattern is compatible with std::regex and the tokenizer's Unicode handling logic.
|
||||
static const std::string CL100K_BASE = R"CL100K('(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s)CL100K";
|
||||
|
||||
// Qwen2.5 pre-tokenizer split pattern (from tokenizer.json)
|
||||
static const std::string QWEN2_5 = R"QWEN('(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+)QWEN";
|
||||
|
||||
}}
|
||||
#endif
|
||||
|
||||
@@ -96,4 +96,59 @@ TEST(Tokenizer_BPE, CatastrophicallyRepetitive_GPT2) {
|
||||
EXPECT_EQ(with_newline, gpt2_tok.decode(gpt2_tok.encode(with_newline)));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Qwen2.5 tests ----
|
||||
// Ground truth generated with:
|
||||
// from transformers import AutoTokenizer
|
||||
// tok = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B")
|
||||
// tok.encode(text)
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_English) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
EXPECT_EQ(tok.encode("Hello world"), (std::vector<int>{9707, 1879}));
|
||||
}
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Chinese) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
// 你好世界
|
||||
EXPECT_EQ(tok.encode("\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xb8\x96\xe7\x95\x8c"),
|
||||
(std::vector<int>{108386, 99489}));
|
||||
}
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Code) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
EXPECT_EQ(tok.encode("def hello(): print('hello')"),
|
||||
(std::vector<int>{750, 23811, 4555, 1173, 492, 14990, 863}));
|
||||
}
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Numbers) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
EXPECT_EQ(tok.encode("2024"), (std::vector<int>{17, 15, 17, 19}));
|
||||
}
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_SpecialTokens) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
// <|im_start|>user\nHello<|im_end|>
|
||||
EXPECT_EQ(tok.encode("<|im_start|>user\nHello<|im_end|>"),
|
||||
(std::vector<int>{151644, 872, 198, 9707, 151645}));
|
||||
}
|
||||
|
||||
TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Roundtrip) {
|
||||
std::string model = _tf("qwen2.5/config.json");
|
||||
Tokenizer tok = Tokenizer::load(model);
|
||||
std::vector<std::string> cases = {
|
||||
"Hello world",
|
||||
"def hello(): print('hello')",
|
||||
"2024",
|
||||
};
|
||||
for (const auto& text : cases) {
|
||||
EXPECT_EQ(tok.decode(tok.encode(text)), text);
|
||||
}
|
||||
}
|
||||
|
||||
}}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
'''
|
||||
This is a sample script to run Qwen2.5 inference in OpenCV using ONNX model.
|
||||
The script loads the Qwen2.5 model and runs inference on a given prompt using
|
||||
the ChatML format (<|im_start|> / <|im_end|> special tokens).
|
||||
|
||||
Model: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct
|
||||
|
||||
Exporting Qwen2.5 model to ONNX:
|
||||
|
||||
1. Install the required dependencies:
|
||||
|
||||
pip install optimum[exporters] torch transformers
|
||||
|
||||
2. Export the model to ONNX:
|
||||
|
||||
optimum-cli export onnx --model Qwen/Qwen2.5-0.5B-Instruct --task causal-lm qwen2.5_instruct_onnx/
|
||||
|
||||
|
||||
Run the script:
|
||||
1. Install the required dependencies:
|
||||
|
||||
pip install numpy
|
||||
|
||||
2. Run the script:
|
||||
|
||||
python qwen_inference.py --model=<path-to-onnx-model> \
|
||||
--tokenizer_path=<path-to-qwen2.5-config.json> \
|
||||
--prompt="What is OpenCV?"
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import argparse
|
||||
import cv2 as cv
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Use this script to run Qwen2.5 inference in OpenCV',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--model', type=str, required=True, help='Path to Qwen2.5 ONNX model file.')
|
||||
parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to Qwen2.5 tokenizer config.json.')
|
||||
parser.add_argument('--prompt', type=str, default='What is OpenCV?', help='User prompt.')
|
||||
parser.add_argument('--max_new_tokens', type=int, default=64, help='Maximum number of new tokens to generate.')
|
||||
parser.add_argument('--seed', type=int, default=0, help='Random seed.')
|
||||
return parser.parse_args()
|
||||
|
||||
def stable_softmax(logits):
|
||||
exp_logits = np.exp(logits - np.max(logits, axis=-1, keepdims=True))
|
||||
return exp_logits / np.sum(exp_logits, axis=-1, keepdims=True)
|
||||
|
||||
def build_chatml_prompt(user_prompt):
|
||||
'''Wrap user prompt in Qwen2.5 ChatML format.'''
|
||||
return '<|im_start|>user\n' + user_prompt + '<|im_end|>\n<|im_start|>assistant\n'
|
||||
|
||||
def qwen_inference(net, prompt, max_new_tokens, tokenizer):
|
||||
|
||||
print("Inferencing Qwen2.5 model...")
|
||||
|
||||
tokens = tokenizer.encode(prompt)
|
||||
tokens = np.array(tokens, dtype=np.int64).reshape(1, -1)
|
||||
|
||||
# Qwen2.5 special token IDs
|
||||
im_end_id = 151645 # <|im_end|>
|
||||
eos_id = 151643 # <|endoftext|>
|
||||
stop_ids = (im_end_id, eos_id)
|
||||
|
||||
for _ in range(max_new_tokens):
|
||||
seq_len = tokens.shape[1]
|
||||
attention_mask = np.ones((1, seq_len), dtype=np.int64)
|
||||
position_ids = np.arange(seq_len, dtype=np.int64).reshape(1, -1)
|
||||
|
||||
net.setInput(tokens, 'input_ids')
|
||||
net.setInput(attention_mask, 'attention_mask')
|
||||
net.setInput(position_ids, 'position_ids')
|
||||
logits = net.forward() # (1, seq_len, vocab_size)
|
||||
logits = logits[:, -1, :] # take last token logits
|
||||
|
||||
new_id = int(np.argmax(logits.reshape(-1)))
|
||||
tokens = np.concatenate((tokens, np.array([[new_id]], dtype=np.int64)), axis=1)
|
||||
|
||||
if new_id in stop_ids:
|
||||
break
|
||||
|
||||
return tokens
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
args = parse_args()
|
||||
np.random.seed(args.seed)
|
||||
|
||||
print("Preparing Qwen2.5 model...")
|
||||
tokenizer = cv.dnn.Tokenizer.load(args.tokenizer_path)
|
||||
|
||||
net = cv.dnn.readNetFromONNX(args.model, cv.dnn.ENGINE_NEW)
|
||||
|
||||
chatml_prompt = build_chatml_prompt(args.prompt)
|
||||
print(f"Prompt:\n{chatml_prompt}")
|
||||
|
||||
tokens = qwen_inference(net, chatml_prompt, args.max_new_tokens, tokenizer)
|
||||
response = tokenizer.decode(tokens[0].tolist())
|
||||
print(f"Response:\n{response}")
|
||||
Reference in New Issue
Block a user