potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 15:23:05 +04:00
Code Packages Releases Wiki Activity

Merge pull request #27534 from JorgeV92:gsoc2025-tokenizer

Browse Source 
GSoC 2025: Add Tokenizer Support to DNN Module #27534

merge with https://github.com/opencv/opencv_extra/pull/1276

### Summary
This pull request introduces initial support for a tokenizer module under `modules/dnn/src/tokenizer` as part of Google Summer of Code 2025 (Project: Tokenization for OpenCV DNN).

### Status
- [x] Project structure in place
- [x] Initial BPE tokenizer loading
- [x] Regex splitting (in progress)
- [x] Encoding logic for GPT-2 tokenizer (in progress)
- [ ] Documentation (to be improved)

### Goals
The goal is to support Hugging Face-compatible tokenization (e.g., GPT-2) natively in C++ to be integrated with DNN inference pipelines. 

The core pipeline lives in `dnn/src/tokenizer/core_bpe.hpp` and `dnn/src/tokenizer/encoding.hpp`. For Unicode handling I’m using `dnn/src/tokenizer/unicode.hpp`, which is adapted from llama.cpp.


### Feedback
Please share early feedback on:
- General design structure
- Integration strategy with `dnn`
- Code organization or naming conventions

### Reference
Project: https://summerofcode.withgoogle.com/programs/2025/projects/79SW6eNK
This commit is contained in:
Jorge Velez
2026-04-06 02:46:13 -05:00
committed by GitHub
parent d0313879da
commit a49a293d3c
14 changed files with 8867 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/src/persistence_json.cpp
+5 -6
View File
@@ -425,13 +425,12 @@ public:
++ptr;
CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP();
key_name += *ptr;
++ptr;
CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP();
} else {
++ptr;
CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP();
if (*ptr != '\\' && *ptr != '"') key_name += *ptr;
} else if (*ptr != '"') {
// normal byte: append current, do NOT skip ahead first
key_name += *ptr;
}
++ptr;
CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP();
} while( cv_isprint(*ptr) && *ptr != '"' );
if( *ptr != '"' )
modules/dnn/CMakeLists.txt
+9 -2
View File
@@ -15,10 +15,8 @@ ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)
include(${CMAKE_CURRENT_LIST_DIR}/cmake/plugin.cmake)
ocv_option(OPENCV_DNN_OPENCL "Build with OpenCL support" HAVE_OPENCL AND NOT APPLE)
if(OPENCV_DNN_OPENCL AND HAVE_OPENCL)
@@ -442,6 +440,15 @@ if(NOT BUILD_PROTOBUF)
endif()
endif()
# Include Tokenizer
file(GLOB extra_tokenizer_src
"${CMAKE_CURRENT_LIST_DIR}/src/tokenizer/*.cpp"
)
list(APPEND include_dirs
"${CMAKE_CURRENT_LIST_DIR}/src/tokenizer"
)
ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs} ${webnn_srcs} ${extra_tokenizer_src})
set(perf_path "${CMAKE_CURRENT_LIST_DIR}/perf")
file(GLOB_RECURSE perf_srcs "${perf_path}/*.cpp")
file(GLOB_RECURSE perf_hdrs "${perf_path}/*.hpp" "${perf_path}/*.h")
modules/dnn/include/opencv2/dnn/dnn.hpp
+54
View File
@@ -2099,6 +2099,60 @@ public:
CV_WRAP int getMaxCandidates() const;
};
/**
* @brief High-level tokenizer wrapper for DNN usage.
*
* Provides a simple API to encode and decode tokens for LLMs.
* Models are loaded via Tokenizer::load().
*
* @code
* using namespace cv::dnn;
* Tokenizer tok = Tokenizer::load("/path/to/model/");
* std::vector<int> ids = tok.encode("hello world");
* std::string text = tok.decode(ids);
* @endcode
*/
class CV_EXPORTS_W_SIMPLE Tokenizer {
public:
/**
* @brief Construct a tokenizer with a given method default BPE.
* For BPE method you normally call Tokenizer::load() to initialize model data.
*/
Tokenizer();
/**
* @brief Load a tokenizer from a model directory.
*
* Expects the directory to contain:
* - `config.json` with field `model_type` with value "gpt2" or "gpt4".
* - `tokenizer.json` produced by the corresponding model family.
*
* The argument is a path prefix; this function concatenates file
* names directly (e.g. `model_dir` + "config.json"), so `model_dir` must
* end with an appropriate path separator.
*
* @param model_config Path to config.json for model.
* @return A Tokenizer ready for use. Throws cv::Exception if files are missing or `model_type` is unsupported.
*/
CV_WRAP static Tokenizer load(CV_WRAP_FILE_PATH const std::string& model_config);
/**
* @brief Encode UTF-8 text to token ids (special tokens currently disabled).
*
* Calls the underlying `CoreBPE::encode` with an empty allowed-special set.
*
* @param text UTF-8 input string.
* @return Vector of token ids (32-bit ids narrowed to int for convenience).
*/
CV_WRAP std::vector<int> encode(const std::string& text);
CV_WRAP std::string decode(const std::vector<int>& tokens);
struct Impl;
private:
Ptr<Impl> impl_;
};
//! @}
CV__DNN_INLINE_NS_END
}
modules/dnn/src/tokenizer/core_bpe.cpp
+260
View File
@@ -0,0 +1,260 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "unicode.hpp"
#include "utils.hpp"
#include "core_bpe.hpp"
#include <regex>
namespace cv { namespace dnn {
CV__DNN_INLINE_NS_BEGIN
static constexpr std::uint32_t RANK_MAX = std::numeric_limits<std::uint32_t>::max();
static constexpr std::size_t SZ_MAX = std::numeric_limits<std::size_t>::max();
static std::uint32_t maybeGetRank(const ByteVecRankMap& ranks, const std::vector<std::uint8_t>& key) {
auto it = ranks.find(key);
return it == ranks.end() ? RANK_MAX : it->second;
}
/*
* This function takes a sequence of bytes and a map of
* mergeable byte pairs (with associated ranks/merge priority), and repeatedly merges the
* lowest-ranked adjacent pairs until no further merges are possible. The result is a vector
* describing the split points and ranks of the final token boundaries.
*
* This function is closely modeled after the original Rust implementation in OpenAI's tiktoken
* library, which can be found here:
* https://github.com/openai/tiktoken/blob/4560a8896f5fb1d35c6f8fd6eee0399f9a1a27ca/src/lib.rs#L17-L73
*
*/
std::vector<std::pair<std::size_t, std::uint32_t>> bytePairMerge(const ByteVecRankMap& ranks,
const std::vector<std::uint8_t>& piece) {
std::vector<std::pair<std::size_t, std::uint32_t>> parts;
parts.reserve(piece.size() + 1);
std::pair<std::uint32_t, std::size_t> minRank{RANK_MAX, SZ_MAX};
for (std::size_t i = 0; i+1 < piece.size(); ++i) {
std::vector<std::uint8_t> key(piece.begin() + i, piece.begin() + i + 2);
std::uint32_t r = maybeGetRank(ranks, key);
if (r < minRank.first) minRank = {r, i};
parts.emplace_back(i, r);
}
parts.emplace_back(piece.size() - 1, RANK_MAX);
parts.emplace_back(piece.size(), RANK_MAX);
auto getRank = [&](const std::vector<std::pair<std::size_t, std::uint32_t>>& p,
std::size_t idx) -> std::uint32_t {
if (idx + 3 < p.size()) {
std::size_t s = p[idx].first;
std::size_t e = p[idx+3].first;
std::vector<std::uint8_t> key(piece.begin()+s, piece.begin()+e);
return maybeGetRank(ranks, key);
}
return RANK_MAX;
};
while (minRank.first != RANK_MAX) {
std::size_t i = minRank.second;
if (i > 0) parts[i-1].second = getRank(parts, i-1);
if (i < parts.size()-2) parts[i].second = getRank(parts, i);
parts.erase(parts.begin() + static_cast<long>(i+1));
minRank = {RANK_MAX, SZ_MAX};
for (std::size_t j = 0; j + 1 < parts.size(); ++j) {
std::uint32_t r = parts[j].second;
if (r < minRank.first) minRank = {r, j};
}
}
return parts;
}
std::vector<std::uint32_t> bytePairEncode(const std::vector<std::uint8_t>& piece,
const ByteVecRankMap& ranks) {
if (piece.size() == 1) {
auto it = ranks.find(piece);
return it == ranks.end() ? std::vector<std::uint32_t>{} : std::vector<std::uint32_t>{it->second};
}
auto merged = bytePairMerge(ranks, piece);
std::vector<std::uint32_t> out;
out.reserve(merged.size()-1);
for (std::size_t i = 0; i+1 < merged.size(); ++i) {
std::size_t s = merged[i].first, e = merged[i+1].first;
out.push_back(ranks.at(std::vector<std::uint8_t>(piece.begin()+s, piece.begin()+e)));
}
return out;
}
std::string CoreBPE::makeSpecialPattern(const std::unordered_map<std::string, std::uint32_t>& special) {
static const std::string meta = R"([.^$|()\[\]{}*+?\\])";
std::string pat;
pat.reserve(special.size() * 10);
bool first = true;
for (auto const& kv : special) {
if (!first) pat.push_back('|');
first = false;
// Escape each character in the token
for (char c : kv.first) {
if (meta.find(c) != std::string::npos)
pat.push_back('\\');
pat.push_back(c);
}
}
return pat;
}
CoreBPE::CoreBPE()
: encoder_(),
specialEncoder_(),
decoder_(),
specialDecoder_(),
pattern_(),
specialPattern_(),
sortedTokenBytes_()
{}
CoreBPE::CoreBPE(ByteVecRankMap encoder,
std::unordered_map<std::string, std::uint32_t> specialEncoder,
const std::string& pattern)
: encoder_(std::move(encoder)),
specialEncoder_(std::move(specialEncoder)),
decoder_(), specialDecoder_(),
pattern_(pattern), specialPattern_(makeSpecialPattern(specialEncoder_)),
sortedTokenBytes_() {
for (auto& kv : encoder_)
decoder_.emplace(kv.second, kv.first);
for (auto& kv : specialEncoder_)
specialDecoder_.emplace(kv.second, std::vector<std::uint8_t>(kv.first.begin(), kv.first.end()));
for (const auto& kv : specialDecoder_) {
std::string str(kv.second.begin(), kv.second.end());
specialStringDecoder_.emplace(str, kv.first);
}
sortedTokenBytes_.reserve(encoder_.size());
for (auto& kv : encoder_) sortedTokenBytes_.push_back(kv.first);
std::sort(sortedTokenBytes_.begin(), sortedTokenBytes_.end());
}
std::optional<std::vector<std::uint8_t>>
CoreBPE::decodeBytes(const std::vector<std::uint32_t>& tokens) const {
std::vector<std::uint8_t> out;
out.reserve(tokens.size() * 2);
for (std::uint32_t t : tokens) {
const std::vector<std::uint8_t>* tokenBytes = nullptr;
auto it = decoder_.find(t);
if (it != decoder_.end()) {
tokenBytes = &it->second;
} else {
auto sit = specialDecoder_.find(t);
if (sit != specialDecoder_.end()) {
tokenBytes = &sit->second;
} else {
return std::nullopt;
}
}
out.insert(out.end(), tokenBytes->begin(), tokenBytes->end());
}
return out;
}
/*
* This function tokenizes input text by handling special tokens and applying Byte Pair Encoding (BPE)
* to ordinary text segments. It searches for allowed special tokens, processes ordinary text with BPE,
* and emits a sequence of token IDs along with the count of tokens in the final processed segment.
*
* The logic and structure of this function are closely modeled after the original Rust implementation
* in OpenAI's tiktoken library, which can be found here:
* https://github.com/openai/tiktoken/blob/4560a8896f5fb1d35c6f8fd6eee0399f9a1a27ca/src/lib.rs#L234-L288
*
*/
std::pair<std::vector<std::uint32_t>, std::size_t>
CoreBPE::encode(const std::string& text,
const std::unordered_set<std::string>& allowedSpecial) const
{
std::vector<std::uint32_t> ret;
std::size_t last_piece_token_len = 0;
size_t start = 0;
// Use unicode_regex_split to find all special token matches in the text
std::vector<std::string> special_regexes = { specialPattern_ };
std::vector<std::pair<size_t, size_t>> special_matches;
// Find all special token matches and their positions
{
std::regex special_re(specialPattern_);
auto words_begin = std::sregex_iterator(text.begin(), text.end(), special_re);
auto words_end = std::sregex_iterator();
for (auto it = words_begin; it != words_end; ++it) {
std::string match_str = it->str();
if (allowedSpecial.count(match_str)) {
special_matches.emplace_back(it->position(), it->position() + match_str.size());
}
}
}
size_t match_idx = 0;
while (start < text.size()) {
// Find the next allowed special token
size_t next_special_start = std::string::npos;
size_t next_special_end = std::string::npos;
std::string matched_special;
while (match_idx < special_matches.size()) {
size_t s = special_matches[match_idx].first;
size_t e = special_matches[match_idx].second;
std::string candidate = text.substr(s, e - s);
if (allowedSpecial.count(candidate) && s >= start) {
next_special_start = s;
next_special_end = e;
matched_special = candidate;
break;
}
++match_idx;
}
size_t end = (next_special_start != std::string::npos) ? next_special_start : text.size();
// Tokenize the ordinary segment [start, end)
if (end > start) {
std::string segment = text.substr(start, end - start);
std::vector<std::string> regexes = { pattern_ };
auto splits = unicode_regex_split(segment, regexes);
for (auto& subUtf8 : splits) {
std::vector<std::uint8_t> piece(subUtf8.begin(), subUtf8.end());
auto it = encoder_.find(piece);
if (it != encoder_.end()) {
last_piece_token_len = 1;
ret.push_back(it->second);
} else {
auto tokens = bytePairEncode(piece, encoder_);
last_piece_token_len = tokens.size();
ret.insert(ret.end(), tokens.begin(), tokens.end());
}
}
}
// If we found a special token, add it and advance
if (next_special_start != std::string::npos) {
std::uint32_t token = specialEncoder_.at(matched_special);
ret.push_back(token);
start = next_special_end;
last_piece_token_len = 0;
++match_idx;
} else {
break;
}
}
return { ret, last_piece_token_len };
}
CV__DNN_INLINE_NS_END
}}
modules/dnn/src/tokenizer/core_bpe.hpp
+166
View File
@@ -0,0 +1,166 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// Portions of this file are inspired by or adapted from the tiktoken Rust
// implementation:
// https://github.com/openai/tiktoken/blob/main/src/lib.rs
//
// This file is part of the OpenCV DNN module for tokenization.
//
////////////////////////////////////////////////////////////////////////////////////////*/
/*M///////////////////////////////////////////////////////////////////////////////////////
// MIT License
//
// Copyright (c) 2022 OpenAI, Shantanu Jain
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////////////*/
#ifndef __OPENCV_DNN_TOKENIZER_CORE_BPE_HPP__
#define __OPENCV_DNN_TOKENIZER_CORE_BPE_HPP__
#include <opencv2/core.hpp>
#include <opencv2/dnn/dnn.hpp>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <optional>
#include <string>
#include <cstdint>
namespace cv { namespace dnn {
CV__DNN_INLINE_NS_BEGIN
/**
* @brief Hasher for byte vectors to enable use in unordered_map.
*
* Computes a simple rolling hash over the bytes.
*
* @note Intended for internal use with Byte Pair Encoding (BPE) maps.
*/
struct ByteVecHash {
std::size_t operator()(const std::vector<std::uint8_t>& v) const noexcept {
std::size_t h = 0;
for (auto b : v) h = h * 31u + static_cast<std::size_t>(b);
return h;
}
};
/**
* @brief Map from raw byte-sequence tokens to their merge rank / token id.
*
* Keys are byte sequences (not Unicode code points). Values are the
* token ids/ranks used by the BPE encoder/decoder.
*/
using ByteVecRankMap = std::unordered_map<std::vector<std::uint8_t>, std::uint32_t, ByteVecHash>;
/**
* @brief Merge-adjacent byte pairs by increasing rank until no mergeable pair remains.
*
* Scans adjacent byte pairs in @p piece, repeatedly splicing out the minimal-rank
* pair (highest merge priority) and updating neighboring ranks, until no pair
* appears in @p ranks. Returns the final segmentation as a list of split
* boundaries and their ranks.
*
* @param ranks Map of mergeable byte pairs (key = 2+ byte token, value = rank/id).
* @param piece Input bytes for a single text span (UTF-8 already flattened to bytes).
* @return Vector of (start_index, rank) pairs describing token boundaries after merging.
* The last element is a sentinel boundary at @c piece.size().
*
* @note This is the low-level merge routine used by BPE; it does not translate
* segments into ids. For that, see bytePairEncode().
* @see bytePairEncode
*/
CV_EXPORTS std::vector<std::pair<std::size_t, std::uint32_t>> bytePairMerge(const ByteVecRankMap& ranks,
const std::vector<std::uint8_t>& piece);
/**
* @brief Encode a byte sequence into token ids using BPE merge rules.
*
* If @p piece is a single byte present in @p ranks, returns that id directly.
* Otherwise, runs the merge loop (bytePairMerge) and maps each resulting segment
* to its id via @p ranks.
*
* @param piece Input bytes (one text span already split by regex).
* @param ranks Map from byte-sequence tokens to ids (includes all singletons 0..255).
* @return Token ids produced by BPE for the given @p piece.
*
* @see bytePairMerge
*/
CV_EXPORTS std::vector<std::uint32_t> bytePairEncode(const std::vector<std::uint8_t>& piece,
const ByteVecRankMap& ranks);
/**
* @brief Core Byte Pair Encoding (BPE) engine (mergeable-ranks model).
*
* Encodes and decodes tokens at the byte level (UTF-8 input is split to bytes),
* with optional support for special tokens that are matched by a separate regex.
*
* The implementation follows the structure of OpenAI’s tiktoken encoders.
*/
class CoreBPE {
public:
CoreBPE();
explicit CoreBPE(ByteVecRankMap encoder,
std::unordered_map<std::string, std::uint32_t> specialEncoder,
const std::string& pattern);
/**
* @brief Encode text with optional special tokens.
*
* Scans @p text for allowed special tokens, emits them as single ids,
* and BPE-encodes the intervening ordinary segments.
*
* @param text UTF-8 input.
* @param allowedSpecial Set of literal special-token strings that may appear and be emitted.
* @return Pair @c (tokens, last_piece_token_len) where:
* - @c tokens is the full token sequence,
* - @c last_piece_token_len is the number of tokens produced by the final ordinary segment
* (0 if the text ended with a special token).
*/
std::pair<std::vector<std::uint32_t>, std::size_t> encode(const std::string& text,
const std::unordered_set<std::string>& allowedSpecial) const;
/**
* @brief Decode a sequence of token ids into raw bytes.
*
* Looks up ids in either the mergeable-token or special-token decoders.
*
* @param tokens Token ids.
* @return Decoded bytes on success, or @c std::nullopt if any id is unknown.
*/
std::optional<std::vector<std::uint8_t>> decodeBytes(const std::vector<std::uint32_t>& tokens) const;
private:
ByteVecRankMap encoder_;
std::unordered_map<std::string, std::uint32_t> specialEncoder_;
std::unordered_map<std::uint32_t, std::vector<std::uint8_t>> decoder_;
std::unordered_map<std::uint32_t, std::vector<std::uint8_t>> specialDecoder_;
std::unordered_map<std::string, std::uint32_t> specialStringDecoder_;
std::string pattern_;
std::string specialPattern_;
std::vector<std::vector<std::uint8_t>> sortedTokenBytes_;
std::string makeSpecialPattern(const std::unordered_map<std::string, std::uint32_t>& special);
};
CV__DNN_INLINE_NS_END
}}
#endif
modules/dnn/src/tokenizer/tokenizer.cpp
+174
View File
@@ -0,0 +1,174 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include <opencv2/dnn/dnn.hpp>
#include "utils.hpp"
#include "unicode.hpp"
#include "core_bpe.hpp"
#include <functional>
namespace cv { namespace dnn {
CV__DNN_INLINE_NS_BEGIN
// Registry of implementations (method -> methodImpl)
using ImplRegestry = std::function<Ptr<Tokenizer::Impl>(const FileStorage& cfg, const std::string& dir)>;
static std::unordered_map<std::string, ImplRegestry>& tokenizerRegistry() {
static std::unordered_map<std::string, ImplRegestry> reg;
return reg;
}
CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path);
struct Tokenizer::Impl {
virtual ~Impl() {}
virtual std::vector<int> encode(const std::string& text) = 0;
virtual std::string decode(const std::vector<int>& tokens) = 0;
};
struct BpeTokenizerImpl : public Tokenizer::Impl {
Ptr<CoreBPE> coreBPE;
explicit BpeTokenizerImpl(CoreBPE core)
: coreBPE(makePtr<CoreBPE>(std::move(core))) {}
std::vector<int> encode(const std::string& text) override {
CV_Assert(coreBPE);
std::vector<uint32_t> tok = coreBPE->encode(text, {}).first;
return std::vector<int>(tok.begin(), tok.end());
}
std::string decode(const std::vector<int>& tokens) override {
CV_Assert(coreBPE);
std::vector<uint32_t> t32(tokens.begin(), tokens.end());
auto opt_bytes = coreBPE->decodeBytes(t32);
if (!opt_bytes)
CV_Error(cv::Error::StsError, "Invalid decode.");
const auto& bytes = *opt_bytes;
return std::string(reinterpret_cast<const char*>(bytes.data()), bytes.size());
}
};
static void registerDefaultTokenizers() {
auto& reg = tokenizerRegistry();
if (reg.find("BPE") == reg.end()) {
reg["BPE"] = [](const FileStorage& cfg, const std::string& dir) -> Ptr<Tokenizer::Impl> {
std::string model_type;
cfg["model_type"] >> model_type;
std::string tok_json = dir + "tokenizer.json";
CoreBPE core;
if (model_type == "gpt2" || model_type == "gpt4") {
core = buildTokenizerGPT(model_type, tok_json);
} else {
CV_Error(cv::Error::StsError, "Unsupported model_type for BPE: " + model_type);
}
return makePtr<BpeTokenizerImpl>(std::move(core));
};
}
}
Tokenizer::Tokenizer() : impl_(nullptr) {}
std::vector<int> Tokenizer::encode(const std::string& text) {
if (!impl_) CV_Error(cv::Error::StsError, "Tokenizer impl null");
return impl_->encode(text);
}
std::string Tokenizer::decode(const std::vector<int>& tokens) {
if (!impl_) CV_Error(cv::Error::StsError, "Tokenizer impl null");
return impl_->decode(tokens);
};
CoreBPE buildTokenizerGPT(const std::string& model_type, const std::string& json_path) {
cv::FileStorage fs(json_path, cv::FileStorage::READ | cv::FileStorage::FORMAT_JSON);
if (!fs.isOpened())
CV_Error(cv::Error::StsError, "Failed to open tokenizer.json: " + json_path);
cv::FileNode model = fs["model"];
CV_CheckFalse(model.empty(), "tokenizer.json missing 'model'");
cv::FileNode vocab = model["vocab"];
CV_CheckFalse(vocab.empty(), "tokenizer.json missing model.vocab");
std::string pattern;
std::unordered_set<std::string> skip_tokens;
if (model_type == "gpt2" || model_type == "r50k_base") {
pattern = R50K_UTF8;
skip_tokens.insert("<|endoftext|>");
} else if (model_type == "gpt4" || model_type == "cl100k_base") {
pattern = CL100K_BASE;
} else {
CV_Error(cv::Error::StsError,
"Unsupported model_type: " + model_type + " (expected gpt2/r50k_base or gpt4/cl100k_base)");
}
auto token_to_bytes = [&](const std::string& token_utf8) -> std::vector<uint8_t> {
std::vector<std::uint8_t> out;
auto cps = unicode_cpts_from_utf8(token_utf8);
out.reserve(cps.size());
for (uint32_t cp : cps) {
const std::string one = unicode_cpt_to_utf8(cp);
out.push_back(unicode_utf8_to_byte(one));
}
return out;
};
ByteVecRankMap mergeableRanks;
mergeableRanks.reserve((size_t)vocab.size());
int max_id = -1;
for (cv::FileNodeIterator it = vocab.begin(); it != vocab.end(); ++it) {
FileNode val = *it;
std::string token = val.name();
if (skip_tokens.find(token) != skip_tokens.end()) continue;
int id = (int)val;
mergeableRanks.emplace(token_to_bytes(token), (uint32_t)id);
if (id > max_id) max_id = id;
}
std::unordered_map<std::string, uint32_t> specialTokens;
FileNode added = fs["added_tokens"];
if (!added.empty()) {
for (auto it = added.begin(); it != added.end(); ++it) {
cv::FileNode t = *it;
bool special = false; t["special"] >> special;
int id = -1; t["id"] >> id;
std::string content; t["content"] >> content;
if (special && id >= 0 && !content.empty()) {
specialTokens.emplace(content, (uint32_t)id);
if (id > max_id) max_id = id;
}
}
}
return CoreBPE(std::move(mergeableRanks), std::move(specialTokens), pattern);
}
Tokenizer Tokenizer::load(const std::string& model_config) {
cv::FileStorage cfg(model_config, cv::FileStorage::READ | cv::FileStorage::FORMAT_JSON);
if (!cfg.isOpened())
CV_Error(cv::Error::StsError, "Could not open config.json: " + model_config);
std::string dir = model_config;
size_t pos = dir.find_last_of("/\\");
dir = (pos == std::string::npos) ? std::string() : dir.substr(0, pos + 1);
std::string methodType = "BPE";
if (!cfg["method"].empty())
cfg["method"] >> methodType;
registerDefaultTokenizers();
auto& reg = tokenizerRegistry();
auto it = reg.find(methodType);
if (it == reg.end())
CV_Error(cv::Error::StsError,
"Unsupported tokenizer method: '" + methodType + "'. Supported: BPE");
Tokenizer tok;
tok.impl_ = it->second(cfg, dir);
return tok;
}
CV__DNN_INLINE_NS_END
}}
modules/dnn/src/tokenizer/unicode-data.cpp
+7039
View File
File diff suppressed because it is too large Load Diff
modules/dnn/src/tokenizer/unicode-data.hpp
+65
View File
@@ -0,0 +1,65 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// Adapted from the llama.cpp Unicode functionality:
// https://github.com/ggml-org/llama.cpp/blob/master/src/unicode-data.cpp
//
// This file is part of the OpenCV DNN module for tokenization.
//
////////////////////////////////////////////////////////////////////////////////////////*/
/*M///////////////////////////////////////////////////////////////////////////////////////
// MIT License
//
// Copyright (c) 2023‑2024 The ggml authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////////////*/
#ifndef __OPENCV_DNN_TOKENIZER_UNICODE_DATA_HPP__
#define __OPENCV_DNN_TOKENIZER_UNICODE_DATA_HPP__
#include <cstdint>
#include <vector>
#include <unordered_map>
#include <unordered_set>
namespace cv { namespace dnn {
struct range_nfd {
uint32_t first;
uint32_t last;
uint32_t nfd;
};
static const uint32_t MAX_CODEPOINTS = 0x110000;
extern const std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;
extern const std::unordered_set<uint32_t> unicode_set_whitespace;
extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase;
extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase;
extern const std::initializer_list<range_nfd> unicode_ranges_nfd;
}}
#endif
modules/dnn/src/tokenizer/unicode.cpp
+774
View File
@@ -0,0 +1,774 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "unicode.hpp"
#include "unicode-data.hpp"
#include <algorithm>
#include <cassert>
#include <codecvt>
#include <cstddef>
#include <cstdint>
#include <locale>
#include <map>
#include <regex>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace cv { namespace dnn {
uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
assert(offset < utf8.size());
if (!(utf8[offset + 0] & 0x80)) {
auto result = utf8[offset + 0];
offset += 1;
return result;
}
if (!(utf8[offset + 0] & 0x40)) {
throw std::invalid_argument("invalid character");
}
if (!(utf8[offset + 0] & 0x20)) {
if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);
offset += 2;
return result;
}
if (!(utf8[offset + 0] & 0x10)) {
if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);
offset += 3;
return result;
}
if (!(utf8[offset + 0] & 0x08)) {
if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);
offset += 4;
return result;
}
throw std::invalid_argument("failed to convert utf8 to codepoint");
}
static std::vector<unicode_cpt_flags> unicode_cpt_flags_array() {
std::vector<unicode_cpt_flags> cpt_flags(MAX_CODEPOINTS, unicode_cpt_flags::UNDEFINED);
assert (unicode_ranges_flags.begin()[0].first == 0);
assert (unicode_ranges_flags.begin()[unicode_ranges_flags.size()-1].first == MAX_CODEPOINTS);
for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {
const auto range_ini = unicode_ranges_flags.begin()[i-1]; // codepoint_ini, flags
const auto range_end = unicode_ranges_flags.begin()[i]; // codepoint_end, flags
for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {
cpt_flags[cpt] = range_ini.second;
}
}
for (auto cpt : unicode_set_whitespace) {
cpt_flags[cpt].is_whitespace = true;
}
for (auto p : unicode_map_lowercase) {
cpt_flags[p.second].is_lowercase = true;
}
for (auto p : unicode_map_uppercase) {
cpt_flags[p.second].is_uppercase = true;
}
for (auto &range : unicode_ranges_nfd) { // start, last, nfd
cpt_flags[range.nfd].is_nfd = true;
}
return cpt_flags;
}
static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {
std::unordered_map<uint8_t, std::string> map;
for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
assert(0 <= ch && ch < 256);
map[ch] = unicode_cpt_to_utf8(ch);
}
for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
assert(0 <= ch && ch < 256);
map[ch] = unicode_cpt_to_utf8(ch);
}
for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
assert(0 <= ch && ch < 256);
map[ch] = unicode_cpt_to_utf8(ch);
}
auto n = 0;
for (int ch = 0; ch < 256; ++ch) {
if (map.find(ch) == map.end()) {
map[ch] = unicode_cpt_to_utf8(256 + n);
++n;
}
}
return map;
}
static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
std::unordered_map<std::string, uint8_t> map;
for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
assert(0 <= ch && ch < 256);
map[unicode_cpt_to_utf8(ch)] = ch;
}
for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
assert(0 <= ch && ch < 256);
map[unicode_cpt_to_utf8(ch)] = ch;
}
for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
assert(0 <= ch && ch < 256);
map[unicode_cpt_to_utf8(ch)] = ch;
}
auto n = 0;
for (int ch = 0; ch < 256; ++ch) {
if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) {
map[unicode_cpt_to_utf8(256 + n)] = ch;
++n;
}
}
return map;
}
static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
#if defined(__clang__)
// disable C++17 deprecation warning for std::codecvt_utf8
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wdeprecated-declarations"
#elif defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
#if defined(__clang__)
# pragma clang diagnostic pop
#elif defined(__GNUC__)
# pragma GCC diagnostic pop
#endif
return conv.from_bytes(s);
}
static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {
std::vector<std::string> bpe_encoded_words;
for (const auto & word : bpe_words) {
std::string text_utf;
auto utf_word = unicode_cpts_from_utf8(word);
for (size_t i = 0; i < utf_word.size(); ++i) {
text_utf += unicode_cpt_to_utf8(utf_word[i]);
}
std::string encoded_token;
for (char & c : text_utf) {
encoded_token += unicode_byte_to_utf8(c);
}
bpe_encoded_words.emplace_back(encoded_token);
}
return bpe_encoded_words;
}
// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
const auto cpts = unicode_cpts_from_utf8(text);
size_t start = 0;
for (auto offset : offsets) {
const size_t offset_ini = start;
const size_t offset_end = start + offset;
assert(offset_end <= cpts.size());
start = offset_end;
static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
auto _get_cpt = [&] (const size_t pos) -> uint32_t {
return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
};
auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {
return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};
};
size_t _prev_end = offset_ini;
auto _add_token = [&] (const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end;
if (len > 0) {
bpe_offsets.push_back(len);
}
_prev_end = end;
//if (len > 0) {
// std::string s = "";
// for(size_t p = end-len; p < end; p++)
// s += unicode_cpt_to_utf8(cpts[p]);
// printf(">>> '%s'\n", s.c_str());
//}
return len;
};
for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
const uint32_t cpt = _get_cpt(pos);
const auto flags = _get_flags(pos);
// regex: 's|'t|'re|'ve|'m|'ll|'d
if (cpt == '\'' && pos+1 < offset_end) {
uint32_t cpt_next = _get_cpt(pos+1);
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
pos += _add_token(pos+2);
continue;
}
if (pos+2 < offset_end) {
uint32_t cpt_next_next = _get_cpt(pos+2);
if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') ||
(cpt_next == 'l' && cpt_next_next == 'l')) {
pos += _add_token(pos+3);
continue;
}
}
}
auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
// regex: <space>?\p{L}+
if (flags2.is_letter) {
pos += (cpt == ' ');
while (flags2.is_letter) {
flags2 = _get_flags(++pos);
}
_add_token(pos);
continue;
}
// regex: <space>?\p{N}+
if (flags2.is_number) {
pos += (cpt == ' ');
while (flags2.is_number) {
flags2 = _get_flags(++pos);
}
_add_token(pos);
continue;
}
// regex: <space>?[^\s\p{L}\p{N}]+
if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
pos += (cpt == ' ');
while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
flags2 = _get_flags(++pos);
}
_add_token(pos);
continue;
}
size_t num_whitespaces = 0;
while (_get_flags(pos+num_whitespaces).is_whitespace) {
num_whitespaces++;
}
// regex: \s+(?!\S)
if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
pos += num_whitespaces - 1;
_add_token(pos);
continue;
}
// regex: \s+
if (num_whitespaces > 0) {
pos += num_whitespaces;
_add_token(pos);
continue;
}
// no matches
_add_token(++pos);
}
}
return bpe_offsets;
}
// LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) {
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
const auto cpts = unicode_cpts_from_utf8(text);
size_t start = 0;
for (auto offset : offsets) {
const size_t offset_ini = start;
const size_t offset_end = start + offset;
assert(offset_end <= cpts.size());
start = offset_end;
static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
auto _get_cpt = [&] (const size_t pos) -> uint32_t {
return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
};
auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {
return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};
};
size_t _prev_end = offset_ini;
auto _add_token = [&] (const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end;
if (len > 0) {
bpe_offsets.push_back(len);
}
_prev_end = end;
//if (len > 0) {
// std::string s = "";
// for(size_t p = end-len; p < end; p++)
// s += unicode_cpt_to_utf8(cpts[p]);
// printf(">>> '%s'\n", s.c_str());
//}
return len;
};
for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
const uint32_t cpt = _get_cpt(pos);
const auto flags = _get_flags(pos);
// regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive
if (cpt == '\'' && pos+1 < offset_end) {
uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1));
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
pos += _add_token(pos+2);
continue;
}
if (pos+2 < offset_end) {
uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));
if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') ||
(cpt_next == 'l' && cpt_next_next == 'l')) {
pos += _add_token(pos+3);
continue;
}
}
}
// regex: [^\r\n\p{L}\p{N}]?\p{L}+
if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) {
if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters
pos++;
while (_get_flags(pos).is_letter) {
pos++;
}
_add_token(pos);
continue;
}
}
// regex: \p{N}{1,3}
if (flags.is_number) {
size_t ini = pos;
while (_get_flags(pos).is_number) {
if (++pos - ini >= 3 ) {
_add_token(pos);
ini = pos;
}
}
_add_token(pos);
continue;
}
// regex: <space>?[^\s\p{L}\p{N}]+[\r\n]*
auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) {
pos += (cpt == ' ');
while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
flags2 = _get_flags(++pos);
}
uint32_t cpt2 = _get_cpt(pos);
while (cpt2 == '\r' || cpt2 == '\n') {
cpt2 = _get_cpt(++pos);
}
_add_token(pos);
continue;
}
size_t num_whitespaces = 0;
size_t last_end_r_or_n = 0;
while (_get_flags(pos+num_whitespaces).is_whitespace) {
uint32_t cpt2 = _get_cpt(pos+num_whitespaces);
if (cpt2 == '\r' || cpt2 == '\n') {
last_end_r_or_n = pos + num_whitespaces + 1;
}
num_whitespaces++;
}
// regex: \s*[\r\n]+
if (last_end_r_or_n > 0) {
pos = last_end_r_or_n;
_add_token(pos);
continue;
}
// regex: \s+(?!\S)
if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
pos += num_whitespaces - 1;
_add_token(pos);
continue;
}
// regex: \s+
if (num_whitespaces > 0) {
pos += num_whitespaces;
_add_token(pos);
continue;
}
// no matches
_add_token(++pos);
}
}
return bpe_offsets;
}
// use std::wregex to split the text
static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) {
std::wregex expr(regex_expr);
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
size_t start = 0;
for (auto offset : offsets) {
std::wcregex_iterator it(wtext.data() + start, wtext.data() + start + offset, expr);
std::wcregex_iterator end;
int64_t start_idx = 0;
while (it != end) {
std::wcmatch& match = const_cast<std::wcmatch&>(*it);
if (match.position() > start_idx) {
bpe_offsets.emplace_back(match.position() - start_idx);
}
bpe_offsets.emplace_back(match.length());
start_idx = match.position() + match.length();
++it;
}
if (start_idx < (int64_t) offset) {
bpe_offsets.emplace_back(offset - start_idx);
}
start += offset;
}
return bpe_offsets;
}
// use std::regex to split the text
static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
std::regex expr(regex_expr);
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
size_t start = 0;
for (auto offset : offsets) {
std::cregex_iterator it(text.data() + start, text.data() + start + offset, expr);
std::cregex_iterator end;
int64_t start_idx = 0;
while (it != end) {
std::cmatch& match = const_cast<std::cmatch&>(*it);
if (match.position() > start_idx) {
bpe_offsets.emplace_back(match.position() - start_idx);
}
bpe_offsets.emplace_back(match.length());
start_idx = match.position() + match.length();
++it;
}
if (start_idx < (int64_t) offset) {
bpe_offsets.emplace_back(offset - start_idx);
}
start += offset;
}
return bpe_offsets;
}
static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
std::vector<size_t> bpe_offsets;
if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {
bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);
} else if (
regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||
regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);
}
return bpe_offsets;
}
//
// interface
//
std::string unicode_cpt_to_utf8(uint32_t cpt) {
std::string result;
if (/* 0x00 <= cpt && */ cpt <= 0x7f) {
result.push_back(cpt);
return result;
}
if (0x80 <= cpt && cpt <= 0x7ff) {
result.push_back(0xc0 | ((cpt >> 6) & 0x1f));
result.push_back(0x80 | (cpt & 0x3f));
return result;
}
if (0x800 <= cpt && cpt <= 0xffff) {
result.push_back(0xe0 | ((cpt >> 12) & 0x0f));
result.push_back(0x80 | ((cpt >> 6) & 0x3f));
result.push_back(0x80 | (cpt & 0x3f));
return result;
}
if (0x10000 <= cpt && cpt <= 0x10ffff) {
result.push_back(0xf0 | ((cpt >> 18) & 0x07));
result.push_back(0x80 | ((cpt >> 12) & 0x3f));
result.push_back(0x80 | ((cpt >> 6) & 0x3f));
result.push_back(0x80 | (cpt & 0x3f));
return result;
}
throw std::invalid_argument("invalid codepoint");
}
std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {
std::vector<uint32_t> result;
result.reserve(utf8.size());
size_t offset = 0;
while (offset < utf8.size()) {
try {
result.push_back(unicode_cpt_from_utf8(utf8, offset));
}
catch (const std::invalid_argument & /*ex*/) {
// Silently ignore invalid UTF-8 input to avoid leaking the exception beyond llama_tokenize
++offset;
result.emplace_back(0xFFFD); // replacement character
}
}
return result;
}
unicode_cpt_flags unicode_cpt_flags_from_cpt(const uint32_t cpt) {
static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED);
static const auto cpt_flags = unicode_cpt_flags_array();
return cpt < cpt_flags.size() ? cpt_flags[cpt] : undef;
}
std::string unicode_byte_to_utf8(uint8_t byte) {
static std::unordered_map<uint8_t, std::string> map = unicode_byte_to_utf8_map();
return map.at(byte);
}
uint8_t unicode_utf8_to_byte(const std::string & utf8) {
static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map();
return map.at(utf8);
}
uint32_t unicode_tolower(uint32_t cpt) {
// binary search
auto it = std::lower_bound(unicode_map_lowercase.begin(), unicode_map_lowercase.end(), cpt,
[](const std::pair<uint32_t, uint32_t> & pair, uint32_t value) {
return pair.first < value;
});
if (it != unicode_map_lowercase.end() && it->first == cpt) {
return it->second;
}
return cpt; // Return the original code point if no lowercase mapping is found
}
std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
// unicode categories
static const std::map<std::string, int> k_ucat_enum = {
{ "\\p{N}", unicode_cpt_flags::NUMBER },
{ "\\p{L}", unicode_cpt_flags::LETTER },
{ "\\p{P}", unicode_cpt_flags::PUNCTUATION },
{ "\\p{M}", unicode_cpt_flags::ACCENT_MARK },
{ "\\p{S}", unicode_cpt_flags::SYMBOL },
};
static const std::map<int, int> k_ucat_cpt = {
{ unicode_cpt_flags::NUMBER, 0xD1 },
{ unicode_cpt_flags::LETTER, 0xD2 },
{ unicode_cpt_flags::PUNCTUATION, 0xD3 },
{ unicode_cpt_flags::ACCENT_MARK, 0xD4 },
{ unicode_cpt_flags::SYMBOL, 0xD5 },
};
static const std::map<int, std::string> k_ucat_map = {
{ unicode_cpt_flags::NUMBER, "\x30-\x39" }, // 0-9
{ unicode_cpt_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z
{ unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
{ unicode_cpt_flags::ACCENT_MARK, "" }, // no sub-128 codepoints
{ unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C" }, // $+<=>^`|
};
// compute collapsed codepoints only if needed by at least one regex
bool need_collapse = false;
for (const auto & regex_expr : regex_exprs) {
// search for unicode categories
for (const auto & ucat : k_ucat_enum) {
if (std::string::npos != regex_expr.find(ucat.first)) {
need_collapse = true;
break;
}
}
}
const auto cpts = unicode_cpts_from_utf8(text);
// generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte
// ref: https://github.com/ggml-org/llama.cpp/pull/6920#issuecomment-2081479935
std::string text_collapsed;
if (need_collapse) {
// collapse all unicode categories
text_collapsed.resize(cpts.size());
for (size_t i = 0; i < cpts.size(); ++i) {
// keep single-byte codepoints as is
if (cpts[i] < 128) {
text_collapsed[i] = cpts[i];
continue;
}
const auto flags = unicode_cpt_flags_from_cpt(cpts[i]);
if (flags.is_whitespace) {
//NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does.
//text_collapsed[i] = (char) 0x85; // <Next Line> as whitespace fallback
text_collapsed[i] = (char) 0x0B; // <vertical tab> as whitespace fallback
} else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) {
text_collapsed[i] = k_ucat_cpt.at(flags.category_flag());
} else {
text_collapsed[i] = (char) 0xD0; // fallback
}
}
}
std::vector<size_t> bpe_offsets = { cpts.size() };
for (const auto & regex_expr : regex_exprs) {
// first, see if we have an efficient custom regex implementation
auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);
if (!tmp.empty()) {
bpe_offsets = std::move(tmp);
continue;
}
// fallback to general-purpose std::regex / std::wregex
try {
// if a unicode category is used in the regex, we use the collapsed text and replace the unicode category
// with the corresponding collapsed representation
bool use_collapsed = false;
for (const auto & ucat : k_ucat_enum) {
if (std::string::npos != regex_expr.find(ucat.first)) {
use_collapsed = true;
break;
}
}
if (use_collapsed) {
// sanity-check that the original regex does not contain any non-ASCII characters
const auto cpts_regex = unicode_cpts_from_utf8(regex_expr);
for (size_t i = 0; i < cpts_regex.size(); ++i) {
if (cpts_regex[i] >= 128) {
throw std::runtime_error("Regex includes both unicode categories and non-ASCII characters - not supported");
}
}
// generate a collapsed representation of the regex
std::string regex_expr_collapsed;
// track if we are inside [], because nested [] are not allowed
bool inside = false;
for (size_t i = 0; i < regex_expr.size(); ++i) {
if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\')) {
regex_expr_collapsed += '[';
inside = true;
continue;
}
if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\') {
regex_expr_collapsed += ']';
inside = false;
continue;
}
if (regex_expr[i + 0] == '\\' && i + 4 < regex_expr.size() &&
regex_expr[i + 1] == 'p' &&
regex_expr[i + 2] == '{' &&
regex_expr[i + 4] == '}') {
const std::string pat = regex_expr.substr(i, 5);
if (k_ucat_enum.find(pat) != k_ucat_enum.end()) {
if (!inside) {
regex_expr_collapsed += '[';
}
regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat));
regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat));
if (!inside) {
regex_expr_collapsed += ']';
}
i += 4;
continue;
}
}
regex_expr_collapsed += regex_expr[i];
}
//printf("text_collapsed: %s\n", text_collapsed.c_str());
//printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str());
bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);
} else {
// no unicode category used, we can use std::wregex directly
const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr);
// std::wregex \s does not mach non-ASCII whitespaces, using 0x0B as fallback
std::wstring wtext(cpts.begin(), cpts.end());
for (size_t i = 0; i < wtext.size(); ++i) {
if (wtext[i] > 0x7F && unicode_cpt_flags_from_cpt(wtext[i]).is_whitespace) {
wtext[i] = 0x0B;
}
}
//printf("text: %s\n", text.c_str());
//printf("regex_expr: %s\n", regex_expr.c_str());
bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);
}
} catch (std::regex_error & e) {
fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str());
fprintf(stderr, "Regex error: %s\n", e.what());
throw std::runtime_error("Failed to process regex");
}
}
std::vector<std::string> bpe_words;
bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size
size_t start = 0;
for (size_t & offset : bpe_offsets) {
bpe_words.emplace_back();
for (size_t i = start; i < start + offset; ++i) {
bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);
}
start += offset;
}
return bpe_words;
return unicode_byte_encoding_process(bpe_words);
}
}}
modules/dnn/src/tokenizer/unicode.hpp
+104
View File
@@ -0,0 +1,104 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// Adapted from the llama.cpp Unicode functionality:
// https://github.com/ggml-org/llama.cpp/blob/master/src/unicode.cpp
//
// This file is part of the OpenCV DNN module for tokenization.
//
////////////////////////////////////////////////////////////////////////////////////////*/
/*M///////////////////////////////////////////////////////////////////////////////////////
// MIT License
//
// Copyright (c) 2023‑2024 The ggml authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////////////*/
#ifndef __OPENCV_DNN_TOKENIZER_UNICODE_HPP__
#define __OPENCV_DNN_TOKENIZER_UNICODE_HPP__
#include <cstdint>
#include <string>
#include <vector>
namespace cv { namespace dnn {
struct unicode_cpt_flags {
enum {
UNDEFINED = 0x0001,
NUMBER = 0x0002, // regex: \p{N}
LETTER = 0x0004, // regex: \p{L}
SEPARATOR = 0x0008, // regex: \p{Z}
ACCENT_MARK = 0x0010, // regex: \p{M}
PUNCTUATION = 0x0020, // regex: \p{P}
SYMBOL = 0x0040, // regex: \p{S}
CONTROL = 0x0080, // regex: \p{C}
MASK_CATEGORIES = 0x00FF,
};
// codepoint type
uint16_t is_undefined : 1;
uint16_t is_number : 1; // regex: \p{N}
uint16_t is_letter : 1; // regex: \p{L}
uint16_t is_separator : 1; // regex: \p{Z}
uint16_t is_accent_mark : 1; // regex: \p{M}
uint16_t is_punctuation : 1; // regex: \p{P}
uint16_t is_symbol : 1; // regex: \p{S}
uint16_t is_control : 1; // regex: \p{C}
// helper flags
uint16_t is_whitespace : 1; // regex: \s
uint16_t is_lowercase : 1;
uint16_t is_uppercase : 1;
uint16_t is_nfd : 1;
// decode from uint16
inline unicode_cpt_flags(const uint16_t flags = 0) {
*reinterpret_cast<uint16_t*>(this) = flags;
}
inline uint16_t as_uint() const {
return *reinterpret_cast<const uint16_t*>(this);
}
inline uint16_t category_flag() const {
return this->as_uint() & MASK_CATEGORIES;
}
};
std::string unicode_cpt_to_utf8 (uint32_t cpt);
uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
unicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);
std::string unicode_byte_to_utf8(uint8_t byte);
uint8_t unicode_utf8_to_byte(const std::string & utf8);
uint32_t unicode_tolower(uint32_t cpt);
std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);
}}
#endif
modules/dnn/src/tokenizer/utils.hpp
+32
View File
@@ -0,0 +1,32 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef __OPENCV_DNN_TOKENIZER_UTILS_HPP__
#define __OPENCV_DNN_TOKENIZER_UTILS_HPP__
#include <string>
namespace cv { namespace dnn {
// R"R50K('(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+$|\s+(?!\S)|\s)R50K"
static const std::string R50K_UTF8 = "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)";
// GPT-4’s cl100k_base split pattern
// NOTE: This pattern is adapted from the original Python regex used for GPT-4's cl100k_base BPE split.
// The original Python pattern is:
// r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"""
//
// This C++ version differs in the following ways:
// 1. Possessive quantifiers (`++`, `*+`, `?+`) are replaced with standard quantifiers (`+`, `*`, `?`)
// because C++ std::regex does not support possessive quantifiers.
// 2. Inline case-insensitive group `(?i:...)` is replaced with a non-capturing group `(?:...)`
// because C++ std::regex does not support inline flags. Case-insensitivity must be handled separately.
// 3. The `$` anchor at the end of `\s++$` is omitted because it's not needed for splitting and may cause issues.
// 4. Unicode classes (`\p{L}`, `\p{N}`) are kept because the tokenizer's implementation handles them via custom llama.cpp logic.
//
// The resulting C++ pattern is compatible with std::regex and the tokenizer's Unicode handling logic.
static const std::string CL100K_BASE = R"CL100K('(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s)CL100K";
}}
#endif
modules/dnn/test/test_tokenizer.cpp
+99
View File
@@ -0,0 +1,99 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "test_precomp.hpp"
namespace opencv_test { namespace {
template<typename TString>
static String _tf(TString filename) {
String basetestdir = getOpenCVExtraDir();
size_t len = basetestdir.size();
if(len > 0 && basetestdir[len-1] != '/' && basetestdir[len-1] != '\\')
return (basetestdir + "/dnn/llm") + filename;
return (basetestdir + "dnn/llm/") + filename;
}
TEST(Tokenizer_BPE, Tokenizer_GPT2_Tokens) {
std::string gpt2_model = _tf("gpt2/config.json");
Tokenizer tok = Tokenizer::load(gpt2_model);
std::vector<int> tokens = tok.encode("hello world");
std::vector<int> expected = {31373, 995};
EXPECT_EQ(tokens, expected);
}
TEST(Tokenizer_BPE, Tokenizer_GPT4) {
std::string gpt4_model = _tf("gpt4/config.json");
Tokenizer tok = Tokenizer::load(gpt4_model);
std::vector<int> tokens = tok.encode("hello world");
std::vector<int> expected = {15339, 1917};
EXPECT_EQ(tokens, expected);
std::string sent = tok.decode({15339, 1917});
std::string expec_str = "hello world";
EXPECT_EQ(sent, expec_str);
}
TEST(Tokenizer_BPE, Tokenizer_GPT2) {
std::string gpt2_model = _tf("gpt2/config.json");
Tokenizer tok = Tokenizer::load(gpt2_model);
auto ids = tok.encode("hello world");
for (auto id : ids) std::cout << id << " ";
std::cout << std::endl;
auto txt = tok.decode(ids);
EXPECT_EQ(txt, "hello world");
// "Long characters" in Chinese
auto ids_j = tok.encode("\xe9\x95\xbf\xe5\xad\x97\xe7\xac\xa6");
std::string word = tok.decode(ids_j);
std::cout << word << std::endl;
}
TEST(Tokenizer_BPE, Tokenizer_GPT2_Model) {
std::string gpt2_model = _tf("gpt2/config.json");
Tokenizer tok = Tokenizer::load(gpt2_model);
auto ids = tok.encode("hello world");
auto text = tok.decode(ids);
EXPECT_EQ(text, "hello world");
}
TEST(Tokenizer_BPE, SimpleRepeated_GPT2) {
Tokenizer gpt2_tok = Tokenizer::load(_tf("gpt2/config.json"));
EXPECT_EQ(gpt2_tok.encode("0"), std::vector<int>({15}));
EXPECT_EQ(gpt2_tok.encode("00"), std::vector<int>({405}));
EXPECT_EQ(gpt2_tok.encode("000"), std::vector<int>({830}));
EXPECT_EQ(gpt2_tok.encode("0000"), std::vector<int>({2388}));
EXPECT_EQ(gpt2_tok.encode("00000"), std::vector<int>({20483}));
EXPECT_EQ(gpt2_tok.encode("000000"), std::vector<int>({10535}));
EXPECT_EQ(gpt2_tok.encode("0000000"), std::vector<int>({24598}));
EXPECT_EQ(gpt2_tok.encode("00000000"), std::vector<int>({8269}));
EXPECT_EQ(gpt2_tok.encode("000000000"), std::vector<int>({10535, 830}));
EXPECT_EQ(gpt2_tok.encode("0000000000"), std::vector<int>({8269, 405}));
EXPECT_EQ(gpt2_tok.encode("00000000000"), std::vector<int>({8269, 830}));
EXPECT_EQ(gpt2_tok.encode("000000000000"), std::vector<int>({8269, 2388}));
EXPECT_EQ(gpt2_tok.encode("0000000000000"), std::vector<int>({8269, 20483}));
EXPECT_EQ(gpt2_tok.encode("00000000000000"), std::vector<int>({8269, 10535}));
EXPECT_EQ(gpt2_tok.encode("000000000000000"), std::vector<int>({8269, 24598}));
EXPECT_EQ(gpt2_tok.encode("0000000000000000"), std::vector<int>({25645}));
EXPECT_EQ(gpt2_tok.encode("00000000000000000"), std::vector<int>({8269, 10535, 830}));
}
TEST(Tokenizer_BPE, CatastrophicallyRepetitive_GPT2) {
Tokenizer gpt2_tok = Tokenizer::load(_tf("gpt2/config.json"));
std::vector<std::string> chars = {"^", "0", "a", "'s", " ", "\n"};
for (const auto& c : chars) {
std::string big_value(c.size() == 1 ? 10000 : 10000 * c.size(), c[0]);
if (c == "'s") big_value = std::string(10000, '\'') + std::string(10000, 's');
EXPECT_EQ(big_value, gpt2_tok.decode(gpt2_tok.encode(big_value)));
std::string with_space = " " + big_value;
EXPECT_EQ(with_space, gpt2_tok.decode(gpt2_tok.encode(with_space)));
std::string with_newline = big_value + "\n";
EXPECT_EQ(with_newline, gpt2_tok.decode(gpt2_tok.encode(with_newline)));
}
}
}}
modules/python/test/test_tokenizer.py
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python
'''
Test for Tokenizer Python bindings
'''
from __future__ import print_function
import cv2 as cv
import os
import json
from tests_common import NewOpenCVTests
def _tf(filename=""):
base = os.environ.get("OPENCV_TEST_DATA_PATH") or os.getcwd()
path = os.path.join(base, "dnn", "llm", filename)
if not os.path.exists(path):
raise FileNotFoundError(
f"Missing test data: {path}. "
"Set OPENCV_TEST_DATA_PATH to the testdata root contains dnn/llm."
)
return path
class TokenizerBindingTest(NewOpenCVTests):
def test_tokenizer_binding(self):
try:
tokenizer = cv.dnn.Tokenizer
print("Tokenizer binding is available.", tokenizer)
gpt2_model = _tf("gpt2/config.json")
tokenizer = cv.dnn.Tokenizer.load(gpt2_model)
print("Tokenizer loaded from:", gpt2_model)
except AttributeError:
self.fail("Tokenizer binding is NOT available.")
def test_tokenizer_gpt2(self):
tok = cv.dnn.Tokenizer.load((_tf("gpt2/config.json")))
ids = tok.encode("hello world")
print(ids)
txt = tok.decode(ids)
self.assertEqual(txt, "hello world")
def test_tokenizer_gpt4(self):
tok = cv.dnn.Tokenizer.load(_tf("gpt4/config.json"))
tokens = tok.encode("hello world")
# expects {15339, 1917}
self.assertEqual(list(tokens), [15339, 1917])
sent = tok.decode([15339, 1917])
self.assertEqual(sent, "hello world")
def test_with_hf_tiktoken(self):
tok = cv.dnn.Tokenizer.load(_tf("gpt2/config.json"))
with open(_tf("gpt2/gpt2_hf_tik_testdata.json"), "r", encoding="utf-8") as f:
golden = json.load(f)
for s in golden["samples"]:
text = s["text"]
expected = s["ids"]
got = tok.encode(text).tolist()
self.assertEqual(
got, expected,
msg=f"Mismatch for sample '{s['name']}'"
)
self.assertEqual(tok.decode(expected), text)
if __name__ == '__main__':
NewOpenCVTests.bootstrap()
samples/dnn/gpt2_inference.py
+19 -50
View File
@@ -10,7 +10,7 @@ To export GPT-2 model to ONNX, you can use the following procedure:
1. Clone fork of Andrej Karpathy's GPT-2 repository:
git clone -b ash/export-gpt2-onnx-dynamic https://github.com/Abdurrahheem/build-nanogpt.git
git clone -b fix-dynamic-axis-export https://github.com/nklskyoy/build-nanogpt
2. Install the required dependencies:
@@ -27,11 +27,10 @@ Run the script:
pip install tiktoken==0.7.0 numpy tqdm
2. Run the script:
python gpt2_inference.py --model=<path-to-onnx-model> --prompt=<use-promt-of-the-same-length-used-while-exporting>
python gpt2_inference.py --model=<path-to-onnx-model> --tokenizer_path=<path-to-tokenizer-config> --prompt=<use-promt-of-the-same-length-used-while-exporting>
'''
import numpy as np
import tiktoken
import argparse
import cv2 as cv
@@ -39,8 +38,9 @@ def parse_args():
parser = argparse.ArgumentParser(description='Use this script to run GPT-2 inference in OpenCV',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model', type=str, required=True, help='Path to GPT-2 model ONNX model file.')
parser.add_argument('--tokenizer_path', type=str, required=True, help='Path to GPT-2 tokenizer config file.')
parser.add_argument("--prompt", type=str, default="Hello, I'm a language model,", help="Prompt to start with.")
parser.add_argument("--max_seq_len", type=int, default=1024, help="Number of tokens to continue.")
parser.add_argument("--max_seq_len", type=int, default=32, help="Number of tokens to continue.")
parser.add_argument("--seed", type=int, default=0, help="Random seed")
return parser.parse_args()
@@ -50,71 +50,40 @@ def stable_softmax(logits):
def gpt2_inference(net, tokens, max_length, tokenizer):
def gpt2_inference(net, prompt, max_length, tokenizer):
print("Inferencing GPT-2 model...")
x = np.array(tokens)
x = np.tile(x, (1, 1)).astype(np.int32)
pos = np.arange(0, len(x), dtype=np.int32)
# warm up
net.setInputsNames(['input_ids', 'position_ids'])
net.setInput(x, 'input_ids')
net.setInput(pos, 'position_ids')
logits = net.forward()
tokens = tokenizer.encode(prompt).reshape(1,-1)
stop_tokens = (50256, ) ## could be extended to include more stop tokens
print("\n", tokenizer.decode(tokens), sep="", end="")
while 0 < max_length and x[:, -1] not in stop_tokens:
while 0 < max_length and tokens[:, -1] not in stop_tokens:
net.setInputsNames(['input_ids', 'position_ids'])
net.setInput(x, 'input_ids')
net.setInput(pos, 'position_ids')
net.setInputsNames(['idx'])
net.setInput(tokens, 'idx')
logits = net.forward()
# logits is assumed to be (B, seq_length, vocab_size) and needs to be the last token's logits
logits = logits[:, -1, :] # (B, vocab_size)
# Get the probabilities using softmax
probs = stable_softmax(logits)
# use hard sampling
new_idx = np.argmax(logits.reshape(-1)).reshape(1,1)
# Do top-k sampling of 50
topk_indices = np.argpartition(probs, -50, axis=-1)[:, -50:]
topk_probs = np.take_along_axis(probs, topk_indices, axis=-1)
# Normalize top-k probabilities
topk_probs /= np.sum(topk_probs, axis=-1, keepdims=True)
# Select a token from the top-k probabilities
sampled_indices = [np.random.choice(topk_indices[i], p=topk_probs[i]) for i in range(len(topk_probs))]
sampled_indices = np.array(sampled_indices).reshape(-1, 1)
# Decode and print the new token
new_word = tokenizer.decode([sampled_indices[0, 0]])
## clean the prints from the previous line
print(new_word, end='', flush=True)
# Append to the sequence
x = np.concatenate((x, sampled_indices), axis=1)
pos = np.arange(0, x.shape[1], dtype=np.int32) # shape (T)
tokens = np.concatenate((tokens, new_idx), axis=1)
max_length -= 1
return tokens
print('\n')
if __name__ == '__main__':
args = parse_args()
print("Preparing GPT-2 model...")
np.random.seed(args.seed)
max_length = args.max_seq_len
prompt = args.prompt
tokenizer_path = args.tokenizer_path
net = cv.dnn.readNet(args.model)
net = cv.dnn.readNetFromONNX(args.model, cv.dnn.ENGINE_NEW)
tokenizer = cv.dnn.Tokenizer.load(tokenizer_path)
enc = tiktoken.get_encoding('gpt2')
tokens = enc.encode(prompt)
gpt2_inference(net, tokens, max_length, enc)
tokens = gpt2_inference(net, prompt, max_length, tokenizer)
print(tokenizer.decode(tokens[0]))