Merge pull request #28837 from varun-jaiswal17:gemma3-tokenizer

Add Gemma3 tokenizer support for dnn #28837

- Adds Gemma3 tokenizer support
- Implements character-level BPE
- Adds 6 tests covering English, phrase, mixed case, numbers, special tokens, and encode/decode
- add gemma3_inference.py

Merge with: 
- **Companion PR**  : https://github.com/opencv/opencv_extra/pull/1346
- forward pass bug in gemma3_inference.py : https://github.com/opencv/opencv/pull/28836

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/dnn/test/test_tokenizer.cpp

@@ -151,4 +151,50 @@ TEST(Tokenizer_BPE, Tokenizer_Qwen2_5_Roundtrip) {
     }
 }
 
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_English) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    EXPECT_EQ(tok.encode("Hello world"), (std::vector<int>{9259, 1902}));
+}
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_Phrase) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    EXPECT_EQ(tok.encode("the quick brown fox"),
+              (std::vector<int>{1437, 3823, 8864, 37423}));
+}
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_Mixed) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    EXPECT_EQ(tok.encode("OpenCV"), (std::vector<int>{7084, 20741}));
+}
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_Numbers) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    EXPECT_EQ(tok.encode("2024"), (std::vector<int>{236778, 236771, 236778, 236812}));
+}
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_SpecialTokens) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    EXPECT_EQ(tok.encode("<bos>Hello<eos>"), (std::vector<int>{2, 9259, 1}));
+}
+
+TEST(Tokenizer_Gemma, Tokenizer_Gemma3_Roundtrip) {
+    std::string model = _tf("gemma3/config.json");
+    Tokenizer tok = Tokenizer::load(model);
+    std::vector<std::string> cases = {
+        "Hello world",
+        "the quick brown fox",
+        "OpenCV",
+        "hello world",
+    };
+    for (const auto& text : cases) {
+        EXPECT_EQ(tok.decode(tok.encode(text)), text);
+    }
+}
+
 }}