diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 9e8ccba0f6..6948c30563 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -14,6 +14,7 @@ In this tutorial, we will introduce the APIs for TextRecognitionModel and TextDetectionModel in detail. --- + ### TextRecognitionModel In the current version, @ref cv::dnn::TextRecognitionModel only supports CNN+RNN+CTC based algorithms, @@ -292,38 +293,13 @@ of these APIs can be found in the DNN module. ## Detailed Sample For more information, please refer to: -- [samples/dnn/scene_text_recognition.cpp](https://github.com/opencv/opencv/blob/5.x/samples/dnn/scene_text_recognition.cpp) -- [samples/dnn/scene_text_detection.cpp](https://github.com/opencv/opencv/blob/5.x/samples/dnn/scene_text_detection.cpp) - [samples/dnn/text_detection.cpp](https://github.com/opencv/opencv/blob/5.x/samples/dnn/text_detection.cpp) -- [samples/dnn/scene_text_spotting.cpp](https://github.com/opencv/opencv/blob/5.x/samples/dnn/scene_text_spotting.cpp) ### Test with an image +Detection models can be downloaded using: +- [samples/dnn/download_models.py](https://github.com/opencv/opencv/blob/5.x/samples/dnn/download_models.py) +All the preprocessing parameters will be loaded from [samples/dnn/models.yml](https://github.com/opencv/opencv/blob/5.x/samples/dnn/models.yml) Examples: ```bash -example_dnn_scene_text_recognition -mp=path/to/crnn_cs.onnx -i=path/to/an/image -rgb=1 -vp=/path/to/alphabet_94.txt -example_dnn_scene_text_detection -mp=path/to/DB_TD500_resnet50.onnx -i=path/to/an/image -ih=736 -iw=736 -example_dnn_scene_text_spotting -dmp=path/to/DB_IC15_resnet50.onnx -rmp=path/to/crnn_cs.onnx -i=path/to/an/image -iw=1280 -ih=736 -rgb=1 -vp=/path/to/alphabet_94.txt -example_dnn_text_detection -dmp=path/to/EAST.pb -rmp=path/to/crnn_cs.onnx -i=path/to/an/image -rgb=1 -vp=path/to/alphabet_94.txt -``` - -### Test on public datasets -Text Recognition: - -The download link for testing images can be found in the **Images for Testing** - - -Examples: -```bash -example_dnn_scene_text_recognition -mp=path/to/crnn.onnx -e=true -edp=path/to/evaluation_data_rec -vp=/path/to/alphabet_36.txt -rgb=0 -example_dnn_scene_text_recognition -mp=path/to/crnn_cs.onnx -e=true -edp=path/to/evaluation_data_rec -vp=/path/to/alphabet_94.txt -rgb=1 -``` - -Text Detection: - -The download links for testing images can be found in the **Images for Testing** - -Examples: -```bash -example_dnn_scene_text_detection -mp=path/to/DB_TD500_resnet50.onnx -e=true -edp=path/to/evaluation_data_det/TD500 -ih=736 -iw=736 -example_dnn_scene_text_detection -mp=path/to/DB_IC15_resnet50.onnx -e=true -edp=path/to/evaluation_data_det/IC15 -ih=736 -iw=1280 -``` +example_dnn_text_detection DB +``` \ No newline at end of file diff --git a/samples/dnn/common.hpp b/samples/dnn/common.hpp index ec31e05d29..92d0c66241 100644 --- a/samples/dnn/common.hpp +++ b/samples/dnn/common.hpp @@ -6,7 +6,7 @@ std::string genArgument(const std::string& argName, const std::string& help, const std::string& modelName, const std::string& zooFile, char key = ' ', std::string defaultVal = ""); -std::string genPreprocArguments(const std::string& modelName, const std::string& zooFile); +std::string genPreprocArguments(const std::string& modelName, const std::string& zooFile, const std::string& prefix); std::string findFile(const std::string& filename); @@ -58,8 +58,13 @@ std::string genArgument(const std::string& argName, const std::string& help, if (!node.empty()) { FileNode value = node[argName]; - if(argName == "sha1"){ - value = node["load_info"][argName]; + if (argName.find("sha1") != std::string::npos) { + std::string prefix = argName.substr(0, argName.find("sha1")); + value = node[prefix+"load_info"][argName]; + } + if (argName.find("download_sha") != std::string::npos) { + std::string prefix = argName.substr(0, argName.find("download_sha")); + value = node[prefix+"load_info"][argName]; } if (!value.empty()) { @@ -149,29 +154,31 @@ std::string findFile(const std::string& filename) std::exit(1); } -std::string genPreprocArguments(const std::string& modelName, const std::string& zooFile) +std::string genPreprocArguments(const std::string& modelName, const std::string& zooFile, const std::string& prefix="") { - return genArgument("model", "Path to a binary file of model contains trained weights. " + return genArgument(prefix + "model", "Path to a binary file of model contains trained weights. " "It could be a file with extensions .caffemodel (Caffe), " ".pb (TensorFlow), .weights (Darknet), .bin (OpenVINO).", modelName, zooFile, 'm') + - genArgument("config", "Path to a text file of model contains network configuration. " + genArgument(prefix + "config", "Path to a text file of model contains network configuration. " "It could be a file with extensions .prototxt (Caffe), .pbtxt (TensorFlow), .cfg (Darknet), .xml (OpenVINO).", modelName, zooFile, 'c') + - genArgument("mean", "Preprocess input image by subtracting mean values. Mean values should be in BGR order and delimited by spaces.", + genArgument(prefix + "mean", "Preprocess input image by subtracting mean values. Mean values should be in BGR order and delimited by spaces.", modelName, zooFile) + - genArgument("std", "Preprocess input image by dividing on a standard deviation.", + genArgument(prefix + "std", "Preprocess input image by dividing on a standard deviation.", modelName, zooFile) + - genArgument("scale", "Preprocess input image by multiplying on a scale factor.", + genArgument(prefix + "scale", "Preprocess input image by multiplying on a scale factor.", modelName, zooFile, ' ', "1.0") + - genArgument("width", "Preprocess input image by resizing to a specific width.", + genArgument(prefix + "width", "Preprocess input image by resizing to a specific width.", modelName, zooFile, ' ', "-1") + - genArgument("height", "Preprocess input image by resizing to a specific height.", + genArgument(prefix + "height", "Preprocess input image by resizing to a specific height.", modelName, zooFile, ' ', "-1") + - genArgument("rgb", "Indicate that model works with RGB input images instead BGR ones.", + genArgument(prefix + "rgb", "Indicate that model works with RGB input images instead BGR ones.", modelName, zooFile)+ - genArgument("labels", "Path to a text file with names of classes to label detected objects.", + genArgument(prefix + "labels", "Path to a text file with names of classes to label detected objects.", modelName, zooFile)+ - genArgument("sha1", "Optional path to hashsum of downloaded model to be loaded from models.yml", + genArgument(prefix + "sha1", "Optional path to hashsum of downloaded model to be loaded from models.yml", + modelName, zooFile)+ + genArgument(prefix + "download_sha", "Optional path to hashsum of downloaded model to be loaded from models.yml", modelName, zooFile); -} +} \ No newline at end of file diff --git a/samples/dnn/common.py b/samples/dnn/common.py index 8634099cbf..4c5cefc8fc 100644 --- a/samples/dnn/common.py +++ b/samples/dnn/common.py @@ -16,8 +16,13 @@ def add_argument(zoo, parser, name, help, required=False, default=None, type=Non node = fs.getNode(modelName) if not node.empty(): value = node.getNode(name) - if name=="sha1": - value = node.getNode("load_info") + if "sha1" in name: + prefix = name.replace("sha1", "") + value = node.getNode(prefix + "load_info") + value = value.getNode(name) + if "download_sha" in name: + prefix = name.replace("download_sha", "") + value = node.getNode(prefix + "load_info") value = value.getNode(name) if not value.empty(): if value.isReal(): @@ -52,7 +57,7 @@ def add_argument(zoo, parser, name, help, required=False, default=None, type=Non action=action, nargs=nargs, type=type) -def add_preproc_args(zoo, parser, sample, alias=None): +def add_preproc_args(zoo, parser, sample, alias=None, prefix=""): aliases = [] if os.path.isfile(zoo): fs = cv.FileStorage(zoo, cv.FILE_STORAGE_READ) @@ -62,35 +67,37 @@ def add_preproc_args(zoo, parser, sample, alias=None): if model.getNode('sample').string() == sample: aliases.append(name) - parser.add_argument('alias', nargs='?', choices=aliases, + parser.add_argument(prefix+'alias', nargs='?', choices=aliases, help='An alias name of model to extract preprocessing parameters from models.yml file.') - add_argument(zoo, parser, 'model', + add_argument(zoo, parser, prefix+'model', help='Path to a binary file of model contains trained weights. ' 'It could be a file with extensions .caffemodel (Caffe), ' '.pb (TensorFlow), .weights (Darknet), .bin (OpenVINO)', alias=alias) - add_argument(zoo, parser, 'config', + add_argument(zoo, parser, prefix+'config', help='Path to a text file of model contains network configuration. ' 'It could be a file with extensions .prototxt (Caffe), .pbtxt or .config (TensorFlow), .cfg (Darknet), .xml (OpenVINO)', alias=alias) - add_argument(zoo, parser, 'mean', nargs='+', type=float, default=[0, 0, 0], + add_argument(zoo, parser, prefix+'mean', nargs='+', type=float, default=[0, 0, 0], help='Preprocess input image by subtracting mean values. ' 'Mean values should be in BGR order.', alias=alias) - add_argument(zoo, parser, 'std', nargs='+', type=float, default=[0, 0, 0], + add_argument(zoo, parser, prefix+'std', nargs='+', type=float, default=[0, 0, 0], help='Preprocess input image by dividing on a standard deviation.', alias=alias) - add_argument(zoo, parser, 'scale', type=float, default=1.0, + add_argument(zoo, parser, prefix+'scale', type=float, default=1.0, help='Preprocess input image by multiplying on a scale factor.', alias=alias) - add_argument(zoo, parser, 'width', type=int, + add_argument(zoo, parser, prefix+'width', type=int, help='Preprocess input image by resizing to a specific width.', alias=alias) - add_argument(zoo, parser, 'height', type=int, + add_argument(zoo, parser, prefix+'height', type=int, help='Preprocess input image by resizing to a specific height.', alias=alias) - add_argument(zoo, parser, 'rgb', action='store_true', + add_argument(zoo, parser, prefix+'rgb', action='store_true', help='Indicate that model works with RGB input images instead BGR ones.', alias=alias) - add_argument(zoo, parser, 'labels', + add_argument(zoo, parser, prefix+'labels', help='Optional path to a text file with names of labels to label detected objects.', alias=alias) - add_argument(zoo, parser, 'postprocessing', type=str, + add_argument(zoo, parser, prefix+'postprocessing', type=str, help='Post-processing kind depends on model topology.', alias=alias) - add_argument(zoo, parser, 'background_label_id', type=int, default=-1, + add_argument(zoo, parser, prefix+'background_label_id', type=int, default=-1, help='An index of background class in predictions. If not negative, exclude such class from list of classes.', alias=alias) - add_argument(zoo, parser, 'sha1', type=str, + add_argument(zoo, parser, prefix+'sha1', type=str, + help='Optional path to hashsum of downloaded model to be loaded from models.yml', alias=alias) + add_argument(zoo, parser, prefix+'download_sha', type=str, help='Optional path to hashsum of downloaded model to be loaded from models.yml', alias=alias) def findModel(filename, sha1): diff --git a/samples/dnn/download_models.py b/samples/dnn/download_models.py index 546a6beab1..351c0f4500 100644 --- a/samples/dnn/download_models.py +++ b/samples/dnn/download_models.py @@ -41,6 +41,9 @@ def getHashsumFromFile(filepath): return hashsum def checkHashsum(expected_sha, filepath, silent=True): + if not os.path.exists(filepath): + print(f"{filepath} does not exist. Skipping hashsum matching") + return False print(' expected SHA1: {}'.format(expected_sha)) actual_sha = getHashsumFromFile(filepath) print(' actual SHA1:{}'.format(actual_sha)) @@ -158,6 +161,8 @@ class Loader(object): if self.archive_member is None: pathDict = dict((os.path.split(elem)[1], os.path.split(elem)[0]) for elem in f.getnames()) self.archive_member = pathDict[requested_file] + if self.archive_member == "": + self.archive_member = requested_file assert self.archive_member in f.getnames() self.save(filepath, f.extractfile(self.archive_member)) except Exception as e: diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index a58eea4cef..46a988abe0 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -290,6 +290,50 @@ u2netp: sample: "segmentation" ################################################################################ +# Text detection models. +################################################################################ + +DB: + load_info: + url: "https://drive.google.com/uc?export=dowload&id=17_ABp79PlFt9yPCxSaarVc_DKTmrSGGf" + sha1: "bef233c28947ef6ec8c663d20a2b326302421fa3" + model: "DB_IC15_resnet50.onnx" + ocr_load_info: + ocr_url: "https://drive.google.com/uc?export=dowload&id=159VavnbvfBQkLIPSAu2SP5Yij1Fy4azw" + ocr_sha1: "c4ab1fb3f13c1c8ffc04f016e72ec85311de4ebe" + ocr_model: "VGG_CTC.onnx" + mean: [122.67891434, 116.66876762, 104.00698793] + scale: 0.00392 + width: 736 + height: 736 + rgb: false + sample: "text_detection" + +East: + load_info: + url: "https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1" + sha1: "fffabf5ac36f37bddf68e34e84b45f5c4247ed06" + download_name: "frozen_east_text_detection.tar.gz" + download_sha: "3ca8233d6edd748f7ed23246c8ca24cbf696bb94" + model: "frozen_east_text_detection.pb" + ocr_load_info: + ocr_url: "https://drive.google.com/uc?export=dowload&id=159VavnbvfBQkLIPSAu2SP5Yij1Fy4azw" + ocr_sha1: "c4ab1fb3f13c1c8ffc04f016e72ec85311de4ebe" + ocr_model: "VGG_CTC.onnx" + mean: [123.68, 116.78, 103.94] + scale: 1.0 + width: 736 + height: 736 + rgb: false + sample: "text_detection" + +OCR: + load_info: + url: "https://drive.google.com/uc?export=dowload&id=159VavnbvfBQkLIPSAu2SP5Yij1Fy4azw" + sha1: "c4ab1fb3f13c1c8ffc04f016e72ec85311de4ebe" + model: "VGG_CTC.onnx" + sample: "text_recognition" + # Edge Detection models. ################################################################################ diff --git a/samples/dnn/scene_text_detection.cpp b/samples/dnn/scene_text_detection.cpp deleted file mode 100644 index 156529937a..0000000000 --- a/samples/dnn/scene_text_detection.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#include -#include - -#include -#include -#include - -using namespace cv; -using namespace cv::dnn; - -std::string keys = - "{ help h | | Print help message. }" - "{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }" - "{ modelPath mp | | Path to a binary .onnx file contains trained DB detector model. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ inputHeight ih |736| image height of the model input. It should be multiple by 32.}" - "{ inputWidth iw |736| image width of the model input. It should be multiple by 32.}" - "{ binaryThreshold bt |0.3| Confidence threshold of the binary map. }" - "{ polygonThreshold pt |0.5| Confidence threshold of polygons. }" - "{ maxCandidate max |200| Max candidates of polygons. }" - "{ unclipRatio ratio |2.0| unclip ratio. }" - "{ evaluate e |false| false: predict with input images; true: evaluate on benchmarks. }" - "{ evalDataPath edp | | Path to benchmarks for evaluation. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"; - -static -void split(const std::string& s, char delimiter, std::vector& elems) -{ - elems.clear(); - size_t prev_pos = 0; - size_t pos = 0; - while ((pos = s.find(delimiter, prev_pos)) != std::string::npos) - { - elems.emplace_back(s.substr(prev_pos, pos - prev_pos)); - prev_pos = pos + 1; - } - if (prev_pos < s.size()) - elems.emplace_back(s.substr(prev_pos, s.size() - prev_pos)); -} - -int main(int argc, char** argv) -{ - // Parse arguments - CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run the official PyTorch implementation (https://github.com/MhLiao/DB) of " - "Real-time Scene Text Detection with Differentiable Binarization (https://arxiv.org/abs/1911.08947)\n" - "The current version of this script is a variant of the original network without deformable convolution"); - if (argc == 1 || parser.has("help")) - { - parser.printMessage(); - return 0; - } - - float binThresh = parser.get("binaryThreshold"); - float polyThresh = parser.get("polygonThreshold"); - uint maxCandidates = parser.get("maxCandidate"); - String modelPath = parser.get("modelPath"); - double unclipRatio = parser.get("unclipRatio"); - int height = parser.get("inputHeight"); - int width = parser.get("inputWidth"); - - if (!parser.check()) - { - parser.printErrors(); - return 1; - } - - // Load the network - CV_Assert(!modelPath.empty()); - TextDetectionModel_DB detector(modelPath); - detector.setBinaryThreshold(binThresh) - .setPolygonThreshold(polyThresh) - .setUnclipRatio(unclipRatio) - .setMaxCandidates(maxCandidates); - - double scale = 1.0 / 255.0; - Size inputSize = Size(width, height); - Scalar mean = Scalar(122.67891434, 116.66876762, 104.00698793); - detector.setInputParams(scale, inputSize, mean); - - // Create a window - static const std::string winName = "TextDetectionModel"; - - if (parser.get("evaluate")) { - // for evaluation - String evalDataPath = parser.get("evalDataPath"); - CV_Assert(!evalDataPath.empty()); - String testListPath = evalDataPath + "/test_list.txt"; - std::ifstream testList; - testList.open(testListPath); - CV_Assert(testList.is_open()); - - // Create a window for showing groundtruth - static const std::string winNameGT = "GT"; - - String testImgPath; - while (std::getline(testList, testImgPath)) { - String imgPath = evalDataPath + "/test_images/" + testImgPath; - std::cout << "Image Path: " << imgPath << std::endl; - - Mat frame = imread(samples::findFile(imgPath), IMREAD_COLOR); - CV_Assert(!frame.empty()); - Mat src = frame.clone(); - - // Inference - std::vector> results; - detector.detect(frame, results); - - polylines(frame, results, true, Scalar(0, 255, 0), 2); - imshow(winName, frame); - - // load groundtruth - String imgName = testImgPath.substr(0, testImgPath.length() - 4); - String gtPath = evalDataPath + "/test_gts/" + imgName + ".txt"; - // std::cout << gtPath << std::endl; - std::ifstream gtFile; - gtFile.open(gtPath); - CV_Assert(gtFile.is_open()); - - std::vector> gts; - String gtLine; - while (std::getline(gtFile, gtLine)) { - size_t splitLoc = gtLine.find_last_of(','); - String text = gtLine.substr(splitLoc+1); - if ( text == "###\r" || text == "1") { - // ignore difficult instances - continue; - } - gtLine = gtLine.substr(0, splitLoc); - - std::vector v; - split(gtLine, ',', v); - - std::vector loc; - std::vector pts; - for (auto && s : v) { - loc.push_back(atoi(s.c_str())); - } - for (size_t i = 0; i < loc.size() / 2; i++) { - pts.push_back(Point(loc[2 * i], loc[2 * i + 1])); - } - gts.push_back(pts); - } - polylines(src, gts, true, Scalar(0, 255, 0), 2); - imshow(winNameGT, src); - - waitKey(); - } - } else { - // Open an image file - CV_Assert(parser.has("inputImage")); - Mat frame = imread(samples::findFile(parser.get("inputImage"))); - CV_Assert(!frame.empty()); - - // Detect - std::vector> results; - detector.detect(frame, results); - - polylines(frame, results, true, Scalar(0, 255, 0), 2); - imshow(winName, frame); - waitKey(); - } - - return 0; -} diff --git a/samples/dnn/scene_text_recognition.cpp b/samples/dnn/scene_text_recognition.cpp deleted file mode 100644 index 29b14441dd..0000000000 --- a/samples/dnn/scene_text_recognition.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include -#include - -#include -#include -#include - -using namespace cv; -using namespace cv::dnn; - -String keys = - "{ help h | | Print help message. }" - "{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }" - "{ modelPath mp | | Path to a binary .onnx file contains trained CRNN text recognition model. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }" - "{ evaluate e |false| false: predict with input images; true: evaluate on benchmarks. }" - "{ evalDataPath edp | | Path to benchmarks for evaluation. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ vocabularyPath vp | alphabet_36.txt | Path to recognition vocabulary. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"; - -String convertForEval(String &input); - -int main(int argc, char** argv) -{ - // Parse arguments - CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run the PyTorch implementation of " - "An End-to-End Trainable Neural Network for Image-based SequenceRecognition and Its Application to Scene Text Recognition " - "(https://arxiv.org/abs/1507.05717)"); - if (argc == 1 || parser.has("help")) - { - parser.printMessage(); - return 0; - } - - String modelPath = parser.get("modelPath"); - String vocPath = parser.get("vocabularyPath"); - int imreadRGB = parser.get("RGBInput"); - - if (!parser.check()) - { - parser.printErrors(); - return 1; - } - - // Load the network - CV_Assert(!modelPath.empty()); - TextRecognitionModel recognizer(modelPath); - - // Load vocabulary - CV_Assert(!vocPath.empty()); - std::ifstream vocFile; - vocFile.open(samples::findFile(vocPath)); - CV_Assert(vocFile.is_open()); - String vocLine; - std::vector vocabulary; - while (std::getline(vocFile, vocLine)) { - vocabulary.push_back(vocLine); - } - recognizer.setVocabulary(vocabulary); - recognizer.setDecodeType("CTC-greedy"); - - // Set parameters - double scale = 1.0 / 127.5; - Scalar mean = Scalar(127.5, 127.5, 127.5); - Size inputSize = Size(100, 32); - recognizer.setInputParams(scale, inputSize, mean); - - if (parser.get("evaluate")) - { - // For evaluation - String evalDataPath = parser.get("evalDataPath"); - CV_Assert(!evalDataPath.empty()); - String gtPath = evalDataPath + "/test_gts.txt"; - std::ifstream evalGts; - evalGts.open(gtPath); - CV_Assert(evalGts.is_open()); - - String gtLine; - int cntRight=0, cntAll=0; - TickMeter timer; - timer.reset(); - - while (std::getline(evalGts, gtLine)) { - size_t splitLoc = gtLine.find_first_of(' '); - String imgPath = evalDataPath + '/' + gtLine.substr(0, splitLoc); - String gt = gtLine.substr(splitLoc+1); - - // Inference - Mat frame = imread(samples::findFile(imgPath), imreadRGB); - CV_Assert(!frame.empty()); - timer.start(); - std::string recognitionResult = recognizer.recognize(frame); - timer.stop(); - - if (gt == convertForEval(recognitionResult)) - cntRight++; - - cntAll++; - } - std::cout << "Accuracy(%): " << (double)(cntRight) / (double)(cntAll) << std::endl; - std::cout << "Average Inference Time(ms): " << timer.getTimeMilli() / (double)(cntAll) << std::endl; - } - else - { - // Create a window - static const std::string winName = "Input Cropped Image"; - - // Open an image file - CV_Assert(parser.has("inputImage")); - Mat frame = imread(samples::findFile(parser.get("inputImage")), imreadRGB); - CV_Assert(!frame.empty()); - - // Recognition - std::string recognitionResult = recognizer.recognize(frame); - - imshow(winName, frame); - std::cout << "Predition: '" << recognitionResult << "'" << std::endl; - waitKey(); - } - - return 0; -} - -// Convert the predictions to lower case, and remove other characters. -// Only for Evaluation -String convertForEval(String & input) -{ - String output; - for (uint i = 0; i < input.length(); i++){ - char ch = input[i]; - if ((int)ch >= 97 && (int)ch <= 122) { - output.push_back(ch); - } else if ((int)ch >= 65 && (int)ch <= 90) { - output.push_back((char)(ch + 32)); - } else { - continue; - } - } - - return output; -} diff --git a/samples/dnn/scene_text_spotting.cpp b/samples/dnn/scene_text_spotting.cpp deleted file mode 100644 index 37796acc60..0000000000 --- a/samples/dnn/scene_text_spotting.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include -#include - -#include -#include -#include - -using namespace cv; -using namespace cv::dnn; - -std::string keys = - "{ help h | | Print help message. }" - "{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }" - "{ detModelPath dmp | | Path to a binary .onnx model for detection. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ recModelPath rmp | | Path to a binary .onnx model for recognition. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ inputHeight ih |736| image height of the model input. It should be multiple by 32.}" - "{ inputWidth iw |736| image width of the model input. It should be multiple by 32.}" - "{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }" - "{ binaryThreshold bt |0.3| Confidence threshold of the binary map. }" - "{ polygonThreshold pt |0.5| Confidence threshold of polygons. }" - "{ maxCandidate max |200| Max candidates of polygons. }" - "{ unclipRatio ratio |2.0| unclip ratio. }" - "{ vocabularyPath vp | alphabet_36.txt | Path to benchmarks for evaluation. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"; - -void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result); -bool sortPts(const Point& p1, const Point& p2); - -int main(int argc, char** argv) -{ - // Parse arguments - CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run an end-to-end inference sample of textDetectionModel and textRecognitionModel APIs\n" - "Use -h for more information"); - if (argc == 1 || parser.has("help")) - { - parser.printMessage(); - return 0; - } - - float binThresh = parser.get("binaryThreshold"); - float polyThresh = parser.get("polygonThreshold"); - uint maxCandidates = parser.get("maxCandidate"); - String detModelPath = parser.get("detModelPath"); - String recModelPath = parser.get("recModelPath"); - String vocPath = parser.get("vocabularyPath"); - double unclipRatio = parser.get("unclipRatio"); - int height = parser.get("inputHeight"); - int width = parser.get("inputWidth"); - int imreadRGB = parser.get("RGBInput"); - - if (!parser.check()) - { - parser.printErrors(); - return 1; - } - - // Load networks - CV_Assert(!detModelPath.empty()); - TextDetectionModel_DB detector(detModelPath); - detector.setBinaryThreshold(binThresh) - .setPolygonThreshold(polyThresh) - .setUnclipRatio(unclipRatio) - .setMaxCandidates(maxCandidates); - - CV_Assert(!recModelPath.empty()); - TextRecognitionModel recognizer(recModelPath); - - // Load vocabulary - CV_Assert(!vocPath.empty()); - std::ifstream vocFile; - vocFile.open(samples::findFile(vocPath)); - CV_Assert(vocFile.is_open()); - String vocLine; - std::vector vocabulary; - while (std::getline(vocFile, vocLine)) { - vocabulary.push_back(vocLine); - } - recognizer.setVocabulary(vocabulary); - recognizer.setDecodeType("CTC-greedy"); - - // Parameters for Detection - double detScale = 1.0 / 255.0; - Size detInputSize = Size(width, height); - Scalar detMean = Scalar(122.67891434, 116.66876762, 104.00698793); - detector.setInputParams(detScale, detInputSize, detMean); - - // Parameters for Recognition - double recScale = 1.0 / 127.5; - Scalar recMean = Scalar(127.5); - Size recInputSize = Size(100, 32); - recognizer.setInputParams(recScale, recInputSize, recMean); - - // Create a window - static const std::string winName = "Text_Spotting"; - - // Input data - Mat frame = imread(samples::findFile(parser.get("inputImage"))); - std::cout << frame.size << std::endl; - - // Inference - std::vector< std::vector > detResults; - detector.detect(frame, detResults); - Mat frame2 = frame.clone(); - - if (detResults.size() > 0) { - // Text Recognition - Mat recInput; - if (!imreadRGB) { - cvtColor(frame, recInput, cv::COLOR_BGR2GRAY); - } else { - recInput = frame; - } - std::vector< std::vector > contours; - for (uint i = 0; i < detResults.size(); i++) - { - const auto& quadrangle = detResults[i]; - CV_CheckEQ(quadrangle.size(), (size_t)4, ""); - - contours.emplace_back(quadrangle); - - std::vector quadrangle_2f; - for (int j = 0; j < 4; j++) - quadrangle_2f.emplace_back(quadrangle[j]); - - // Transform and Crop - Mat cropped; - fourPointsTransform(recInput, &quadrangle_2f[0], cropped); - - std::string recognitionResult = recognizer.recognize(cropped); - std::cout << i << ": '" << recognitionResult << "'" << std::endl; - - putText(frame2, recognitionResult, quadrangle[3], FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 0, 255), 2); - } - polylines(frame2, contours, true, Scalar(0, 255, 0), 2); - } else { - std::cout << "No Text Detected." << std::endl; - } - imshow(winName, frame2); - waitKey(); - - return 0; -} - -void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result) -{ - const Size outputSize = Size(100, 32); - - Point2f targetVertices[4] = { - Point(0, outputSize.height - 1), - Point(0, 0), - Point(outputSize.width - 1, 0), - Point(outputSize.width - 1, outputSize.height - 1) - }; - Mat rotationMatrix = getPerspectiveTransform(vertices, targetVertices); - - warpPerspective(frame, result, rotationMatrix, outputSize); - -#if 0 - imshow("roi", result); - waitKey(); -#endif -} - -bool sortPts(const Point& p1, const Point& p2) -{ - return p1.x < p2.x; -} diff --git a/samples/dnn/text_detection.cpp b/samples/dnn/text_detection.cpp index d97ec7bc9b..9ede4c9975 100644 --- a/samples/dnn/text_detection.cpp +++ b/samples/dnn/text_detection.cpp @@ -1,21 +1,28 @@ /* - Text detection model: https://github.com/argman/EAST - Download link: https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1 + Text detection model (EAST): https://github.com/argman/EAST + Download link for EAST model: https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1 - Text recognition models can be downloaded directly here: + DB detector model: + https://drive.google.com/uc?export=download&id=17_ABp79PlFt9yPCxSaarVc_DKTmrSGGf + + CRNN Text recognition model sourced from: https://github.com/meijieru/crnn.pytorch + How to convert from .pb to .onnx: + Using classes from: https://github.com/meijieru/crnn.pytorch/blob/master/models/crnn.py + + Additional converted ONNX text recognition models available for direct download: Download link: https://drive.google.com/drive/folders/1cTbQ3nuZG-EKWak6emD_s8_hHXWz7lAr?usp=sharing - and doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown + These models are taken from: https://github.com/clovaai/deep-text-recognition-benchmark - How to convert from pb to onnx: - Using classes from here: https://github.com/meijieru/crnn.pytorch/blob/master/models/crnn.py + Importing and using the CRNN model in PyTorch: import torch from models.crnn import CRNN + model = CRNN(32, 1, 37, 256) model.load_state_dict(torch.load('crnn.pth')) dummy_input = torch.randn(1, 1, 32, 100) torch.onnx.export(model, dummy_input, "crnn.onnx", verbose=True) - For more information, please refer to doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown and doc/tutorials/dnn/dnn_OCR/dnn_OCR.markdown + Usage: ./example_dnn_text_detection DB */ #include #include @@ -24,154 +31,234 @@ #include #include +#include "common.hpp" + using namespace cv; +using namespace std; using namespace cv::dnn; -const char* keys = - "{ help h | | Print help message. }" - "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera.}" - "{ detModel dmp | | Path to a binary .pb file contains trained detector network.}" - "{ width | 320 | Preprocess input image by resizing to a specific width. It should be a multiple of 32. }" - "{ height | 320 | Preprocess input image by resizing to a specific height. It should be a multiple of 32. }" - "{ thr | 0.5 | Confidence threshold. }" - "{ nms | 0.4 | Non-maximum suppression threshold. }" - "{ recModel rmp | | Path to a binary .onnx file contains trained CRNN text recognition model. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}" - "{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }" - "{ vocabularyPath vp | alphabet_36.txt | Path to benchmarks for evaluation. " - "Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"; +const string about = "Use this script for Text Detection and Recognition using OpenCV. \n\n" + "Firstly, download required models using `download_models.py` (if not already done). Set environment variable OPENCV_DOWNLOAD_CACHE_DIR to point to the directory where models are downloaded. Also, point OPENCV_SAMPLES_DATA_PATH to opencv/samples/data.\n" + "To run:\n" + "\t Example: ./example_dnn_text_detection modelName(i.e. DB or East) --ocr_model=\n\n" + "Detection model path can also be specified using --model argument. \n\n" + "Download ocr model using: python download_models.py OCR \n\n"; -void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result); +// Command-line keys to parse the input arguments +string keys = + "{ help h | | Print help message. }" + "{ input i | right.jpg | Path to an input image. }" + "{ @alias | | An alias name of model to extract preprocessing parameters from models.yml file. }" + "{ zoo | ../dnn/models.yml | An optional path to file with preprocessing parameters }" + "{ ocr_model | | Path to a binary .onnx model for recognition. }" + "{ model | | Path to detection model file. }" + "{ thr | 0.5 | Confidence threshold for EAST detector. }" + "{ nms | 0.4 | Non-maximum suppression threshold for EAST detector. }" + "{ binaryThreshold bt | 0.3 | Confidence threshold for the binary map in DB detector. }" + "{ polygonThreshold pt | 0.5 | Confidence threshold for polygons in DB detector. }" + "{ maxCandidate max | 200 | Max candidates for polygons in DB detector. }" + "{ unclipRatio ratio | 2.0 | Unclip ratio for DB detector. }" + "{ vocabularyPath vp | alphabet_36.txt | Path to vocabulary file. }"; -int main(int argc, char** argv) -{ - // Parse command line arguments. +// Function prototype for the four-point perspective transform +static void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result); +static void processFrame( + const Mat& frame, + const vector>& detResults, + const std::string& ocr_model, + bool imreadRGB, + Mat& board, + FontFace& fontFace, + int fontSize, + int fontWeight, + const vector& vocabulary +); + +int main(int argc, char** argv) { + // Setting up command-line parser with the specified keys CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run TensorFlow implementation (https://github.com/argman/EAST) of " - "EAST: An Efficient and Accurate Scene Text Detector (https://arxiv.org/abs/1704.03155v2)"); - if (argc == 1 || parser.has("help")) - { - parser.printMessage(); - return 0; - } + if (!parser.has("@alias") || parser.has("help")) + { + cout << about << endl; + parser.printMessage(); + return -1; + } + const string modelName = parser.get("@alias"); + const string zooFile = findFile(parser.get("zoo")); + + keys += genPreprocArguments(modelName, zooFile, ""); + keys += genPreprocArguments(modelName, zooFile, "ocr_"); + parser = CommandLineParser(argc, argv, keys); + parser.about(about); + + // Parsing command-line arguments + + String sha1 = parser.get("download_sha"); + if (sha1.empty()){ + sha1 = parser.get("sha1"); + } + String ocr_sha1 = parser.get("ocr_sha1"); + String detModelPath = findModel(parser.get("model"), sha1); + String ocr = findModel(parser.get("ocr_model"), ocr_sha1); + int height = parser.get("height"); + int width = parser.get("width"); + bool imreadRGB = parser.get("rgb"); + String vocPath = parser.get("vocabularyPath"); + float binThresh = parser.get("binaryThreshold"); + float polyThresh = parser.get("polygonThreshold"); + double unclipRatio = parser.get("unclipRatio"); + uint maxCandidates = parser.get("maxCandidate"); float confThreshold = parser.get("thr"); float nmsThreshold = parser.get("nms"); - int width = parser.get("width"); - int height = parser.get("height"); - int imreadRGB = parser.get("RGBInput"); - String detModelPath = parser.get("detModel"); - String recModelPath = parser.get("recModel"); - String vocPath = parser.get("vocabularyPath"); + Scalar mean = parser.get("mean"); - if (!parser.check()) - { + // Ensuring the provided arguments are valid + if (!parser.check()) { parser.printErrors(); return 1; } - // Load networks. - CV_Assert(!detModelPath.empty() && !recModelPath.empty()); - TextDetectionModel_EAST detector(detModelPath); - detector.setConfidenceThreshold(confThreshold) - .setNMSThreshold(nmsThreshold); + // Asserting detection model path is provided + CV_Assert(!detModelPath.empty()); - TextRecognitionModel recognizer(recModelPath); + vector> detResults; + // Reading the input image + Mat frame = imread(samples::findFile(parser.get("input"))); + Mat board(frame.size(), frame.type(), Scalar(255, 255, 255)); + int stdSize = 20; + int stdWeight = 400; + int stdImgSize = 512; + int imgWidth = min(frame.rows, frame.cols); + int size = (stdSize*imgWidth)/stdImgSize; + int weight = (stdWeight*imgWidth)/stdImgSize; + FontFace fontFace("sans"); - // Load vocabulary + // Initializing and configuring the text detection model based on the provided config + if (modelName == "East") { + // EAST Detector initialization + TextDetectionModel_EAST detector(detModelPath); + detector.setConfidenceThreshold(confThreshold) + .setNMSThreshold(nmsThreshold); + // Setting input parameters specific to EAST model + detector.setInputParams(1.0, Size(width, height), mean, true); + // Performing text detection + detector.detect(frame, detResults); + } + else if (modelName == "DB") { + // DB Detector initialization + TextDetectionModel_DB detector(detModelPath); + detector.setBinaryThreshold(binThresh) + .setPolygonThreshold(polyThresh) + .setUnclipRatio(unclipRatio) + .setMaxCandidates(maxCandidates); + // Setting input parameters specific to DB model + detector.setInputParams(1.0 / 255.0, Size(width, height), mean); + // Performing text detection + detector.detect(frame, detResults); + } + else { + cout << "[ERROR]: Unsupported file config for the detector model. Valid values: east/db" << endl; + return 1; + } + + // Reading and storing vocabulary for text recognition CV_Assert(!vocPath.empty()); - std::ifstream vocFile; + ifstream vocFile; vocFile.open(samples::findFile(vocPath)); CV_Assert(vocFile.is_open()); - String vocLine; - std::vector vocabulary; - while (std::getline(vocFile, vocLine)) { + std::string vocLine; + vector vocabulary; + while (getline(vocFile, vocLine)) { vocabulary.push_back(vocLine); } - recognizer.setVocabulary(vocabulary); - recognizer.setDecodeType("CTC-greedy"); - // Parameters for Recognition - double recScale = 1.0 / 127.5; - Scalar recMean = Scalar(127.5, 127.5, 127.5); - Size recInputSize = Size(100, 32); - recognizer.setInputParams(recScale, recInputSize, recMean); - - // Parameters for Detection - double detScale = 1.0; - Size detInputSize = Size(width, height); - Scalar detMean = Scalar(123.68, 116.78, 103.94); - bool swapRB = true; - detector.setInputParams(detScale, detInputSize, detMean, swapRB); - - // Open a video file or an image file or a camera stream. - VideoCapture cap; - bool openSuccess = parser.has("input") ? cap.open(parser.get("input")) : cap.open(0); - CV_Assert(openSuccess); - - static const std::string kWinName = "EAST: An Efficient and Accurate Scene Text Detector"; - - Mat frame; - while (waitKey(1) < 0) - { - cap >> frame; - if (frame.empty()) - { - waitKey(); - break; - } - - std::cout << frame.size << std::endl; - - // Detection - std::vector< std::vector > detResults; - detector.detect(frame, detResults); - Mat frame2 = frame.clone(); - if (detResults.size() > 0) { - // Text Recognition - Mat recInput; - if (!imreadRGB) { - cvtColor(frame, recInput, cv::COLOR_BGR2GRAY); - } else { - recInput = frame; - } - std::vector< std::vector > contours; - for (uint i = 0; i < detResults.size(); i++) - { - const auto& quadrangle = detResults[i]; - CV_CheckEQ(quadrangle.size(), (size_t)4, ""); - - contours.emplace_back(quadrangle); - - std::vector quadrangle_2f; - for (int j = 0; j < 4; j++) - quadrangle_2f.emplace_back(quadrangle[j]); - - Mat cropped; - fourPointsTransform(recInput, &quadrangle_2f[0], cropped); - - std::string recognitionResult = recognizer.recognize(cropped); - std::cout << i << ": '" << recognitionResult << "'" << std::endl; - - putText(frame2, recognitionResult, quadrangle[3], FONT_HERSHEY_SIMPLEX, 1.5, Scalar(0, 0, 255), 2); - } - polylines(frame2, contours, true, Scalar(0, 255, 0), 2); - } - imshow(kWinName, frame2); - } + processFrame(frame, detResults, ocr, imreadRGB, board, fontFace, size, weight, vocabulary); return 0; } -void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result) -{ +// Performs a perspective transform for a four-point region +static void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result) { const Size outputSize = Size(100, 32); - + // Defining target vertices for the perspective transform Point2f targetVertices[4] = { Point(0, outputSize.height - 1), - Point(0, 0), Point(outputSize.width - 1, 0), + Point(0, 0), + Point(outputSize.width - 1, 0), Point(outputSize.width - 1, outputSize.height - 1) }; + // Computing the perspective transform matrix Mat rotationMatrix = getPerspectiveTransform(vertices, targetVertices); - + // Applying the perspective transform to the region warpPerspective(frame, result, rotationMatrix, outputSize); } + +void processFrame( + const Mat& frame, + const vector>& detResults, + const std::string& ocr_model, + bool imreadRGB, + Mat& board, + FontFace& fontFace, + int fontSize, + int fontWeight, + const vector& vocabulary +) { + if (detResults.size() > 0) { + // Text Recognition + Mat recInput; + if (!imreadRGB) { + cvtColor(frame, recInput, cv::COLOR_BGR2GRAY); + } else { + recInput = frame; + } + + vector> contours; + for (uint i = 0; i < detResults.size(); i++) { + const auto& quadrangle = detResults[i]; + CV_CheckEQ(quadrangle.size(), (size_t)4, ""); + + contours.emplace_back(quadrangle); + + vector quadrangle_2f; + for (int j = 0; j < 4; j++) + quadrangle_2f.emplace_back(detResults[i][j]); + + // Cropping the detected text region using a four-point transform + Mat cropped; + fourPointsTransform(recInput, &quadrangle_2f[0], cropped); + + if(!ocr_model.empty()){ + TextRecognitionModel recognizer(ocr_model); + recognizer.setVocabulary(vocabulary); + recognizer.setDecodeType("CTC-greedy"); + + // Setting input parameters for the recognition model + double recScale = 1.0 / 127.5; + Scalar recMean = Scalar(127.5); + Size recInputSize = Size(100, 32); + recognizer.setInputParams(recScale, recInputSize, recMean); + // Recognizing text from the cropped image + string recognitionResult = recognizer.recognize(cropped); + cout << i << ": '" << recognitionResult << "'" << endl; + + // Displaying the recognized text on the image + putText(board, recognitionResult, Point(detResults[i][1].x, detResults[i][0].y), Scalar(0, 0, 0), fontFace, fontSize, fontWeight); + } + else{ + cout << "[WARN] Please pass the path to the ocr model using --ocr_model to get the recognised text." << endl; + } + } + // Drawing detected text regions on the image + polylines(board, contours, true, Scalar(200, 255, 200), 1); + polylines(frame, contours, true, Scalar(0, 255, 0), 1); + } else { + cout << "No Text Detected." << endl; + } + + // Displaying the final image with detected and recognized text + Mat stacked; + hconcat(frame, board, stacked); + imshow("Text Detection and Recognition", stacked); + waitKey(0); +} diff --git a/samples/dnn/text_detection.py b/samples/dnn/text_detection.py index db0ea197bd..a880ffc09f 100644 --- a/samples/dnn/text_detection.py +++ b/samples/dnn/text_detection.py @@ -1,15 +1,19 @@ ''' - Text detection model: https://github.com/argman/EAST - Download link: https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1 + Text detection model (EAST): https://github.com/argman/EAST + Download link for EAST model: https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1 - CRNN Text recognition model taken from here: https://github.com/meijieru/crnn.pytorch - How to convert from pb to onnx: - Using classes from here: https://github.com/meijieru/crnn.pytorch/blob/master/models/crnn.py + DB detector model: + https://drive.google.com/uc?export=download&id=17_ABp79PlFt9yPCxSaarVc_DKTmrSGGf - More converted onnx text recognition models can be downloaded directly here: + CRNN Text recognition model sourced from: https://github.com/meijieru/crnn.pytorch + How to convert from .pb to .onnx: + Using classes from: https://github.com/meijieru/crnn.pytorch/blob/master/models/crnn.py + + Additional converted ONNX text recognition models available for direct download: Download link: https://drive.google.com/drive/folders/1cTbQ3nuZG-EKWak6emD_s8_hHXWz7lAr?usp=sharing - And these models taken from here:https://github.com/clovaai/deep-text-recognition-benchmark + These models are taken from: https://github.com/clovaai/deep-text-recognition-benchmark + Importing and using the CRNN model in PyTorch: import torch from models.crnn import CRNN @@ -17,39 +21,62 @@ model.load_state_dict(torch.load('crnn.pth')) dummy_input = torch.randn(1, 1, 32, 100) torch.onnx.export(model, dummy_input, "crnn.onnx", verbose=True) + + Usage: python text_detection.py DB --ocr_model= + ''' - - -# Import required modules -import numpy as np -import cv2 as cv -import math +import os +import cv2 import argparse +import numpy as np +from common import * + +def help(): + print( + ''' + Use this script for Text Detection and Recognition using OpenCV. + + Firstly, download required models using `download_models.py` (if not already done). Set environment variable OPENCV_DOWNLOAD_CACHE_DIR to specify where models should be downloaded. Also, point OPENCV_SAMPLES_DATA_PATH to opencv/samples/data. + + Example: python download_models.py East + python download_models.py OCR + + To run: + Example: python text_detection.py modelName(i.e. DB or East) + + Detection model path can also be specified using --model argument and ocr model can be specified using --ocr_model. + ''' + ) ############ Add argument parser for command line arguments ############ -parser = argparse.ArgumentParser( - description="Use this script to run TensorFlow implementation (https://github.com/argman/EAST) of " - "EAST: An Efficient and Accurate Scene Text Detector (https://arxiv.org/abs/1704.03155v2)" - "The OCR model can be obtained from converting the pretrained CRNN model to .onnx format from the github repository https://github.com/meijieru/crnn.pytorch" - "Or you can download trained OCR model directly from https://drive.google.com/drive/folders/1cTbQ3nuZG-EKWak6emD_s8_hHXWz7lAr?usp=sharing") -parser.add_argument('--input', - help='Path to input image or video file. Skip this argument to capture frames from a camera.') -parser.add_argument('--model', '-m', required=True, - help='Path to a binary .pb file contains trained detector network.') -parser.add_argument('--ocr', default="crnn.onnx", - help="Path to a binary .pb or .onnx file contains trained recognition network", ) -parser.add_argument('--width', type=int, default=320, - help='Preprocess input image by resizing to a specific width. It should be multiple by 32.') -parser.add_argument('--height', type=int, default=320, - help='Preprocess input image by resizing to a specific height. It should be multiple by 32.') -parser.add_argument('--thr', type=float, default=0.5, - help='Confidence threshold.') -parser.add_argument('--nms', type=float, default=0.4, - help='Non-maximum suppression threshold.') -args = parser.parse_args() +def get_args_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--input', default='right.jpg', + help='Path to input image or video file. Skip this argument to capture frames from a camera.') + parser.add_argument('--zoo', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models.yml'), + help='An optional path to file with preprocessing parameters.') + parser.add_argument('--thr', type=float, default=0.5, + help='Confidence threshold.') + parser.add_argument('--nms', type=float, default=0.4, + help='Non-maximum suppression threshold.') + parser.add_argument('--binary_threshold', type=float, default=0.3, + help='Confidence threshold for the binary map in DB detector. ') + parser.add_argument('--polygon_threshold', type=float, default=0.5, + help='Confidence threshold for polygons in DB detector.') + parser.add_argument('--max_candidate', type=int, default=200, + help='Max candidates for polygons in DB detector.') + parser.add_argument('--unclip_ratio', type=float, default=2.0, + help='Unclip ratio for DB detector.') + parser.add_argument('--vocabulary_path', default='alphabet_36.txt', + help='Path to vocabulary file.') + args, _ = parser.parse_known_args() - -############ Utility functions ############ + add_preproc_args(args.zoo, parser, 'text_detection', prefix="") + add_preproc_args(args.zoo, parser, 'text_recognition', prefix="ocr_") + parser = argparse.ArgumentParser(parents=[parser], + description='Text Detection and Recognition using OpenCV.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + return parser.parse_args() def fourPointsTransform(frame, vertices): vertices = np.asarray(vertices) @@ -60,179 +87,110 @@ def fourPointsTransform(frame, vertices): [outputSize[0] - 1, 0], [outputSize[0] - 1, outputSize[1] - 1]], dtype="float32") - rotationMatrix = cv.getPerspectiveTransform(vertices, targetVertices) - result = cv.warpPerspective(frame, rotationMatrix, outputSize) + rotationMatrix = cv2.getPerspectiveTransform(vertices, targetVertices) + result = cv2.warpPerspective(frame, rotationMatrix, outputSize) return result - -def decodeText(scores): - text = "" - alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" - for i in range(scores.shape[0]): - c = np.argmax(scores[i][0]) - if c != 0: - text += alphabet[c - 1] - else: - text += '-' - - # adjacent same letters as well as background text must be removed to get the final output - char_list = [] - for i in range(len(text)): - if text[i] != '-' and (not (i > 0 and text[i] == text[i - 1])): - char_list.append(text[i]) - return ''.join(char_list) - - -def decodeBoundingBoxes(scores, geometry, scoreThresh): - detections = [] - confidences = [] - - ############ CHECK DIMENSIONS AND SHAPES OF geometry AND scores ############ - assert len(scores.shape) == 4, "Incorrect dimensions of scores" - assert len(geometry.shape) == 4, "Incorrect dimensions of geometry" - assert scores.shape[0] == 1, "Invalid dimensions of scores" - assert geometry.shape[0] == 1, "Invalid dimensions of geometry" - assert scores.shape[1] == 1, "Invalid dimensions of scores" - assert geometry.shape[1] == 5, "Invalid dimensions of geometry" - assert scores.shape[2] == geometry.shape[2], "Invalid dimensions of scores and geometry" - assert scores.shape[3] == geometry.shape[3], "Invalid dimensions of scores and geometry" - height = scores.shape[2] - width = scores.shape[3] - for y in range(0, height): - - # Extract data from scores - scoresData = scores[0][0][y] - x0_data = geometry[0][0][y] - x1_data = geometry[0][1][y] - x2_data = geometry[0][2][y] - x3_data = geometry[0][3][y] - anglesData = geometry[0][4][y] - for x in range(0, width): - score = scoresData[x] - - # If score is lower than threshold score, move to next x - if (score < scoreThresh): - continue - - # Calculate offset - offsetX = x * 4.0 - offsetY = y * 4.0 - angle = anglesData[x] - - # Calculate cos and sin of angle - cosA = math.cos(angle) - sinA = math.sin(angle) - h = x0_data[x] + x2_data[x] - w = x1_data[x] + x3_data[x] - - # Calculate offset - offset = ([offsetX + cosA * x1_data[x] + sinA * x2_data[x], offsetY - sinA * x1_data[x] + cosA * x2_data[x]]) - - # Find points for rectangle - p1 = (-sinA * h + offset[0], -cosA * h + offset[1]) - p3 = (-cosA * w + offset[0], sinA * w + offset[1]) - center = (0.5 * (p1[0] + p3[0]), 0.5 * (p1[1] + p3[1])) - detections.append((center, (w, h), -1 * angle * 180.0 / math.pi)) - confidences.append(float(score)) - - # Return detections and confidences - return [detections, confidences] - - def main(): - # Read and store arguments - confThreshold = args.thr - nmsThreshold = args.nms - inpWidth = args.width - inpHeight = args.height - modelDetector = args.model - modelRecognition = args.ocr + args = get_args_parser() + if args.alias is None or hasattr(args, 'help'): + help() + exit(1) - # Load network - detector = cv.dnn.readNet(modelDetector) - recognizer = cv.dnn.readNet(modelRecognition) + if args.download_sha is not None: + args.model = findModel(args.model, args.download_sha) + else: + args.model = findModel(args.model, args.sha1) - # Create a new named window - kWinName = "EAST: An Efficient and Accurate Scene Text Detector" - cv.namedWindow(kWinName, cv.WINDOW_NORMAL) - outNames = [] - outNames.append("feature_fusion/Conv_7/Sigmoid") - outNames.append("feature_fusion/concat_3") + args.ocr_model = findModel(args.ocr_model, args.ocr_sha1) + args.input = findFile(args.input) + args.vocabulary_path = findFile(args.vocabulary_path) - # Open a video file or an image file or a camera stream - cap = cv.VideoCapture(args.input if args.input else 0) + frame = cv2.imread(args.input) + board = np.ones_like(frame)*255 - tickmeter = cv.TickMeter() - while cv.waitKey(1) < 0: - # Read frame - hasFrame, frame = cap.read() - if not hasFrame: - cv.waitKey() - break + stdSize = 0.8 + stdWeight = 2 + stdImgSize = 512 + imgWidth = min(frame.shape[:2]) + fontSize = (stdSize*imgWidth)/stdImgSize + fontThickness = max(1,(stdWeight*imgWidth)//stdImgSize) - # Get frame height and width - height_ = frame.shape[0] - width_ = frame.shape[1] - rW = width_ / float(inpWidth) - rH = height_ / float(inpHeight) + if(args.alias == "DB"): + # DB Detector initialization + detector = cv2.dnn_TextDetectionModel_DB(args.model) + detector.setBinaryThreshold(args.binary_threshold) + detector.setPolygonThreshold(args.polygon_threshold) + detector.setUnclipRatio(args.unclip_ratio) + detector.setMaxCandidates(args.max_candidate) + # Setting input parameters specific to the DB model + detector.setInputParams(scale=args.scale, size=(args.width, args.height), mean=args.mean) + # Performing text detection + detResults = detector.detect(frame) + elif(args.alias == "East"): + # EAST Detector initialization + detector = cv2.dnn_TextDetectionModel_EAST(args.model) + detector.setConfidenceThreshold(args.thr) + detector.setNMSThreshold(args.nms) + # Setting input parameters specific to EAST model + detector.setInputParams(scale=args.scale, size=(args.width, args.height), mean=args.mean, swapRB=True) + # Perfroming text detection + detResults = detector.detect(frame) - # Create a 4D blob from frame. - blob = cv.dnn.blobFromImage(frame, 1.0, (inpWidth, inpHeight), (123.68, 116.78, 103.94), True, False) + # Open the vocabulary file and read lines into a list + with open(args.vocabulary_path, 'r') as voc_file: + vocabulary = [line.strip() for line in voc_file] - # Run the detection model - detector.setInput(blob) + if args.ocr_model is None: + print("[ERROR] Please pass the path to the ocr model using --ocr_model to run the sample") + exit(1) + # Initialize the text recognition model with the specified model path + recognizer = cv2.dnn_TextRecognitionModel(args.ocr_model) - tickmeter.start() - outs = detector.forward(outNames) - tickmeter.stop() + # Set the vocabulary for the model + recognizer.setVocabulary(vocabulary) - # Get scores and geometry - scores = outs[0] - geometry = outs[1] - [boxes, confidences] = decodeBoundingBoxes(scores, geometry, confThreshold) + # Set the decoding method to 'CTC-greedy' + recognizer.setDecodeType("CTC-greedy") - # Apply NMS - indices = cv.dnn.NMSBoxesRotated(boxes, confidences, confThreshold, nmsThreshold) - for i in indices: - # get 4 corners of the rotated rect - vertices = cv.boxPoints(boxes[i]) - # scale the bounding box coordinates based on the respective ratios - for j in range(4): - vertices[j][0] *= rW - vertices[j][1] *= rH + recScale = 1.0 / 127.5 + recMean = (127.5, 127.5, 127.5) + recInputSize = (100, 32) + recognizer.setInputParams(scale=recScale, size=recInputSize, mean=recMean) + if len(detResults) > 0: + recInput = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if not args.rgb else frame.copy() + contours = [] - # get cropped image using perspective transform - if modelRecognition: - cropped = fourPointsTransform(frame, vertices) - cropped = cv.cvtColor(cropped, cv.COLOR_BGR2GRAY) + for i, (quadrangle, _) in enumerate(zip(detResults[0], detResults[1])): + if isinstance(quadrangle, np.ndarray): + quadrangle = np.array(quadrangle).astype(np.float32) - # Create a 4D blob from cropped image - blob = cv.dnn.blobFromImage(cropped, size=(100, 32), mean=127.5, scalefactor=1 / 127.5) - recognizer.setInput(blob) + if quadrangle is None or len(quadrangle) != 4: + print("Skipping a quadrangle with incorrect points or transformation failed.") + continue - # Run the recognition model - tickmeter.start() - result = recognizer.forward() - tickmeter.stop() + contours.append(np.array(quadrangle, dtype=np.int32)) + cropped = fourPointsTransform(recInput, quadrangle) + recognitionResult = recognizer.recognize(cropped) + print(f"{i}: '{recognitionResult}'") - # decode the result into text - wordRecognized = decodeText(result) - cv.putText(frame, wordRecognized, (int(vertices[1][0]), int(vertices[1][1])), cv.FONT_HERSHEY_SIMPLEX, - 0.5, (255, 0, 0)) + try: + text_origin = (int(quadrangle[1][0]), int(quadrangle[0][1])) + cv2.putText(board, recognitionResult, text_origin, cv2.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness) + except Exception as e: + print("Failed to write text on the frame:", e) + else: + print("Skipping a detection with invalid format:", quadrangle) - for j in range(4): - p1 = (int(vertices[j][0]), int(vertices[j][1])) - p2 = (int(vertices[(j + 1) % 4][0]), int(vertices[(j + 1) % 4][1])) - cv.line(frame, p1, p2, (0, 255, 0), 1) + cv2.polylines(frame, contours, True, (0, 255, 0), 1) + cv2.polylines(board, contours, True, (200, 255, 200), 1) + else: + print("No Text Detected.") - # Put efficiency information - label = 'Inference time: %.2f ms' % (tickmeter.getTimeMilli()) - cv.putText(frame, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) - - # Display the frame - cv.imshow(kWinName, frame) - tickmeter.reset() + stacked = cv2.hconcat([frame, board]) + cv2.imshow("Text Detection and Recognition", stacked) + cv2.waitKey(0) if __name__ == "__main__":