mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 15:23:05 +04:00
Merge pull request #17570 from HannibalAPE:text_det_recog_demo
[GSoC] High Level API and Samples for Scene Text Detection and Recognition * APIs and samples for scene text detection and recognition * update APIs and tutorial for Text Detection and Recognition * API updates: (1) put decodeType into struct Voc (2) optimize the post-processing of DB * sample update: (1) add transformation into scene_text_spotting.cpp (2) modify text_detection.cpp with API update * update tutorial * simplify text recognition API update tutorial * update impl usage in recognize() and detect() * dnn: refactoring public API of TextRecognitionModel/TextDetectionModel * update provided models update opencv.bib * dnn: adjust text rectangle angle * remove points ordering operation in model.cpp * update gts of DB test in test_model.cpp * dnn: ensure to keep text rectangle angle - avoid 90/180 degree turns * dnn(text): use quadrangle result in TextDetectionModel API * dnn: update Text Detection API (1) keep points' order consistent with (bl, tl, tr, br) in unclip (2) update contourScore with boundingRect
This commit is contained in:
Wenqing Zhang
GitHub
@@ -113,6 +113,155 @@ public:
|
||||
model.segment(frame, mask);
|
||||
normAssert(mask, exp, "", norm, norm);
|
||||
}
|
||||
|
||||
void testTextRecognitionModel(const std::string& weights, const std::string& cfg,
|
||||
const std::string& imgPath, const std::string& seq,
|
||||
const std::string& decodeType, const std::vector<std::string>& vocabulary,
|
||||
const Size& size = {-1, -1}, Scalar mean = Scalar(),
|
||||
double scale = 1.0, bool swapRB = false, bool crop = false)
|
||||
{
|
||||
checkBackend();
|
||||
|
||||
Mat frame = imread(imgPath, IMREAD_GRAYSCALE);
|
||||
|
||||
TextRecognitionModel model(weights, cfg);
|
||||
model.setDecodeType(decodeType)
|
||||
.setVocabulary(vocabulary)
|
||||
.setInputSize(size).setInputMean(mean).setInputScale(scale)
|
||||
.setInputSwapRB(swapRB).setInputCrop(crop);
|
||||
|
||||
model.setPreferableBackend(backend);
|
||||
model.setPreferableTarget(target);
|
||||
|
||||
std::string result = model.recognize(frame);
|
||||
EXPECT_EQ(result, seq) << "Full frame: " << imgPath;
|
||||
|
||||
std::vector<Rect> rois;
|
||||
rois.push_back(Rect(0, 0, frame.cols, frame.rows));
|
||||
rois.push_back(Rect(0, 0, frame.cols, frame.rows)); // twice
|
||||
std::vector<std::string> results;
|
||||
model.recognize(frame, rois, results);
|
||||
EXPECT_EQ((size_t)2u, results.size()) << "ROI: " << imgPath;
|
||||
EXPECT_EQ(results[0], seq) << "ROI[0]: " << imgPath;
|
||||
EXPECT_EQ(results[1], seq) << "ROI[1]: " << imgPath;
|
||||
}
|
||||
|
||||
void testTextDetectionModelByDB(const std::string& weights, const std::string& cfg,
|
||||
const std::string& imgPath, const std::vector<std::vector<Point>>& gt,
|
||||
float binThresh, float polyThresh,
|
||||
uint maxCandidates, double unclipRatio,
|
||||
const Size& size = {-1, -1}, Scalar mean = Scalar(),
|
||||
double scale = 1.0, bool swapRB = false, bool crop = false)
|
||||
{
|
||||
checkBackend();
|
||||
|
||||
Mat frame = imread(imgPath);
|
||||
|
||||
TextDetectionModel_DB model(weights, cfg);
|
||||
model.setBinaryThreshold(binThresh)
|
||||
.setPolygonThreshold(polyThresh)
|
||||
.setUnclipRatio(unclipRatio)
|
||||
.setMaxCandidates(maxCandidates)
|
||||
.setInputSize(size).setInputMean(mean).setInputScale(scale)
|
||||
.setInputSwapRB(swapRB).setInputCrop(crop);
|
||||
|
||||
model.setPreferableBackend(backend);
|
||||
model.setPreferableTarget(target);
|
||||
|
||||
// 1. Check common TextDetectionModel API through RotatedRect
|
||||
std::vector<cv::RotatedRect> results;
|
||||
model.detectTextRectangles(frame, results);
|
||||
|
||||
EXPECT_GT(results.size(), (size_t)0);
|
||||
|
||||
std::vector< std::vector<Point> > contours;
|
||||
for (size_t i = 0; i < results.size(); i++)
|
||||
{
|
||||
const RotatedRect& box = results[i];
|
||||
Mat contour;
|
||||
boxPoints(box, contour);
|
||||
std::vector<Point> contour2i(4);
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
contour2i[i].x = cvRound(contour.at<float>(i, 0));
|
||||
contour2i[i].y = cvRound(contour.at<float>(i, 1));
|
||||
}
|
||||
contours.push_back(contour2i);
|
||||
}
|
||||
#if 0 // test debug
|
||||
Mat result = frame.clone();
|
||||
drawContours(result, contours, -1, Scalar(0, 0, 255), 1);
|
||||
imshow("result", result); // imwrite("result.png", result);
|
||||
waitKey(0);
|
||||
#endif
|
||||
normAssertTextDetections(gt, contours, "", 0.05f);
|
||||
|
||||
// 2. Check quadrangle-based API
|
||||
// std::vector< std::vector<Point> > contours;
|
||||
model.detect(frame, contours);
|
||||
|
||||
#if 0 // test debug
|
||||
Mat result = frame.clone();
|
||||
drawContours(result, contours, -1, Scalar(0, 0, 255), 1);
|
||||
imshow("result_contours", result); // imwrite("result_contours.png", result);
|
||||
waitKey(0);
|
||||
#endif
|
||||
normAssertTextDetections(gt, contours, "", 0.05f);
|
||||
}
|
||||
|
||||
void testTextDetectionModelByEAST(const std::string& weights, const std::string& cfg,
|
||||
const std::string& imgPath, const std::vector<RotatedRect>& gt,
|
||||
float confThresh, float nmsThresh,
|
||||
const Size& size = {-1, -1}, Scalar mean = Scalar(),
|
||||
double scale = 1.0, bool swapRB = false, bool crop = false)
|
||||
{
|
||||
const double EPS_PIXELS = 3;
|
||||
|
||||
checkBackend();
|
||||
|
||||
Mat frame = imread(imgPath);
|
||||
|
||||
TextDetectionModel_EAST model(weights, cfg);
|
||||
model.setConfidenceThreshold(confThresh)
|
||||
.setNMSThreshold(nmsThresh)
|
||||
.setInputSize(size).setInputMean(mean).setInputScale(scale)
|
||||
.setInputSwapRB(swapRB).setInputCrop(crop);
|
||||
|
||||
model.setPreferableBackend(backend);
|
||||
model.setPreferableTarget(target);
|
||||
|
||||
std::vector<cv::RotatedRect> results;
|
||||
model.detectTextRectangles(frame, results);
|
||||
|
||||
EXPECT_EQ(results.size(), (size_t)1);
|
||||
for (size_t i = 0; i < results.size(); i++)
|
||||
{
|
||||
const RotatedRect& box = results[i];
|
||||
#if 0 // test debug
|
||||
Mat contour;
|
||||
boxPoints(box, contour);
|
||||
std::vector<Point> contour2i(4);
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
contour2i[i].x = cvRound(contour.at<float>(i, 0));
|
||||
contour2i[i].y = cvRound(contour.at<float>(i, 1));
|
||||
}
|
||||
std::vector< std::vector<Point> > contours;
|
||||
contours.push_back(contour2i);
|
||||
|
||||
Mat result = frame.clone();
|
||||
drawContours(result, contours, -1, Scalar(0, 0, 255), 1);
|
||||
imshow("result", result); //imwrite("result.png", result);
|
||||
waitKey(0);
|
||||
#endif
|
||||
const RotatedRect& gtBox = gt[i];
|
||||
EXPECT_NEAR(box.center.x, gtBox.center.x, EPS_PIXELS);
|
||||
EXPECT_NEAR(box.center.y, gtBox.center.y, EPS_PIXELS);
|
||||
EXPECT_NEAR(box.size.width, gtBox.size.width, EPS_PIXELS);
|
||||
EXPECT_NEAR(box.size.height, gtBox.size.height, EPS_PIXELS);
|
||||
EXPECT_NEAR(box.angle, gtBox.angle, 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TEST_P(Test_Model, Classify)
|
||||
@@ -446,6 +595,77 @@ TEST_P(Test_Model, Segmentation)
|
||||
testSegmentationModel(weights_file, config_file, inp, exp, norm, size, mean, scale, swapRB);
|
||||
}
|
||||
|
||||
TEST_P(Test_Model, TextRecognition)
|
||||
{
|
||||
if (target == DNN_TARGET_OPENCL_FP16)
|
||||
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
|
||||
|
||||
std::string imgPath = _tf("text_rec_test.png");
|
||||
std::string weightPath = _tf("onnx/models/crnn.onnx", false);
|
||||
std::string seq = "welcome";
|
||||
|
||||
Size size{100, 32};
|
||||
double scale = 1.0 / 127.5;
|
||||
Scalar mean = Scalar(127.5);
|
||||
std::string decodeType = "CTC-greedy";
|
||||
std::vector<std::string> vocabulary = {"0","1","2","3","4","5","6","7","8","9",
|
||||
"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"};
|
||||
|
||||
testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale);
|
||||
}
|
||||
|
||||
TEST_P(Test_Model, TextDetectionByDB)
|
||||
{
|
||||
if (target == DNN_TARGET_OPENCL_FP16)
|
||||
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
|
||||
|
||||
std::string imgPath = _tf("text_det_test1.png");
|
||||
std::string weightPath = _tf("onnx/models/DB_TD500_resnet50.onnx", false);
|
||||
|
||||
// GroundTruth
|
||||
std::vector<std::vector<Point>> gt = {
|
||||
{ Point(142, 193), Point(136, 164), Point(213, 150), Point(219, 178) },
|
||||
{ Point(136, 165), Point(122, 114), Point(319, 71), Point(330, 122) }
|
||||
};
|
||||
|
||||
Size size{736, 736};
|
||||
double scale = 1.0 / 255.0;
|
||||
Scalar mean = Scalar(122.67891434, 116.66876762, 104.00698793);
|
||||
|
||||
float binThresh = 0.3;
|
||||
float polyThresh = 0.5;
|
||||
uint maxCandidates = 200;
|
||||
double unclipRatio = 2.0;
|
||||
|
||||
testTextDetectionModelByDB(weightPath, "", imgPath, gt, binThresh, polyThresh, maxCandidates, unclipRatio, size, mean, scale);
|
||||
}
|
||||
|
||||
TEST_P(Test_Model, TextDetectionByEAST)
|
||||
{
|
||||
if (target == DNN_TARGET_OPENCL_FP16)
|
||||
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
|
||||
|
||||
std::string imgPath = _tf("text_det_test2.jpg");
|
||||
std::string weightPath = _tf("frozen_east_text_detection.pb", false);
|
||||
|
||||
// GroundTruth
|
||||
std::vector<RotatedRect> gt = {
|
||||
RotatedRect(Point2f(657.55f, 409.5f), Size2f(316.84f, 62.45f), -4.79)
|
||||
};
|
||||
|
||||
// Model parameters
|
||||
Size size{320, 320};
|
||||
double scale = 1.0;
|
||||
Scalar mean = Scalar(123.68, 116.78, 103.94);
|
||||
bool swapRB = true;
|
||||
|
||||
// Detection algorithm parameters
|
||||
float confThresh = 0.5;
|
||||
float nmsThresh = 0.4;
|
||||
|
||||
testTextDetectionModelByEAST(weightPath, "", imgPath, gt, confThresh, nmsThresh, size, mean, scale, swapRB);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(/**/, Test_Model, dnnBackendsAndTargets());
|
||||
|
||||
}} // namespace
|
||||
|
||||
Reference in New Issue
Block a user