From f212c163e311bfc516ccca6cc00a64002a7a30ec Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Wed, 19 Feb 2025 18:37:49 +0100 Subject: [PATCH] have two detectMarkers functions for python backwards compatibility using multiple dictionaries for refinement (function split not necessary as it's backwards compatible) --- .../opencv2/objdetect/aruco_detector.hpp | 33 +- .../misc/python/test/test_objdetect_aruco.py | 8 +- .../objdetect/src/aruco/aruco_detector.cpp | 626 ++++++++++-------- .../objdetect/test/test_arucodetection.cpp | 2 +- .../objdetect/test/test_boarddetection.cpp | 3 + 5 files changed, 394 insertions(+), 278 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index d38fb1753e..281d772db8 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -287,7 +287,7 @@ public: /** @brief ArucoDetector constructor for multiple dictionaries * - * @param dictionaries indicates the type of markers that will be searched + * @param dictionaries indicates the type of markers that will be searched. Empty dictionaries will throw an error. * @param detectorParams marker detection parameters * @param refineParams marker refine detection parameters */ @@ -306,10 +306,8 @@ public: * The identifiers have the same order than the markers in the imgPoints array. * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a * correct codification. Useful for debugging purposes. - * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the - * list of corresponding dictionaries. * - * Performs marker detection in the input image. Only markers included in the specific dictionaries + * Performs marker detection in the input image. Only markers included in the first specified dictionary * are searched. For each detected marker, it returns the 2D position of its corner in the image * and its corresponding identifier. * Note that this function does not perform pose estimation. @@ -318,7 +316,7 @@ public: * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard */ CV_WRAP void detectMarkers(InputArray image, OutputArrayOfArrays corners, OutputArray ids, - OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + OutputArrayOfArrays rejectedImgPoints = noArray()) const; /** @brief Refine not detected markers based on the already detected and the board layout * @@ -350,6 +348,31 @@ public: InputArray cameraMatrix = noArray(), InputArray distCoeffs = noArray(), OutputArray recoveredIdxs = noArray()) const; + /** @brief Basic marker detection + * + * @param image input image + * @param corners vector of detected marker corners. For each marker, its four corners + * are provided, (e.g std::vector > ). For N detected markers, + * the dimensions of this array is Nx4. The order of the corners is clockwise. + * @param ids vector of identifiers of the detected markers. The identifier is of type int + * (e.g. std::vector). For N detected markers, the size of ids is also N. + * The identifiers have the same order than the markers in the imgPoints array. + * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a + * correct codification. Useful for debugging purposes. + * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the + * list of corresponding dictionaries. + * + * Performs marker detection in the input image. Only markers included in the specific dictionaries + * are searched. For each detected marker, it returns the 2D position of its corner in the image + * and its corresponding identifier. + * Note that this function does not perform pose estimation. + * @note The function does not correct lens distortion or takes it into account. It's recommended to undistort + * input image with corresponding camera model, if camera parameters are known + * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard + */ + CV_WRAP void detectMarkersMultiDict(InputArray image, OutputArrayOfArrays corners, OutputArray ids, + OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + CV_WRAP const Dictionary& getDictionary(int index = 0) const; CV_WRAP void setDictionary(const Dictionary& dictionary, int index = 0); CV_WRAP const std::vector& getDictionaries() const; diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index a6991f364d..305d2bdca9 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -156,7 +156,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array([[offset, offset],[marker_size+offset-1.0,offset], [marker_size+offset-1.0,marker_size+offset-1.0], [offset, marker_size+offset-1.0]], dtype=np.float32) - corners, ids, rejected, _ = aruco_detector.detectMarkers(img_marker) + corners, ids, rejected = aruco_detector.detectMarkers(img_marker) self.assertEqual(1, len(ids)) self.assertEqual(id, ids[0]) @@ -171,7 +171,7 @@ class aruco_objdetect_test(NewOpenCVTests): board = cv.aruco.GridBoard(board_size, 5.0, 1.0, aruco_dict) board_image = board.generateImage((board_size[0]*50, board_size[1]*50), marginSize=10) - corners, ids, rejected, _ = aruco_detector.detectMarkers(board_image) + corners, ids, rejected = aruco_detector.detectMarkers(board_image) self.assertEqual(board_size[0]*board_size[1], len(ids)) part_corners, part_ids, part_rejected = corners[:-1], ids[:-1], list(rejected) @@ -203,7 +203,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array(board.getObjPoints())[:, :, 0:2]*cell_size # detect corners - markerCorners, markerIds, _, _ = aruco_detector.detectMarkers(image) + markerCorners, markerIds, _ = aruco_detector.detectMarkers(image) # test refine rejected = [markerCorners[-1]] @@ -474,7 +474,7 @@ class aruco_objdetect_test(NewOpenCVTests): img_marker2 = np.pad(img_marker2, pad_width=offset, mode='constant', constant_values=255) img_markers = np.concatenate((img_marker1, img_marker2), axis=1) - corners, ids, rejected, dictIndices = aruco_detector.detectMarkers(img_markers) + corners, ids, rejected, dictIndices = aruco_detector.detectMarkersMultiDict(img_markers) self.assertEqual(2, len(ids)) self.assertEqual(id, ids[0]) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 1fb2592b21..881b627d00 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -641,6 +641,11 @@ static inline void findCornerInPyrImage(const float scale_init, const int closes } } +enum class DictionaryMode { + Single, + Multi +}; + struct ArucoDetector::ArucoDetectorImpl { /// dictionaries indicates the types of markers that will be searched std::vector dictionaries; @@ -657,6 +662,252 @@ struct ArucoDetector::ArucoDetectorImpl { detectorParams(_detectorParams), refineParams(_refineParams) { CV_Assert(!dictionaries.empty()); } + + /* + * @brief Detect markers either using multiple or just first dictionary + */ + void detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices, DictionaryMode dictMode) { + CV_Assert(!_image.empty()); + + CV_Assert(detectorParams.markerBorderBits > 0); + // check that the parameters are set correctly if Aruco3 is used + CV_Assert(!(detectorParams.useAruco3Detection == true && + detectorParams.minSideLengthCanonicalImg == 0 && + detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); + + Mat grey; + _convertToGrey(_image, grey); + + // Aruco3 functionality is the extension of Aruco. + // The description can be found in: + // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. + // if Aruco3 functionality if not wanted + // change some parameters to be sure to turn it off + if (!detectorParams.useAruco3Detection) { + detectorParams.minMarkerLengthRatioOriginalImg = 0.0; + detectorParams.minSideLengthCanonicalImg = 0; + } + else { + // always turn on corner refinement in case of Aruco3, due to upsampling + detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + } + + /// Step 0: equation (2) from paper [1] + const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / + (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* + detectorParams.minMarkerLengthRatioOriginalImg)); + + /// Step 1: create image pyramid. Section 3.4. in [1] + vector grey_pyramid; + int closest_pyr_image_idx = 0, num_levels = 0; + //// Step 1.1: resize image with equation (1) from paper [1] + if (detectorParams.useAruco3Detection) { + const float scale_pyr = 2.f; + const float img_area = static_cast(grey.rows*grey.cols); + const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* + detectorParams.minSideLengthCanonicalImg); + // find max level + num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); + // the closest pyramid image to the downsampled segmentation image + // will later be used as start index for corner upsampling + const float scale_img_area = img_area * fxfy * fxfy; + closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); + } + buildPyramid(grey, grey_pyramid, num_levels); + + // resize to segmentation image + // in this reduces size the contours will be detected + if (fxfy != 1.f) + resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); + + /// STEP 2: Detect marker candidates + vector > candidates; + vector > contours; + vector ids; + + /// STEP 2.a Detect marker candidates :: using AprilTag + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ + _apriltag(grey, detectorParams, candidates, contours); + } + /// STEP 2.b Detect marker candidates :: traditional way + else { + detectCandidates(grey, candidates, contours); + } + + /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS + vector dictIndices; + vector> rejectedImgPoints; + if (DictionaryMode::Single == dictMode) { + Dictionary& dictionary = dictionaries.at(0); + auto selectedCandidates = filterTooCloseCandidates(candidates, contours, dictionary.markerSize); + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, + ids, dictionary, rejectedImgPoints); + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); + } else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + } + }); + } + } else if (DictionaryMode::Multi == dictMode) { + unordered_set uniqueMarkerSizes; + for (const Dictionary& dictionary : dictionaries) { + uniqueMarkerSizes.insert(dictionary.markerSize); + } + + // create at max 4 marker candidate trees for each dictionary size + vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; + for (int markerSize : uniqueMarkerSizes) { + // min marker size is 4, so subtract 4 to get index + const auto dictionarySizeIndex = markerSize - 4; + // copy candidates + vector> candidatesCopy = candidates; + vector > contoursCopy = contours; + candidatesPerDictionarySize[dictionarySizeIndex] = filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); + } + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + int dictIndex = 0; + for (const Dictionary& currentDictionary : dictionaries) { + const auto dictionarySizeIndex = currentDictionary.markerSize - 4; + // temporary variable to store the current candidates + vector> currentCandidates; + identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, + ids, currentDictionary, rejectedImgPoints); + if (_dictIndices.needed()) { + dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); + } + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); + } + else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + } + }); + } + candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); + dictIndex++; + } + + // Clean up rejectedImgPoints by comparing to itself and all candidates + const float epsilon = 0.000001; + auto compareCandidates = [epsilon](std::vector a, std::vector b) { + for (int i = 0; i < 4; i++) { + if (std::abs(a[i].x - b[i].x) > epsilon || std::abs(a[i].y - b[i].y) > epsilon) { + return false; + } + } + return true; + }; + std::sort(rejectedImgPoints.begin(), rejectedImgPoints.end(), [](const vector& a, const vector&b){ + float avgX = (a[0].x + a[1].x + a[2].x + a[3].x)*.25f; + float avgY = (a[0].y + a[1].y + a[2].y + a[3].y)*.25f; + float aDist = avgX*avgX + avgY*avgY; + avgX = (b[0].x + b[1].x + b[2].x + b[3].x)*.25f; + avgY = (b[0].y + b[1].y + b[2].y + b[3].y)*.25f; + float bDist = avgX*avgX + avgY*avgY; + return aDist < bDist; + }); + auto last = std::unique(rejectedImgPoints.begin(), rejectedImgPoints.end(), compareCandidates); + rejectedImgPoints.erase(last, rejectedImgPoints.end()); + + for (auto it = rejectedImgPoints.begin(); it != rejectedImgPoints.end();) { + bool erased = false; + for (const auto& candidate : candidates) { + if (compareCandidates(candidate, *it)) { + it = rejectedImgPoints.erase(it); + erased = true; + break; + } + } + if (!erased) { + it++; + } + } + } + + /// STEP 3, Optional : Corner refinement :: use contour container + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ + + if (!ids.empty()) { + + // do corner refinement using the contours for each detected markers + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + for (int i = range.start; i < range.end; i++) { + _refineCandidateLines(contours[i], candidates[i]); + } + }); + } + } + + if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + + // scale to orignal size, this however will lead to inaccurate detections! + for (auto &vecPoints : candidates) + for (auto &point : vecPoints) + point *= 1.f/fxfy; + } + + // copy to output arrays + _copyVector2Output(candidates, _corners); + Mat(ids).copyTo(_ids); + if(_rejectedImgPoints.needed()) { + _copyVector2Output(rejectedImgPoints, _rejectedImgPoints); + } + if (_dictIndices.needed()) { + Mat(dictIndices).copyTo(_dictIndices); + } + } + /** * @brief Detect square candidates in the input image */ @@ -771,9 +1022,8 @@ struct ArucoDetector::ArucoDetectorImpl { */ void identifyCandidates(const Mat& grey, const vector& image_pyr, vector& selectedContours, vector >& accepted, vector >& contours, - vector& ids, const Dictionary& currentDictionary, OutputArrayOfArrays _rejected = noArray()) { + vector& ids, const Dictionary& currentDictionary, vector>& rejected) const { size_t ncandidates = selectedContours.size(); - vector > rejected; vector idsTmp(ncandidates, -1); vector rotated(ncandidates, 0); @@ -853,11 +1103,6 @@ struct ArucoDetector::ArucoDetectorImpl { rejected.push_back(selectedContours[i].corners); } } - - // parse output - if(_rejected.needed()) { - _copyVector2Output(rejected, _rejected); - } } }; @@ -875,169 +1120,13 @@ ArucoDetector::ArucoDetector(const std::vector &_dictionaries, } void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints) const { + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, noArray(), DictionaryMode::Single); +} + +void ArucoDetector::detectMarkersMultiDict(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices) const { - CV_Assert(!_image.empty()); - DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - - CV_Assert(detectorParams.markerBorderBits > 0); - // check that the parameters are set correctly if Aruco3 is used - CV_Assert(!(detectorParams.useAruco3Detection == true && - detectorParams.minSideLengthCanonicalImg == 0 && - detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); - - Mat grey; - _convertToGrey(_image, grey); - - // Aruco3 functionality is the extension of Aruco. - // The description can be found in: - // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. - // if Aruco3 functionality if not wanted - // change some parameters to be sure to turn it off - if (!detectorParams.useAruco3Detection) { - detectorParams.minMarkerLengthRatioOriginalImg = 0.0; - detectorParams.minSideLengthCanonicalImg = 0; - } - else { - // always turn on corner refinement in case of Aruco3, due to upsampling - detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - } - - /// Step 0: equation (2) from paper [1] - const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / - (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* - detectorParams.minMarkerLengthRatioOriginalImg)); - - /// Step 1: create image pyramid. Section 3.4. in [1] - vector grey_pyramid; - int closest_pyr_image_idx = 0, num_levels = 0; - //// Step 1.1: resize image with equation (1) from paper [1] - if (detectorParams.useAruco3Detection) { - const float scale_pyr = 2.f; - const float img_area = static_cast(grey.rows*grey.cols); - const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* - detectorParams.minSideLengthCanonicalImg); - // find max level - num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); - // the closest pyramid image to the downsampled segmentation image - // will later be used as start index for corner upsampling - const float scale_img_area = img_area * fxfy * fxfy; - closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); - } - buildPyramid(grey, grey_pyramid, num_levels); - - // resize to segmentation image - // in this reduces size the contours will be detected - if (fxfy != 1.f) - resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); - - /// STEP 2: Detect marker candidates - vector > candidates; - vector > contours; - vector ids; - - /// STEP 2.a Detect marker candidates :: using AprilTag - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ - _apriltag(grey, detectorParams, candidates, contours); - } - /// STEP 2.b Detect marker candidates :: traditional way - else { - arucoDetectorImpl->detectCandidates(grey, candidates, contours); - } - - /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS - unordered_set uniqueMarkerSizes; - for (const Dictionary& dictionary : arucoDetectorImpl->dictionaries) { - uniqueMarkerSizes.insert(dictionary.markerSize); - } - - // create at max 4 marker candidate trees for each dictionary size - vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; - for (int markerSize : uniqueMarkerSizes) { - // min marker size is 4, so subtract 4 to get index - const auto dictionarySizeIndex = markerSize - 4; - // copy candidates - vector> candidatesCopy = candidates; - vector > contoursCopy = contours; - candidatesPerDictionarySize[dictionarySizeIndex] = arucoDetectorImpl->filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); - } - candidates.clear(); - contours.clear(); - - /// STEP 2: Check candidate codification (identify markers) - size_t dictIndex = 0; - vector dictIndices; - for (const Dictionary& currentDictionary : arucoDetectorImpl->dictionaries) { - const auto dictionarySizeIndex = currentDictionary.markerSize - 4; - // temporary variable to store the current candidates - vector> currentCandidates; - arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, - ids, currentDictionary, _rejectedImgPoints); - if (_dictIndices.needed()) { - dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); - } - - /// STEP 3: Corner refinement :: use corner subpix - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners - parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { - const int begin = range.start; - const int end = range.end; - - for (int i = begin; i < end; i++) { - if (detectorParams.useAruco3Detection) { - const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; - findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); - } - else { - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - } - }); - } - candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); - dictIndex++; - } - - /// STEP 3, Optional : Corner refinement :: use contour container - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ - - if (!ids.empty()) { - - // do corner refinement using the contours for each detected markers - parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { - for (int i = range.start; i < range.end; i++) { - _refineCandidateLines(contours[i], candidates[i]); - } - }); - } - } - - if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - - // scale to orignal size, this however will lead to inaccurate detections! - for (auto &vecPoints : candidates) - for (auto &point : vecPoints) - point *= 1.f/fxfy; - } - - // copy to output arrays - _copyVector2Output(candidates, _corners); - Mat(ids).copyTo(_ids); - if (_dictIndices.needed()) { - Mat(dictIndices).copyTo(_dictIndices); - } + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, _dictIndices, DictionaryMode::Multi); } /** @@ -1151,7 +1240,6 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board InputOutputArrayOfArrays _rejectedCorners, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _recoveredIdxs) const { DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionaries[0]; RefineParameters& refineParams = arucoDetectorImpl->refineParams; CV_Assert(refineParams.minRepDistance > 0); @@ -1174,10 +1262,6 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board // list of missing markers indicating if they have been assigned to a candidate vector alreadyIdentified(_rejectedCorners.total(), false); - // maximum bits that can be corrected - int maxCorrectionRecalculated = - int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); - Mat grey; _convertToGrey(_image, grey); @@ -1193,106 +1277,112 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } vector recoveredIdxs; // original indexes of accepted markers in _rejectedCorners - // for each missing marker, try to find a correspondence - for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { + for (const auto& dictionary : arucoDetectorImpl->dictionaries) { + // maximum bits that can be corrected + int maxCorrectionRecalculated = + int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); - // best match at the moment - int closestCandidateIdx = -1; - double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; - Mat closestRotatedMarker; + // for each missing marker, try to find a correspondence + for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { - for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { - if(alreadyIdentified[j]) continue; + // best match at the moment + int closestCandidateIdx = -1; + double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; + Mat closestRotatedMarker; - // check distance - double minDistance = closestCandidateDistance + 1; - bool valid = false; - int validRot = 0; - for(int c = 0; c < 4; c++) { // first corner in rejected candidate - double currentMaxDistance = 0; - for(int k = 0; k < 4; k++) { - Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; - Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; - double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; - currentMaxDistance = max(currentMaxDistance, cornerDist); + for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { + if(alreadyIdentified[j]) continue; + + // check distance + double minDistance = closestCandidateDistance + 1; + bool valid = false; + int validRot = 0; + for(int c = 0; c < 4; c++) { // first corner in rejected candidate + double currentMaxDistance = 0; + for(int k = 0; k < 4; k++) { + Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; + Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; + double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; + currentMaxDistance = max(currentMaxDistance, cornerDist); + } + // if distance is better than current best distance + if(currentMaxDistance < closestCandidateDistance) { + valid = true; + validRot = c; + minDistance = currentMaxDistance; + } + if(!refineParams.checkAllOrders) break; } - // if distance is better than current best distance - if(currentMaxDistance < closestCandidateDistance) { - valid = true; - validRot = c; - minDistance = currentMaxDistance; + + if(!valid) continue; + + // apply rotation + Mat rotatedMarker; + if(refineParams.checkAllOrders) { + rotatedMarker = Mat(4, 1, CV_32FC2); + for(int c = 0; c < 4; c++) + rotatedMarker.ptr()[c] = + _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; + } + else rotatedMarker = _rejectedCorners.getMat(j); + + // last filter, check if inner code is close enough to the assigned marker code + int codeDistance = 0; + // if errorCorrectionRate, dont check code + if(refineParams.errorCorrectionRate >= 0) { + + // extract bits + Mat bits = _extractBits( + grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, + detectorParams.perspectiveRemovePixelPerCell, + detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); + + Mat onlyBits = + bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) + .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); + + codeDistance = + dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); + } + + // if everythin is ok, assign values to current best match + if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { + closestCandidateIdx = j; + closestCandidateDistance = minDistance; + closestRotatedMarker = rotatedMarker; } - if(!refineParams.checkAllOrders) break; } - if(!valid) continue; + // if at least one good match, we have rescue the missing marker + if(closestCandidateIdx >= 0) { - // apply rotation - Mat rotatedMarker; - if(refineParams.checkAllOrders) { - rotatedMarker = Mat(4, 1, CV_32FC2); - for(int c = 0; c < 4; c++) - rotatedMarker.ptr()[c] = - _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; + // subpixel refinement + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && + detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + + std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, closestRotatedMarker, + Size(cornerRefinementWinSize, cornerRefinementWinSize), + Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + + // remove from rejected + alreadyIdentified[closestCandidateIdx] = true; + + // add to detected + finalAcceptedCorners.push_back(closestRotatedMarker); + finalAcceptedIds.push_back(undetectedMarkersIds[i]); + + // add the original index of the candidate + recoveredIdxs.push_back(closestCandidateIdx); } - else rotatedMarker = _rejectedCorners.getMat(j); - - // last filter, check if inner code is close enough to the assigned marker code - int codeDistance = 0; - // if errorCorrectionRate, dont check code - if(refineParams.errorCorrectionRate >= 0) { - - // extract bits - Mat bits = _extractBits( - grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, - detectorParams.perspectiveRemovePixelPerCell, - detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); - - Mat onlyBits = - bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) - .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); - - codeDistance = - dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); - } - - // if everythin is ok, assign values to current best match - if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { - closestCandidateIdx = j; - closestCandidateDistance = minDistance; - closestRotatedMarker = rotatedMarker; - } - } - - // if at least one good match, we have rescue the missing marker - if(closestCandidateIdx >= 0) { - - // subpixel refinement - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && - detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - - std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, closestRotatedMarker, - Size(cornerRefinementWinSize, cornerRefinementWinSize), - Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - - // remove from rejected - alreadyIdentified[closestCandidateIdx] = true; - - // add to detected - finalAcceptedCorners.push_back(closestRotatedMarker); - finalAcceptedIds.push_back(undetectedMarkersIds[i]); - - // add the original index of the candidate - recoveredIdxs.push_back(closestCandidateIdx); } } @@ -1346,13 +1436,13 @@ void ArucoDetector::read(const FileNode &fn) { } const Dictionary& ArucoDetector::getDictionary(int index) const { - CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); return arucoDetectorImpl->dictionaries[index]; } void ArucoDetector::setDictionary(const Dictionary& dictionary, int index) { // special case: if index is 0, we add the dictionary to the list to preserve the old behavior - CV_Assert(index == 0 || static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index == 0 || (index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size())); if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { arucoDetectorImpl->dictionaries.push_back(dictionary); } else { @@ -1373,7 +1463,7 @@ void ArucoDetector::addDictionary(const Dictionary& dictionary) { } void ArucoDetector::removeDictionary(int index) { - CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); // disallow no dictionaries CV_Assert(arucoDetectorImpl->dictionaries.size() > 1ul); arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 511f71e74b..c3138312a4 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -708,7 +708,7 @@ TEST(CV_ArucoMultiDict, multiMarkerDetection) vector markerIds; vector > rejectedImgPts; vector dictIds; - detector.detectMarkers(img, markerCorners, markerIds, rejectedImgPts, dictIds); + detector.detectMarkersMultiDict(img, markerCorners, markerIds, rejectedImgPts, dictIds); ASSERT_EQ(markerIds.size(), 4u); ASSERT_EQ(dictIds.size(), 4u); for (size_t i = 0; i < dictIds.size(); ++i) { diff --git a/modules/objdetect/test/test_boarddetection.cpp b/modules/objdetect/test/test_boarddetection.cpp index 1fa6e11994..ac767f24a7 100644 --- a/modules/objdetect/test/test_boarddetection.cpp +++ b/modules/objdetect/test/test_boarddetection.cpp @@ -142,6 +142,9 @@ class CV_ArucoRefine : public cvtest::BaseTest { params.useAruco3Detection = true; aruco::RefineParameters refineParams(10.f, 3.f, true); detector = aruco::ArucoDetector(dictionary, params, refineParams); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_250)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_4X4_250)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); } protected: