From 74b83cfce59318c90683fb5daa73fc8d4bea118e Mon Sep 17 00:00:00 2001 From: Oliver Schreer Date: Fri, 22 May 2015 10:39:01 +0200 Subject: [PATCH 1/6] Modified and improved the method for chessboard detection. It is now faster and detects chessboards under difficult lighting condition as well as when the chessboard has strong out of plane rotations --- .../include/opencv2/calib3d/calib3d.hpp | 9 + modules/calib3d/src/calibinit.cpp | 652 +++++++++++++----- modules/calib3d/src/checkchessboard.cpp | 98 ++- 3 files changed, 585 insertions(+), 174 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp index b3da45edd5..517372edc5 100644 --- a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp @@ -46,3 +46,12 @@ #endif #include "opencv2/calib3d.hpp" +// Performs a fast check if a chessboard is in the input image. This is a workaround to +// a problem of cvFindChessboardCorners being slow on images with no chessboard. +// This method works using a binary image as input +// - src: input binary image +// - size: chessboard size +// Returns 1 if a chessboard can be in this image and findChessboardCorners should be called, +// 0 if there is no chessboard, -1 in case of error +CVAPI(int) cvCheckChessboardBinary(IplImage* src, CvSize size); + diff --git a/modules/calib3d/src/calibinit.cpp b/modules/calib3d/src/calibinit.cpp index d6b4923d11..0f4676da97 100644 --- a/modules/calib3d/src/calibinit.cpp +++ b/modules/calib3d/src/calibinit.cpp @@ -59,11 +59,22 @@ \************************************************************************************/ +/************************************************************************************\ + This version adds a new and improved variant of chessboard corner detection + that works better in poor lighting condition. It is based on work from + Oliver Schreer and Stefano Masneri. This method works faster than the previous + one and reverts back to the older method in case no chessboard detection is + possible. Overall performance improves also because now the method avoids + performing the same computation multiple times when not necessary. + +\************************************************************************************/ + #include "precomp.hpp" #include "opencv2/imgproc/imgproc_c.h" #include "opencv2/calib3d/calib3d_c.h" #include "circlesgrid.hpp" #include +#include //#define ENABLE_TRIM_COL_ROW @@ -191,6 +202,202 @@ static void icvRemoveQuadFromGroup(CvCBQuad **quads, int count, CvCBQuad *q0); static int icvCheckBoardMonotony( CvPoint2D32f* corners, CvSize pattern_size ); +/***************************************************************************************************/ +//COMPUTE INTENSITY HISTOGRAM OF INPUT IMAGE +static int icvGetIntensityHistogram( unsigned char* pucImage, int iSizeCols, int iSizeRows, std::vector& piHist ); +//SMOOTH HISTOGRAM USING WINDOW OF SIZE 2*iWidth+1 +static int icvSmoothHistogram( const std::vector& piHist, std::vector& piHistSmooth, int iWidth ); +//COMPUTE FAST HISTOGRAM GRADIENT +static int icvGradientOfHistogram( const std::vector& piHist, std::vector& piHistGrad ); +//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSTY HISTOGRAM +static bool icvBinarizationHistogramBased( unsigned char* pucImg, int iCols, int iRows ); +/***************************************************************************************************/ +int icvGetIntensityHistogram( unsigned char* pucImage, int iSizeCols, int iSizeRows, std::vector& piHist ) +{ + int iVal; + + // sum up all pixel in row direction and divide by number of columns + for ( int j=0; j& piHist, std::vector& piHistSmooth, int iWidth ) +{ + int iIdx; + for ( int i=0; i<256; i++) + { + int iSmooth = 0; + for ( int ii=-iWidth; ii<=iWidth; ii++) + { + iIdx = i+ii; + if (iIdx > 0 && iIdx < 256) + { + iSmooth += piHist[iIdx]; + } + } + piHistSmooth[i] = iSmooth/(2*iWidth+1); + } + return 0; +} +/***************************************************************************************************/ +int icvGradientOfHistogram( const std::vector& piHist, std::vector& piHistGrad ) +{ + piHistGrad[0] = 0; + for ( int i=1; i<255; i++) + { + piHistGrad[i] = piHist[i-1] - piHist[i+1]; + if ( abs(piHistGrad[i]) < 100 ) + { + if ( piHistGrad[i-1] == 0) + piHistGrad[i] = -100; + else + piHistGrad[i] = piHistGrad[i-1]; + } + } + return 0; +} +/***************************************************************************************************/ +bool icvBinarizationHistogramBased( unsigned char* pucImg, int iCols, int iRows ) +{ + int iMaxPix = iCols*iRows; + int iMaxPix1 = iMaxPix/100; + const int iNumBins = 256; + std::vector piHistIntensity(iNumBins, 0); + std::vector piHistSmooth(iNumBins, 0); + std::vector piHistGrad(iNumBins, 0); + std::vector piAccumSum(iNumBins, 0); + std::vector piMaxPos(20, 0); + int iThresh = 0; + int iIdx; + int iWidth = 1; + + icvGetIntensityHistogram( pucImg, iCols, iRows, piHistIntensity ); + + // get accumulated sum starting from bright + piAccumSum[iNumBins-1] = piHistIntensity[iNumBins-1]; + for ( int i=iNumBins-2; i>=0; i-- ) + { + piAccumSum[i] = piHistIntensity[i] + piAccumSum[i+1]; + } + + // first smooth the distribution + icvSmoothHistogram( piHistIntensity, piHistSmooth, iWidth ); + + // compute gradient + icvGradientOfHistogram( piHistSmooth, piHistGrad ); + + // check for zeros + int iCntMaxima = 0; + for ( int i=iNumBins-2; (i>2) && (iCntMaxima<20); i--) + { + if ( (piHistGrad[i-1] < 0) && (piHistGrad[i] > 0) ) + { + piMaxPos[iCntMaxima] = i; + iCntMaxima++; + } + } + + iIdx = 0; + int iSumAroundMax = 0; + for ( int i=0; i= 3 + { + // CHECKING THRESHOLD FOR WHITE + int iIdxAccSum = 0, iAccum = 0; + for (int i=iNumBins-1; i>0; i--) + { + iAccum += piHistIntensity[i]; + // iMaxPix/18 is about 5,5%, minimum required number of pixels required for white part of chessboard + if ( iAccum > (iMaxPix/18) ) + { + iIdxAccSum = i; + break; + } + } + + int iIdxBGMax = 0; + int iBrightMax = piMaxPos[0]; + // printf("iBrightMax = %d\n", iBrightMax); + for ( int n=0; n= 250 && iIdxBGMax < iCntMaxima ) + { + iIdxBGMax++; + iMaxVal = piHistIntensity[piMaxPos[iIdxBGMax]]; + } + + for ( int n=iIdxBGMax + 1; n= iMaxVal ) + { + iMaxVal = piHistIntensity[piMaxPos[n]]; + iIdxBGMax = n; + } + } + + //SETTING THRESHOLD FOR BINARIZATION + int iDist2 = (iBrightMax - piMaxPos[iIdxBGMax])/2; + iThresh = iBrightMax - iDist2; + PRINTF("THRESHOLD SELECTED = %d, BRIGHTMAX = %d, DARKMAX = %d\n", iThresh, iBrightMax, piMaxPos[iIdxBGMax]); + } + + + if ( iThresh > 0 ) + { + for ( int jj=0; jj storage; CvMat stub, *img = (CvMat*)arr; + cImgSeg = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 1 ); + memcpy( cImgSeg->imageData, cvPtr1D( img, 0), img->rows*img->cols ); + + CvMat stub2, *thresh_img_new; + thresh_img_new = cvGetMat( cImgSeg, &stub2, 0, 0 ); int expected_corners_num = (pattern_size.width/2+1)*(pattern_size.height/2+1); @@ -255,7 +468,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, if( out_corner_count ) *out_corner_count = 0; - IplImage _img; int quad_count = 0, group_idx = 0, dilations = 0; img = cvGetMat( img, &stub ); @@ -300,209 +512,303 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, if( flags & CV_CALIB_CB_FAST_CHECK) { - cvGetImage(img, &_img); - int check_chessboard_result = cvCheckChessboard(&_img, pattern_size); - if(check_chessboard_result <= 0) + //perform new method for checking chessboard using a binary image. + //image is binarised using a threshold dependent on the image histogram + icvBinarizationHistogramBased( (unsigned char*) cImgSeg->imageData, cImgSeg->width, cImgSeg->height ); + check_chessboard_result = cvCheckChessboardBinary(cImgSeg, pattern_size); + if(check_chessboard_result <= 0) //fall back to the old method { - return 0; + IplImage _img; + cvGetImage(img, &_img); + check_chessboard_result = cvCheckChessboard(&_img, pattern_size); + if(check_chessboard_result <= 0) + { + return 0; + } } } + // empiric threshold level + // thresholding performed here and not inside the cycle to save processing time + int thresh_level; + if ( !(flags & CV_CALIB_CB_ADAPTIVE_THRESH) ) + { + double mean = cvAvg( img ).val[0]; + thresh_level = cvRound( mean - 10 ); + thresh_level = MAX( thresh_level, 10 ); + cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); + } // Try our standard "1" dilation, but if the pattern is not found, iterate the whole procedure with higher dilations. // This is necessary because some squares simply do not separate properly with a single dilation. However, // we want to use the minimum number of dilations possible since dilations cause the squares to become smaller, // making it difficult to detect smaller squares. - for( k = 0; k < 6; k++ ) + for( dilations = min_dilations; dilations <= max_dilations; dilations++ ) { + if (found) + break; // already found it + + cvFree(&quads); + cvFree(&corners); + + //USE BINARY IMAGE COMPUTED USING icvBinarizationHistogramBased METHOD + cvDilate( thresh_img_new, thresh_img_new, 0, 1 ); + + // So we can find rectangles that go to the edge, we draw a white line around the image edge. + // Otherwise FindContours will miss those clipped rectangle contours. + // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... + cvRectangle( thresh_img_new, cvPoint(0,0), cvPoint(thresh_img_new->cols-1, thresh_img_new->rows-1), CV_RGB(255,255,255), 3, 8); + quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img_new, flags, &max_quad_buf_size ); + PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); + + if( quad_count <= 0 ) + { + continue; + } + + // Find quad's neighbors + icvFindQuadNeighbors( quads, quad_count ); + + // allocate extra for adding in icvOrderFoundQuads + cvFree(&quad_group); + cvFree(&corner_group); + quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); + corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + + for( group_idx = 0; ; group_idx++ ) + { + int count = 0; + count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); + + int icount = count; + if( count == 0 ) + break; + + // order the quad corners globally + // maybe delete or add some + PRINTF("Starting ordering of inner quads\n"); + count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, + pattern_size, storage ); + PRINTF("Orig count: %d After ordering: %d\n", icount, count); + + if (count == 0) + continue; // haven't found inner quads + + // If count is more than it should be, this will remove those quads + // which cause maximum deviation from a nice square pattern. + count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); + PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); + + count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); + PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); + + int n = count > 0 ? pattern_size.width * pattern_size.height : -count; + n = MIN( n, pattern_size.width * pattern_size.height ); + float sum_dist = 0; + int total = 0; + + for(int i = 0; i < n; i++ ) + { + int ni = 0; + float avgi = corner_group[i]->meanDist(&ni); + sum_dist += avgi*ni; + total += ni; + } + prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); + + if( count > 0 || (out_corner_count && -count > *out_corner_count) ) + { + // copy corners to output array + for(int i = 0; i < n; i++ ) + out_corners[i] = corner_group[i]->pt; + + if( out_corner_count ) + *out_corner_count = n; + + if( count == pattern_size.width*pattern_size.height && + icvCheckBoardMonotony( out_corners, pattern_size )) + { + found = 1; + break; + } + } + } + }//dilations + + // revert to old, slower, method if detection failed + if (!found) + { + for( k = 0; k < 6; k++ ) + { int max_quad_buf_size = 0; for( dilations = min_dilations; dilations <= max_dilations; dilations++ ) { - if (found) - break; // already found it + if (found) + break; // already found it - cvFree(&quads); - cvFree(&corners); + cvFree(&quads); + cvFree(&corners); - /*if( k == 1 ) - { - //Pattern was not found using binarization - // Run multi-level quads extraction - // In case one-level binarization did not give enough number of quads - CV_CALL( quad_count = icvGenerateQuadsEx( &quads, &corners, storage, img, thresh_img, dilations, flags )); - PRINTF("EX quad count: %d/%d\n", quad_count, expected_corners_num); - } - else*/ - { - // convert the input grayscale image to binary (black-n-white) - if( flags & CV_CALIB_CB_ADAPTIVE_THRESH ) - { - int block_size = cvRound(prev_sqr_size == 0 ? - MIN(img->cols,img->rows)*(k%2 == 0 ? 0.2 : 0.1): prev_sqr_size*2)|1; + // convert the input grayscale image to binary (black-n-white) + if( flags & CV_CALIB_CB_ADAPTIVE_THRESH ) + { + int block_size = cvRound(prev_sqr_size == 0 ? + MIN(img->cols,img->rows)*(k%2 == 0 ? 0.2 : 0.1): prev_sqr_size*2)|1; - // convert to binary - cvAdaptiveThreshold( img, thresh_img, 255, - CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, block_size, (k/2)*5 ); - if (dilations > 0) - cvDilate( thresh_img, thresh_img, 0, dilations-1 ); - } - else - { - // Make dilation before the thresholding. - // It splits chessboard corners - //cvDilate( img, thresh_img, 0, 1 ); - - // empiric threshold level - double mean = cvAvg( img ).val[0]; - int thresh_level = cvRound( mean - 10 ); - thresh_level = MAX( thresh_level, 10 ); - - cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); - cvDilate( thresh_img, thresh_img, 0, dilations ); - } + // convert to binary + cvAdaptiveThreshold( img, thresh_img, 255, + CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, block_size, (k/2)*5 ); + if (dilations > 0) + cvDilate( thresh_img, thresh_img, 0, dilations-1 ); + } + //if flag CV_CALIB_CB_ADAPTIVE_THRESH is not set it doesn't make sense + //to iterate over k + else + { + k = 6; + cvDilate( thresh_img, thresh_img, 0, 1 ); + } #ifdef DEBUG_CHESSBOARD - cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); + cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); #endif - // So we can find rectangles that go to the edge, we draw a white line around the image edge. - // Otherwise FindContours will miss those clipped rectangle contours. - // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... - cvRectangle( thresh_img, cvPoint(0,0), cvPoint(thresh_img->cols-1, - thresh_img->rows-1), CV_RGB(255,255,255), 3, 8); + // So we can find rectangles that go to the edge, we draw a white line around the image edge. + // Otherwise FindContours will miss those clipped rectangle contours. + // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... + cvRectangle( thresh_img, cvPoint(0,0), cvPoint(thresh_img->cols-1, + thresh_img->rows-1), CV_RGB(255,255,255), 3, 8); - quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img, flags, &max_quad_buf_size); + quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img, flags, &max_quad_buf_size); + PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); - PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); - } +#ifdef DEBUG_CHESSBOARD + cvCopy(dbg_img, dbg1_img); + cvNamedWindow("all_quads", 1); + // copy corners to temp array + for(int i = 0; i < quad_count; i++ ) + { + for (int z=0; z<4; z++) + { + CvPoint2D32f pt1, pt2; + CvScalar color = CV_RGB(30,255,30); + pt1 = quads[i].corners[z]->pt; + pt2 = quads[i].corners[(z+1)%4]->pt; + pt2.x = (pt1.x + pt2.x)/2; + pt2.y = (pt1.y + pt2.y)/2; + if (z>0) + color = CV_RGB(200,200,0); + cvLine( dbg1_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); + } + } + + + cvShowImage("all_quads", (IplImage*)dbg1_img); + cvWaitKey(); +#endif + + if( quad_count <= 0 ) + { + continue; + } + + // Find quad's neighbors + icvFindQuadNeighbors( quads, quad_count ); + + // allocate extra for adding in icvOrderFoundQuads + cvFree(&quad_group); + cvFree(&corner_group); + quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); + corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + + for( group_idx = 0; ; group_idx++ ) + { + int count = 0; + count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); + + int icount = count; + if( count == 0 ) + break; + + // order the quad corners globally + // maybe delete or add some + PRINTF("Starting ordering of inner quads\n"); + count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, pattern_size, max_quad_buf_size, storage ); + + PRINTF("Orig count: %d After ordering: %d\n", icount, count); #ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img, dbg1_img); - cvNamedWindow("all_quads", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - for (int k=0; k<4; k++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - pt1 = quads[i].corners[k]->pt; - pt2 = quads[i].corners[(k+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (k>0) - color = CV_RGB(200,200,0); - cvLine( dbg1_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - - - cvShowImage("all_quads", (IplImage*)dbg1_img); - cvWaitKey(); + cvCopy(dbg_img,dbg2_img); + cvNamedWindow("connected_group", 1); + // copy corners to temp array + for(int i = 0; i < quad_count; i++ ) + { + if (quads[i].group_idx == group_idx) + for (int z=0; z<4; z++) + { + CvPoint2D32f pt1, pt2; + CvScalar color = CV_RGB(30,255,30); + if (quads[i].ordered) + color = CV_RGB(255,30,30); + pt1 = quads[i].corners[z]->pt; + pt2 = quads[i].corners[(z+1)%4]->pt; + pt2.x = (pt1.x + pt2.x)/2; + pt2.y = (pt1.y + pt2.y)/2; + if (z>0) + color = CV_RGB(200,200,0); + cvLine( dbg2_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); + } + } + cvShowImage("connected_group", (IplImage*)dbg2_img); + cvWaitKey(); #endif - if( quad_count <= 0 ) - continue; + if (count == 0) + continue; // haven't found inner quads - // Find quad's neighbors - icvFindQuadNeighbors( quads, quad_count ); - // allocate extra for adding in icvOrderFoundQuads - cvFree(&quad_group); - cvFree(&corner_group); - quad_group = (CvCBQuad**)cvAlloc( sizeof(quad_group[0]) * max_quad_buf_size); - corner_group = (CvCBCorner**)cvAlloc( sizeof(corner_group[0]) * max_quad_buf_size * 4 ); + // If count is more than it should be, this will remove those quads + // which cause maximum deviation from a nice square pattern. + count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); + PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); - for( group_idx = 0; ; group_idx++ ) + count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); + PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); + + int n = count > 0 ? pattern_size.width * pattern_size.height : -count; + n = MIN( n, pattern_size.width * pattern_size.height ); + float sum_dist = 0; + int total = 0; + + for(int i = 0; i < n; i++ ) { - int count = 0; - count = icvFindConnectedQuads( quads, quad_count, quad_group, group_idx, storage ); - - int icount = count; - if( count == 0 ) - break; - - // order the quad corners globally - // maybe delete or add some - PRINTF("Starting ordering of inner quads\n"); - count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, - pattern_size, max_quad_buf_size, storage ); - PRINTF("Orig count: %d After ordering: %d\n", icount, count); - - -#ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img,dbg2_img); - cvNamedWindow("connected_group", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - if (quads[i].group_idx == group_idx) - for (int k=0; k<4; k++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - if (quads[i].ordered) - color = CV_RGB(255,30,30); - pt1 = quads[i].corners[k]->pt; - pt2 = quads[i].corners[(k+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (k>0) - color = CV_RGB(200,200,0); - cvLine( dbg2_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - cvShowImage("connected_group", (IplImage*)dbg2_img); - cvWaitKey(); -#endif - - if (count == 0) - continue; // haven't found inner quads - - - // If count is more than it should be, this will remove those quads - // which cause maximum deviation from a nice square pattern. - count = icvCleanFoundConnectedQuads( count, quad_group, pattern_size ); - PRINTF("Connected group: %d orig count: %d cleaned: %d\n", group_idx, icount, count); - - count = icvCheckQuadGroup( quad_group, count, corner_group, pattern_size ); - PRINTF("Connected group: %d count: %d cleaned: %d\n", group_idx, icount, count); - - { - int n = count > 0 ? pattern_size.width * pattern_size.height : -count; - n = MIN( n, pattern_size.width * pattern_size.height ); - float sum_dist = 0; - int total = 0; - - for(int i = 0; i < n; i++ ) - { - int ni = 0; - float avgi = corner_group[i]->meanDist(&ni); - sum_dist += avgi*ni; - total += ni; - } - prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); - - if( count > 0 || (out_corner_count && -count > *out_corner_count) ) - { - // copy corners to output array - for(int i = 0; i < n; i++ ) - out_corners[i] = corner_group[i]->pt; - - if( out_corner_count ) - *out_corner_count = n; - - if( count == pattern_size.width*pattern_size.height && - icvCheckBoardMonotony( out_corners, pattern_size )) - { - found = 1; - break; - } - } - } + int ni = 0; + float avgi = corner_group[i]->meanDist(&ni); + sum_dist += avgi*ni; + total += ni; } + prev_sqr_size = cvRound(sum_dist/MAX(total, 1)); + + if( count > 0 || (out_corner_count && -count > *out_corner_count) ) + { + // copy corners to output array + for(int i = 0; i < n; i++ ) + out_corners[i] = corner_group[i]->pt; + + if( out_corner_count ) + *out_corner_count = n; + + if( count == pattern_size.width*pattern_size.height && icvCheckBoardMonotony( out_corners, pattern_size )) + { + found = 1; + break; + } + } + } }//dilations - }// + }// for k = 0 -> 6 + } + if( found ) found = icvCheckBoardMonotony( out_corners, pattern_size ); @@ -559,6 +865,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvFree(&corners); cvFree(&quad_group); cvFree(&corner_group); + cvFree(&cImgSeg); throw; } @@ -566,6 +873,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvFree(&corners); cvFree(&quad_group); cvFree(&corner_group); + cvFree(&cImgSeg); return found; } diff --git a/modules/calib3d/src/checkchessboard.cpp b/modules/calib3d/src/checkchessboard.cpp index 715fe73ef8..5ecd37faab 100644 --- a/modules/calib3d/src/checkchessboard.cpp +++ b/modules/calib3d/src/checkchessboard.cpp @@ -59,8 +59,8 @@ static void icvGetQuadrangleHypotheses(CvSeq* contours, std::vector >& quads, int class_id) { - const float min_aspect_ratio = 0.3f; - const float max_aspect_ratio = 3.0f; + const float min_aspect_ratio = 0.2f; + const float max_aspect_ratio = 5.0f; const float min_box_size = 10.0f; for(CvSeq* seq = contours; seq != NULL; seq = seq->h_next) @@ -205,3 +205,97 @@ int cvCheckChessboard(IplImage* src, CvSize size) return result; } + +// does a fast check if a chessboard is in the input image. This is a workaround to +// a problem of cvFindChessboardCorners being slow on images with no chessboard +// - src: input binary image +// - size: chessboard size +// Returns 1 if a chessboard can be in this image and findChessboardCorners should be called, +// 0 if there is no chessboard, -1 in case of error +int cvCheckChessboardBinary(IplImage* src, CvSize size) +{ + if(src->nChannels > 1) + { + cvError(CV_BadNumChannels, "cvCheckChessboard", "supports single-channel images only", + __FILE__, __LINE__); + } + + if(src->depth != 8) + { + cvError(CV_BadDepth, "cvCheckChessboard", "supports depth=8 images only", + __FILE__, __LINE__); + } + + CvMemStorage* storage = cvCreateMemStorage(); + + IplImage* white = cvCloneImage(src); + IplImage* black = cvCloneImage(src); + IplImage* thresh = cvCreateImage(cvGetSize(src), IPL_DEPTH_8U, 1); + + int result = 0; + + for ( int erosion_count = 0; erosion_count <= 3; erosion_count++ ) + { + if ( 1 == result ) + break; + + if ( 0 != erosion_count ) // first iteration keeps original images + { + cvErode(white, white, NULL, 1); + cvDilate(black, black, NULL, 1); + } + + cvThreshold(white, thresh, 128, 255, CV_THRESH_BINARY); + + CvSeq* first = 0; + std::vector > quads; + cvFindContours(thresh, storage, &first, sizeof(CvContour), CV_RETR_CCOMP); + icvGetQuadrangleHypotheses(first, quads, 1); + + cvThreshold(black, thresh, 128, 255, CV_THRESH_BINARY_INV); + cvFindContours(thresh, storage, &first, sizeof(CvContour), CV_RETR_CCOMP); + icvGetQuadrangleHypotheses(first, quads, 0); + + const size_t min_quads_count = size.width*size.height/2; + std::sort(quads.begin(), quads.end(), less_pred); + + // now check if there are many hypotheses with similar sizes + // do this by floodfill-style algorithm + const float size_rel_dev = 0.4f; + + for(size_t i = 0; i < quads.size(); i++) + { + size_t j = i + 1; + for(; j < quads.size(); j++) + { + if(quads[j].first/quads[i].first > 1.0f + size_rel_dev) + { + break; + } + } + + if(j + 1 > min_quads_count + i) + { + // check the number of black and white squares + std::vector counts; + countClasses(quads, i, j, counts); + const int black_count = cvRound(ceil(size.width/2.0)*ceil(size.height/2.0)); + const int white_count = cvRound(floor(size.width/2.0)*floor(size.height/2.0)); + if(counts[0] < black_count*0.75 || + counts[1] < white_count*0.75) + { + continue; + } + result = 1; + break; + } + } + } + + cvReleaseImage(&thresh); + cvReleaseImage(&white); + cvReleaseImage(&black); + cvReleaseMemStorage(&storage); + + return result; +} \ No newline at end of file From 2a350b11ddbbf56814eb2b4616e5fbb69598ee21 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 13 Jul 2016 11:45:32 +0300 Subject: [PATCH 2/6] Fixed several merge issues --- .../include/opencv2/calib3d/calib3d.hpp | 9 ------ modules/calib3d/src/calibinit.cpp | 31 ++++++++++--------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp index 517372edc5..b3da45edd5 100644 --- a/modules/calib3d/include/opencv2/calib3d/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d/calib3d.hpp @@ -46,12 +46,3 @@ #endif #include "opencv2/calib3d.hpp" -// Performs a fast check if a chessboard is in the input image. This is a workaround to -// a problem of cvFindChessboardCorners being slow on images with no chessboard. -// This method works using a binary image as input -// - src: input binary image -// - size: chessboard size -// Returns 1 if a chessboard can be in this image and findChessboardCorners should be called, -// 0 if there is no chessboard, -1 in case of error -CVAPI(int) cvCheckChessboardBinary(IplImage* src, CvSize size); - diff --git a/modules/calib3d/src/calibinit.cpp b/modules/calib3d/src/calibinit.cpp index 0f4676da97..398be938fe 100644 --- a/modules/calib3d/src/calibinit.cpp +++ b/modules/calib3d/src/calibinit.cpp @@ -202,6 +202,8 @@ static void icvRemoveQuadFromGroup(CvCBQuad **quads, int count, CvCBQuad *q0); static int icvCheckBoardMonotony( CvPoint2D32f* corners, CvSize pattern_size ); +int cvCheckChessboardBinary(IplImage* src, CvSize size); + /***************************************************************************************************/ //COMPUTE INTENSITY HISTOGRAM OF INPUT IMAGE static int icvGetIntensityHistogram( unsigned char* pucImage, int iSizeCols, int iSizeRows, std::vector& piHist ); @@ -515,7 +517,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, //perform new method for checking chessboard using a binary image. //image is binarised using a threshold dependent on the image histogram icvBinarizationHistogramBased( (unsigned char*) cImgSeg->imageData, cImgSeg->width, cImgSeg->height ); - check_chessboard_result = cvCheckChessboardBinary(cImgSeg, pattern_size); + int check_chessboard_result = cvCheckChessboardBinary(cImgSeg, pattern_size); if(check_chessboard_result <= 0) //fall back to the old method { IplImage _img; @@ -528,16 +530,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, } } - // empiric threshold level - // thresholding performed here and not inside the cycle to save processing time - int thresh_level; - if ( !(flags & CV_CALIB_CB_ADAPTIVE_THRESH) ) - { - double mean = cvAvg( img ).val[0]; - thresh_level = cvRound( mean - 10 ); - thresh_level = MAX( thresh_level, 10 ); - cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); - } // Try our standard "1" dilation, but if the pattern is not found, iterate the whole procedure with higher dilations. // This is necessary because some squares simply do not separate properly with a single dilation. However, // we want to use the minimum number of dilations possible since dilations cause the squares to become smaller, @@ -550,6 +542,8 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvFree(&quads); cvFree(&corners); + int max_quad_buf_size = 0; + //USE BINARY IMAGE COMPUTED USING icvBinarizationHistogramBased METHOD cvDilate( thresh_img_new, thresh_img_new, 0, 1 ); @@ -586,8 +580,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, // order the quad corners globally // maybe delete or add some PRINTF("Starting ordering of inner quads\n"); - count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, - pattern_size, storage ); + count = icvOrderFoundConnectedQuads(count, quad_group, &quad_count, &quads, &corners, pattern_size, max_quad_buf_size, storage ); PRINTF("Orig count: %d After ordering: %d\n", icount, count); if (count == 0) @@ -637,6 +630,16 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, // revert to old, slower, method if detection failed if (!found) { + // empiric threshold level + // thresholding performed here and not inside the cycle to save processing time + int thresh_level; + if ( !(flags & CV_CALIB_CB_ADAPTIVE_THRESH) ) + { + double mean = cvAvg( img ).val[0]; + thresh_level = cvRound( mean - 10 ); + thresh_level = MAX( thresh_level, 10 ); + cvThreshold( img, thresh_img, thresh_level, 255, CV_THRESH_BINARY ); + } for( k = 0; k < 6; k++ ) { int max_quad_buf_size = 0; @@ -669,7 +672,7 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, } #ifdef DEBUG_CHESSBOARD - cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); + cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); #endif // So we can find rectangles that go to the edge, we draw a white line around the image edge. From 8215d20dbf4a380dab75c397e39c9c3e59a75de8 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 13 Jul 2016 12:11:44 +0300 Subject: [PATCH 3/6] fixup! Fixed several merge issues --- modules/calib3d/src/checkchessboard.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/calib3d/src/checkchessboard.cpp b/modules/calib3d/src/checkchessboard.cpp index 5ecd37faab..5f954617d3 100644 --- a/modules/calib3d/src/checkchessboard.cpp +++ b/modules/calib3d/src/checkchessboard.cpp @@ -57,6 +57,8 @@ # endif #endif +int cvCheckChessboardBinary(IplImage* src, CvSize size); + static void icvGetQuadrangleHypotheses(CvSeq* contours, std::vector >& quads, int class_id) { const float min_aspect_ratio = 0.2f; From b000663a8807faff57867164d53be7630db6a218 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 13 Jul 2016 15:45:48 +0300 Subject: [PATCH 4/6] Cleanup --- modules/calib3d/src/calibinit.cpp | 164 ++++++------------------------ 1 file changed, 33 insertions(+), 131 deletions(-) diff --git a/modules/calib3d/src/calibinit.cpp b/modules/calib3d/src/calibinit.cpp index 398be938fe..55fadcb4f0 100644 --- a/modules/calib3d/src/calibinit.cpp +++ b/modules/calib3d/src/calibinit.cpp @@ -79,14 +79,7 @@ //#define ENABLE_TRIM_COL_ROW //#define DEBUG_CHESSBOARD -#ifdef DEBUG_CHESSBOARD -# include "opencv2/opencv_modules.hpp" -# ifdef HAVE_OPENCV_HIGHGUI -# include "opencv2/highgui.hpp" -# else -# undef DEBUG_CHESSBOARD -# endif -#endif + #ifdef DEBUG_CHESSBOARD static int PRINTF( const char* fmt, ... ) { @@ -400,38 +393,6 @@ bool icvBinarizationHistogramBased( unsigned char* pucImg, int iCols, int iRows return true; } -#if 0 -static void -icvCalcAffineTranf2D32f(CvPoint2D32f* pts1, CvPoint2D32f* pts2, int count, CvMat* affine_trans) -{ - int i, j; - int real_count = 0; - for( j = 0; j < count; j++ ) - { - if( pts1[j].x >= 0 ) real_count++; - } - if(real_count < 3) return; - cv::Ptr xy = cvCreateMat( 2*real_count, 6, CV_32FC1 ); - cv::Ptr uv = cvCreateMat( 2*real_count, 1, CV_32FC1 ); - //estimate affine transfromation - for( i = 0, j = 0; j < count; j++ ) - { - if( pts1[j].x >= 0 ) - { - CV_MAT_ELEM( *xy, float, i*2+1, 2 ) = CV_MAT_ELEM( *xy, float, i*2, 0 ) = pts2[j].x; - CV_MAT_ELEM( *xy, float, i*2+1, 3 ) = CV_MAT_ELEM( *xy, float, i*2, 1 ) = pts2[j].y; - CV_MAT_ELEM( *xy, float, i*2, 2 ) = CV_MAT_ELEM( *xy, float, i*2, 3 ) = CV_MAT_ELEM( *xy, float, i*2, 5 ) = \ - CV_MAT_ELEM( *xy, float, i*2+1, 0 ) = CV_MAT_ELEM( *xy, float, i*2+1, 1 ) = CV_MAT_ELEM( *xy, float, i*2+1, 4 ) = 0; - CV_MAT_ELEM( *xy, float, i*2, 4 ) = CV_MAT_ELEM( *xy, float, i*2+1, 5 ) = 1; - CV_MAT_ELEM( *uv, float, i*2, 0 ) = pts1[j].x; - CV_MAT_ELEM( *uv, float, i*2+1, 0 ) = pts1[j].y; - i++; - } - } - - cvSolve( xy, uv, affine_trans, CV_SVD ); -} -#endif CV_IMPL int cvFindChessboardCorners( const void* arr, CvSize pattern_size, @@ -449,11 +410,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, const int min_dilations = 0; const int max_dilations = 7; cv::Ptr norm_img, thresh_img; -#ifdef DEBUG_CHESSBOARD - cv::Ptr dbg_img; - cv::Ptr dbg1_img; - cv::Ptr dbg2_img; -#endif cv::Ptr storage; CvMat stub, *img = (CvMat*)arr; @@ -487,12 +443,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, storage.reset(cvCreateMemStorage(0)); thresh_img.reset(cvCreateMat( img->rows, img->cols, CV_8UC1 )); -#ifdef DEBUG_CHESSBOARD - dbg_img = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 3 ); - dbg1_img = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 3 ); - dbg2_img = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 3 ); -#endif - if( CV_MAT_CN(img->type) != 1 || (flags & CV_CALIB_CB_NORMALIZE_IMAGE) ) { // equalize the input image histogram - @@ -627,9 +577,12 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, } }//dilations + PRINTF("Chessboard detection result 0: %d\n", found); + // revert to old, slower, method if detection failed if (!found) { + PRINTF("Fallback to old algorithm\n"); // empiric threshold level // thresholding performed here and not inside the cycle to save processing time int thresh_level; @@ -671,10 +624,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, cvDilate( thresh_img, thresh_img, 0, 1 ); } -#ifdef DEBUG_CHESSBOARD - cvCvtColor(thresh_img,dbg_img,CV_GRAY2BGR); -#endif - // So we can find rectangles that go to the edge, we draw a white line around the image edge. // Otherwise FindContours will miss those clipped rectangle contours. // The border color will be the image mean, because otherwise we risk screwing up filters like cvSmooth()... @@ -684,31 +633,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, quad_count = icvGenerateQuads( &quads, &corners, storage, thresh_img, flags, &max_quad_buf_size); PRINTF("Quad count: %d/%d\n", quad_count, expected_corners_num); -#ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img, dbg1_img); - cvNamedWindow("all_quads", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - for (int z=0; z<4; z++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - pt1 = quads[i].corners[z]->pt; - pt2 = quads[i].corners[(z+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (z>0) - color = CV_RGB(200,200,0); - cvLine( dbg1_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - - - cvShowImage("all_quads", (IplImage*)dbg1_img); - cvWaitKey(); -#endif - if( quad_count <= 0 ) { continue; @@ -739,33 +663,6 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, PRINTF("Orig count: %d After ordering: %d\n", icount, count); - -#ifdef DEBUG_CHESSBOARD - cvCopy(dbg_img,dbg2_img); - cvNamedWindow("connected_group", 1); - // copy corners to temp array - for(int i = 0; i < quad_count; i++ ) - { - if (quads[i].group_idx == group_idx) - for (int z=0; z<4; z++) - { - CvPoint2D32f pt1, pt2; - CvScalar color = CV_RGB(30,255,30); - if (quads[i].ordered) - color = CV_RGB(255,30,30); - pt1 = quads[i].corners[z]->pt; - pt2 = quads[i].corners[(z+1)%4]->pt; - pt2.x = (pt1.x + pt2.x)/2; - pt2.y = (pt1.y + pt2.y)/2; - if (z>0) - color = CV_RGB(200,200,0); - cvLine( dbg2_img, cvPointFrom32f(pt1), cvPointFrom32f(pt2), color, 3, 8); - } - } - cvShowImage("connected_group", (IplImage*)dbg2_img); - cvWaitKey(); -#endif - if (count == 0) continue; // haven't found inner quads @@ -812,10 +709,13 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, }// for k = 0 -> 6 } + PRINTF("Chessboard detection result 1: %d\n", found); if( found ) found = icvCheckBoardMonotony( out_corners, pattern_size ); + PRINTF("Chessboard detection result 2: %d\n", found); + // check that none of the found corners is too close to the image boundary if( found ) { @@ -830,36 +730,38 @@ int cvFindChessboardCorners( const void* arr, CvSize pattern_size, found = k == pattern_size.width*pattern_size.height; } - if( found && pattern_size.height % 2 == 0 && pattern_size.width % 2 == 0 ) + PRINTF("Chessboard detection result 3: %d\n", found); + + if( found ) { + if ( pattern_size.height % 2 == 0 && pattern_size.width % 2 == 0 ) + { int last_row = (pattern_size.height-1)*pattern_size.width; double dy0 = out_corners[last_row].y - out_corners[0].y; if( dy0 < 0 ) { - int n = pattern_size.width*pattern_size.height; - for(int i = 0; i < n/2; i++ ) - { - CvPoint2D32f temp; - CV_SWAP(out_corners[i], out_corners[n-i-1], temp); - } + int n = pattern_size.width*pattern_size.height; + for(int i = 0; i < n/2; i++ ) + { + CvPoint2D32f temp; + CV_SWAP(out_corners[i], out_corners[n-i-1], temp); + } } - } - - if( found ) - { - cv::Ptr gray; - if( CV_MAT_CN(img->type) != 1 ) - { - gray.reset(cvCreateMat(img->rows, img->cols, CV_8UC1)); - cvCvtColor(img, gray, CV_BGR2GRAY); - } - else - { - gray.reset(cvCloneMat(img)); - } - int wsize = 2; - cvFindCornerSubPix( gray, out_corners, pattern_size.width*pattern_size.height, - cvSize(wsize, wsize), cvSize(-1,-1), cvTermCriteria(CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 15, 0.1)); + } + cv::Ptr gray; + if( CV_MAT_CN(img->type) != 1 ) + { + gray.reset(cvCreateMat(img->rows, img->cols, CV_8UC1)); + cvCvtColor(img, gray, CV_BGR2GRAY); + } + else + { + gray.reset(cvCloneMat(img)); + } + int wsize = 2; + cvFindCornerSubPix( gray, out_corners, pattern_size.width*pattern_size.height, + cvSize(wsize, wsize), cvSize(-1,-1), + cvTermCriteria(CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 15, 0.1)); } } catch(...) From 3bfe76e24ccfb46d07bb8158f2adbc38220f28f7 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 13 Jul 2016 15:46:48 +0300 Subject: [PATCH 5/6] Ported PR #4187 to master --- modules/calib3d/test/test_chesscorners_timing.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/calib3d/test/test_chesscorners_timing.cpp b/modules/calib3d/test/test_chesscorners_timing.cpp index 61287ab671..570c125c2a 100644 --- a/modules/calib3d/test/test_chesscorners_timing.cpp +++ b/modules/calib3d/test/test_chesscorners_timing.cpp @@ -113,11 +113,7 @@ void CV_ChessboardDetectorTimingTest::run( int start_from ) if( img2.empty() ) { ts->printf( cvtest::TS::LOG, "one of chessboard images can't be read: %s\n", filename.c_str() ); - if( max_idx == 1 ) - { - code = cvtest::TS::FAIL_MISSING_TEST_DATA; - goto _exit_; - } + code = cvtest::TS::FAIL_MISSING_TEST_DATA; continue; } From 95da5ec245db6d038c43550b10cbb2c4964f82cd Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 13 Jul 2016 16:03:47 +0300 Subject: [PATCH 6/6] Restored check chessboard parameters --- modules/calib3d/src/checkchessboard.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/calib3d/src/checkchessboard.cpp b/modules/calib3d/src/checkchessboard.cpp index 5f954617d3..88c6baf107 100644 --- a/modules/calib3d/src/checkchessboard.cpp +++ b/modules/calib3d/src/checkchessboard.cpp @@ -61,8 +61,8 @@ int cvCheckChessboardBinary(IplImage* src, CvSize size); static void icvGetQuadrangleHypotheses(CvSeq* contours, std::vector >& quads, int class_id) { - const float min_aspect_ratio = 0.2f; - const float max_aspect_ratio = 5.0f; + const float min_aspect_ratio = 0.3f; + const float max_aspect_ratio = 3.0f; const float min_box_size = 10.0f; for(CvSeq* seq = contours; seq != NULL; seq = seq->h_next)