diff --git a/3rdparty/ippicv/ippicv.cmake b/3rdparty/ippicv/ippicv.cmake index 1c711d8847..744c45882e 100644 --- a/3rdparty/ippicv/ippicv.cmake +++ b/3rdparty/ippicv/ippicv.cmake @@ -2,9 +2,10 @@ function(download_ippicv root_var) set(${root_var} "" PARENT_SCOPE) # Commit SHA in the opencv_3rdparty repo - set(IPPICV_COMMIT "c7c6d527dde5fee7cb914ee9e4e20f7436aab3a1") + set(IPPICV_COMMIT "fd27188235d85e552de31425e7ea0f53ba73ba53") # Define actual ICV versions if(APPLE) + set(IPPICV_COMMIT "0cc4aa06bf2bef4b05d237c69a5a96b9cd0cb85a") set(OPENCV_ICV_PLATFORM "macosx") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_mac") set(OPENCV_ICV_NAME "ippicv_2021.9.1_mac_intel64_20230919_general.tgz") @@ -13,21 +14,21 @@ function(download_ippicv root_var) set(OPENCV_ICV_PLATFORM "linux") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_lnx") if(X86_64) - set(OPENCV_ICV_NAME "ippicv_2021.10.1_lnx_intel64_20231206_general.tgz") - set(OPENCV_ICV_HASH "90884d3b9508f31f6a154165591b8b0b") + set(OPENCV_ICV_NAME "ippicv_2021.11.0_lnx_intel64_20240201_general.tgz") + set(OPENCV_ICV_HASH "0f2745ff705ecae31176dad437608f6f") else() - set(OPENCV_ICV_NAME "ippicv_2021.10.1_lnx_ia32_20231206_general.tgz") - set(OPENCV_ICV_HASH "d9510f3ce08f6074aac472a5c19a3b53") + set(OPENCV_ICV_NAME "ippicv_2021.11.0_lnx_ia32_20240201_general.tgz") + set(OPENCV_ICV_HASH "63e381bf08076ca34fd5264203043a45") endif() elseif(WIN32 AND NOT ARM) set(OPENCV_ICV_PLATFORM "windows") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_win") if(X86_64) - set(OPENCV_ICV_NAME "ippicv_2021.10.1_win_intel64_20231206_general.zip") - set(OPENCV_ICV_HASH "2d5f137d4dd8a5205cc1edb5616fb3da") + set(OPENCV_ICV_NAME "ippicv_2021.11.0_win_intel64_20240201_general.zip") + set(OPENCV_ICV_HASH "59d154bf54a1e3eea20d7248f81a2a8e") else() - set(OPENCV_ICV_NAME "ippicv_2021.10.1_win_ia32_20231206_general.zip") - set(OPENCV_ICV_HASH "63c41a943e93ca87541b71ab67f207b5") + set(OPENCV_ICV_NAME "ippicv_2021.11.0_win_ia32_20240201_general.zip") + set(OPENCV_ICV_HASH "7a6d8ac5825c02fea6cbfc1201b521b5") endif() else() return() diff --git a/3rdparty/libjpeg-turbo/CMakeLists.txt b/3rdparty/libjpeg-turbo/CMakeLists.txt index 6e508c8860..a995707852 100644 --- a/3rdparty/libjpeg-turbo/CMakeLists.txt +++ b/3rdparty/libjpeg-turbo/CMakeLists.txt @@ -56,8 +56,7 @@ if(MSVC_IDE AND CMAKE_GENERATOR_PLATFORM MATCHES "arm64") set(CPU_TYPE arm64) endif() -OCV_OPTION(ENABLE_LIBJPEG_TURBO_SIMD "Include SIMD extensions for libjpeg-turbo, if available for this platform" (NOT CV_DISABLE_OPTIMIZATION) - VISIBLE_IF BUILD_JPEG) +OCV_OPTION(ENABLE_LIBJPEG_TURBO_SIMD "Include SIMD extensions for libjpeg-turbo, if available for this platform" (NOT CV_DISABLE_OPTIMIZATION)) option(WITH_ARITH_ENC "Include arithmetic encoding support when emulating the libjpeg v6b API/ABI" TRUE) option(WITH_ARITH_DEC "Include arithmetic decoding support when emulating the libjpeg v6b API/ABI" TRUE) set(WITH_SIMD 1) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0789a090d..56767f2ec7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1424,7 +1424,7 @@ status(" ZLib:" ZLIB_FOUND THEN "${ZLIB_LIBRARIES} (ver ${ZLIB_VERSION_STRI if(WITH_JPEG OR HAVE_JPEG) if(NOT HAVE_JPEG) status(" JPEG:" NO) - elseif(BUILD_JPEG) + elseif(BUILD_JPEG OR NOT JPEG_FOUND) status(" JPEG:" "build-${JPEG_LIBRARY} (ver ${JPEG_LIB_VERSION})") if(ENABLE_LIBJPEG_TURBO_SIMD) status(" SIMD Support Request:" "YES") diff --git a/doc/tutorials/objdetect/aruco_board_detection/aruco_board_detection.markdown b/doc/tutorials/objdetect/aruco_board_detection/aruco_board_detection.markdown index 87f5fb8d63..56eb1a75d1 100644 --- a/doc/tutorials/objdetect/aruco_board_detection/aruco_board_detection.markdown +++ b/doc/tutorials/objdetect/aruco_board_detection/aruco_board_detection.markdown @@ -2,8 +2,7 @@ Detection of ArUco boards {#tutorial_aruco_board_detection} ========================= @prev_tutorial{tutorial_aruco_detection} -@next_tutorial{tutorial_barcode_detect_and_decode} - +@next_tutorial{tutorial_charuco_detection} | | | | -: | :- | diff --git a/doc/tutorials/objdetect/aruco_calibration/aruco_calibration.markdown b/doc/tutorials/objdetect/aruco_calibration/aruco_calibration.markdown new file mode 100644 index 0000000000..e36e335d4f --- /dev/null +++ b/doc/tutorials/objdetect/aruco_calibration/aruco_calibration.markdown @@ -0,0 +1,88 @@ +Calibration with ArUco and ChArUco {#tutorial_aruco_calibration} +================================== + +@prev_tutorial{tutorial_charuco_diamond_detection} +@next_tutorial{tutorial_aruco_faq} + +The ArUco module can also be used to calibrate a camera. Camera calibration consists in obtaining the +camera intrinsic parameters and distortion coefficients. This parameters remain fixed unless the camera +optic is modified, thus camera calibration only need to be done once. + +Camera calibration is usually performed using the OpenCV `cv::calibrateCamera()` function. This function +requires some correspondences between environment points and their projection in the camera image from +different viewpoints. In general, these correspondences are obtained from the corners of chessboard +patterns. See `cv::calibrateCamera()` function documentation or the OpenCV calibration tutorial for +more detailed information. + +Using the ArUco module, calibration can be performed based on ArUco markers corners or ChArUco corners. +Calibrating using ArUco is much more versatile than using traditional chessboard patterns, since it +allows occlusions or partial views. + +As it can be stated, calibration can be done using both, marker corners or ChArUco corners. However, +it is highly recommended using the ChArUco corners approach since the provided corners are much +more accurate in comparison to the marker corners. Calibration using a standard Board should only be +employed in those scenarios where the ChArUco boards cannot be employed because of any kind of restriction. + +Calibration with ChArUco Boards +------------------------------- + +To calibrate using a ChArUco board, it is necessary to detect the board from different viewpoints, in the +same way that the standard calibration does with the traditional chessboard pattern. However, due to the +benefits of using ChArUco, occlusions and partial views are allowed, and not all the corners need to be +visible in all the viewpoints. + +![ChArUco calibration viewpoints](images/charucocalibration.jpg) + +The example of using `cv::calibrateCamera()` for cv::aruco::CharucoBoard: + +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp CalibrationWithCharucoBoard1 +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp CalibrationWithCharucoBoard2 +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp CalibrationWithCharucoBoard3 + +The ChArUco corners and ChArUco identifiers captured on each viewpoint are stored in the vectors +`allCharucoCorners` and `allCharucoIds`, one element per viewpoint. + +The `calibrateCamera()` function will fill the `cameraMatrix` and `distCoeffs` arrays with the +camera calibration parameters. It will return the reprojection error obtained from the calibration. +The elements in `rvecs` and `tvecs` will be filled with the estimated pose of the camera +(respect to the ChArUco board) in each of the viewpoints. + +Finally, the `calibrationFlags` parameter determines some of the options for the calibration. + +A full working example is included in the `calibrate_camera_charuco.cpp` inside the +`samples/cpp/tutorial_code/objectDetection` folder. + +The samples now take input via commandline via the `cv::CommandLineParser`. For this file the example +parameters will look like: +@code{.cpp} + "camera_calib.txt" -w=5 -h=7 -sl=0.04 -ml=0.02 -d=10 + -v=path/img_%02d.jpg +@endcode + +The camera calibration parameters from `opencv/samples/cpp/tutorial_code/objectDetection/tutorial_camera_charuco.yml` +were obtained by the `img_00.jpg-img_03.jpg` placed from this +[folder](https://github.com/opencv/opencv_contrib/tree/4.6.0/modules/aruco/tutorials/aruco_calibration/images). + +Calibration with ArUco Boards +----------------------------- + +As it has been stated, it is recommended the use of ChAruco boards instead of ArUco boards for camera +calibration, since ChArUco corners are more accurate than marker corners. However, in some special cases +it must be required to use calibration based on ArUco boards. As in the previous case, it requires +the detections of an ArUco board from different viewpoints. + +![ArUco calibration viewpoints](images/arucocalibration.jpg) + +The example of using `cv::calibrateCamera()` for cv::aruco::GridBoard: + +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp CalibrationWithArucoBoard1 +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp CalibrationWithArucoBoard2 +@snippet samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp CalibrationWithArucoBoard3 + +A full working example is included in the `calibrate_camera.cpp` inside the `samples/cpp/tutorial_code/objectDetection` folder. + +The samples now take input via commandline via the `cv::CommandLineParser`. For this file the example +parameters will look like: +@code{.cpp} + "camera_calib.txt" -w=5 -h=7 -l=100 -s=10 -d=10 -v=path/aruco_videos_or_images +@endcode diff --git a/doc/tutorials/objdetect/aruco_calibration/images/arucocalibration.jpg b/doc/tutorials/objdetect/aruco_calibration/images/arucocalibration.jpg new file mode 100644 index 0000000000..9a86015049 Binary files /dev/null and b/doc/tutorials/objdetect/aruco_calibration/images/arucocalibration.jpg differ diff --git a/doc/tutorials/objdetect/aruco_calibration/images/charucocalibration.jpg b/doc/tutorials/objdetect/aruco_calibration/images/charucocalibration.jpg new file mode 100644 index 0000000000..ed45382cd0 Binary files /dev/null and b/doc/tutorials/objdetect/aruco_calibration/images/charucocalibration.jpg differ diff --git a/doc/tutorials/objdetect/aruco_faq/aruco_faq.markdown b/doc/tutorials/objdetect/aruco_faq/aruco_faq.markdown new file mode 100644 index 0000000000..c17ed2f059 --- /dev/null +++ b/doc/tutorials/objdetect/aruco_faq/aruco_faq.markdown @@ -0,0 +1,191 @@ +Aruco module FAQ {#tutorial_aruco_faq} +================ + +@prev_tutorial{tutorial_aruco_calibration} +@next_tutorial{tutorial_barcode_detect_and_decode} + +This is a compilation of questions that can be useful for those that want to use the aruco module. + +- I only want to label some objects, what should I use? + +In this case, you only need single ArUco markers. You can place one or several markers with different +ids in each of the object you want to identify. + + +- Which algorithm is used for marker detection? + +The aruco module is based on the original ArUco library. A full description of the detection process +can be found in: + +> S. Garrido-Jurado, R. Muñoz-Salinas, F. J. Madrid-Cuevas, and M. J. Marín-Jiménez. 2014. +> "Automatic generation and detection of highly reliable fiducial markers under occlusion". +> Pattern Recogn. 47, 6 (June 2014), 2280-2292. DOI=10.1016/j.patcog.2014.01.005 + + +- My markers are not being detected correctly, what can I do? + +There can be many factors that avoid the correct detection of markers. You probably need to adjust +some of the parameters in the `cv::aruco::DetectorParameters` object. The first thing you can do is +checking if your markers are returned as rejected candidates by the `cv::aruco::ArucoDetector::detectMarkers()` +function. Depending on this, you should try to modify different parameters. + +If you are using a ArUco board, you can also try the `cv::aruco::ArucoDetector::refineDetectedMarkers()` function. +If you are [using big markers](https://github.com/opencv/opencv_contrib/issues/2811) (400x400 pixels and more), try +increasing `cv::aruco::DetectorParameters::adaptiveThreshWinSizeMax` value. +Also avoid [narrow borders around the ArUco marker](https://github.com/opencv/opencv_contrib/issues/2492) +(5% or less of the marker perimeter, adjusted by `cv::aruco::DetectorParameters::minMarkerDistanceRate`) +around markers. + + +- What are the benefits of ArUco boards? What are the drawbacks? + +Using a board of markers you can obtain the camera pose from a set of markers, instead of a single one. +This way, the detection is able to handle occlusion of partial views of the Board, since only one +marker is necessary to obtain the pose. + +Furthermore, as in most cases you are using more corners for pose estimation, it will be more +accurate than using a single marker. + +The main drawback is that a Board is not as versatile as a single marker. + + + +- What are the benefits of ChArUco boards over ArUco boards? And the drawbacks? + +ChArUco boards combines chessboards with ArUco boards. Thanks to this, the corners provided by +ChArUco boards are more accurate than those provided by ArUco Boards (or single markers). + +The main drawback is that ChArUco boards are not as versatile as ArUco board. For instance, +a ChArUco board is a planar board with a specific marker layout while the ArUco boards can have +any layout, even in 3d. Furthermore, the markers in the ChArUco board are usually smaller and +more difficult to detect. + + +- I do not need pose estimation, should I use ChArUco boards? + +No. The main goal of ChArUco boards is provide high accurate corners for pose estimation or camera +calibration. + + +- Should all the markers in an ArUco board be placed in the same plane? + +No, the marker corners in a ArUco board can be placed anywhere in its 3d coordinate system. + + +- Should all the markers in an ChArUco board be placed in the same plane? + +Yes, all the markers in a ChArUco board need to be in the same plane and their layout is fixed by +the chessboard shape. + + +- What is the difference between a `cv::aruco::Board` object and a `cv::aruco::GridBoard` object? + +The `cv::aruco::GridBoard` class is a specific type of board that inherits from `cv::aruco::Board` class. +A `cv::aruco::GridBoard` object is a board whose markers are placed in the same plane and in a grid layout. + + +- What are Diamond markers? + +Diamond markers are very similar to a ChArUco board of 3x3 squares. However, contrary to ChArUco boards, +the detection of diamonds is based on the relative position of the markers. +They are useful when you want to provide a conceptual meaning to any (or all) of the markers in +the diamond. An example is using one of the marker to provide the diamond scale. + + +- Do I need to detect marker before board detection, ChArUco board detection or Diamond detection? + +Yes, the detection of single markers is a basic tool in the aruco module. It is done using the +`cv::aruco::DetectorParameters::detectMarkers()` function. The rest of functionalities receives +a list of detected markers from this function. + + +- I want to calibrate my camera, can I use this module? + +Yes, the aruco module provides functionalities to calibrate the camera using both, ArUco boards and +ChArUco boards. + + +- Should I calibrate using a ChArUco board or an ArUco board? + +It is highly recommended the calibration using ChArUco board due to the high accuracy. + + +- Should I use a predefined dictionary or generate my own dictionary? + +In general, it is easier to use one of the predefined dictionaries. However, if you need a bigger +dictionary (in terms of number of markers or number of bits) you should generate your own dictionary. +Dictionary generation is also useful if you want to maximize the inter-marker distance to achieve +a better error correction during the identification step. + +- I am generating my own dictionary but it takes too long + +Dictionary generation should only be done once at the beginning of your application and it should take +some seconds. If you are generating the dictionary on each iteration of your detection loop, you are +doing it wrong. + +Furthermore, it is recommendable to save the dictionary to a file with `cv::aruco::Dictionary::writeDictionary()` +and read it with `cv::aruco::Dictionary::readDictionary()` on every execution, so you don't need +to generate it. + + +- I would like to use some markers of the original ArUco library that I have already printed, can I use them? + +Yes, one of the predefined dictionary is `cv::aruco::DICT_ARUCO_ORIGINAL`, which detects the marker +of the original ArUco library with the same identifiers. + + +- Can I use the Board configuration file of the original ArUco library in this module? + +Not directly, you will need to adapt the information of the ArUco file to the aruco module Board format. + + +- Can I use this module to detect the markers of other libraries based on binary fiducial markers? + +Probably yes, however you will need to port the dictionary of the original library to the aruco module format. + + +- Do I need to store the Dictionary information in a file so I can use it in different executions? + +If you are using one of the predefined dictionaries, it is not necessary. Otherwise, it is recommendable +that you save it to file. + + +- Do I need to store the Board information in a file so I can use it in different executions? + +If you are using a `cv::aruco::GridBoard` or a `cv::aruco::CharucoBoard` you only need to store +the board measurements that are provided to the `cv::aruco::GridBoard::GridBoard()` constructor or +in or `cv::aruco::CharucoBoard` constructor. If you manually modify the marker ids of the boards, +or if you use a different type of board, you should save your board object to file. + +- Does the aruco module provide functions to save the Dictionary or Board to file? + +You can use `cv::aruco::Dictionary::writeDictionary()` and `cv::aruco::Dictionary::readDictionary()` +for `cv::aruco::Dictionary`. The data member of board classes are public and can be easily stored. + + +- Alright, but how can I render a 3d model to create an augmented reality application? + +To do so, you will need to use an external rendering engine library, such as OpenGL. The aruco module +only provides the functionality to obtain the camera pose, i.e. the rotation and traslation vectors, +which is necessary to create the augmented reality effect. However, you will need to adapt the rotation +and traslation vectors from the OpenCV format to the format accepted by your 3d rendering library. +The original ArUco library contains examples of how to do it for OpenGL and Ogre3D. + + +- I have use this module in my research work, how can I cite it? + +You can cite the original ArUco library: + +> S. Garrido-Jurado, R. Muñoz-Salinas, F. J. Madrid-Cuevas, and M. J. Marín-Jiménez. 2014. +> "Automatic generation and detection of highly reliable fiducial markers under occlusion". +> Pattern Recogn. 47, 6 (June 2014), 2280-2292. DOI=10.1016/j.patcog.2014.01.005 + +- Pose estimation markers are not being detected correctly, what can I do? + +It is important to remark that the estimation of the pose using only 4 coplanar points is subject to ambiguity. +In general, the ambiguity can be solved, if the camera is near to the marker. +However, as the marker becomes small, the errors in the corner estimation grows and ambiguity comes +as a problem. Try increasing the size of the marker you're using, and you can also try non-symmetrical +(aruco_dict_utils.cpp) markers to avoid collisions. Use multiple markers (ArUco/ChArUco/Diamonds boards) +and pose estimation with solvePnP() with the `cv::SOLVEPNP_IPPE_SQUARE` option. +More in [this issue](https://github.com/opencv/opencv/issues/8813). diff --git a/doc/tutorials/objdetect/barcode_detect_and_decode/barcode_detect_and_decode.markdown b/doc/tutorials/objdetect/barcode_detect_and_decode/barcode_detect_and_decode.markdown index 4f2f0df381..7b7d8b4a0e 100644 --- a/doc/tutorials/objdetect/barcode_detect_and_decode/barcode_detect_and_decode.markdown +++ b/doc/tutorials/objdetect/barcode_detect_and_decode/barcode_detect_and_decode.markdown @@ -3,7 +3,7 @@ Barcode Recognition {#tutorial_barcode_detect_and_decode} @tableofcontents -@prev_tutorial{tutorial_aruco_board_detection} +@prev_tutorial{tutorial_aruco_faq} | | | | -: | :- | diff --git a/doc/tutorials/objdetect/charuco_detection/charuco_detection.markdown b/doc/tutorials/objdetect/charuco_detection/charuco_detection.markdown new file mode 100644 index 0000000000..c1039376ea --- /dev/null +++ b/doc/tutorials/objdetect/charuco_detection/charuco_detection.markdown @@ -0,0 +1,265 @@ +Detection of ChArUco Boards {#tutorial_charuco_detection} +=========================== + +@prev_tutorial{tutorial_aruco_board_detection} +@next_tutorial{tutorial_charuco_diamond_detection} + +ArUco markers and boards are very useful due to their fast detection and their versatility. +However, one of the problems of ArUco markers is that the accuracy of their corner positions is not +too high, even after applying subpixel refinement. + +On the contrary, the corners of chessboard patterns can be refined more accurately since each corner +is surrounded by two black squares. However, finding a chessboard pattern is not as versatile as +finding an ArUco board: it has to be completely visible and occlusions are not permitted. + +A ChArUco board tries to combine the benefits of these two approaches: + +![Charuco definition](images/charucodefinition.png) + +The ArUco part is used to interpolate the position of the chessboard corners, so that it has the +versatility of marker boards, since it allows occlusions or partial views. Moreover, since the +interpolated corners belong to a chessboard, they are very accurate in terms of subpixel accuracy. + +When high precision is necessary, such as in camera calibration, Charuco boards are a better option +than standard ArUco boards. + +Goal +---- + +In this tutorial you will learn: + +- How to create a charuco board ? +- How to detect the charuco corners without performing camera calibration ? +- How to detect the charuco corners with camera calibration and pose estimation ? + +Source code +----------- + +You can find this code in `samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp` + +Here's a sample code of how to achieve all the stuff enumerated at the goal list. + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp charuco_detect_board_full_sample + +ChArUco Board Creation +---------------------- + +The aruco module provides the `cv::aruco::CharucoBoard` class that represents a Charuco Board and +which inherits from the `cv::aruco::Board` class. + +This class, as the rest of ChArUco functionalities, are defined in: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp charucohdr + +To define a `cv::aruco::CharucoBoard`, it is necessary: + +- Number of chessboard squares in X and Y directions. +- Length of square side. +- Length of marker side. +- The dictionary of the markers. +- Ids of all the markers. + +As for the `cv::aruco::GridBoard` objects, the aruco module provides to create `cv::aruco::CharucoBoard` +easily. This object can be easily created from these parameters using the `cv::aruco::CharucoBoard` +constructor: + +@snippet samples/cpp/tutorial_code/objectDetection/create_board_charuco.cpp create_charucoBoard + +- The first parameter is the number of squares in X and Y direction respectively. +- The second and third parameters are the length of the squares and the markers respectively. They can + be provided in any unit, having in mind that the estimated pose for this board would be measured + in the same units (usually meters are used). +- Finally, the dictionary of the markers is provided. + +The ids of each of the markers are assigned by default in ascending order and starting on 0, like in +`cv::aruco::GridBoard` constructor. This can be easily customized by accessing to the ids vector +through `board.ids`, like in the `cv::aruco::Board` parent class. + +Once we have our `cv::aruco::CharucoBoard` object, we can create an image to print it. There are +two ways to do this: +1. By using the script `doc/patter_tools/gen_pattern.py `, see @subpage tutorial_camera_calibration_pattern. +2. By using the function `cv::aruco::CharucoBoard::generateImage()`. + +The function `cv::aruco::CharucoBoard::generateImage()` is provided in cv::aruco::CharucoBoard class +and can be called by using the following code: +@snippet samples/cpp/tutorial_code/objectDetection/create_board_charuco.cpp generate_charucoBoard + +- The first parameter is the size of the output image in pixels. If this is not proportional +to the board dimensions, it will be centered on the image. +- The second parameter is the output image with the charuco board. +- The third parameter is the (optional) margin in pixels, so none of the markers are touching the + image border. +- Finally, the size of the marker border, similarly to `cv::aruco::generateImageMarker()` function. + The default value is 1. + +The output image will be something like this: + +![](images/charucoboard.png) + +A full working example is included in the `create_board_charuco.cpp` inside the `samples/cpp/tutorial_code/objectDetection/`. + +The samples `create_board_charuco.cpp` now take input via commandline via the `cv::CommandLineParser`. +For this file the example +parameters will look like: +@code{.cpp} + "_output_path_/chboard.png" -w=5 -h=7 -sl=100 -ml=60 -d=10 +@endcode + + +ChArUco Board Detection +----------------------- + +When you detect a ChArUco board, what you are actually detecting is each of the chessboard corners +of the board. + +Each corner on a ChArUco board has a unique identifier (id) assigned. These ids go from 0 to the total +number of corners in the board. +The steps of charuco board detection can be broken down to the following steps: + +- **Taking input Image** + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp inputImg + +The original image where the markers are to be detected. The image is necessary to perform subpixel +refinement in the ChArUco corners. + +- **Reading the camera calibration Parameters(only for detection with camera calibration)** + +@snippet samples/cpp/tutorial_code/objectDetection/aruco_samples_utility.hpp camDistCoeffs + +The parameters of `readCameraParameters` are: +- The first parameter is the path to the camera intrinsic matrix and distortion coefficients. +- The second and third parameters are cameraMatrix and distCoeffs. + +This function takes these parameters as input and returns a boolean value of whether the camera +calibration parameters are valid or not. For detection of charuco corners without calibration, +this step is not required. + +- **Detecting the markers and interpolation of charuco corners from markers** + +The detection of the ChArUco corners is based on the previous detected markers. +So that, first markers are detected, and then ChArUco corners are interpolated from markers. +The method that detect the ChArUco corners is `cv::aruco::CharucoDetector::detectBoard()`. + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp interpolateCornersCharuco + +The parameters of detectBoard are: +- `image` - Input image. +- `charucoCorners` - output list of image positions of the detected corners. +- `charucoIds` - output ids for each of the detected corners in `charucoCorners`. +- `markerCorners` - input/output vector of detected marker corners. +- `markerIds` - input/output vector of identifiers of the detected markers + +If markerCorners and markerIds are empty, the function will detect aruco markers and ids. + +If calibration parameters are provided, the ChArUco corners are interpolated by, first, estimating +a rough pose from the ArUco markers and, then, reprojecting the ChArUco corners back to the image. + +On the other hand, if calibration parameters are not provided, the ChArUco corners are interpolated +by calculating the corresponding homography between the ChArUco plane and the ChArUco image projection. + +The main problem of using homography is that the interpolation is more sensible to image distortion. +Actually, the homography is only performed using the closest markers of each ChArUco corner to reduce +the effect of distortion. + +When detecting markers for ChArUco boards, and specially when using homography, it is recommended to +disable the corner refinement of markers. The reason of this is that, due to the proximity of the +chessboard squares, the subpixel process can produce important deviations in the corner positions and +these deviations are propagated to the ChArUco corner interpolation, producing poor results. + +@note To avoid deviations, the margin between chessboard square and aruco marker should be greater +than 70% of one marker module. + +Furthermore, only those corners whose two surrounding markers have be found are returned. If any of +the two surrounding markers has not been detected, this usually means that there is some occlusion +or the image quality is not good in that zone. In any case, it is preferable not to consider that +corner, since what we want is to be sure that the interpolated ChArUco corners are very accurate. + +After the ChArUco corners have been interpolated, a subpixel refinement is performed. + +Once we have interpolated the ChArUco corners, we would probably want to draw them to see if their +detections are correct. This can be easily done using the `cv::aruco::drawDetectedCornersCharuco()` +function: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp drawDetectedCornersCharuco + +- `imageCopy` is the image where the corners will be drawn (it will normally be the same image where + the corners were detected). +- The `outputImage` will be a clone of `inputImage` with the corners drawn. +- `charucoCorners` and `charucoIds` are the detected Charuco corners from the `cv::aruco::CharucoDetector::detectBoard()` + function. +- Finally, the last parameter is the (optional) color we want to draw the corners with, of type `cv::Scalar`. + +For this image: + +![Image with Charuco board](images/choriginal.jpg) + +The result will be: + +![Charuco board detected](images/chcorners.jpg) + +In the presence of occlusion. like in the following image, although some corners are clearly visible, +not all their surrounding markers have been detected due occlusion and, thus, they are not interpolated: + +![Charuco detection with occlusion](images/chocclusion.jpg) + +Sample video: + +@youtube{Nj44m_N_9FY} + +A full working example is included in the `detect_board_charuco.cpp` inside the +`samples/cpp/tutorial_code/objectDetection/`. + +The samples `detect_board_charuco.cpp` now take input via commandline via the `cv::CommandLineParser`. +For this file the example parameters will look like: +@code{.cpp} + -w=5 -h=7 -sl=0.04 -ml=0.02 -d=10 -v=/path_to_opencv/opencv/doc/tutorials/objdetect/charuco_detection/images/choriginal.jpg +@endcode + +ChArUco Pose Estimation +----------------------- + +The final goal of the ChArUco boards is finding corners very accurately for a high precision calibration +or pose estimation. + +The aruco module provides a function to perform ChArUco pose estimation easily. As in the +`cv::aruco::GridBoard`, the coordinate system of the `cv::aruco::CharucoBoard` is placed in +the board plane with the Z axis pointing in, and centered in the bottom left corner of the board. + +@note After OpenCV 4.6.0, there was an incompatible change in the coordinate systems of the boards, +now the coordinate systems are placed in the boards plane with the Z axis pointing in the plane +(previously the axis pointed out the plane). +`objPoints` in CW order correspond to the Z-axis pointing in the plane. +`objPoints` in CCW order correspond to the Z-axis pointing out the plane. +See PR https://github.com/opencv/opencv_contrib/pull/3174 + + +To perform pose estimation for charuco boards, you should use `cv::aruco::CharucoBoard::matchImagePoints()` +and `cv::solvePnP()`: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp poseCharuco + +- The `charucoCorners` and `charucoIds` parameters are the detected charuco corners from the + `cv::aruco::CharucoDetector::detectBoard()` function. +- The `cameraMatrix` and `distCoeffs` are the camera calibration parameters which are necessary + for pose estimation. +- Finally, the `rvec` and `tvec` parameters are the output pose of the Charuco Board. +- `cv::solvePnP()` returns true if the pose was correctly estimated and false otherwise. + The main reason of failing is that there are not enough corners for pose estimation or + they are in the same line. + +The axis can be drawn using `cv::drawFrameAxes()` to check the pose is correctly estimated. +The result would be: (X:red, Y:green, Z:blue) + +![Charuco Board Axis](images/chaxis.jpg) + +A full working example is included in the `detect_board_charuco.cpp` inside the +`samples/cpp/tutorial_code/objectDetection/`. + +The samples `detect_board_charuco.cpp` now take input via commandline via the `cv::CommandLineParser`. +For this file the example parameters will look like: +@code{.cpp} + -w=5 -h=7 -sl=0.04 -ml=0.02 -d=10 + -v=/path_to_opencv/opencv/doc/tutorials/objdetect/charuco_detection/images/choriginal.jpg + -c=/path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/tutorial_camera_charuco.yml +@endcode diff --git a/doc/tutorials/objdetect/charuco_detection/images/charucoboard.png b/doc/tutorials/objdetect/charuco_detection/images/charucoboard.png new file mode 100644 index 0000000000..ee9fe3e0db Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/charucoboard.png differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/charucodefinition.png b/doc/tutorials/objdetect/charuco_detection/images/charucodefinition.png new file mode 100644 index 0000000000..44684f310c Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/charucodefinition.png differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/chaxis.jpg b/doc/tutorials/objdetect/charuco_detection/images/chaxis.jpg new file mode 100644 index 0000000000..a00ba6134c Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/chaxis.jpg differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/chcorners.jpg b/doc/tutorials/objdetect/charuco_detection/images/chcorners.jpg new file mode 100644 index 0000000000..1eca446421 Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/chcorners.jpg differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/chocclusion.jpg b/doc/tutorials/objdetect/charuco_detection/images/chocclusion.jpg new file mode 100644 index 0000000000..e4860fc832 Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/chocclusion.jpg differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/chocclusion_original.jpg b/doc/tutorials/objdetect/charuco_detection/images/chocclusion_original.jpg new file mode 100644 index 0000000000..037a8eb128 Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/chocclusion_original.jpg differ diff --git a/doc/tutorials/objdetect/charuco_detection/images/choriginal.jpg b/doc/tutorials/objdetect/charuco_detection/images/choriginal.jpg new file mode 100644 index 0000000000..3ca7c3149f Binary files /dev/null and b/doc/tutorials/objdetect/charuco_detection/images/choriginal.jpg differ diff --git a/doc/tutorials/objdetect/charuco_diamond_detection/charuco_diamond_detection.markdown b/doc/tutorials/objdetect/charuco_diamond_detection/charuco_diamond_detection.markdown new file mode 100644 index 0000000000..04ae79ded0 --- /dev/null +++ b/doc/tutorials/objdetect/charuco_diamond_detection/charuco_diamond_detection.markdown @@ -0,0 +1,143 @@ +Detection of Diamond Markers {#tutorial_charuco_diamond_detection} +============================== + +@prev_tutorial{tutorial_charuco_detection} +@next_tutorial{tutorial_aruco_calibration} + +A ChArUco diamond marker (or simply diamond marker) is a chessboard composed by 3x3 squares and 4 ArUco markers inside the white squares. +It is similar to a ChArUco board in appearance, however they are conceptually different. + +![Diamond marker examples](images/diamondmarkers.jpg) + +In both, ChArUco board and Diamond markers, their detection is based on the previous detected ArUco +markers. In the ChArUco case, the used markers are selected by directly looking their identifiers. This means +that if a marker (included in the board) is found on a image, it will be automatically assumed to belong to the board. Furthermore, +if a marker board is found more than once in the image, it will produce an ambiguity since the system wont +be able to know which one should be used for the Board. + +On the other hand, the detection of Diamond marker is not based on the identifiers. Instead, their detection +is based on the relative position of the markers. As a consequence, marker identifiers can be repeated in the +same diamond or among different diamonds, and they can be detected simultaneously without ambiguity. However, +due to the complexity of finding marker based on their relative position, the diamond markers are limited to +a size of 3x3 squares and 4 markers. + +As in a single ArUco marker, each Diamond marker is composed by 4 corners and a identifier. The four corners +correspond to the 4 chessboard corners in the marker and the identifier is actually an array of 4 numbers, which are +the identifiers of the four ArUco markers inside the diamond. + +Diamond markers are useful in those scenarios where repeated markers should be allowed. For instance: + +- To increase the number of identifiers of single markers by using diamond marker for labeling. They would allow +up to N^4 different ids, being N the number of markers in the used dictionary. + +- Give to each of the four markers a conceptual meaning. For instance, one of the four marker ids could be +used to indicate the scale of the marker (i.e. the size of the square), so that the same diamond can be found +in the environment with different sizes just by changing one of the four markers and the user does not need +to manually indicate the scale of each of them. This case is included in the `detect_diamonds.cpp` file inside +the samples folder of the module. + +Furthermore, as its corners are chessboard corners, they can be used for accurate pose estimation. + +The diamond functionalities are included in `` + + +ChArUco Diamond Creation +------ + +The image of a diamond marker can be easily created using the `cv::aruco::CharucoBoard::generateImage()` function. +For instance: + +@snippet samples/cpp/tutorial_code/objectDetection/create_diamond.cpp generate_diamond + +This will create a diamond marker image with a square size of 200 pixels and a marker size of 120 pixels. +The marker ids are given in the second parameter as a `cv::Vec4i` object. The order of the marker ids +in the diamond layout are the same as in a standard ChArUco board, i.e. top, left, right and bottom. + +The image produced will be: + +![Diamond marker](images/diamondmarker.png) + +A full working example is included in the `create_diamond.cpp` inside the `samples/cpp/tutorial_code/objectDetection/`. + +The samples `create_diamond.cpp` now take input via commandline via the `cv::CommandLineParser`. For this file the example +parameters will look like: +@code{.cpp} + "_path_/mydiamond.png" -sl=200 -ml=120 -d=10 -ids=0,1,2,3 +@endcode + +ChArUco Diamond Detection +------ + +As in most cases, the detection of diamond markers requires a previous detection of ArUco markers. +After detecting markers, diamond are detected using the `cv::aruco::CharucoDetector::detectDiamonds()` function: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp detect_diamonds + +The `cv::aruco::CharucoDetector::detectDiamonds()` function receives the original image and the previous detected marker corners and ids. +If markerCorners and markerIds are empty, the function will detect aruco markers and ids. +The input image is necessary to perform subpixel refinement in the ChArUco corners. +It also receives the rate between the square size and the marker sizes which is required for both, detecting the diamond +from the relative positions of the markers and interpolating the ChArUco corners. + +The function returns the detected diamonds in two parameters. The first parameter, `diamondCorners`, is an array containing +all the four corners of each detected diamond. Its format is similar to the detected corners by the `cv::aruco::ArucoDetector::detectMarkers()` +function and, for each diamond, the corners are represented in the same order than in the ArUco markers, i.e. clockwise order +starting with the top-left corner. The second returned parameter, `diamondIds`, contains all the ids of the returned +diamond corners in `diamondCorners`. Each id is actually an array of 4 integers that can be represented with `cv::Vec4i`. + +The detected diamond can be visualized using the function `cv::aruco::drawDetectedDiamonds()` which simply receives the image and the diamond +corners and ids: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp draw_diamonds + +The result is the same that the one produced by `cv::aruco::drawDetectedMarkers()`, but printing the four ids of the diamond: + +![Detected diamond markers](images/detecteddiamonds.jpg) + +A full working example is included in the `detect_diamonds.cpp` inside the `samples/cpp/tutorial_code/objectDetection/`. + +The samples `detect_diamonds.cpp` now take input via commandline via the `cv::CommandLineParser`. For this file the example +parameters will look like: +@code{.cpp} + -dp=path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/detector_params.yml -sl=0.4 -ml=0.25 -refine=3 + -v=path_to_opencv/opencv/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarkers.jpg + -cd=path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/tutorial_dict.yml +@endcode + +ChArUco Diamond Pose Estimation +------ + +Since a ChArUco diamond is represented by its four corners, its pose can be estimated in the same way than in a single ArUco marker, +i.e. using the `cv::solvePnP()` function. For instance: + +@snippet samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp diamond_pose_estimation +@snippet samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp draw_diamond_pose_estimation + +The function will obtain the rotation and translation vector for each of the diamond marker and store them +in `rvecs` and `tvecs`. Note that the diamond corners are a chessboard square corners and thus, the square length +has to be provided for pose estimation, and not the marker length. Camera calibration parameters are also required. + +Finally, an axis can be drawn to check the estimated pose is correct using `drawFrameAxes()`: + +![Detected diamond axis](images/diamondsaxis.jpg) + +The coordinate system of the diamond pose will be in the center of the marker with the Z axis pointing out, +as in a simple ArUco marker pose estimation. + +Sample video: + +@youtube{OqKpBnglH7k} + +Also ChArUco diamond pose can be estimated as ChArUco board: +@snippet samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp diamond_pose_estimation_as_charuco + +A full working example is included in the `detect_diamonds.cpp` inside the `samples/cpp/tutorial_code/objectDetection/`. + +The samples `detect_diamonds.cpp` now take input via commandline via the `cv::CommandLineParser`. For this file the example +parameters will look like: +@code{.cpp} + -dp=path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/detector_params.yml -sl=0.4 -ml=0.25 -refine=3 + -v=path_to_opencv/opencv/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarkers.jpg + -cd=path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/tutorial_dict.yml + -c=path_to_opencv/opencv/samples/cpp/tutorial_code/objectDetection/tutorial_camera_params.yml +@endcode diff --git a/doc/tutorials/objdetect/charuco_diamond_detection/images/detecteddiamonds.jpg b/doc/tutorials/objdetect/charuco_diamond_detection/images/detecteddiamonds.jpg new file mode 100644 index 0000000000..32b6eb0b92 Binary files /dev/null and b/doc/tutorials/objdetect/charuco_diamond_detection/images/detecteddiamonds.jpg differ diff --git a/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarker.png b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarker.png new file mode 100644 index 0000000000..6a490806fc Binary files /dev/null and b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarker.png differ diff --git a/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarkers.jpg b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarkers.jpg new file mode 100644 index 0000000000..7d270ad648 Binary files /dev/null and b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondmarkers.jpg differ diff --git a/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondsaxis.jpg b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondsaxis.jpg new file mode 100644 index 0000000000..20e144036d Binary files /dev/null and b/doc/tutorials/objdetect/charuco_diamond_detection/images/diamondsaxis.jpg differ diff --git a/doc/tutorials/objdetect/table_of_content_objdetect.markdown b/doc/tutorials/objdetect/table_of_content_objdetect.markdown index 9456165d43..a79b29dd7a 100644 --- a/doc/tutorials/objdetect/table_of_content_objdetect.markdown +++ b/doc/tutorials/objdetect/table_of_content_objdetect.markdown @@ -3,4 +3,8 @@ Object Detection (objdetect module) {#tutorial_table_of_content_objdetect} - @subpage tutorial_aruco_detection - @subpage tutorial_aruco_board_detection +- @subpage tutorial_charuco_detection +- @subpage tutorial_charuco_diamond_detection +- @subpage tutorial_aruco_calibration +- @subpage tutorial_aruco_faq - @subpage tutorial_barcode_detect_and_decode diff --git a/modules/calib/src/calibinit.cpp b/modules/calib/src/calibinit.cpp index 489451a10c..e4e934ba95 100644 --- a/modules/calib/src/calibinit.cpp +++ b/modules/calib/src/calibinit.cpp @@ -1666,8 +1666,7 @@ void ChessBoardDetector::findQuadNeighbors() continue; const float dist = normL2Sqr(pt - all_quads_pts[neighbor_idx]); - if (dist < min_dist && - dist <= cur_quad.edge_len * thresh_scale && + if (dist <= cur_quad.edge_len * thresh_scale && dist <= q_k.edge_len * thresh_scale) { // check edge lengths, make sure they're compatible @@ -1684,6 +1683,7 @@ void ChessBoardDetector::findQuadNeighbors() closest_corner_idx = j; closest_quad = &q_k; min_dist = dist; + break; } } diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index 5c9ed53075..a85d2d39fe 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -933,7 +933,9 @@ inline hfloat hfloatFromBits(ushort w) { #endif } +#if !defined(__OPENCV_BUILD) && !(defined __STDCPP_FLOAT16_T__) && !(defined __ARM_NEON) typedef hfloat float16_t; +#endif class bfloat { diff --git a/modules/core/misc/objc/common/Mat.h b/modules/core/misc/objc/common/Mat.h index 42d4689e8f..04fc8da82c 100644 --- a/modules/core/misc/objc/common/Mat.h +++ b/modules/core/misc/objc/common/Mat.h @@ -15,7 +15,7 @@ #import #ifdef AVAILABLE_IMGCODECS -#if TARGET_OS_IPHONE +#if TARGET_OS_IPHONE || TARGET_OS_VISION #import #elif TARGET_OS_MAC #import @@ -197,7 +197,7 @@ CV_EXPORTS @interface Mat : NSObject - (instancetype)initWithCGImage:(CGImageRef)image; - (instancetype)initWithCGImage:(CGImageRef)image alphaExist:(BOOL)alphaExist; -#if TARGET_OS_IPHONE +#if TARGET_OS_IPHONE || TARGET_OS_VISION - (UIImage*)toUIImage; - (instancetype)initWithUIImage:(UIImage*)image; diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 80ada0b930..54387a4772 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -951,7 +951,7 @@ template int putData(NSArray* indices, cv::Mat* mat, int return [MatConverters convertCGImageRefToMat:image alphaExist:alphaExist]; } -#if TARGET_OS_IPHONE +#if TARGET_OS_IPHONE || TARGET_OS_VISION -(UIImage*)toUIImage { return [MatConverters converMatToUIImage:self]; diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py index 7cbc0645de..8a1e4580c9 100644 --- a/modules/core/misc/python/package/mat_wrapper/__init__.py +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -4,15 +4,16 @@ import numpy as np import cv2 as cv from typing import TYPE_CHECKING, Any -# Same as cv2.typing.NumPyArrayGeneric, but avoids circular dependencies +# Same as cv2.typing.NumPyArrayNumeric, but avoids circular dependencies if TYPE_CHECKING: - _NumPyArrayGeneric = np.ndarray[Any, np.dtype[np.generic]] + _NumPyArrayNumeric = np.ndarray[Any, np.dtype[np.integer[Any] | np.floating[Any]]] else: - _NumPyArrayGeneric = np.ndarray + _NumPyArrayNumeric = np.ndarray # NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html -class Mat(_NumPyArrayGeneric): + +class Mat(_NumPyArrayNumeric): ''' cv.Mat wrapper for numpy array. diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 12cb3a7588..1bd8a22da2 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -98,9 +98,11 @@ char* doubleToString( char* buf, size_t bufSize, double value, bool explicitZero } else { - static const char* fmt = "%.16e"; + // binary64 has 52 bit fraction with hidden bit. + // 53 * log_10(2) is 15.955. So "%.16f" should be fine, but its test fails. + snprintf( buf, bufSize, "%.17g", value ); + char* ptr = buf; - snprintf( buf, bufSize, fmt, value ); if( *ptr == '+' || *ptr == '-' ) ptr++; for( ; cv_isdigit(*ptr); ptr++ ) @@ -140,11 +142,21 @@ char* floatToString( char* buf, size_t bufSize, float value, bool halfprecision, } else { - char* ptr = buf; if (halfprecision) - snprintf(buf, bufSize, "%.4e", value); + { + // bfloat16 has 7 bit fraction with hidden bit. + // binary16 has 10 bit fraction with hidden bit. + // 11 * log_10(2) is 3.311. So "%.4f" should be fine, but its test fails. + snprintf(buf, bufSize, "%.5g", value); + } else - snprintf(buf, bufSize, "%.8e", value); + { + // binray32 has 23 bit fraction with hidden bit. + // 24 * log_10(2) is 7.225. So "%.8f" should be fine, but its test fails. + snprintf(buf, bufSize, "%.9g", value); + } + + char* ptr = buf; if( *ptr == '+' || *ptr == '-' ) ptr++; for( ; cv_isdigit(*ptr); ptr++ ) diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index d638a3edfc..f316a3940e 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -1191,11 +1191,7 @@ TEST(Core_InputOutput, FileStorage_DMatch) EXPECT_NO_THROW(fs << "d" << d); cv::String fs_result = fs.releaseAndGetString(); -#if defined _MSC_VER && _MSC_VER <= 1800 /* MSVC 2013 and older */ - EXPECT_STREQ(fs_result.c_str(), "%YAML:1.0\n---\nd: [ 1, 2, 3, -1.5000000000000000e+000 ]\n"); -#else - EXPECT_STREQ(fs_result.c_str(), "%YAML:1.0\n---\nd: [ 1, 2, 3, -1.5000000000000000e+00 ]\n"); -#endif + EXPECT_STREQ(fs_result.c_str(), "%YAML:1.0\n---\nd: [ 1, 2, 3, -1.5 ]\n"); cv::FileStorage fs_read(fs_result, cv::FileStorage::READ | cv::FileStorage::MEMORY); @@ -1222,25 +1218,14 @@ TEST(Core_InputOutput, FileStorage_DMatch_vector) EXPECT_NO_THROW(fs << "dv" << dv); cv::String fs_result = fs.releaseAndGetString(); -#if defined _MSC_VER && _MSC_VER <= 1800 /* MSVC 2013 and older */ EXPECT_STREQ(fs_result.c_str(), "%YAML:1.0\n" "---\n" "dv:\n" -" - [ 1, 2, 3, -1.5000000000000000e+000 ]\n" -" - [ 2, 3, 4, 1.5000000000000000e+000 ]\n" -" - [ 3, 2, 1, 5.0000000000000000e-001 ]\n" +" - [ 1, 2, 3, -1.5 ]\n" +" - [ 2, 3, 4, 1.5 ]\n" +" - [ 3, 2, 1, 0.5 ]\n" ); -#else - EXPECT_STREQ(fs_result.c_str(), -"%YAML:1.0\n" -"---\n" -"dv:\n" -" - [ 1, 2, 3, -1.5000000000000000e+00 ]\n" -" - [ 2, 3, 4, 1.5000000000000000e+00 ]\n" -" - [ 3, 2, 1, 5.0000000000000000e-01 ]\n" -); -#endif cv::FileStorage fs_read(fs_result, cv::FileStorage::READ | cv::FileStorage::MEMORY); @@ -1280,33 +1265,18 @@ TEST(Core_InputOutput, FileStorage_DMatch_vector_vector) EXPECT_NO_THROW(fs << "dvv" << dvv); cv::String fs_result = fs.releaseAndGetString(); #ifndef OPENCV_TRAITS_ENABLE_DEPRECATED -#if defined _MSC_VER && _MSC_VER <= 1800 /* MSVC 2013 and older */ EXPECT_STREQ(fs_result.c_str(), "%YAML:1.0\n" "---\n" "dvv:\n" " -\n" -" - [ 1, 2, 3, -1.5000000000000000e+000 ]\n" -" - [ 2, 3, 4, 1.5000000000000000e+000 ]\n" -" - [ 3, 2, 1, 5.0000000000000000e-001 ]\n" +" - [ 1, 2, 3, -1.5 ]\n" +" - [ 2, 3, 4, 1.5 ]\n" +" - [ 3, 2, 1, 0.5 ]\n" " -\n" -" - [ 3, 2, 1, 5.0000000000000000e-001 ]\n" -" - [ 1, 2, 3, -1.5000000000000000e+000 ]\n" +" - [ 3, 2, 1, 0.5 ]\n" +" - [ 1, 2, 3, -1.5 ]\n" ); -#else - EXPECT_STREQ(fs_result.c_str(), -"%YAML:1.0\n" -"---\n" -"dvv:\n" -" -\n" -" - [ 1, 2, 3, -1.5000000000000000e+00 ]\n" -" - [ 2, 3, 4, 1.5000000000000000e+00 ]\n" -" - [ 3, 2, 1, 5.0000000000000000e-01 ]\n" -" -\n" -" - [ 3, 2, 1, 5.0000000000000000e-01 ]\n" -" - [ 1, 2, 3, -1.5000000000000000e+00 ]\n" -); -#endif #endif // OPENCV_TRAITS_ENABLE_DEPRECATED cv::FileStorage fs_read(fs_result, cv::FileStorage::READ | cv::FileStorage::MEMORY); @@ -1988,5 +1958,53 @@ TEST(Core_InputOutput, FileStorage_invalid_path_regression_21448_JSON) fs.release(); } +// see https://github.com/opencv/opencv/issues/25073 +typedef testing::TestWithParam< std::string > Core_InputOutput_regression_25073; + +TEST_P(Core_InputOutput_regression_25073, my_double) +{ + cv::String res = ""; + double my_double = 0.5; + + FileStorage fs( GetParam(), cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + EXPECT_NO_THROW( fs << "my_double" << my_double ); + EXPECT_NO_THROW( fs << "my_int" << 5 ); + EXPECT_NO_THROW( res = fs.releaseAndGetString() ); + EXPECT_NE( res.find("0.5"), String::npos ) << res; // Found "0.5" + EXPECT_EQ( res.find("5.0"), String::npos ) << res; // Not Found "5.000000000000000000e-01" + fs.release(); +} + +TEST_P(Core_InputOutput_regression_25073, my_float) +{ + cv::String res = ""; + float my_float = 0.5; + + FileStorage fs( GetParam(), cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + EXPECT_NO_THROW( fs << "my_float" << my_float ); + EXPECT_NO_THROW( fs << "my_int" << 5 ); + EXPECT_NO_THROW( res = fs.releaseAndGetString() ); + EXPECT_NE( res.find("0.5"), String::npos ) << res; // Found "0.5" + EXPECT_EQ( res.find("5.0"), String::npos ) << res; // Not Found "5.00000000e-01", + fs.release(); +} + +TEST_P(Core_InputOutput_regression_25073, my_hfloat) +{ + cv::String res = ""; + cv::hfloat my_hfloat(0.5); + + FileStorage fs( GetParam(), cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + EXPECT_NO_THROW( fs << "my_hfloat" << my_hfloat ); + EXPECT_NO_THROW( fs << "my_int" << 5 ); + EXPECT_NO_THROW( res = fs.releaseAndGetString() ); + EXPECT_NE( res.find("0.5"), String::npos ) << res; // Found "0.5". + EXPECT_EQ( res.find("5.0"), String::npos ) << res; // Not Found "5.0000e-01". + fs.release(); +} + +INSTANTIATE_TEST_CASE_P( /*nothing*/, + Core_InputOutput_regression_25073, + Values("test.json", "test.xml", "test.yml") ); }} // namespace diff --git a/modules/dnn/cmake/plugin.cmake b/modules/dnn/cmake/plugin.cmake index 055d21efc3..df603b7c7a 100644 --- a/modules/dnn/cmake/plugin.cmake +++ b/modules/dnn/cmake/plugin.cmake @@ -50,6 +50,7 @@ function(ocv_create_builtin_dnn_plugin name target) endforeach() if(WIN32) + add_definitions(-D_USE_MATH_DEFINES) set(OPENCV_PLUGIN_VERSION "${OPENCV_DLLVERSION}" CACHE STRING "") if(CMAKE_CXX_SIZEOF_DATA_PTR EQUAL 8) set(OPENCV_PLUGIN_ARCH "_64" CACHE STRING "") diff --git a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp index 22d7d5194a..1734dccc63 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp @@ -494,10 +494,9 @@ void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, const int convMR_fp16, const int convNR_fp16) { - typedef __fp16 float16_t; - const float16_t* a = (const float16_t*)_a; - const float16_t* b = (const float16_t*)_b; - float16_t* c = (float16_t*)_c; + const __fp16* a = (const __fp16*)_a; + const __fp16* b = (const __fp16*)_b; + __fp16* c = (__fp16*)_c; CV_Assert(convMR_fp16 == 8 && convNR_fp16 == 24); float16x8_t c00 = vdupq_n_f16(0), c01 = c00, c02 = c00; @@ -638,12 +637,11 @@ void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16) { - typedef __fp16 float16_t; CV_Assert(convNR_FP16 == 24); // CONV_NR_FP16 = 24 - const float16_t* a = (const float16_t*)_a; - const float16_t* b = (const float16_t*)_b; + const __fp16* a = (const __fp16*)_a; + const __fp16* b = (const __fp16*)_b; - const float16_t bias = (float16_t)_bias; + const __fp16 bias = (__fp16)_bias; float16x8_t c0 = vdupq_n_f16(bias), c1 = c0, c2 = c0; diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp index d19cec64de..46e220e69f 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp @@ -85,7 +85,7 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu // works at FP 16. CONV_WINO_ATOM = CONV_WINO_ATOM_F16; CONV_WINO_NATOMS = CONV_WINO_NATOMS_F16; - esz = sizeof(float16_t); + esz = sizeof(__fp16); } #endif diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp index d1f1610280..e44d0f8004 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp @@ -435,10 +435,9 @@ void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, int Cg, int iblock, const int winoIblock, const int winoKblock, const int winoAtomF16, const int winoNatomF16) { - typedef __fp16 float16_t; - const float16_t* inwptr = (const float16_t*)_inwptr; - const float16_t* wptr = (const float16_t*)_wptr; - float16_t* outbuf = (float16_t*)_outbuf; + const __fp16* inwptr = (const __fp16*)_inwptr; + const __fp16* wptr = (const __fp16*)_wptr; + __fp16* outbuf = (__fp16*)_outbuf; CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF16 == 8); @@ -591,8 +590,7 @@ void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, i void winofunc_BtXB_8x8_F16(const float * inptr, int inpstep, char * _outptr, int Cg, const int winoIblock, const int winoAtomF16) { - typedef __fp16 float16_t; - float16_t* outptr = (float16_t*)_outptr; + __fp16* outptr = (__fp16*)_outptr; float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4); float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4); float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4); @@ -757,8 +755,7 @@ void winofunc_AtXA_8x8_F16(const char* _inptr, int inpstep, float * bpptr, int bpstep, float* outptr, int outstep, float bias, float minval, float maxval, bool ifMinMaxAct) { - typedef __fp16 float16_t; - const float16_t* inptr = (const float16_t*)_inptr; + const __fp16* inptr = (const __fp16*)_inptr; float32x4_t x00 = vcvt_f32_f16(vld1_f16(inptr)), x01 = vcvt_f32_f16(vld1_f16(inptr + 4)); float32x4_t x10 = vcvt_f32_f16(vld1_f16(inptr + inpstep)), x11 = vcvt_f32_f16(vld1_f16(inptr + inpstep + 4)); diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.cpp b/modules/dnn/src/layers/cpu_kernels/convolution.cpp index 7bbcf1e8e8..33fb62a47b 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.cpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.cpp @@ -26,7 +26,7 @@ void convBlockMR1_F32(int np, const float* a, const float* b, float *c, const fl #ifdef CONV_ARM_FP16 // Fast convert float 32 to float16 -static inline void _cvt32f16f(const float* src, float16_t* dst, int len) +static inline void _cvt32f16f(const float* src, __fp16* dst, int len) { int j = 0; const int VECSZ = 4; @@ -60,7 +60,7 @@ static inline void _cvt32f16f(const float* src, float16_t* dst, int len) vst1_f16(dst_FP16 + j, hv); } for( ; j < len; j++ ) - dst[j] = float16_t(src[j]); + dst[j] = __fp16(src[j]); } #endif @@ -74,12 +74,12 @@ float* FastConv::getWeightsWino() return alignPtr(weightsWinoBuf.data(), VEC_ALIGN); } -float16_t* FastConv::getWeightsFP16() +hfloat* FastConv::getWeightsFP16() { return alignPtr(weightsBuf_FP16.data(), VEC_ALIGN); } -float16_t* FastConv::getWeightsWinoFP16() +hfloat* FastConv::getWeightsWinoFP16() { return alignPtr(weightsWinoBuf_FP16.data(), VEC_ALIGN); } @@ -209,7 +209,7 @@ Ptr initFastConv( if (conv->useFP16) { conv->weightsBuf_FP16.resize(nweights + VEC_ALIGN); - auto weightsPtr_FP16 = conv->getWeightsFP16(); + auto weightsPtr_FP16 = (__fp16*)conv->getWeightsFP16(); parallel_for_(Range(0, C), [&](const Range& r0){ for(int c = r0.start; c < r0.end; c++) @@ -269,11 +269,11 @@ Ptr initFastConv( float* wptrWino = nullptr; #ifdef CONV_ARM_FP16 - float16_t* wptrWino_FP16 = nullptr; + __fp16* wptrWino_FP16 = nullptr; if (conv->useFP16) { conv->weightsWinoBuf_FP16.resize(nweights + VEC_ALIGN); - wptrWino_FP16 = conv->getWeightsWinoFP16(); + wptrWino_FP16 = (__fp16*)conv->getWeightsWinoFP16(); } else #endif @@ -323,7 +323,7 @@ Ptr initFastConv( #ifdef CONV_ARM_FP16 if (conv->useFP16) { - float16_t* wptr = wptrWino_FP16 + (g*Kg_nblocks + ki) * Cg *CONV_WINO_KBLOCK*CONV_WINO_AREA + + __fp16* wptr = wptrWino_FP16 + (g*Kg_nblocks + ki) * Cg *CONV_WINO_KBLOCK*CONV_WINO_AREA + (c*CONV_WINO_KBLOCK + dk)*CONV_WINO_ATOM_F16; for (int i = 0; i < CONV_WINO_NATOMS_F16; i++, wptr += Cg * CONV_WINO_KBLOCK * CONV_WINO_ATOM_F16) @@ -331,7 +331,7 @@ Ptr initFastConv( CV_Assert(wptrWino_FP16 <= wptr && wptr + CONV_WINO_ATOM_F16 <= wptrWino_FP16 + nweights); for (int j = 0; j < CONV_WINO_ATOM_F16; j++) { - wptr[j] = (float16_t)kernelTm[i * CONV_WINO_ATOM_F16 + j]; + wptr[j] = (__fp16)kernelTm[i * CONV_WINO_ATOM_F16 + j]; } } } @@ -367,12 +367,12 @@ Ptr initFastConv( int numStripsMR_FP16 = (Kg + CONV_MR_FP16 - 1) / CONV_MR_FP16; int Kg_aligned_FP16 = numStripsMR_FP16 * CONV_MR_FP16; size_t nweights_FP16 = ngroups * Kg_aligned_FP16 * DkHkWkCg; - float16_t* weightsPtr_FP16 = nullptr; + __fp16* weightsPtr_FP16 = nullptr; if (conv->useFP16) { conv->weightsBuf_FP16.resize(nweights_FP16 + VEC_ALIGN); - weightsPtr_FP16 = conv->getWeightsFP16(); + weightsPtr_FP16 = (__fp16*)conv->getWeightsFP16(); } else #endif @@ -394,7 +394,7 @@ Ptr initFastConv( int startK = si * CONV_MR_FP16; CV_Assert(startK < Kg_aligned_FP16); - float16_t* packed_wptr = weightsPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16); + __fp16* packed_wptr = weightsPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16); int dk = Kg - startK < CONV_MR_FP16 ? Kg - startK : CONV_MR_FP16; // check if we need zero padding. int k_idx = g*Kg + startK; @@ -405,9 +405,9 @@ Ptr initFastConv( const float* wptr = srcWeights + wstep * k_idx + c*Hk*Wk*Dk + hwd; int k = 0; for(; k < dk; k++, wptr += wstep) - packed_wptr[k] = (float16_t)(*wptr); + packed_wptr[k] = (__fp16)(*wptr); for(; k < CONV_MR_FP16; k++) - packed_wptr[k] = (float16_t)0.f; + packed_wptr[k] = (__fp16)0.f; } } }}); @@ -467,8 +467,8 @@ static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0, float* inptrInC = (float* )inptrIn; #ifdef CONV_ARM_FP16 - float16_t* inpbufC_FP16 = (float16_t *)inpbufC; - if (esz == sizeof(float16_t)) + __fp16* inpbufC_FP16 = (__fp16 *)inpbufC; + if (esz == sizeof(__fp16)) { if (stride_w == 1) { @@ -565,16 +565,16 @@ static inline void packData2(char *& inpbuf, float*& inptrIn, int& in_w, int& x0 float* inptrInC = inptrIn; #ifdef CONV_ARM_FP16 - float16_t* inpbufC_FP16 = (float16_t *)inpbufC; - if (esz == sizeof(float16_t)) + __fp16* inpbufC_FP16 = (__fp16 *)inpbufC; + if (esz == sizeof(__fp16)) { for (int k = 0; k < ksize; k++) { int k1 = ofstab[k]; float v0 = inptrInC[k1]; float v1 = inptrInC[k1 + stride_w]; - inpbufC_FP16[k*CONV_NR_FP16] = (float16_t)v0; - inpbufC_FP16[k*CONV_NR_FP16+1] = (float16_t)v1; + inpbufC_FP16[k*CONV_NR_FP16] = (__fp16)v0; + inpbufC_FP16[k*CONV_NR_FP16+1] = (__fp16)v1; } } else #endif @@ -630,7 +630,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta if (useFP16) { for (int c = 0; c < Cg; c++, inptr += inp_planesize, inpbuf += CONV_NR_esz) - _cvt32f16f(inptr, (float16_t *)inpbuf, CONV_NR); + _cvt32f16f(inptr, (__fp16 *)inpbuf, CONV_NR); } else #endif @@ -644,7 +644,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta { for (int c = 0; c < Cg; c++, inptr += inp_planesize, inpbuf += CONV_NR_esz) { - _cvt32f16f(inptr, (float16_t *)inpbuf, slice_len); + _cvt32f16f(inptr, (__fp16 *)inpbuf, slice_len); } } else @@ -704,11 +704,11 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta #ifdef CONV_ARM_FP16 if (useFP16) { - float16_t* inpbufC = (float16_t *)inpbuf + s0; + __fp16* inpbufC = (__fp16 *)inpbuf + s0; for (int w = w0; w < w1; w++) { int imgofs = w*dilation_w; - inpbufC[w*CONV_NR] = (float16_t)inptrInC[imgofs]; + inpbufC[w*CONV_NR] = (__fp16)inptrInC[imgofs]; } } else @@ -765,14 +765,14 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta #ifdef CONV_ARM_FP16 if (useFP16) { - float16_t* inpbufC = (float16_t *)inpbuf + s0; + __fp16* inpbufC = (__fp16 *)inpbuf + s0; for (int h = h0; h < h1; h++) { for (int w = w0; w < w1; w++) { int imgofs = h*(dilation_h*Wi) + w*dilation_w; - inpbufC[(h*Wk + w)*CONV_NR] = (float16_t)inptrInC[imgofs]; + inpbufC[(h*Wk + w)*CONV_NR] = (__fp16)inptrInC[imgofs]; } } } @@ -838,7 +838,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta #ifdef CONV_ARM_FP16 if (useFP16) { - float16_t* inpbufC = (float16_t* )inpbuf + s0; + __fp16* inpbufC = (__fp16* )inpbuf + s0; for ( int d = d0; d < d1; d++) { @@ -847,7 +847,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta for (int w = w0; w < w1; w++) { int imgofs = d*dilation_d*HWi + h*(dilation_h*Wi) + w*dilation_w; - inpbufC[((d*Hk + h)*Wk + w)*CONV_NR] = (float16_t)inptrInC[imgofs]; + inpbufC[((d*Hk + h)*Wk + w)*CONV_NR] = (__fp16)inptrInC[imgofs]; } } } @@ -889,7 +889,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta { float* inpbuf_ki = (float* )inpbuf + k * CONV_NR * Cg + i; #ifdef CONV_ARM_FP16 - float16_t * inpbuf_ki_FP16 = (float16_t *)inpbuf + k * CONV_NR * Cg + i; + __fp16 * inpbuf_ki_FP16 = (__fp16 *)inpbuf + k * CONV_NR * Cg + i; #endif int zi = z0 * stride_d + dz - pad_front; @@ -1053,7 +1053,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta if (useFP16) { for (int c = 0; c < Cg; c++, inpbuf_ki_FP16 += CONV_NR, inptr_ki += inp_planesize) - inpbuf_ki_FP16[0] = (float16_t)(*inptr_ki); + inpbuf_ki_FP16[0] = (__fp16)(*inptr_ki); } else #endif @@ -1069,7 +1069,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta if (useFP16) { for (int c = 0; c < Cg; c++, inpbuf_ki_FP16 += CONV_NR) - inpbuf_ki_FP16[0] = (float16_t)0.f; + inpbuf_ki_FP16[0] = (__fp16)0.f; } else #endif @@ -1257,7 +1257,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co // works at FP 16. CONV_NR = CONV_NR_FP16; CONV_MR = CONV_MR_FP16; - esz = sizeof(float16_t); + esz = sizeof(__fp16); } #endif @@ -1511,7 +1511,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co char *wptr = weights + (k0_block * DkHkWkCg + c0 * CONV_MR) * esz; float *cptr = cbuf_task + stripe * CONV_NR; - float16_t* cptr_f16 = (float16_t*)cbuf_task + stripe*CONV_NR; + hfloat* cptr_f16 = (hfloat*)cbuf_task + stripe*CONV_NR; for (int k = k0_block; k < k1_block; k += CONV_MR, wptr += DkHkWkCg * CONV_MR * esz, cptr += CONV_MR * ldc, cptr_f16 += CONV_MR * ldc) { @@ -1547,7 +1547,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co size_t outofs = ((n * ngroups + g) * Kg + k0_block) * out_planesize + zyx0; const float *cptr = cbuf_task; - const float16_t *cptr_fp16 = (const float16_t *)cbuf_task; + const hfloat *cptr_fp16 = (const hfloat *)cbuf_task; float *outptr = out + outofs; const float *pbptr = fusedAddPtr0 ? fusedAddPtr0 + outofs : 0; diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.hpp b/modules/dnn/src/layers/cpu_kernels/convolution.hpp index e9f169bbaf..5c8055337c 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.hpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.hpp @@ -62,10 +62,10 @@ struct FastConv float* getWeights(); float* getWeightsWino(); - std::vector weightsBuf_FP16; - std::vector weightsWinoBuf_FP16; - float16_t* getWeightsFP16(); - float16_t* getWeightsWinoFP16(); + std::vector weightsBuf_FP16; + std::vector weightsWinoBuf_FP16; + hfloat* getWeightsFP16(); + hfloat* getWeightsWinoFP16(); int conv_type; int conv_dim; // Flag for conv1d, conv2d, or conv3d. diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index f4af272490..0f9e808817 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -1745,12 +1745,12 @@ Mat getMatFromTensor(const opencv_onnx::TensorProto& tensor_proto) #endif const ::google::protobuf::RepeatedField field = tensor_proto.int32_data(); - AutoBuffer aligned_val; + AutoBuffer aligned_val; size_t sz = tensor_proto.int32_data().size(); aligned_val.allocate(sz); - float16_t* bufPtr = aligned_val.data(); + hfloat* bufPtr = aligned_val.data(); - float16_t *fp16Ptr = (float16_t *)field.data(); + hfloat *fp16Ptr = (hfloat *)field.data(); for (int i = 0; i < sz; i++) { bufPtr[i] = fp16Ptr[i*2 + offset]; @@ -1762,11 +1762,11 @@ Mat getMatFromTensor(const opencv_onnx::TensorProto& tensor_proto) char* val = const_cast(tensor_proto.raw_data().c_str()); #if CV_STRONG_ALIGNMENT // Aligned pointer is required. - AutoBuffer aligned_val; - if (!isAligned(val)) + AutoBuffer aligned_val; + if (!isAligned(val)) { size_t sz = tensor_proto.raw_data().size(); - aligned_val.allocate(divUp(sz, sizeof(float16_t))); + aligned_val.allocate(divUp(sz, sizeof(hfloat))); memcpy(aligned_val.data(), val, sz); val = (char*)aligned_val.data(); } diff --git a/modules/features2d/misc/java/test/AKAZEDescriptorExtractorTest.java b/modules/features2d/misc/java/test/AKAZEDescriptorExtractorTest.java index 69b12d00b1..a64b6ae4ad 100644 --- a/modules/features2d/misc/java/test/AKAZEDescriptorExtractorTest.java +++ b/modules/features2d/misc/java/test/AKAZEDescriptorExtractorTest.java @@ -58,7 +58,7 @@ public class AKAZEDescriptorExtractorTest extends OpenCVTestCase { extractor.write(filename); - String truth = "%YAML:1.0\n---\nformat: 3\nname: \"Feature2D.AKAZE\"\ndescriptor: 5\ndescriptor_channels: 3\ndescriptor_size: 0\nthreshold: 1.0000000474974513e-03\noctaves: 4\nsublevels: 4\ndiffusivity: 1\nmax_points: -1\n"; + String truth = "%YAML:1.0\n---\nformat: 3\nname: \"Feature2D.AKAZE\"\ndescriptor: 5\ndescriptor_channels: 3\ndescriptor_size: 0\nthreshold: 0.0010000000474974513\noctaves: 4\nsublevels: 4\ndiffusivity: 1\nmax_points: -1\n"; String actual = readFile(filename); actual = actual.replaceAll("e([+-])0(\\d\\d)", "e$1$2"); // NOTE: workaround for different platforms double representation assertEquals(truth, actual); diff --git a/modules/features2d/misc/java/test/GFTTFeatureDetectorTest.java b/modules/features2d/misc/java/test/GFTTFeatureDetectorTest.java index 86e42cbc1d..d21d4f2475 100644 --- a/modules/features2d/misc/java/test/GFTTFeatureDetectorTest.java +++ b/modules/features2d/misc/java/test/GFTTFeatureDetectorTest.java @@ -58,7 +58,7 @@ public class GFTTFeatureDetectorTest extends OpenCVTestCase { detector.write(filename); - String truth = "%YAML:1.0\n---\nname: \"Feature2D.GFTTDetector\"\nnfeatures: 1000\nqualityLevel: 1.0000000000000000e-02\nminDistance: 1.\nblockSize: 3\ngradSize: 3\nuseHarrisDetector: 0\nk: 4.0000000000000001e-02\n"; + String truth = "%YAML:1.0\n---\nname: \"Feature2D.GFTTDetector\"\nnfeatures: 1000\nqualityLevel: 0.01\nminDistance: 1.\nblockSize: 3\ngradSize: 3\nuseHarrisDetector: 0\nk: 0.040000000000000001\n"; String actual = readFile(filename); actual = actual.replaceAll("e([+-])0(\\d\\d)", "e$1$2"); // NOTE: workaround for different platforms double representation assertEquals(truth, actual); diff --git a/modules/features2d/misc/java/test/KAZEDescriptorExtractorTest.java b/modules/features2d/misc/java/test/KAZEDescriptorExtractorTest.java index 69ca35e015..d33ee24f49 100644 --- a/modules/features2d/misc/java/test/KAZEDescriptorExtractorTest.java +++ b/modules/features2d/misc/java/test/KAZEDescriptorExtractorTest.java @@ -57,7 +57,7 @@ public class KAZEDescriptorExtractorTest extends OpenCVTestCase { extractor.write(filename); - String truth = "%YAML:1.0\n---\nformat: 3\nname: \"Feature2D.KAZE\"\nextended: 0\nupright: 0\nthreshold: 1.0000000474974513e-03\noctaves: 4\nsublevels: 4\ndiffusivity: 1\n"; + String truth = "%YAML:1.0\n---\nformat: 3\nname: \"Feature2D.KAZE\"\nextended: 0\nupright: 0\nthreshold: 0.0010000000474974513\noctaves: 4\nsublevels: 4\ndiffusivity: 1\n"; String actual = readFile(filename); actual = actual.replaceAll("e([+-])0(\\d\\d)", "e$1$2"); // NOTE: workaround for different platforms double representation assertEquals(truth, actual); diff --git a/modules/features2d/misc/java/test/MSERFeatureDetectorTest.java b/modules/features2d/misc/java/test/MSERFeatureDetectorTest.java index 7f5f1c1849..956e0600e3 100644 --- a/modules/features2d/misc/java/test/MSERFeatureDetectorTest.java +++ b/modules/features2d/misc/java/test/MSERFeatureDetectorTest.java @@ -61,7 +61,7 @@ public class MSERFeatureDetectorTest extends OpenCVTestCase { detector.write(filename); - String truth = "%YAML:1.0\n---\nname: \"Feature2D.MSER\"\ndelta: 5\nminArea: 60\nmaxArea: 14400\nmaxVariation: 2.5000000000000000e-01\nminDiversity: 2.0000000000000001e-01\nmaxEvolution: 200\nareaThreshold: 1.0100000000000000e+00\nminMargin: 3.0000000000000001e-03\nedgeBlurSize: 5\npass2Only: 0\n"; + String truth = "%YAML:1.0\n---\nname: \"Feature2D.MSER\"\ndelta: 5\nminArea: 60\nmaxArea: 14400\nmaxVariation: 0.25\nminDiversity: 0.20000000000000001\nmaxEvolution: 200\nareaThreshold: 1.01\nminMargin: 0.0030000000000000001\nedgeBlurSize: 5\npass2Only: 0\n"; String actual = readFile(filename); actual = actual.replaceAll("e([+-])0(\\d\\d)", "e$1$2"); // NOTE: workaround for different platforms double representation assertEquals(truth, actual); diff --git a/modules/features2d/misc/java/test/ORBDescriptorExtractorTest.java b/modules/features2d/misc/java/test/ORBDescriptorExtractorTest.java index 6bc9bb6299..a1a96491f5 100644 --- a/modules/features2d/misc/java/test/ORBDescriptorExtractorTest.java +++ b/modules/features2d/misc/java/test/ORBDescriptorExtractorTest.java @@ -111,7 +111,7 @@ public class ORBDescriptorExtractorTest extends OpenCVTestCase { extractor.write(filename); - String truth = "%YAML:1.0\n---\nname: \"Feature2D.ORB\"\nnfeatures: 500\nscaleFactor: 1.2000000476837158e+00\nnlevels: 8\nedgeThreshold: 31\nfirstLevel: 0\nwta_k: 2\nscoreType: 0\npatchSize: 31\nfastThreshold: 20\n"; + String truth = "%YAML:1.0\n---\nname: \"Feature2D.ORB\"\nnfeatures: 500\nscaleFactor: 1.2000000476837158\nnlevels: 8\nedgeThreshold: 31\nfirstLevel: 0\nwta_k: 2\nscoreType: 0\npatchSize: 31\nfastThreshold: 20\n"; // String truth = "%YAML:1.0\n---\n"; String actual = readFile(filename); actual = actual.replaceAll("e\\+000", "e+00"); // NOTE: workaround for different platforms double representation diff --git a/modules/features2d/misc/java/test/SIFTDescriptorExtractorTest.java b/modules/features2d/misc/java/test/SIFTDescriptorExtractorTest.java index 63a59aa58c..dcd8564c3c 100644 --- a/modules/features2d/misc/java/test/SIFTDescriptorExtractorTest.java +++ b/modules/features2d/misc/java/test/SIFTDescriptorExtractorTest.java @@ -100,7 +100,7 @@ public class SIFTDescriptorExtractorTest extends OpenCVTestCase { extractor.write(filename); - String truth = "%YAML:1.0\n---\nname: \"Feature2D.SIFT\"\nnfeatures: 0\nnOctaveLayers: 3\ncontrastThreshold: 4.0000000000000001e-02\nedgeThreshold: 10.\nsigma: 1.6000000000000001e+00\ndescriptorType: 5\n"; + String truth = "%YAML:1.0\n---\nname: \"Feature2D.SIFT\"\nnfeatures: 0\nnOctaveLayers: 3\ncontrastThreshold: 0.040000000000000001\nedgeThreshold: 10.\nsigma: 1.6000000000000001\ndescriptorType: 5\n"; String actual = readFile(filename); actual = actual.replaceAll("e([+-])0(\\d\\d)", "e$1$2"); // NOTE: workaround for different platforms double representation assertEquals(truth, actual); diff --git a/modules/features2d/misc/java/test/SIMPLEBLOBFeatureDetectorTest.java b/modules/features2d/misc/java/test/SIMPLEBLOBFeatureDetectorTest.java index a67a0e8c3a..75817ca6b1 100644 --- a/modules/features2d/misc/java/test/SIMPLEBLOBFeatureDetectorTest.java +++ b/modules/features2d/misc/java/test/SIMPLEBLOBFeatureDetectorTest.java @@ -133,8 +133,7 @@ public class SIMPLEBLOBFeatureDetectorTest extends OpenCVTestCase { String filename = OpenCVTestRunner.getTempFileName("xml"); detector.write(filename); - - String truth = "\n\n3\n10.\n50.\n220.\n2\n10.\n1\n0\n1\n25.\n5000.\n0\n8.0000001192092896e-01\n3.4028234663852886e+38\n1\n1.0000000149011612e-01\n3.4028234663852886e+38\n1\n9.4999998807907104e-01\n3.4028234663852886e+38\n0\n\n"; + String truth = "\n\n3\n10.\n50.\n220.\n2\n10.\n1\n0\n1\n25.\n5000.\n0\n0.80000001192092896\n3.4028234663852886e+38\n1\n0.10000000149011612\n3.4028234663852886e+38\n1\n0.94999998807907104\n3.4028234663852886e+38\n0\n\n"; assertEquals(truth, readFile(filename)); } } diff --git a/modules/imgcodecs/src/ios_conversions.mm b/modules/imgcodecs/src/ios_conversions.mm index 7992325d77..5fea686643 100644 --- a/modules/imgcodecs/src/ios_conversions.mm +++ b/modules/imgcodecs/src/ios_conversions.mm @@ -40,7 +40,7 @@ // //M*/ #include -#if TARGET_OS_IOS && !TARGET_OS_MACCATALYST +#if (TARGET_OS_IOS || TARGET_OS_VISION) && !TARGET_OS_MACCATALYST #import #include "apple_conversions.h" diff --git a/modules/imgproc/doc/colors.markdown b/modules/imgproc/doc/colors.markdown index 9881ca793f..9827f5e62d 100644 --- a/modules/imgproc/doc/colors.markdown +++ b/modules/imgproc/doc/colors.markdown @@ -6,8 +6,8 @@ See cv::cvtColor and cv::ColorConversionCodes @todo document other conversion modes @anchor color_convert_rgb_gray -RGB \emoji arrow_right GRAY ------------------------------- +RGB <-> GRAY +------------ Transformations within RGB space like adding/removing the alpha channel, reversing the channel order, conversion to/from 16-bit RGB color (R5:G6:B5 or R5:G5:B5), as well as conversion to/from grayscale using: @@ -22,8 +22,8 @@ More advanced channel reordering can also be done with cv::mixChannels. @see cv::COLOR_BGR2GRAY, cv::COLOR_RGB2GRAY, cv::COLOR_GRAY2BGR, cv::COLOR_GRAY2RGB @anchor color_convert_rgb_xyz -RGB \emoji arrow_right CIE XYZ.Rec 709 with D65 white point --------------------------------------------------------------- +RGB <-> CIE XYZ.Rec 709 with D65 white point +-------------------------------------------- \f[\begin{bmatrix} X \\ Y \\ Z \end{bmatrix} \leftarrow \begin{bmatrix} 0.412453 & 0.357580 & 0.180423 \\ 0.212671 & 0.715160 & 0.072169 \\ 0.019334 & 0.119193 & 0.950227 \end{bmatrix} \cdot \begin{bmatrix} R \\ G \\ B @@ -37,8 +37,8 @@ RGB \emoji arrow_right CIE XYZ.Rec 709 with D65 white point @see cv::COLOR_BGR2XYZ, cv::COLOR_RGB2XYZ, cv::COLOR_XYZ2BGR, cv::COLOR_XYZ2RGB @anchor color_convert_rgb_ycrcb -RGB \emoji arrow_right YCrCb JPEG (or YCC) ---------------------------------------------- +RGB <-> YCrCb JPEG (or YCC) +--------------------------- \f[Y \leftarrow 0.299 \cdot R + 0.587 \cdot G + 0.114 \cdot B\f] \f[Cr \leftarrow (R-Y) \cdot 0.713 + delta\f] \f[Cb \leftarrow (B-Y) \cdot 0.564 + delta\f] @@ -51,8 +51,8 @@ Y, Cr, and Cb cover the whole value range. @see cv::COLOR_BGR2YCrCb, cv::COLOR_RGB2YCrCb, cv::COLOR_YCrCb2BGR, cv::COLOR_YCrCb2RGB @anchor color_convert_rgb_hsv -RGB \emoji arrow_right HSV ------------------------------ +RGB <-> HSV +----------- In case of 8-bit and 16-bit images, R, G, and B are converted to the floating-point format and scaled to fit the 0 to 1 range. @@ -73,8 +73,8 @@ The values are then converted to the destination data type: @see cv::COLOR_BGR2HSV, cv::COLOR_RGB2HSV, cv::COLOR_HSV2BGR, cv::COLOR_HSV2RGB @anchor color_convert_rgb_hls -RGB \emoji arrow_right HLS ------------------------------ +RGB <-> HLS +----------- In case of 8-bit and 16-bit images, R, G, and B are converted to the floating-point format and scaled to fit the 0 to 1 range. @@ -98,8 +98,8 @@ The values are then converted to the destination data type: @see cv::COLOR_BGR2HLS, cv::COLOR_RGB2HLS, cv::COLOR_HLS2BGR, cv::COLOR_HLS2RGB @anchor color_convert_rgb_lab -RGB \emoji arrow_right CIE L\*a\*b\* ---------------------------------------- +RGB <-> CIE L\*a\*b\* +--------------------- In case of 8-bit and 16-bit images, R, G, and B are converted to the floating-point format and scaled to fit the 0 to 1 range. @@ -123,8 +123,8 @@ are then converted to the destination data type: @see cv::COLOR_BGR2Lab, cv::COLOR_RGB2Lab, cv::COLOR_Lab2BGR, cv::COLOR_Lab2RGB @anchor color_convert_rgb_luv -RGB \emoji arrow_right CIE L\*u\*v\* ---------------------------------------- +RGB <-> CIE L\*u\*v\* +--------------------- In case of 8-bit and 16-bit images, R, G, and B are converted to the floating-point format and scaled to fit 0 to 1 range. @@ -150,8 +150,8 @@ sources on the web, primarily from the Charles Poynton site RGB +------------ The Bayer pattern is widely used in CCD and CMOS cameras. It enables you to get color pictures from a single plane where R, G, and B pixels (sensors of a particular component) are interleaved as follows: diff --git a/modules/imgproc/src/bilateral_filter.dispatch.cpp b/modules/imgproc/src/bilateral_filter.dispatch.cpp index 501af5b244..4ccec12496 100644 --- a/modules/imgproc/src/bilateral_filter.dispatch.cpp +++ b/modules/imgproc/src/bilateral_filter.dispatch.cpp @@ -415,6 +415,9 @@ void bilateralFilter( InputArray _src, OutputArray _dst, int d, Mat src = _src.getMat(), dst = _dst.getMat(); + CALL_HAL(bilateralFilter, cv_hal_bilateralFilter, src.data, src.step, dst.data, dst.step, src.cols, src.rows, src.depth(), + src.channels(), d, sigmaColor, sigmaSpace, borderType); + CV_IPP_RUN_FAST(ipp_bilateralFilter(src, dst, d, sigmaColor, sigmaSpace, borderType)); if( src.depth() == CV_8U ) diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index c066f3d6f3..5c6497bd80 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -763,6 +763,29 @@ inline int hal_ni_medianBlur(const uchar* src_data, size_t src_step, uchar* dst_ #define cv_hal_medianBlur hal_ni_medianBlur //! @endcond +/** + @brief Calculate bilateral filter. See https://homepages.inf.ed.ac.uk/rbf/CVonline/LOCAL_COPIES/MANDUCHI1/Bilateral_Filtering.html + @param src_data Source image data + @param src_step Source image step + @param dst_data Destination image data + @param dst_step Destination image step + @param width Source image width + @param height Source image height + @param depth Depths of source and destination image. Should support CV_8U and CV_32F + @param cn Number of channels + @param d Diameter of each pixel neighborhood that is used during filtering. If it is non-positive, it is computed from sigmaSpace + @param sigma_color Filter sigma in the color space + @param sigma_space Filter sigma in the coordinate space. When d>0, it specifies the neighborhood size regardless of sigmaSpace. Otherwise, d is proportional to sigmaSpace + @param border_type border mode used to extrapolate pixels outside of the image +*/ +inline int hal_ni_bilateralFilter(const uchar* src_data, size_t src_step, uchar* dst_data, size_t dst_step, + int width, int height, int depth, int cn, int d, double sigma_color, double sigma_space, int border_type) +{ return CV_HAL_ERROR_NOT_IMPLEMENTED; } + +//! @cond IGNORED +#define cv_hal_bilateralFilter hal_ni_bilateralFilter +//! @endcond + /** @brief Calculates adaptive threshold @param src_data Source image data diff --git a/modules/objdetect/test/test_aruco_tutorial.cpp b/modules/objdetect/test/test_aruco_tutorial.cpp new file mode 100644 index 0000000000..1af91bc637 --- /dev/null +++ b/modules/objdetect/test/test_aruco_tutorial.cpp @@ -0,0 +1,246 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "opencv2/objdetect/aruco_detector.hpp" + +namespace opencv_test { namespace { + + +TEST(CV_ArucoTutorial, can_find_singlemarkersoriginal) +{ + string img_path = cvtest::findDataFile("aruco/singlemarkersoriginal.jpg"); + Mat image = imread(img_path); + aruco::ArucoDetector detector(aruco::getPredefinedDictionary(aruco::DICT_6X6_250)); + + vector ids; + vector > corners, rejected; + const size_t N = 6ull; + // corners of ArUco markers with indices goldCornersIds + const int goldCorners[N][8] = { {359,310, 404,310, 410,350, 362,350}, {427,255, 469,256, 477,289, 434,288}, + {233,273, 190,273, 196,241, 237,241}, {298,185, 334,186, 335,212, 297,211}, + {425,163, 430,186, 394,186, 390,162}, {195,155, 230,155, 227,178, 190,178} }; + const int goldCornersIds[N] = { 40, 98, 62, 23, 124, 203}; + map mapGoldCorners; + for (size_t i = 0; i < N; i++) + mapGoldCorners[goldCornersIds[i]] = goldCorners[i]; + + detector.detectMarkers(image, corners, ids, rejected); + + ASSERT_EQ(N, ids.size()); + for (size_t i = 0; i < N; i++) + { + int arucoId = ids[i]; + ASSERT_EQ(4ull, corners[i].size()); + ASSERT_TRUE(mapGoldCorners.find(arucoId) != mapGoldCorners.end()); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2]), corners[i][j].x, 1.f); + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2 + 1]), corners[i][j].y, 1.f); + } + } +} + +TEST(CV_ArucoTutorial, can_find_gboriginal) +{ + string imgPath = cvtest::findDataFile("aruco/gboriginal.jpg"); + Mat image = imread(imgPath); + string dictPath = cvtest::findDataFile("aruco/tutorial_dict.yml"); + aruco::Dictionary dictionary; + + FileStorage fs(dictPath, FileStorage::READ); + dictionary.aruco::Dictionary::readDictionary(fs.root()); // set marker from tutorial_dict.yml + aruco::DetectorParameters detectorParams; + + aruco::ArucoDetector detector(dictionary, detectorParams); + + vector ids; + vector > corners, rejected; + const size_t N = 35ull; + // corners of ArUco markers with indices 0, 1, ..., 34 + const int goldCorners[N][8] = { {252,74, 286,81, 274,102, 238,95}, {295,82, 330,89, 319,111, 282,104}, + {338,91, 375,99, 365,121, 327,113}, {383,100, 421,107, 412,130, 374,123}, + {429,109, 468,116, 461,139, 421,132}, {235,100, 270,108, 257,130, 220,122}, + {279,109, 316,117, 304,140, 266,133}, {324,119, 362,126, 352,150, 313,143}, + {371,128, 410,136, 400,161, 360,152}, {418,139, 459,145, 451,170, 410,163}, + {216,128, 253,136, 239,161, 200,152}, {262,138, 300,146, 287,172, 248,164}, + {309,148, 349,156, 337,183, 296,174}, {358,158, 398,167, 388,194, 346,185}, + {407,169, 449,176, 440,205, 397,196}, {196,158, 235,168, 218,195, 179,185}, + {243,170, 283,178, 269,206, 228,197}, {293,180, 334,190, 321,218, 279,209}, + {343,192, 385,200, 374,230, 330,220}, {395,203, 438,211, 429,241, 384,233}, + {174,192, 215,201, 197,231, 156,221}, {223,204, 265,213, 249,244, 207,234}, + {275,215, 317,225, 303,257, 259,246}, {327,227, 371,238, 359,270, 313,259}, + {381,240, 426,249, 416,282, 369,273}, {151,228, 193,238, 173,271, 130,260}, + {202,241, 245,251, 228,285, 183,274}, {255,254, 300,264, 284,299, 238,288}, + {310,267, 355,278, 342,314, 295,302}, {366,281, 413,290, 402,327, 353,317}, + {125,267, 168,278, 147,314, 102,303}, {178,281, 223,293, 204,330, 157,317}, + {233,296, 280,307, 263,346, 214,333}, {291,310, 338,322, 323,363, 274,349}, + {349,325, 399,336, 386,378, 335,366} }; + map mapGoldCorners; + for (int i = 0; i < static_cast(N); i++) + mapGoldCorners[i] = goldCorners[i]; + + detector.detectMarkers(image, corners, ids, rejected); + + ASSERT_EQ(N, ids.size()); + for (size_t i = 0; i < N; i++) + { + int arucoId = ids[i]; + ASSERT_EQ(4ull, corners[i].size()); + ASSERT_TRUE(mapGoldCorners.find(arucoId) != mapGoldCorners.end()); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j*2]), corners[i][j].x, 1.f); + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j*2+1]), corners[i][j].y, 1.f); + } + } +} + +TEST(CV_ArucoTutorial, can_find_choriginal) +{ + string imgPath = cvtest::findDataFile("aruco/choriginal.jpg"); + Mat image = imread(imgPath); + aruco::ArucoDetector detector(aruco::getPredefinedDictionary(aruco::DICT_6X6_250)); + + vector< int > ids; + vector< vector< Point2f > > corners, rejected; + const size_t N = 17ull; + // corners of aruco markers with indices goldCornersIds + const int goldCorners[N][8] = { {268,77, 290,80, 286,97, 263,94}, {360,90, 382,93, 379,111, 357,108}, + {211,106, 233,109, 228,127, 205,123}, {306,120, 328,124, 325,142, 302,138}, + {402,135, 425,139, 423,157, 400,154}, {247,152, 271,155, 267,174, 242,171}, + {347,167, 371,171, 369,191, 344,187}, {185,185, 209,189, 203,210, 178,206}, + {288,201, 313,206, 309,227, 284,223}, {393,218, 418,222, 416,245, 391,241}, + {223,240, 250,244, 244,268, 217,263}, {333,258, 359,262, 356,286, 329,282}, + {152,281, 179,285, 171,312, 143,307}, {267,300, 294,305, 289,331, 261,327}, + {383,319, 410,324, 408,351, 380,347}, {194,347, 223,352, 216,382, 186,377}, + {315,368, 345,373, 341,403, 310,398} }; + map mapGoldCorners; + for (int i = 0; i < static_cast(N); i++) + mapGoldCorners[i] = goldCorners[i]; + + detector.detectMarkers(image, corners, ids, rejected); + + ASSERT_EQ(N, ids.size()); + for (size_t i = 0; i < N; i++) + { + int arucoId = ids[i]; + ASSERT_EQ(4ull, corners[i].size()); + ASSERT_TRUE(mapGoldCorners.find(arucoId) != mapGoldCorners.end()); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2]), corners[i][j].x, 1.f); + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2 + 1]), corners[i][j].y, 1.f); + } + } +} + +TEST(CV_ArucoTutorial, can_find_chocclusion) +{ + string imgPath = cvtest::findDataFile("aruco/chocclusion_original.jpg"); + Mat image = imread(imgPath); + aruco::ArucoDetector detector(aruco::getPredefinedDictionary(aruco::DICT_6X6_250)); + + vector< int > ids; + vector< vector< Point2f > > corners, rejected; + const size_t N = 13ull; + // corners of aruco markers with indices goldCornersIds + const int goldCorners[N][8] = { {301,57, 322,62, 317,79, 295,73}, {391,80, 413,85, 408,103, 386,97}, + {242,79, 264,85, 256,102, 234,96}, {334,103, 357,109, 352,126, 329,121}, + {428,129, 451,134, 448,152, 425,146}, {274,128, 296,134, 290,153, 266,147}, + {371,154, 394,160, 390,180, 366,174}, {208,155, 232,161, 223,181, 199,175}, + {309,182, 333,188, 327,209, 302,203}, {411,210, 436,216, 432,238, 407,231}, + {241,212, 267,219, 258,242, 232,235}, {167,244, 194,252, 183,277, 156,269}, + {202,314, 230,322, 220,349, 191,341} }; + map mapGoldCorners; + const int goldCornersIds[N] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 15}; + for (int i = 0; i < static_cast(N); i++) + mapGoldCorners[goldCornersIds[i]] = goldCorners[i]; + + detector.detectMarkers(image, corners, ids, rejected); + + ASSERT_EQ(N, ids.size()); + for (size_t i = 0; i < N; i++) + { + int arucoId = ids[i]; + ASSERT_EQ(4ull, corners[i].size()); + ASSERT_TRUE(mapGoldCorners.find(arucoId) != mapGoldCorners.end()); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2]), corners[i][j].x, 1.f); + EXPECT_NEAR(static_cast(mapGoldCorners[arucoId][j * 2 + 1]), corners[i][j].y, 1.f); + } + } +} + +TEST(CV_ArucoTutorial, can_find_diamondmarkers) +{ + string imgPath = cvtest::findDataFile("aruco/diamondmarkers.jpg"); + Mat image = imread(imgPath); + + string dictPath = cvtest::findDataFile("aruco/tutorial_dict.yml"); + aruco::Dictionary dictionary; + FileStorage fs(dictPath, FileStorage::READ); + dictionary.aruco::Dictionary::readDictionary(fs.root()); // set marker from tutorial_dict.yml + + string detectorPath = cvtest::findDataFile("aruco/detector_params.yml"); + fs = FileStorage(detectorPath, FileStorage::READ); + aruco::DetectorParameters detectorParams; + detectorParams.readDetectorParameters(fs.root()); + detectorParams.cornerRefinementMethod = aruco::CORNER_REFINE_APRILTAG; + + aruco::CharucoBoard charucoBoard(Size(3, 3), 0.4f, 0.25f, dictionary); + aruco::CharucoDetector detector(charucoBoard, aruco::CharucoParameters(), detectorParams); + + vector ids; + vector > corners, diamondCorners; + vector diamondIds; + const size_t N = 12ull; + // corner indices of ArUco markers + const int goldCornersIds[N] = { 4, 12, 11, 3, 12, 10, 12, 10, 10, 11, 2, 11 }; + map counterGoldCornersIds; + for (int i = 0; i < static_cast(N); i++) + counterGoldCornersIds[goldCornersIds[i]]++; + + const size_t diamondsN = 3; + // corners of diamonds with Vec4i indices + const float goldDiamondCorners[diamondsN][8] = {{195.6f,150.9f, 213.5f,201.2f, 136.4f,215.3f, 122.4f,163.5f}, + {501.1f,171.3f, 501.9f,208.5f, 446.2f,199.8f, 447.8f,163.3f}, + {343.4f,361.2f, 359.7f,328.7f, 400.8f,344.6f, 385.7f,378.4f}}; + auto comp = [](const Vec4i& a, const Vec4i& b) { + for (int i = 0; i < 3; i++) + if (a[i] != b[i]) return a[i] < b[i]; + return a[3] < b[3]; + }; + map goldDiamonds(comp); + goldDiamonds[Vec4i(10, 4, 11, 12)] = goldDiamondCorners[0]; + goldDiamonds[Vec4i(10, 3, 11, 12)] = goldDiamondCorners[1]; + goldDiamonds[Vec4i(10, 2, 11, 12)] = goldDiamondCorners[2]; + + detector.detectDiamonds(image, diamondCorners, diamondIds, corners, ids); + map counterRes; + + ASSERT_EQ(N, ids.size()); + for (size_t i = 0; i < N; i++) + { + int arucoId = ids[i]; + counterRes[arucoId]++; + } + + ASSERT_EQ(counterGoldCornersIds, counterRes); // check the number of ArUco markers + ASSERT_EQ(goldDiamonds.size(), diamondIds.size()); // check the number of diamonds + + for (size_t i = 0; i < goldDiamonds.size(); i++) + { + Vec4i diamondId = diamondIds[i]; + ASSERT_TRUE(goldDiamonds.find(diamondId) != goldDiamonds.end()); + for (int j = 0; j < 4; j++) + { + EXPECT_NEAR(goldDiamonds[diamondId][j * 2], diamondCorners[i][j].x, 0.5f); + EXPECT_NEAR(goldDiamonds[diamondId][j * 2 + 1], diamondCorners[i][j].y, 0.5f); + } + } +} + +}} // namespace diff --git a/modules/photo/src/inpaint.cpp b/modules/photo/src/inpaint.cpp index b4448f45dd..0fe3550bb8 100644 --- a/modules/photo/src/inpaint.cpp +++ b/modules/photo/src/inpaint.cpp @@ -46,6 +46,7 @@ // */ #include +#include #include "precomp.hpp" #include "opencv2/core/core_c.h" @@ -55,6 +56,16 @@ #define CV_MAT_ELEM_PTR_FAST( mat, row, col, pix_size ) \ ((mat).data.ptr + (size_t)(mat).step*(row) + (pix_size)*(col)) +template +typename std::enable_if::value, T>::type round_cast(float val) { + return cv::saturate_cast(val); +} + +template +typename std::enable_if::value, T>::type round_cast(float val) { + return cv::saturate_cast(val + 0.5); +} + inline float min4( float a, float b, float c, float d ) { @@ -339,8 +350,8 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu } } for (color=0; color<=2; color++) { - sat = (float)((Ia[color]/s[color]+(Jx[color]+Jy[color])/(sqrt(Jx[color]*Jx[color]+Jy[color]*Jy[color])+1.0e-20f)+0.5f)); - CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast(sat); + sat = (float)(Ia[color]/s[color]+(Jx[color]+Jy[color])/(sqrt(Jx[color]*Jx[color]+Jy[color]*Jy[color])+1.0e-20f)); + CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = round_cast(sat); } CV_MAT_ELEM(*f,uchar,i,j) = BAND; @@ -449,9 +460,9 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu } } } - sat = (float)((Ia/s+(Jx+Jy)/(sqrt(Jx*Jx+Jy*Jy)+1.0e-20f)+0.5f)); + sat = (float)(Ia/s+(Jx+Jy)/(sqrt(Jx*Jx+Jy*Jy)+1.0e-20f)); { - CV_MAT_ELEM(*out,data_type,i-1,j-1) = cv::saturate_cast(sat); + CV_MAT_ELEM(*out,data_type,i-1,j-1) = round_cast(sat); } } diff --git a/modules/photo/test/test_inpaint.cpp b/modules/photo/test/test_inpaint.cpp index 58806cbbc2..1536490693 100644 --- a/modules/photo/test/test_inpaint.cpp +++ b/modules/photo/test/test_inpaint.cpp @@ -116,9 +116,9 @@ void CV_InpaintTest::run( int ) TEST(Photo_Inpaint, regression) { CV_InpaintTest test; test.safe_run(); } -typedef testing::TestWithParam > formats; +typedef testing::TestWithParam > formats; -TEST_P(formats, 1c) +TEST_P(formats, basic) { const int type = get<0>(GetParam()); Mat src(100, 100, type); @@ -126,18 +126,18 @@ TEST_P(formats, 1c) Mat ref = src.clone(); Mat dst, mask = Mat::zeros(src.size(), CV_8U); - circle(src, Point(50, 50), 5, Scalar(200), 6); - circle(mask, Point(50, 50), 5, Scalar(200), 6); + circle(src, Point(50, 50), 5, Scalar::all(200), 6); + circle(mask, Point(50, 50), 5, Scalar::all(200), 6); inpaint(src, mask, dst, 10, INPAINT_NS); Mat dst2; inpaint(src, mask, dst2, 10, INPAINT_TELEA); - ASSERT_LE(cv::norm(dst, ref, NORM_INF), 3.); - ASSERT_LE(cv::norm(dst2, ref, NORM_INF), 3.); + ASSERT_EQ(cv::norm(dst, ref, NORM_INF), 0.); + ASSERT_EQ(cv::norm(dst2, ref, NORM_INF), 0.); } -INSTANTIATE_TEST_CASE_P(Photo_Inpaint, formats, testing::Values(CV_32F, CV_16U, CV_8U)); +INSTANTIATE_TEST_CASE_P(Photo_Inpaint, formats, testing::Values(CV_32FC1, CV_16UC1, CV_8UC1, CV_8UC3)); TEST(Photo_InpaintBorders, regression) { diff --git a/modules/python/src2/typing_stubs_generation/predefined_types.py b/modules/python/src2/typing_stubs_generation/predefined_types.py index 0514a14dda..f5ba0bc29e 100644 --- a/modules/python/src2/typing_stubs_generation/predefined_types.py +++ b/modules/python/src2/typing_stubs_generation/predefined_types.py @@ -34,7 +34,10 @@ _PREDEFINED_TYPES = ( PrimitiveTypeNode.str_("char"), PrimitiveTypeNode.str_("String"), PrimitiveTypeNode.str_("c_string"), - ConditionalAliasTypeNode.numpy_array_("NumPyArrayGeneric"), + ConditionalAliasTypeNode.numpy_array_( + "NumPyArrayNumeric", + dtype="numpy.integer[_typing.Any] | numpy.floating[_typing.Any]" + ), ConditionalAliasTypeNode.numpy_array_("NumPyArrayFloat32", dtype="numpy.float32"), ConditionalAliasTypeNode.numpy_array_("NumPyArrayFloat64", dtype="numpy.float64"), NoneTypeNode("void"), @@ -42,7 +45,7 @@ _PREDEFINED_TYPES = ( AliasTypeNode.union_( "Mat", items=(ASTNodeTypeNode("Mat", module_name="cv2.mat_wrapper"), - AliasRefTypeNode("NumPyArrayGeneric")), + AliasRefTypeNode("NumPyArrayNumeric")), export_name="MatLike" ), AliasTypeNode.sequence_("MatShape", PrimitiveTypeNode.int_()), diff --git a/modules/videoio/src/cap_avfoundation.mm b/modules/videoio/src/cap_avfoundation.mm index 0853e57255..2df990392f 100644 --- a/modules/videoio/src/cap_avfoundation.mm +++ b/modules/videoio/src/cap_avfoundation.mm @@ -221,18 +221,18 @@ cv::Ptr cv::create_AVFoundation_capture_file(const std::strin } -#if !TARGET_OS_VISION cv::Ptr cv::create_AVFoundation_capture_cam(int index) { +#if !TARGET_OS_VISION CvCaptureCAM* retval = new CvCaptureCAM(index); if (retval->didStart()) return cv::makePtr(retval); delete retval; +#endif return 0; } -#endif cv::Ptr cv::create_AVFoundation_writer(const std::string& filename, int fourcc, double fps, const cv::Size &frameSize, diff --git a/platforms/ios/PrivacyInfo.xcprivacy b/platforms/ios/PrivacyInfo.xcprivacy new file mode 100644 index 0000000000..f369de3813 --- /dev/null +++ b/platforms/ios/PrivacyInfo.xcprivacy @@ -0,0 +1,31 @@ + + + + + NSPrivacyTracking + + NSPrivacyCollectedDataTypes + + NSPrivacyTrackingDomains + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + 0A2A.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategorySystemBootTime + NSPrivacyAccessedAPITypeReasons + + 35F9.1 + + + + + diff --git a/platforms/ios/build_framework.py b/platforms/ios/build_framework.py index 4a5194a4a6..2c513e4ee2 100755 --- a/platforms/ios/build_framework.py +++ b/platforms/ios/build_framework.py @@ -46,6 +46,9 @@ from cv_build_utils import execute, print_error, get_xcode_major, get_xcode_sett IPHONEOS_DEPLOYMENT_TARGET='9.0' # default, can be changed via command line options or environment variable +CURRENT_FILE_DIR = os.path.dirname(__file__) + + class Builder: def __init__(self, opencv, contrib, dynamic, bitcodedisabled, exclude, disable, enablenonfree, targets, debug, debug_info, framework_name, run_tests, build_docs, swiftdisabled): self.opencv = os.path.abspath(opencv) @@ -477,6 +480,9 @@ class Builder: s = os.path.join(*l[0]) d = os.path.join(framework_dir, *l[1]) os.symlink(s, d) + # Copy Apple privacy manifest + shutil.copyfile(os.path.join(CURRENT_FILE_DIR, "PrivacyInfo.xcprivacy"), + os.path.join(resdir, "PrivacyInfo.xcprivacy")) def copy_samples(self, outdir): return diff --git a/samples/cpp/tutorial_code/objectDetection/aruco_samples_utility.hpp b/samples/cpp/tutorial_code/objectDetection/aruco_samples_utility.hpp index 3b28a91977..05c52e1133 100644 --- a/samples/cpp/tutorial_code/objectDetection/aruco_samples_utility.hpp +++ b/samples/cpp/tutorial_code/objectDetection/aruco_samples_utility.hpp @@ -45,4 +45,47 @@ inline static bool saveCameraParams(const std::string &filename, cv::Size imageS return true; } +inline static cv::aruco::DetectorParameters readDetectorParamsFromCommandLine(cv::CommandLineParser &parser) { + cv::aruco::DetectorParameters detectorParams; + if (parser.has("dp")) { + cv::FileStorage fs(parser.get("dp"), cv::FileStorage::READ); + bool readOk = detectorParams.readDetectorParameters(fs.root()); + if(!readOk) { + throw std::runtime_error("Invalid detector parameters file\n"); + } + } + return detectorParams; +} + +inline static void readCameraParamsFromCommandLine(cv::CommandLineParser &parser, cv::Mat& camMatrix, cv::Mat& distCoeffs) { + //! [camDistCoeffs] + if(parser.has("c")) { + bool readOk = readCameraParameters(parser.get("c"), camMatrix, distCoeffs); + if(!readOk) { + throw std::runtime_error("Invalid camera file\n"); + } + } + //! [camDistCoeffs] +} + +inline static cv::aruco::Dictionary readDictionatyFromCommandLine(cv::CommandLineParser &parser) { + cv::aruco::Dictionary dictionary; + if (parser.has("cd")) { + cv::FileStorage fs(parser.get("cd"), cv::FileStorage::READ); + bool readOk = dictionary.readDictionary(fs.root()); + if(!readOk) { + throw std::runtime_error("Invalid dictionary file\n"); + } + } + else { + int dictionaryId = parser.has("d") ? parser.get("d"): cv::aruco::DICT_4X4_50; + if (!parser.has("d")) { + std::cout << "The default DICT_4X4_50 dictionary has been selected, you could " + "select the specific dictionary using flags -d or -cd." << std::endl; + } + dictionary = cv::aruco::getPredefinedDictionary(dictionaryId); + } + return dictionary; +} + } diff --git a/samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp b/samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp new file mode 100644 index 0000000000..b415477019 --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/calibrate_camera.cpp @@ -0,0 +1,188 @@ +#include +#include +#include +#include +#include +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace std; +using namespace cv; + + +namespace { +const char* about = + "Calibration using a ArUco Planar Grid board\n" + " To capture a frame for calibration, press 'c',\n" + " If input comes from video, press any key for next frame\n" + " To finish capturing, press 'ESC' key and calibration starts.\n"; +const char* keys = + "{w | | Number of squares in X direction }" + "{h | | Number of squares in Y direction }" + "{l | | Marker side length (in meters) }" + "{s | | Separation between two consecutive markers in the grid (in meters) }" + "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{cd | | Input file with custom dictionary }" + "{@outfile |cam.yml| Output file with calibrated camera parameters }" + "{v | | Input from video file, if ommited, input comes from camera }" + "{ci | 0 | Camera id if input doesnt come from video (-v) }" + "{dp | | File of marker detector parameters }" + "{rs | false | Apply refind strategy }" + "{zt | false | Assume zero tangential distortion }" + "{a | | Fix aspect ratio (fx/fy) to this value }" + "{pc | false | Fix the principal point at the center }"; +} + + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + + if(argc < 6) { + parser.printMessage(); + return 0; + } + + int markersX = parser.get("w"); + int markersY = parser.get("h"); + float markerLength = parser.get("l"); + float markerSeparation = parser.get("s"); + string outputFile = parser.get(0); + + int calibrationFlags = 0; + float aspectRatio = 1; + if(parser.has("a")) { + calibrationFlags |= CALIB_FIX_ASPECT_RATIO; + aspectRatio = parser.get("a"); + } + if(parser.get("zt")) calibrationFlags |= CALIB_ZERO_TANGENT_DIST; + if(parser.get("pc")) calibrationFlags |= CALIB_FIX_PRINCIPAL_POINT; + + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); + + bool refindStrategy = parser.get("rs"); + int camId = parser.get("ci"); + String video; + + if(parser.has("v")) { + video = parser.get("v"); + } + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + VideoCapture inputVideo; + int waitTime; + if(!video.empty()) { + inputVideo.open(video); + waitTime = 0; + } else { + inputVideo.open(camId); + waitTime = 10; + } + + //! [CalibrationWithArucoBoard1] + // Create board object and ArucoDetector + aruco::GridBoard gridboard(Size(markersX, markersY), markerLength, markerSeparation, dictionary); + aruco::ArucoDetector detector(dictionary, detectorParams); + + // Collected frames for calibration + vector>> allMarkerCorners; + vector> allMarkerIds; + Size imageSize; + + while(inputVideo.grab()) { + Mat image, imageCopy; + inputVideo.retrieve(image); + + vector markerIds; + vector> markerCorners, rejectedMarkers; + + // Detect markers + detector.detectMarkers(image, markerCorners, markerIds, rejectedMarkers); + + // Refind strategy to detect more markers + if(refindStrategy) { + detector.refineDetectedMarkers(image, gridboard, markerCorners, markerIds, rejectedMarkers); + } + //! [CalibrationWithArucoBoard1] + + // Draw results + image.copyTo(imageCopy); + + if(!markerIds.empty()) { + aruco::drawDetectedMarkers(imageCopy, markerCorners, markerIds); + } + + putText(imageCopy, "Press 'c' to add current frame. 'ESC' to finish and calibrate", + Point(10, 20), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255, 0, 0), 2); + imshow("out", imageCopy); + + // Wait for key pressed + char key = (char)waitKey(waitTime); + + if(key == 27) { + break; + } + + //! [CalibrationWithArucoBoard2] + if(key == 'c' && !markerIds.empty()) { + cout << "Frame captured" << endl; + allMarkerCorners.push_back(markerCorners); + allMarkerIds.push_back(markerIds); + imageSize = image.size(); + } + } + //! [CalibrationWithArucoBoard2] + + if(allMarkerIds.empty()) { + throw std::runtime_error("Not enough captures for calibration\n"); + } + + //! [CalibrationWithArucoBoard3] + Mat cameraMatrix, distCoeffs; + + if(calibrationFlags & CALIB_FIX_ASPECT_RATIO) { + cameraMatrix = Mat::eye(3, 3, CV_64F); + cameraMatrix.at(0, 0) = aspectRatio; + } + + // Prepare data for calibration + vector objectPoints; + vector imagePoints; + vector processedObjectPoints, processedImagePoints; + size_t nFrames = allMarkerCorners.size(); + + for(size_t frame = 0; frame < nFrames; frame++) { + Mat currentImgPoints, currentObjPoints; + + gridboard.matchImagePoints(allMarkerCorners[frame], allMarkerIds[frame], currentObjPoints, currentImgPoints); + + if(currentImgPoints.total() > 0 && currentObjPoints.total() > 0) { + processedImagePoints.push_back(currentImgPoints); + processedObjectPoints.push_back(currentObjPoints); + } + } + + // Calibrate camera + double repError = calibrateCamera(processedObjectPoints, processedImagePoints, imageSize, cameraMatrix, distCoeffs, + noArray(), noArray(), noArray(), noArray(), noArray(), calibrationFlags); + //! [CalibrationWithArucoBoard3] + bool saveOk = saveCameraParams(outputFile, imageSize, aspectRatio, calibrationFlags, + cameraMatrix, distCoeffs, repError); + + if(!saveOk) { + throw std::runtime_error("Cannot save output file\n"); + } + + cout << "Rep Error: " << repError << endl; + cout << "Calibration saved to " << outputFile << endl; + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp b/samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp new file mode 100644 index 0000000000..5bea807db9 --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/calibrate_camera_charuco.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace std; +using namespace cv; + +namespace { +const char* about = + "Calibration using a ChArUco board\n" + " To capture a frame for calibration, press 'c',\n" + " If input comes from video, press any key for next frame\n" + " To finish capturing, press 'ESC' key and calibration starts.\n"; +const char* keys = + "{w | | Number of squares in X direction }" + "{h | | Number of squares in Y direction }" + "{sl | | Square side length (in meters) }" + "{ml | | Marker side length (in meters) }" + "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{cd | | Input file with custom dictionary }" + "{@outfile |cam.yml| Output file with calibrated camera parameters }" + "{v | | Input from video file, if ommited, input comes from camera }" + "{ci | 0 | Camera id if input doesnt come from video (-v) }" + "{dp | | File of marker detector parameters }" + "{rs | false | Apply refind strategy }" + "{zt | false | Assume zero tangential distortion }" + "{a | | Fix aspect ratio (fx/fy) to this value }" + "{pc | false | Fix the principal point at the center }" + "{sc | false | Show detected chessboard corners after calibration }"; +} + + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + + if(argc < 7) { + parser.printMessage(); + return 0; + } + + int squaresX = parser.get("w"); + int squaresY = parser.get("h"); + float squareLength = parser.get("sl"); + float markerLength = parser.get("ml"); + string outputFile = parser.get(0); + + bool showChessboardCorners = parser.get("sc"); + + int calibrationFlags = 0; + float aspectRatio = 1; + if(parser.has("a")) { + calibrationFlags |= CALIB_FIX_ASPECT_RATIO; + aspectRatio = parser.get("a"); + } + if(parser.get("zt")) calibrationFlags |= CALIB_ZERO_TANGENT_DIST; + if(parser.get("pc")) calibrationFlags |= CALIB_FIX_PRINCIPAL_POINT; + + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + + bool refindStrategy = parser.get("rs"); + int camId = parser.get("ci"); + String video; + + if(parser.has("v")) { + video = parser.get("v"); + } + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + VideoCapture inputVideo; + int waitTime; + if(!video.empty()) { + inputVideo.open(video); + waitTime = 0; + } else { + inputVideo.open(camId); + waitTime = 10; + } + + aruco::CharucoParameters charucoParams; + if(refindStrategy) { + charucoParams.tryRefineMarkers = true; + } + + //! [CalibrationWithCharucoBoard1] + // Create charuco board object and CharucoDetector + aruco::CharucoBoard board(Size(squaresX, squaresY), squareLength, markerLength, dictionary); + aruco::CharucoDetector detector(board, charucoParams, detectorParams); + + // Collect data from each frame + vector allCharucoCorners, allCharucoIds; + + vector> allImagePoints; + vector> allObjectPoints; + + vector allImages; + Size imageSize; + + while(inputVideo.grab()) { + Mat image, imageCopy; + inputVideo.retrieve(image); + + vector markerIds; + vector> markerCorners; + Mat currentCharucoCorners, currentCharucoIds; + vector currentObjectPoints; + vector currentImagePoints; + + // Detect ChArUco board + detector.detectBoard(image, currentCharucoCorners, currentCharucoIds); + //! [CalibrationWithCharucoBoard1] + + // Draw results + image.copyTo(imageCopy); + if(!markerIds.empty()) { + aruco::drawDetectedMarkers(imageCopy, markerCorners); + } + + if(currentCharucoCorners.total() > 3) { + aruco::drawDetectedCornersCharuco(imageCopy, currentCharucoCorners, currentCharucoIds); + } + + putText(imageCopy, "Press 'c' to add current frame. 'ESC' to finish and calibrate", + Point(10, 20), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255, 0, 0), 2); + + imshow("out", imageCopy); + + // Wait for key pressed + char key = (char)waitKey(waitTime); + + if(key == 27) { + break; + } + + //! [CalibrationWithCharucoBoard2] + if(key == 'c' && currentCharucoCorners.total() > 3) { + // Match image points + board.matchImagePoints(currentCharucoCorners, currentCharucoIds, currentObjectPoints, currentImagePoints); + + if(currentImagePoints.empty() || currentObjectPoints.empty()) { + cout << "Point matching failed, try again." << endl; + continue; + } + + cout << "Frame captured" << endl; + + allCharucoCorners.push_back(currentCharucoCorners); + allCharucoIds.push_back(currentCharucoIds); + allImagePoints.push_back(currentImagePoints); + allObjectPoints.push_back(currentObjectPoints); + allImages.push_back(image); + + imageSize = image.size(); + } + } + //! [CalibrationWithCharucoBoard2] + + if(allCharucoCorners.size() < 4) { + cerr << "Not enough corners for calibration" << endl; + return 0; + } + + //! [CalibrationWithCharucoBoard3] + Mat cameraMatrix, distCoeffs; + + if(calibrationFlags & CALIB_FIX_ASPECT_RATIO) { + cameraMatrix = Mat::eye(3, 3, CV_64F); + cameraMatrix.at(0, 0) = aspectRatio; + } + + // Calibrate camera using ChArUco + double repError = calibrateCamera(allObjectPoints, allImagePoints, imageSize, cameraMatrix, distCoeffs, + noArray(), noArray(), noArray(), noArray(), noArray(), calibrationFlags); + //! [CalibrationWithCharucoBoard3] + + bool saveOk = saveCameraParams(outputFile, imageSize, aspectRatio, calibrationFlags, + cameraMatrix, distCoeffs, repError); + + if(!saveOk) { + cerr << "Cannot save output file" << endl; + return 0; + } + + cout << "Rep Error: " << repError << endl; + cout << "Calibration saved to " << outputFile << endl; + + // Show interpolated charuco corners for debugging + if(showChessboardCorners) { + for(size_t frame = 0; frame < allImages.size(); frame++) { + Mat imageCopy = allImages[frame].clone(); + + if(allCharucoCorners[frame].total() > 0) { + aruco::drawDetectedCornersCharuco(imageCopy, allCharucoCorners[frame], allCharucoIds[frame]); + } + + imshow("out", imageCopy); + char key = (char)waitKey(0); + if(key == 27) { + break; + } + } + } + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/create_board.cpp b/samples/cpp/tutorial_code/objectDetection/create_board.cpp index ead9f73857..b1864ffc19 100644 --- a/samples/cpp/tutorial_code/objectDetection/create_board.cpp +++ b/samples/cpp/tutorial_code/objectDetection/create_board.cpp @@ -23,7 +23,6 @@ const char* keys = "{si | false | show generated image }"; } - int main(int argc, char *argv[]) { CommandLineParser parser(argc, argv, keys); parser.about(about); @@ -57,25 +56,7 @@ int main(int argc, char *argv[]) { imageSize.height = markersY * (markerLength + markerSeparation) - markerSeparation + 2 * margins; - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50); - if (parser.has("d")) { - int dictionaryId = parser.get("d"); - dictionary = aruco::getPredefinedDictionary(aruco::PredefinedDictionaryType(dictionaryId)); - } - else if (parser.has("cd")) { - FileStorage fs(parser.get("cd"), FileStorage::READ); - bool readOk = dictionary.readDictionary(fs.root()); - if(!readOk) - { - std::cerr << "Invalid dictionary file" << std::endl; - return 0; - } - } - else { - std::cerr << "Dictionary not specified" << std::endl; - return 0; - } - + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); aruco::GridBoard board(Size(markersX, markersY), float(markerLength), float(markerSeparation), dictionary); // show created board @@ -90,6 +71,5 @@ int main(int argc, char *argv[]) { } imwrite(out, boardImage); - return 0; } diff --git a/samples/cpp/tutorial_code/objectDetection/create_board_charuco.cpp b/samples/cpp/tutorial_code/objectDetection/create_board_charuco.cpp new file mode 100644 index 0000000000..b76708817a --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/create_board_charuco.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace cv; + +namespace { +const char* about = "Create a ChArUco board image"; +//! [charuco_detect_board_keys] +const char* keys = + "{@outfile |res.png| Output image }" + "{w | 5 | Number of squares in X direction }" + "{h | 7 | Number of squares in Y direction }" + "{sl | 100 | Square side length (in pixels) }" + "{ml | 60 | Marker side length (in pixels) }" + "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{cd | | Input file with custom dictionary }" + "{m | | Margins size (in pixels). Default is (squareLength-markerLength) }" + "{bb | 1 | Number of bits in marker borders }" + "{si | false | show generated image }"; +} +//! [charuco_detect_board_keys] + + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + if (argc == 1) { + parser.printMessage(); + } + + int squaresX = parser.get("w"); + int squaresY = parser.get("h"); + int squareLength = parser.get("sl"); + int markerLength = parser.get("ml"); + int margins = squareLength - markerLength; + if(parser.has("m")) { + margins = parser.get("m"); + } + + int borderBits = parser.get("bb"); + bool showImage = parser.get("si"); + + std::string pathOutImg = parser.get(0); + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + //! [create_charucoBoard] + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + cv::aruco::CharucoBoard board(Size(squaresX, squaresY), (float)squareLength, (float)markerLength, dictionary); + //! [create_charucoBoard] + + // show created board + //! [generate_charucoBoard] + Mat boardImage; + Size imageSize; + imageSize.width = squaresX * squareLength + 2 * margins; + imageSize.height = squaresY * squareLength + 2 * margins; + board.generateImage(imageSize, boardImage, margins, borderBits); + //! [generate_charucoBoard] + + if(showImage) { + imshow("board", boardImage); + waitKey(0); + } + + if (pathOutImg != "") + imwrite(pathOutImg, boardImage); + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/create_diamond.cpp b/samples/cpp/tutorial_code/objectDetection/create_diamond.cpp new file mode 100644 index 0000000000..0db00e7e02 --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/create_diamond.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace std; +using namespace cv; + +namespace { +const char* about = "Create a ChArUco marker image"; +const char* keys = + "{@outfile | res.png | Output image }" + "{sl | 100 | Square side length (in pixels) }" + "{ml | 60 | Marker side length (in pixels) }" + "{cd | | Input file with custom dictionary }" + "{d | 10 | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{ids |0, 1, 2, 3 | Four ids for the ChArUco marker: id1,id2,id3,id4 }" + "{m | 0 | Margins size (in pixels) }" + "{bb | 1 | Number of bits in marker borders }" + "{si | false | show generated image }"; +} + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + + int squareLength = parser.get("sl"); + int markerLength = parser.get("ml"); + string idsString = parser.get("ids"); + int margins = parser.get("m"); + int borderBits = parser.get("bb"); + bool showImage = parser.get("si"); + string out = parser.get(0); + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + istringstream ss(idsString); + vector splittedIds; + string token; + while(getline(ss, token, ',')) + splittedIds.push_back(token); + if(splittedIds.size() < 4) { + throw std::runtime_error("Incorrect ids format\n"); + } + Vec4i ids; + for(int i = 0; i < 4; i++) + ids[i] = atoi(splittedIds[i].c_str()); + + //! [generate_diamond] + vector diamondIds = {ids[0], ids[1], ids[2], ids[3]}; + aruco::CharucoBoard charucoBoard(Size(3, 3), (float)squareLength, (float)markerLength, dictionary, diamondIds); + Mat markerImg; + charucoBoard.generateImage(Size(3*squareLength + 2*margins, 3*squareLength + 2*margins), markerImg, margins, borderBits); + //! [generate_diamond] + + if(showImage) { + imshow("board", markerImg); + waitKey(0); + } + + if (out != "") + imwrite(out, markerImg); + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/create_marker.cpp b/samples/cpp/tutorial_code/objectDetection/create_marker.cpp index 57b08b0ef7..560e51c974 100644 --- a/samples/cpp/tutorial_code/objectDetection/create_marker.cpp +++ b/samples/cpp/tutorial_code/objectDetection/create_marker.cpp @@ -10,13 +10,13 @@ const char* about = "Create an ArUco marker image"; //! [aruco_create_markers_keys] const char* keys = - "{@outfile | | Output image }" - "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "{@outfile |res.png| Output image }" + "{d | 0 | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" "{cd | | Input file with custom dictionary }" - "{id | | Marker id in the dictionary }" + "{id | 0 | Marker id in the dictionary }" "{ms | 200 | Marker size in pixels }" "{bb | 1 | Number of bits in marker borders }" "{si | false | show generated image }"; @@ -28,11 +28,6 @@ int main(int argc, char *argv[]) { CommandLineParser parser(argc, argv, keys); parser.about(about); - if(argc < 4) { - parser.printMessage(); - return 0; - } - int markerId = parser.get("id"); int borderBits = parser.get("bb"); int markerSize = parser.get("ms"); @@ -45,23 +40,7 @@ int main(int argc, char *argv[]) { return 0; } - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50); - if (parser.has("d")) { - int dictionaryId = parser.get("d"); - dictionary = aruco::getPredefinedDictionary(aruco::PredefinedDictionaryType(dictionaryId)); - } - else if (parser.has("cd")) { - FileStorage fs(parser.get("cd"), FileStorage::READ); - bool readOk = dictionary.readDictionary(fs.root()); - if(!readOk) { - std::cerr << "Invalid dictionary file" << std::endl; - return 0; - } - } - else { - std::cerr << "Dictionary not specified" << std::endl; - return 0; - } + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); Mat markerImg; aruco::generateImageMarker(dictionary, markerId, markerSize, markerImg, borderBits); diff --git a/samples/cpp/tutorial_code/objectDetection/detect_board.cpp b/samples/cpp/tutorial_code/objectDetection/detect_board.cpp index ffea660e76..a9c74f2fdc 100644 --- a/samples/cpp/tutorial_code/objectDetection/detect_board.cpp +++ b/samples/cpp/tutorial_code/objectDetection/detect_board.cpp @@ -30,46 +30,6 @@ const char* keys = } //! [aruco_detect_board_keys] -static void readDetectorParamsFromCommandLine(CommandLineParser &parser, aruco::DetectorParameters& detectorParams) { - if(parser.has("dp")) { - FileStorage fs(parser.get("dp"), FileStorage::READ); - bool readOk = detectorParams.readDetectorParameters(fs.root()); - if(!readOk) { - cerr << "Invalid detector parameters file" << endl; - throw -1; - } - } -} - -static void readCameraParamsFromCommandLine(CommandLineParser &parser, Mat& camMatrix, Mat& distCoeffs) { - if(parser.has("c")) { - bool readOk = readCameraParameters(parser.get("c"), camMatrix, distCoeffs); - if(!readOk) { - cerr << "Invalid camera file" << endl; - throw -1; - } - } -} - -static void readDictionatyFromCommandLine(CommandLineParser &parser, aruco::Dictionary& dictionary) { - if (parser.has("d")) { - int dictionaryId = parser.get("d"); - dictionary = aruco::getPredefinedDictionary(aruco::PredefinedDictionaryType(dictionaryId)); - } - else if (parser.has("cd")) { - FileStorage fs(parser.get("cd"), FileStorage::READ); - bool readOk = dictionary.readDictionary(fs.root()); - if(!readOk) { - cerr << "Invalid dictionary file" << endl; - throw -1; - } - } - else { - cerr << "Dictionary not specified" << endl; - throw -1; - } -} - int main(int argc, char *argv[]) { CommandLineParser parser(argc, argv, keys); parser.about(about); @@ -91,10 +51,8 @@ int main(int argc, char *argv[]) { Mat camMatrix, distCoeffs; readCameraParamsFromCommandLine(parser, camMatrix, distCoeffs); - - aruco::DetectorParameters detectorParams; - detectorParams.cornerRefinementMethod = aruco::CORNER_REFINE_SUBPIX; // do corner refinement in markers - readDetectorParamsFromCommandLine(parser, detectorParams); + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); String video; if(parser.has("v")) { @@ -106,9 +64,6 @@ int main(int argc, char *argv[]) { return 0; } - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50); - readDictionatyFromCommandLine(parser, dictionary); - aruco::ArucoDetector detector(dictionary, detectorParams); VideoCapture inputVideo; int waitTime; @@ -181,9 +136,8 @@ int main(int argc, char *argv[]) { // Draw results image.copyTo(imageCopy); - if(!ids.empty()) { + if(!ids.empty()) aruco::drawDetectedMarkers(imageCopy, corners, ids); - } if(showRejected && !rejected.empty()) aruco::drawDetectedMarkers(imageCopy, rejected, noArray(), Scalar(100, 0, 255)); diff --git a/samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp b/samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp new file mode 100644 index 0000000000..c02318d6eb --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/detect_board_charuco.cpp @@ -0,0 +1,144 @@ +#include +//! [charucohdr] +#include +//! [charucohdr] +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace std; +using namespace cv; + +namespace { +const char* about = "Pose estimation using a ChArUco board"; +const char* keys = + "{w | | Number of squares in X direction }" + "{h | | Number of squares in Y direction }" + "{sl | | Square side length (in meters) }" + "{ml | | Marker side length (in meters) }" + "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{cd | | Input file with custom dictionary }" + "{c | | Output file with calibrated camera parameters }" + "{v | | Input from video or image file, if ommited, input comes from camera }" + "{ci | 0 | Camera id if input doesnt come from video (-v) }" + "{dp | | File of marker detector parameters }" + "{rs | | Apply refind strategy }"; +} + + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + + if(argc < 6) { + parser.printMessage(); + return 0; + } + + //! [charuco_detect_board_full_sample] + int squaresX = parser.get("w"); + int squaresY = parser.get("h"); + float squareLength = parser.get("sl"); + float markerLength = parser.get("ml"); + bool refine = parser.has("rs"); + int camId = parser.get("ci"); + + string video; + if(parser.has("v")) { + video = parser.get("v"); + } + + Mat camMatrix, distCoeffs; + readCameraParamsFromCommandLine(parser, camMatrix, distCoeffs); + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + VideoCapture inputVideo; + int waitTime = 0; + if(!video.empty()) { + inputVideo.open(video); + } else { + inputVideo.open(camId); + waitTime = 10; + } + + float axisLength = 0.5f * ((float)min(squaresX, squaresY) * (squareLength)); + + // create charuco board object + aruco::CharucoBoard charucoBoard(Size(squaresX, squaresY), squareLength, markerLength, dictionary); + + // create charuco detector + aruco::CharucoParameters charucoParams; + charucoParams.tryRefineMarkers = refine; // if tryRefineMarkers, refineDetectedMarkers() will be used in detectBoard() + charucoParams.cameraMatrix = camMatrix; // cameraMatrix can be used in detectBoard() + charucoParams.distCoeffs = distCoeffs; // distCoeffs can be used in detectBoard() + aruco::CharucoDetector charucoDetector(charucoBoard, charucoParams, detectorParams); + + double totalTime = 0; + int totalIterations = 0; + + while(inputVideo.grab()) { + //! [inputImg] + Mat image, imageCopy; + inputVideo.retrieve(image); + //! [inputImg] + + double tick = (double)getTickCount(); + + vector markerIds, charucoIds; + vector > markerCorners; + vector charucoCorners; + Vec3d rvec, tvec; + + //! [interpolateCornersCharuco] + // detect markers and charuco corners + charucoDetector.detectBoard(image, charucoCorners, charucoIds, markerCorners, markerIds); + //! [interpolateCornersCharuco] + + //! [poseCharuco] + // estimate charuco board pose + bool validPose = false; + if(camMatrix.total() != 0 && distCoeffs.total() != 0 && charucoIds.size() >= 4) { + Mat objPoints, imgPoints; + charucoBoard.matchImagePoints(charucoCorners, charucoIds, objPoints, imgPoints); + validPose = solvePnP(objPoints, imgPoints, camMatrix, distCoeffs, rvec, tvec); + } + //! [poseCharuco] + + double currentTime = ((double)getTickCount() - tick) / getTickFrequency(); + totalTime += currentTime; + totalIterations++; + if(totalIterations % 30 == 0) { + cout << "Detection Time = " << currentTime * 1000 << " ms " + << "(Mean = " << 1000 * totalTime / double(totalIterations) << " ms)" << endl; + } + + // draw results + image.copyTo(imageCopy); + if(markerIds.size() > 0) { + aruco::drawDetectedMarkers(imageCopy, markerCorners); + } + + if(charucoIds.size() > 0) { + //! [drawDetectedCornersCharuco] + aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, cv::Scalar(255, 0, 0)); + //! [drawDetectedCornersCharuco] + } + + if(validPose) + cv::drawFrameAxes(imageCopy, camMatrix, distCoeffs, rvec, tvec, axisLength); + + imshow("out", imageCopy); + if(waitKey(waitTime) == 27) break; + } + //! [charuco_detect_board_full_sample] + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp b/samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp new file mode 100644 index 0000000000..f6a6236b2a --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/detect_diamonds.cpp @@ -0,0 +1,187 @@ +#include +#include +#include +#include +#include "aruco_samples_utility.hpp" + +using namespace std; +using namespace cv; + + +namespace { +const char* about = "Detect ChArUco markers"; +const char* keys = + "{sl | 100 | Square side length (in meters) }" + "{ml | 60 | Marker side length (in meters) }" + "{d | 10 | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " + "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," + "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16}" + "{cd | | Input file with custom dictionary }" + "{c | | Output file with calibrated camera parameters }" + "{as | | Automatic scale. The provided number is multiplied by the last" + "diamond id becoming an indicator of the square length. In this case, the -sl and " + "-ml are only used to know the relative length relation between squares and markers }" + "{v | | Input from video file, if ommited, input comes from camera }" + "{ci | 0 | Camera id if input doesnt come from video (-v) }" + "{dp | | File of marker detector parameters }" + "{refine | | Corner refinement: CORNER_REFINE_NONE=0, CORNER_REFINE_SUBPIX=1," + "CORNER_REFINE_CONTOUR=2, CORNER_REFINE_APRILTAG=3}"; + +const string refineMethods[4] = { + "None", + "Subpixel", + "Contour", + "AprilTag" +}; + +} + +int main(int argc, char *argv[]) { + CommandLineParser parser(argc, argv, keys); + parser.about(about); + + float squareLength = parser.get("sl"); + float markerLength = parser.get("ml"); + bool estimatePose = parser.has("c"); + bool autoScale = parser.has("as"); + float autoScaleFactor = autoScale ? parser.get("as") : 1.f; + + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); + Mat camMatrix, distCoeffs; + readCameraParamsFromCommandLine(parser, camMatrix, distCoeffs); + + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); + if (parser.has("refine")) { + // override cornerRefinementMethod read from config file + int user_method = parser.get("refine"); + if (user_method < 0 || user_method >= 4) + { + std::cout << "Corner refinement method should be in range 0..3" << std::endl; + return 0; + } + detectorParams.cornerRefinementMethod = user_method; + } + std::cout << "Corner refinement method: " << refineMethods[detectorParams.cornerRefinementMethod] << std::endl; + + int camId = parser.get("ci"); + String video; + + if(parser.has("v")) { + video = parser.get("v"); + } + + if(!parser.check()) { + parser.printErrors(); + return 0; + } + + VideoCapture inputVideo; + int waitTime; + if(!video.empty()) { + inputVideo.open(video); + waitTime = 0; + } else { + inputVideo.open(camId); + waitTime = 10; + } + + double totalTime = 0; + int totalIterations = 0; + + aruco::CharucoBoard charucoBoard(Size(3, 3), squareLength, markerLength, dictionary); + aruco::CharucoDetector detector(charucoBoard, aruco::CharucoParameters(), detectorParams); + + while(inputVideo.grab()) { + Mat image, imageCopy; + inputVideo.retrieve(image); + + double tick = (double)getTickCount(); + + //! [detect_diamonds] + vector markerIds; + vector diamondIds; + vector > markerCorners, diamondCorners; + vector rvecs, tvecs; + + detector.detectDiamonds(image, diamondCorners, diamondIds, markerCorners, markerIds); + //! [detect_diamonds] + + //! [diamond_pose_estimation] + // estimate diamond pose + size_t N = diamondIds.size(); + if(estimatePose && N > 0) { + cv::Mat objPoints(4, 1, CV_32FC3); + rvecs.resize(N); + tvecs.resize(N); + if(!autoScale) { + // set coordinate system + objPoints.ptr(0)[0] = Vec3f(-squareLength/2.f, squareLength/2.f, 0); + objPoints.ptr(0)[1] = Vec3f(squareLength/2.f, squareLength/2.f, 0); + objPoints.ptr(0)[2] = Vec3f(squareLength/2.f, -squareLength/2.f, 0); + objPoints.ptr(0)[3] = Vec3f(-squareLength/2.f, -squareLength/2.f, 0); + // Calculate pose for each marker + for (size_t i = 0ull; i < N; i++) + solvePnP(objPoints, diamondCorners.at(i), camMatrix, distCoeffs, rvecs.at(i), tvecs.at(i)); + //! [diamond_pose_estimation] + /* //! [diamond_pose_estimation_as_charuco] + for (size_t i = 0ull; i < N; i++) { // estimate diamond pose as Charuco board + Mat objPoints_b, imgPoints; + // The coordinate system of the diamond is placed in the board plane centered in the bottom left corner + vector charucoIds = {0, 1, 3, 2}; // if CCW order, Z axis pointing in the plane + // vector charucoIds = {0, 2, 3, 1}; // if CW order, Z axis pointing out the plane + charucoBoard.matchImagePoints(diamondCorners[i], charucoIds, objPoints_b, imgPoints); + solvePnP(objPoints_b, imgPoints, camMatrix, distCoeffs, rvecs[i], tvecs[i]); + } + //! [diamond_pose_estimation_as_charuco] */ + } + else { + // if autoscale, extract square size from last diamond id + for(size_t i = 0; i < N; i++) { + float sqLenScale = autoScaleFactor * float(diamondIds[i].val[3]); + vector > currentCorners; + vector currentRvec, currentTvec; + currentCorners.push_back(diamondCorners[i]); + // set coordinate system + objPoints.ptr(0)[0] = Vec3f(-sqLenScale/2.f, sqLenScale/2.f, 0); + objPoints.ptr(0)[1] = Vec3f(sqLenScale/2.f, sqLenScale/2.f, 0); + objPoints.ptr(0)[2] = Vec3f(sqLenScale/2.f, -sqLenScale/2.f, 0); + objPoints.ptr(0)[3] = Vec3f(-sqLenScale/2.f, -sqLenScale/2.f, 0); + solvePnP(objPoints, diamondCorners.at(i), camMatrix, distCoeffs, rvecs.at(i), tvecs.at(i)); + } + } + } + + + double currentTime = ((double)getTickCount() - tick) / getTickFrequency(); + totalTime += currentTime; + totalIterations++; + if(totalIterations % 30 == 0) { + cout << "Detection Time = " << currentTime * 1000 << " ms " + << "(Mean = " << 1000 * totalTime / double(totalIterations) << " ms)" << endl; + } + + + // draw results + image.copyTo(imageCopy); + if(markerIds.size() > 0) + aruco::drawDetectedMarkers(imageCopy, markerCorners); + + //! [draw_diamonds] + if(diamondIds.size() > 0) { + aruco::drawDetectedDiamonds(imageCopy, diamondCorners, diamondIds); + //! [draw_diamonds] + + //! [draw_diamond_pose_estimation] + if(estimatePose) { + for(size_t i = 0u; i < diamondIds.size(); i++) + cv::drawFrameAxes(imageCopy, camMatrix, distCoeffs, rvecs[i], tvecs[i], squareLength*1.1f); + } + //! [draw_diamond_pose_estimation] + } + imshow("out", imageCopy); + char key = (char)waitKey(waitTime); + if(key == 27) break; + } + return 0; +} diff --git a/samples/cpp/tutorial_code/objectDetection/detect_markers.cpp b/samples/cpp/tutorial_code/objectDetection/detect_markers.cpp index 720fb8ddae..f220b84565 100644 --- a/samples/cpp/tutorial_code/objectDetection/detect_markers.cpp +++ b/samples/cpp/tutorial_code/objectDetection/detect_markers.cpp @@ -11,7 +11,7 @@ const char* about = "Basic marker detection"; //! [aruco_detect_markers_keys] const char* keys = - "{d | | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," + "{d | 0 | dictionary: DICT_4X4_50=0, DICT_4X4_100=1, DICT_4X4_250=2," "DICT_4X4_1000=3, DICT_5X5_50=4, DICT_5X5_100=5, DICT_5X5_250=6, DICT_5X5_1000=7, " "DICT_6X6_50=8, DICT_6X6_100=9, DICT_6X6_250=10, DICT_6X6_1000=11, DICT_7X7_50=12," "DICT_7X7_100=13, DICT_7X7_250=14, DICT_7X7_1000=15, DICT_ARUCO_ORIGINAL = 16," @@ -25,37 +25,41 @@ const char* keys = "{r | | show rejected candidates too }" "{refine | | Corner refinement: CORNER_REFINE_NONE=0, CORNER_REFINE_SUBPIX=1," "CORNER_REFINE_CONTOUR=2, CORNER_REFINE_APRILTAG=3}"; -} + //! [aruco_detect_markers_keys] +const string refineMethods[4] = { + "None", + "Subpixel", + "Contour", + "AprilTag" +}; + +} + int main(int argc, char *argv[]) { CommandLineParser parser(argc, argv, keys); parser.about(about); - if(argc < 2) { - parser.printMessage(); - return 0; - } - bool showRejected = parser.has("r"); bool estimatePose = parser.has("c"); float markerLength = parser.get("l"); - cv::aruco::DetectorParameters detectorParams; - if(parser.has("dp")) { - cv::FileStorage fs(parser.get("dp"), FileStorage::READ); - bool readOk = detectorParams.readDetectorParameters(fs.root()); - if(!readOk) { - cerr << "Invalid detector parameters file" << endl; - return 0; - } - } + aruco::DetectorParameters detectorParams = readDetectorParamsFromCommandLine(parser); + aruco::Dictionary dictionary = readDictionatyFromCommandLine(parser); if (parser.has("refine")) { // override cornerRefinementMethod read from config file - detectorParams.cornerRefinementMethod = parser.get("refine"); + int user_method = parser.get("refine"); + if (user_method < 0 || user_method >= 4) + { + std::cout << "Corner refinement method should be in range 0..3" << std::endl; + return 0; + } + detectorParams.cornerRefinementMethod = user_method; } - std::cout << "Corner refinement method (0: None, 1: Subpixel, 2:contour, 3: AprilTag 2): " << (int)detectorParams.cornerRefinementMethod << std::endl; + + std::cout << "Corner refinement method: " << refineMethods[detectorParams.cornerRefinementMethod] << std::endl; int camId = parser.get("ci"); @@ -69,33 +73,11 @@ int main(int argc, char *argv[]) { return 0; } - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50); - if (parser.has("d")) { - int dictionaryId = parser.get("d"); - dictionary = aruco::getPredefinedDictionary(aruco::PredefinedDictionaryType(dictionaryId)); - } - else if (parser.has("cd")) { - cv::FileStorage fs(parser.get("cd"), FileStorage::READ); - bool readOk = dictionary.readDictionary(fs.root()); - if(!readOk) { - std::cerr << "Invalid dictionary file" << std::endl; - return 0; - } - } - else { - std::cerr << "Dictionary not specified" << std::endl; - return 0; - } - //! [aruco_pose_estimation1] - cv::Mat camMatrix, distCoeffs; + Mat camMatrix, distCoeffs; if(estimatePose) { // You can read camera parameters from tutorial_camera_params.yml - bool readOk = readCameraParameters(parser.get("c"), camMatrix, distCoeffs); - if(!readOk) { - cerr << "Invalid camera file" << endl; - return 0; - } + readCameraParamsFromCommandLine(parser, camMatrix, distCoeffs); } //! [aruco_pose_estimation1] //! [aruco_detect_markers] diff --git a/samples/cpp/tutorial_code/objectDetection/detector_params.yml b/samples/cpp/tutorial_code/objectDetection/detector_params.yml new file mode 100644 index 0000000000..1155c25126 --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/detector_params.yml @@ -0,0 +1,30 @@ +%YAML:1.0 +adaptiveThreshWinSizeMin: 3 +adaptiveThreshWinSizeMax: 23 +adaptiveThreshWinSizeStep: 10 +adaptiveThreshWinSize: 21 +adaptiveThreshConstant: 7 +minMarkerPerimeterRate: 0.03 +maxMarkerPerimeterRate: 4.0 +polygonalApproxAccuracyRate: 0.05 +minCornerDistanceRate: 0.05 +minDistanceToBorder: 3 +minMarkerDistance: 10.0 +minMarkerDistanceRate: 0.05 +cornerRefinementMethod: 0 +cornerRefinementWinSize: 5 +cornerRefinementMaxIterations: 30 +cornerRefinementMinAccuracy: 0.1 +markerBorderBits: 1 +perspectiveRemovePixelPerCell: 8 +perspectiveRemoveIgnoredMarginPerCell: 0.13 +maxErroneousBitsInBorderRate: 0.04 +minOtsuStdDev: 5.0 +errorCorrectionRate: 0.6 + +# new aruco 3 functionality +useAruco3Detection: 0 +minSideLengthCanonicalImg: 32 # 16, 32, 64 --> tau_c from the paper +minMarkerLengthRatioOriginalImg: 0.02 # range [0,0.2] --> tau_i from the paper +cameraMotionSpeed: 0.1 # range [0,1) --> tau_s from the paper +useGlobalThreshold: 0 diff --git a/samples/cpp/tutorial_code/objectDetection/tutorial_camera_charuco.yml b/samples/cpp/tutorial_code/objectDetection/tutorial_camera_charuco.yml new file mode 100644 index 0000000000..f1ded5993a --- /dev/null +++ b/samples/cpp/tutorial_code/objectDetection/tutorial_camera_charuco.yml @@ -0,0 +1,21 @@ +%YAML:1.0 +--- +calibration_time: "Wed 08 Dec 2021 05:13:09 PM MSK" +image_width: 640 +image_height: 480 +flags: 0 +camera_matrix: !!opencv-matrix + rows: 3 + cols: 3 + dt: d + data: [ 4.5251072219637672e+02, 0., 3.1770297317353277e+02, 0., + 4.5676707935146891e+02, 2.7775155919135995e+02, 0., 0., 1. ] +distortion_coefficients: !!opencv-matrix + rows: 1 + cols: 5 + dt: d + data: [ 1.2136925618707872e-01, -1.0854664722560681e+00, + 1.1786843796668460e-04, -4.6240686046485508e-04, + 2.9542589406810080e+00 ] +avg_reprojection_error: 1.8234905535936044e-01 +info: "The camera calibration parameters were obtained by img_00.jpg-img_03.jpg from aruco/tutorials/aruco_calibration/images"