From 3bcab8db0a887bca715dc425c6100c5d75e3e9a3 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov <2536374+asmorkalov@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:53:16 +0300 Subject: [PATCH] Merge pull request #26221 from asmorkalov:as/refactor_multiview_interface Reworked multiview calibration interface #26221 - Use InputArray / OutputArray - Use enum for camera type - Sort parameters according guidelines - Made more outputs optional - Introduce flags and added tests for intrinsics and extrinsics guess. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../multiview_calibration.markdown | 2 +- modules/calib/include/opencv2/calib.hpp | 27 +- modules/calib/src/multiview_calibration.cpp | 260 ++++++++++-------- modules/calib/test/test_cameracalibration.cpp | 6 +- modules/calib/test/test_fisheye.cpp | 8 +- modules/calib/test/test_multiview_calib.cpp | 198 +++++++++---- samples/cpp/multiview_calibration_sample.cpp | 21 +- samples/python/multiview_calibration.py | 54 ++-- 8 files changed, 362 insertions(+), 214 deletions(-) diff --git a/doc/tutorials/calib3d/camera_multiview_calibration/multiview_calibration.markdown b/doc/tutorials/calib3d/camera_multiview_calibration/multiview_calibration.markdown index 07d2f2c81f..d81fa4178e 100644 --- a/doc/tutorials/calib3d/camera_multiview_calibration/multiview_calibration.markdown +++ b/doc/tutorials/calib3d/camera_multiview_calibration/multiview_calibration.markdown @@ -162,7 +162,7 @@ Details Of The Algorithm 3. Output of intrinsic calibration also includes rotation, translation vectors (transform of pattern points to camera frame), and errors per frame. For each frame, the index of the camera with the lowest error among all cameras is saved. 2. Otherwise, if intrinsics are known, then the proposed algorithm runs perspective-n-point estimation (@ref cv::solvePnP, @ref cv::fisheye::solvePnP) to estimate rotation and translation vectors, and reprojection error for each frame. 2. **Initialization of relative camera pose**. - 1. If the initial relative poses are not assumed known (the `useExtrinsicsGuess` is set false), then the relative camera extrinsics are found by traversing a spanning tree and estimating pairwise relative camera pose. + 1. If the initial relative poses are not assumed known (CALIB_USE_EXTRINSIC_GUESS flag not set), then the relative camera extrinsics are found by traversing a spanning tree and estimating pairwise relative camera pose. 1. **Miminal spanning tree establishment**. Assume that cameras can be represented as nodes of a connected graph. An edge between two cameras is created if there is any concurrent observation over all frames. If the graph does not connect all cameras (i.e., exists a camera that has no overlap with other cameras) then calibration is not possible. Otherwise, the next step consists of finding the [maximum spanning tree](https://en.wikipedia.org/wiki/Minimum_spanning_tree) (MST) of this graph. The MST captures all the best pairwise camera connections. The weight of edges across all frames is a weighted combination of multiple factors: * (Major) The number of pattern points detected in both images (cameras) * Ratio of area of convex hull of projected points in the image to the image resolution. diff --git a/modules/calib/include/opencv2/calib.hpp b/modules/calib/include/opencv2/calib.hpp index 8a85dde977..245829bdc1 100644 --- a/modules/calib/include/opencv2/calib.hpp +++ b/modules/calib/include/opencv2/calib.hpp @@ -449,7 +449,7 @@ enum { CALIB_USE_INTRINSIC_GUESS = 0x00001, //!< Use user provided intrinsics as // for stereo rectification CALIB_ZERO_DISPARITY = 0x00400, //!< Deprecated synonim of @ref STEREO_ZERO_DISPARITY. See @ref stereoRectify. CALIB_USE_LU = (1 << 17), //!< use LU instead of SVD decomposition for solving. much faster but potentially less precise - CALIB_USE_EXTRINSIC_GUESS = (1 << 22), //!< For stereo calibration only. Use user provided extrinsics (R, T) as initial point for optimization + CALIB_USE_EXTRINSIC_GUESS = (1 << 22), //!< For stereo and multi-view calibration. Use user provided extrinsics (R, T) as initial point for optimization // fisheye only flags CALIB_RECOMPUTE_EXTRINSIC = (1 << 23), //!< For fisheye model only. Recompute board position on each calibration iteration CALIB_CHECK_COND = (1 << 24), //!< For fisheye model only. Check SVD decomposition quality for each frame during extrinsics estimation @@ -1236,16 +1236,14 @@ CV_EXPORTS_W double registerCameras( InputArrayOfArrays objectPoints1, @param[in] imagePoints Detected pattern points on camera images. Expected shape: NUM_CAMERAS x NUM_FRAMES x NUM_POINTS x 2. This function supports partial observation of the calibration pattern. To enable this, set the unobserved image points to be invalid points (eg. (-1., -1.)). -@param[in] imageSize Images resolution. +@param[in] imageSize Images resolution array for each camera. @param[in] detectionMask Pattern detection mask. Each value defines if i-camera observes the calibration pattern in j-th frame. Expected size: NUM_CAMERAS x NUM_FRAMES. Expected type: CV_8U. -@param[in] isFisheye indicates whether i-th camera is fisheye. In case the input data contains -a mix of pinhole and fisheye cameras Rational distortion model is used. See @ref CALIB_RATIONAL_MODEL -for details. Expected type: CV_8U. -@param[in] useIntrinsicsGuess Use user-specified intrinsic parameters (internal camera matrix and distortion). -If true intrinsics are not estimated during calibration. +@param[in] models indicates camera models for each camera: cv::CALIB_MODEL_PINHOLE or cv::CALIB_MODEL_PINHOLE. +Current implementation does not support mix of different camera models. Expected type: CV_8U. @param[in] flagsForIntrinsics Flags used for each camera intrinsics calibration. Use per-camera call and the `useIntrinsicsGuess` flag to get custom intrinsics calibration for each camera. +@param[in] flags Common multiview calibration flags. cv::CALIB_USE_INTRINSIC_GUESS and cv::CALIB_USE_EXTRINSIC_GUESS are supported. See @ref CALIB_USE_INTRINSIC_GUESS and other `CALIB_` constants. Expected shape: NUM_CAMERAS x 1. Supported data type: CV_32S. @param[out] Rs Rotation vectors relative to camera 0, where Rs[0] = 0. Output size: NUM_CAMERAS x 3 x 3. @param[out] Ts Estimated translation vectors relative to camera 0, where Ts[0] = 0. Output size: NUM_CAMERAS x 3 x 1. @@ -1255,6 +1253,7 @@ See @ref CALIB_USE_INTRINSIC_GUESS and other `CALIB_` constants. Expected shape: @param[out] distortions Distortion coefficients. Output size: NUM_CAMERAS x NUM_PARAMS. @param[out] perFrameErrors RMSE value for each visible frame, (-1 for non-visible). Output size: NUM_CAMERAS x NUM_FRAMES. @param[out] initializationPairs Pairs with camera indices that were used for initial pairwise stereo calibration. + Output size: (NUM_CAMERAS-1) x 2. @ref tutorial_multiview_camera_calibration provides a detailed tutorial of using this function. Please refer to it for more information. @@ -1288,12 +1287,14 @@ points in all the available views from all cameras. @sa findChessboardCorners, findCirclesGrid, calibrateCamera, fisheye::calibrate, registerCameras */ -CV_EXPORTS_W double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector> &imagePoints, - const std::vector &imageSize, InputArray detectionMask, - OutputArrayOfArrays Rs, OutputArrayOfArrays Ts, CV_IN_OUT std::vector &Ks, CV_IN_OUT std::vector &distortions, - OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0, InputArray isFisheye, - OutputArray perFrameErrors, OutputArray initializationPairs, - bool useIntrinsicsGuess=false, InputArray flagsForIntrinsics=noArray()); +CV_EXPORTS_W double calibrateMultiview ( + InputArrayOfArrays objPoints, const std::vector> &imagePoints, + const std::vector& imageSize, InputArray detectionMask, InputArray models, + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, + int flags = 0, InputArray flagsForIntrinsics=noArray(), + OutputArrayOfArrays rvecs0=noArray(), OutputArrayOfArrays tvecs0=noArray(), + OutputArray perFrameErrors=noArray(), OutputArray initializationPairs=noArray()); /** @brief Computes Hand-Eye calibration: \f$_{}^{g}\textrm{T}_c\f$ diff --git a/modules/calib/src/multiview_calibration.cpp b/modules/calib/src/multiview_calibration.cpp index 142f94d9a0..680fb5ec87 100644 --- a/modules/calib/src/multiview_calibration.cpp +++ b/modules/calib/src/multiview_calibration.cpp @@ -63,7 +63,7 @@ static double robustWrapper (const Mat& ptsErrors, Mat& weights, const RobustFun } static double computeReprojectionMSE(const Mat &obj_points_, const Mat &img_points_, const Matx33d &K, const Mat &distortion, - const Mat &rvec, const Mat &tvec, InputArray rvec2, InputArray tvec2, bool is_fisheye) { + const Mat &rvec, const Mat &tvec, InputArray rvec2, InputArray tvec2, int model) { Mat r, t; if (!rvec2.empty() && !tvec2.empty()) { composeRT(rvec, tvec, rvec2, tvec2, r, t); @@ -71,11 +71,15 @@ static double computeReprojectionMSE(const Mat &obj_points_, const Mat &img_poin r = rvec; t = tvec; } Mat tmpImagePoints, obj_points = obj_points_, img_points = img_points_; - if (is_fisheye) { + if (model == cv::CALIB_MODEL_FISHEYE) { obj_points = obj_points.reshape(3); // must be 3 channels fisheye::projectPoints(obj_points, tmpImagePoints, r, t, K, distortion); - } else + } else if (model == cv::CALIB_MODEL_PINHOLE) { projectPoints(obj_points, r, t, K, distortion, tmpImagePoints); + } else { + CV_Error(Error::StsBadArg, "Unsupported camera model!"); + } + if (img_points.channels() != tmpImagePoints.channels()) img_points = img_points.reshape(tmpImagePoints.channels()); if (img_points.rows != tmpImagePoints.rows) @@ -257,12 +261,24 @@ static void thresholdPatternCameraAngles (int NUM_PATTERN_PTS, double THR_PATTER } static void pairwiseStereoCalibration (const std::vector> &pairs, - const std::vector &is_fisheye_vec, const std::vector &objPoints_norm, + const cv::Mat &models, const std::vector &objPoints_norm, const std::vector> &imagePoints, const std::vector> &overlaps, const std::vector> &detection_mask_mat, const std::vector &Ks, const std::vector &distortions, std::vector &Rs_vec, std::vector &Ts_vec, - Mat &flags) { - const int NUM_FRAMES = (int) objPoints_norm.size(); + Mat &intrinsic_flags, int extrinsic_flags = 0) { + const int NUM_FRAMES = (int)objPoints_norm.size(); + const int NUM_CAMERAS = (int)detection_mask_mat.size(); + std::vector Rs_prior; + std::vector Ts_prior; + if (extrinsic_flags & cv::CALIB_USE_EXTRINSIC_GUESS) { + Rs_prior.resize(NUM_CAMERAS); + Ts_prior.resize(NUM_CAMERAS); + for (int i = 0; i < NUM_CAMERAS; i++) { + Rs_vec[i].copyTo(Rs_prior[i]); + Ts_vec[i].copyTo(Ts_prior[i]); + } + } + for (const auto &pair : pairs) { const int c1 = pair.first, c2 = pair.second, overlap = overlaps[c1][c2]; // prepare image points of two cameras and grid points @@ -270,7 +286,8 @@ static void pairwiseStereoCalibration (const std::vector> &pa grid_points.reserve(overlap); image_points1.reserve(overlap); image_points2.reserve(overlap); - const bool are_fisheye_cams = is_fisheye_vec[c1] && is_fisheye_vec[c2]; + const bool are_fisheye_cams = models.at(c1) == cv::CALIB_MODEL_FISHEYE && + models.at(c2) == cv::CALIB_MODEL_FISHEYE; for (int f = 0; f < NUM_FRAMES; f++) { if (detection_mask_mat[c1][f] && detection_mask_mat[c2][f]) { grid_points.emplace_back((are_fisheye_cams && objPoints_norm[f].channels() != 3) ? @@ -283,23 +300,30 @@ static void pairwiseStereoCalibration (const std::vector> &pa } Matx33d R; Vec3d T; + + if (extrinsic_flags & cv::CALIB_USE_EXTRINSIC_GUESS) { + R = Rs_prior[c2] * Rs_prior[c1].t(); + T = -R * Ts_prior[c1] + Ts_prior[c2]; + } + // image size does not matter since intrinsics are used if (are_fisheye_cams) { + extrinsic_flags |= CALIB_FIX_INTRINSIC; fisheye::stereoCalibrate(grid_points, image_points1, image_points2, Ks[c1], distortions[c1], Ks[c2], distortions[c2], - Size(), R, T, CALIB_FIX_INTRINSIC); + Size(), R, T, extrinsic_flags); } else { - int flags_extrinsics = CALIB_FIX_INTRINSIC; - if ((flags.at(c1) & CALIB_RATIONAL_MODEL) || (flags.at(c2) & CALIB_RATIONAL_MODEL)) - flags_extrinsics += CALIB_RATIONAL_MODEL; - if ((flags.at(c1) & CALIB_THIN_PRISM_MODEL) || (flags.at(c2) & CALIB_THIN_PRISM_MODEL)) - flags_extrinsics += CALIB_THIN_PRISM_MODEL; + extrinsic_flags |= CALIB_FIX_INTRINSIC; + if ((intrinsic_flags.at(c1) & CALIB_RATIONAL_MODEL) || (intrinsic_flags.at(c2) & CALIB_RATIONAL_MODEL)) + extrinsic_flags |= CALIB_RATIONAL_MODEL; + if ((intrinsic_flags.at(c1) & CALIB_THIN_PRISM_MODEL) || (intrinsic_flags.at(c2) & CALIB_THIN_PRISM_MODEL)) + extrinsic_flags |= CALIB_THIN_PRISM_MODEL; stereoCalibrate(grid_points, image_points1, image_points2, Ks[c1], distortions[c1], Ks[c2], distortions[c2], - Size(), R, T, noArray(), noArray(), noArray(), flags_extrinsics); + Size(), R, T, noArray(), noArray(), noArray(), extrinsic_flags); } // R_0 = I @@ -319,7 +343,7 @@ static void optimizeLM (std::vector ¶m, const RobustFunction &robust const std::vector &valid_frames, const std::vector> &detection_mask_mat, const std::vector &objPoints_norm, const std::vector> &imagePoints, const std::vector &Ks, const std::vector &distortions, - const std::vector &is_fisheye_vec, int NUM_PATTERN_PTS) { + const Mat& models, int NUM_PATTERN_PTS) { const int NUM_FRAMES = (int) objPoints_norm.size(), NUM_CAMERAS = (int)detection_mask_mat.size(); int iters_lm = 0, cnt_valid_frame = 0; auto lmcallback = [&](InputOutputArray _param, OutputArray JtErr_, OutputArray JtJ_, double& errnorm) { @@ -358,7 +382,7 @@ static void optimizeLM (std::vector ¶m, const RobustFunction &robust Mat imgpt_ik = imagePoints[k][i].reshape(2, 1); imgpt_ik.convertTo(imgpt_ik, CV_64FC2); - if (is_fisheye_vec[k]) { + if (models.at(k)) { if( JtJ_.needed() || JtErr_.needed() ) { Mat jacobian; // of size num_points*2 x 15 (2 + 2 + 1 + 4 + 3 + 3; // f, c, alpha, k, om, T) fisheye::projectPoints(objpt_i, tmpImagePoints, om[1], T[1], Ks[k], distortions[k], 0, jacobian); @@ -482,43 +506,55 @@ static void checkConnected (const std::vector> &detection_mask } } -//TODO: use Input/OutputArrays for imagePoints, imageSize(?), Ks, distortions -double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector> &imagePoints, - const std::vector &imageSize, InputArray detectionMask, - OutputArrayOfArrays Rs, OutputArrayOfArrays Ts, std::vector &Ks, std::vector &distortions, - OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0, InputArray isFisheye, - OutputArray perFrameErrors, OutputArray initializationPairs, bool useIntrinsicsGuess, InputArray flagsForIntrinsics) { +//TODO: use Input/OutputArrays for imagePoints(?) +double calibrateMultiview( + InputArrayOfArrays objPoints, const std::vector> &imagePoints, + const std::vector& imageSize, InputArray detectionMask, InputArray models, + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, + int flags, InputArray flagsForIntrinsics, + OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0, + OutputArray perFrameErrors, OutputArray initializationPairs) { - CV_CheckEQ((int)objPoints.empty(), 0, "Objects points must not be empty!"); - CV_CheckEQ((int)imagePoints.empty(), 0, "Image points must not be empty!"); - CV_CheckEQ((int)imageSize.empty(), 0, "Image size per camera must not be empty!"); - CV_CheckEQ((int)detectionMask.empty(), 0, "detectionMask matrix must not be empty!"); - CV_CheckEQ((int)isFisheye.empty(), 0, "Fisheye mask must not be empty!"); + CV_CheckFalse(objPoints.empty(), "Objects points must not be empty!"); + CV_CheckFalse(imagePoints.empty(), "Image points must not be empty!"); + CV_CheckFalse(imageSize.empty(), "Image size per camera must not be empty!"); + CV_CheckFalse(detectionMask.empty(), "detectionMask matrix must not be empty!"); + CV_CheckFalse(models.empty(), "Fisheye mask must not be empty!"); + + Mat detection_mask_ = detectionMask.getMat(); + Mat models_mat = models.getMat(); - Mat detection_mask_ = detectionMask.getMat(), is_fisheye_mat = isFisheye.getMat(); CV_CheckEQ(detection_mask_.type(), CV_8U, "detectionMask must be of type CV_8U"); - CV_CheckEQ(is_fisheye_mat.type(), CV_8U, "isFisheye must be of type CV_8U"); + CV_CheckEQ(models_mat.type(), CV_8U, "models must be of type CV_8U"); bool is_fisheye = false; bool is_pinhole = false; - for (int i = 0; i < (int)is_fisheye_mat.total(); i++) { - if (is_fisheye_mat.at(i)) { + for (int i = 0; i < (int)models_mat.total(); i++) { + if (models_mat.at(i) == cv::CALIB_MODEL_FISHEYE) { is_fisheye = true; - } else { + } else if (models_mat.at(i) == cv::CALIB_MODEL_PINHOLE) { is_pinhole = true; + } else { + CV_Error(Error::StsBadArg, "Unsupported camera model"); } } CV_CheckEQ(is_fisheye && is_pinhole, false, "Mix of pinhole and fisheye cameras is not supported for now"); // equal number of cameras CV_Assert(imageSize.size() == imagePoints.size()); - CV_Assert(detection_mask_.rows == std::max(isFisheye.rows(), isFisheye.cols())); + CV_Assert(detection_mask_.rows == std::max(models.rows(), models.cols())); CV_Assert(detection_mask_.rows == (int)imageSize.size()); CV_Assert(detection_mask_.cols == std::max(objPoints.rows(), objPoints.cols())); // equal number of frames CV_Assert(Rs.isMatVector() == Ts.isMatVector()); - if (useIntrinsicsGuess) { - CV_Assert(Ks.size() == distortions.size() && Ks.size() == imageSize.size()); + if (flags & cv::CALIB_USE_INTRINSIC_GUESS) { + CV_Assert(Ks.isMatVector() && distortions.isMatVector()); + CV_Assert(Ks.total() == distortions.total() && Ks.total() == imageSize.size()); + } + if (flags & cv::CALIB_USE_EXTRINSIC_GUESS) { + CV_Assert(Rs.isMatVector() && Ts.isMatVector()); + CV_Assert(Rs.total() == Ts.total() && Rs.total() == imageSize.size()); } // normalize object points const Mat obj_pts_0 = objPoints.getMat(0); @@ -555,11 +591,9 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector valid_frames(NUM_FRAMES, false); // process input and count all visible frames and points - std::vector is_fisheye_vec(NUM_CAMERAS); std::vector> detection_mask_mat(NUM_CAMERAS, std::vector(NUM_FRAMES)); - const auto * const detection_mask_ptr = detection_mask_.data, * const is_fisheye_ptr = is_fisheye_mat.data; + const auto * const detection_mask_ptr = detection_mask_.data; for (int c = 0; c < NUM_CAMERAS; c++) { - is_fisheye_vec[c] = is_fisheye_ptr[c] != 0; int num_visible_frames = 0; for (int f = 0; f < NUM_FRAMES; f++) { detection_mask_mat[c][f] = detection_mask_ptr[c*NUM_FRAMES + f] != 0; @@ -589,11 +623,13 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector camera_rt_best(NUM_FRAMES, -1); std::vector camera_rt_errors(NUM_FRAMES, std::numeric_limits::max()); const double WARNING_RMSE = 15.; - if (!useIntrinsicsGuess) { + if ((flags & cv::CALIB_USE_INTRINSIC_GUESS) == 0) { + Ks.create(NUM_CAMERAS, 1, CV_64F); + distortions.create(NUM_CAMERAS, 1, CV_64F); + // calibrate each camera independently to find intrinsic parameters - K and distortion coefficients - distortions = std::vector(NUM_CAMERAS); - Ks = std::vector(NUM_CAMERAS); for (int camera = 0; camera < NUM_CAMERAS; camera++) { + Mat K, dist; Mat rvecs, tvecs; std::vector obj_points_, img_points_; std::vector errors_per_view; @@ -601,25 +637,25 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ? objPoints_norm[f].reshape(3): objPoints_norm[f]); - img_points_.emplace_back((is_fisheye_vec[camera] && imagePoints[camera][f].channels() != 2) ? + img_points_.emplace_back((models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ? imagePoints[camera][f].reshape(2) : imagePoints[camera][f]); } } double repr_err; - if (is_fisheye_vec[camera]) { + if (models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE) { repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera], - Ks[camera], distortions[camera], rvecs, tvecs, flagsForIntrinsics_mat.at(camera)); + K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at(camera)); // calibrate does not compute error per view, so compute it manually errors_per_view = std::vector(obj_points_.size()); for (int f = 0; f < (int) obj_points_.size(); f++) { double err2 = multiview::computeReprojectionMSE(obj_points_[f], - img_points_[f], Ks[camera], distortions[camera], rvecs.row(f), tvecs.row(f), noArray(), noArray(), true); + img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE); errors_per_view[f] = sqrt(err2); } } else { - repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], Ks[camera], distortions[camera], + repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist, rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at(camera)); } CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!"); @@ -637,6 +673,11 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector(k)); if (camera_rt_errors[i] > err2) { camera_rt_errors[i] = err2; camera_rt_best[i] = k; @@ -656,6 +697,10 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector Ks_vec, distortions_vec; + Ks.getMatVector(Ks_vec); + distortions.getMatVector(distortions_vec); + std::vector> is_valid_angle2pattern; multiview::thresholdPatternCameraAngles(NUM_PATTERN_PTS, THR_PATTERN_CAMERA_ANGLES, objPoints_norm, rvecs_all, opt_axes, is_valid_angle2pattern); @@ -687,8 +732,8 @@ double calibrateMultiview (InputArrayOfArrays objPoints, const std::vector(NUM_FRAMES, 3); - } - cnt_valid_frame = 0; + bool is_mat_vec = rvecs0.needed() && rvecs0.isMatVector(); + if (is_mat_vec) { + rvecs0.create(NUM_FRAMES, 1, CV_64F); + } else { + rvecs0_ = Mat_(NUM_FRAMES, 3); + } + cnt_valid_frame = 0; + for (int f = 0; f < NUM_FRAMES; f++) { + if (!valid_frames[f]) continue; + if (is_mat_vec) + rvecs0.create(3, 1, CV_64F, f, true); + Mat store = is_mat_vec ? rvecs0.getMat(f) : rvecs0_.row(f); + memcpy(store.ptr(), params + (cnt_valid_frame + NUM_CAMERAS - 1)*6, 3*sizeof(double)); + cnt_valid_frame += 1; + } + if (!is_mat_vec && rvecs0.needed()) + rvecs0_.copyTo(rvecs0); + + is_mat_vec = tvecs0.needed() && tvecs0.isMatVector(); + if (is_mat_vec) { + tvecs0.create(NUM_FRAMES, 1, CV_64F); + } else { + tvecs0_ = Mat_(NUM_FRAMES, 3); + } + cnt_valid_frame = 0; + for (int f = 0; f < NUM_FRAMES; f++) { + if (!valid_frames[f]) continue; + if (is_mat_vec) + tvecs0.create(3, 1, CV_64F, f, true); + Mat store = is_mat_vec ? tvecs0.getMat(f) : tvecs0_.row(f); + memcpy(store.ptr(), params + (cnt_valid_frame + NUM_CAMERAS - 1)*6+3, 3*sizeof(double)); + store *= scale_3d_pts; + cnt_valid_frame += 1; + } + if (!is_mat_vec && tvecs0.needed()) + tvecs0_.copyTo(tvecs0); + double sum_errors = 0, cnt_errors = 0; + + const bool rvecs_mat_vec = rvecs0.needed() && rvecs0.isMatVector(), tvecs_mat_vec = tvecs0.needed() && tvecs0.isMatVector(); + const bool r_mat_vec = Rs.isMatVector(), t_mat_vec = Ts.isMatVector(); + Mat errs = Mat_(NUM_CAMERAS, NUM_FRAMES); + auto * errs_ptr = (double *) errs.data; + for (int c = 0; c < NUM_CAMERAS; c++) { + const Mat rvec = r_mat_vec ? Rs.getMat(c) : Rs.getMat().row(c).t(); + const Mat tvec = t_mat_vec ? Ts.getMat(c) : Ts.getMat().row(c).t(); for (int f = 0; f < NUM_FRAMES; f++) { - if (!valid_frames[f]) continue; - if (is_mat_vec) - rvecs0.create(3, 1, CV_64F, f, true); - Mat store = is_mat_vec ? rvecs0.getMat(f) : rvecs0_.row(f); - memcpy(store.ptr(), params + (cnt_valid_frame + NUM_CAMERAS - 1)*6, 3*sizeof(double)); - cnt_valid_frame += 1; + if (detection_mask_mat[c][f]) { + const Mat rvec0 = rvecs_mat_vec ? rvecs0.getMat(f) : rvecs0_.row(f).t(); + const Mat tvec0 = tvecs_mat_vec ? tvecs0.getMat(f) : tvecs0_.row(f).t(); + const double err2 = multiview::computeReprojectionMSE(objPoints.getMat(f), imagePoints[c][f], Ks_vec[c], + distortions_vec[c], rvec0, tvec0, rvec, tvec, models_mat.at(c)); + (*errs_ptr++) = sqrt(err2); + sum_errors += err2; + cnt_errors += 1; + } else (*errs_ptr++) = -1.0; } - if (!is_mat_vec && rvecs0.needed()) - rvecs0_.copyTo(rvecs0); } - if (tvecs0.needed() || perFrameErrors.needed()) { - const bool is_mat_vec = tvecs0.needed() && tvecs0.isMatVector(); - if (is_mat_vec) { - tvecs0.create(NUM_FRAMES, 1, CV_64F); - } else { - tvecs0_ = Mat_(NUM_FRAMES, 3); - } - cnt_valid_frame = 0; - for (int f = 0; f < NUM_FRAMES; f++) { - if (!valid_frames[f]) continue; - if (is_mat_vec) - tvecs0.create(3, 1, CV_64F, f, true); - Mat store = is_mat_vec ? tvecs0.getMat(f) : tvecs0_.row(f); - memcpy(store.ptr(), params + (cnt_valid_frame + NUM_CAMERAS - 1)*6+3, 3*sizeof(double)); - store *= scale_3d_pts; - cnt_valid_frame += 1; - } - if (!is_mat_vec && tvecs0.needed()) - tvecs0_.copyTo(tvecs0); - } - double sum_errors = 0, cnt_errors = 0; if (perFrameErrors.needed()) { - const bool rvecs_mat_vec = rvecs0.needed() && rvecs0.isMatVector(), tvecs_mat_vec = tvecs0.needed() && tvecs0.isMatVector(); - const bool r_mat_vec = Rs.isMatVector(), t_mat_vec = Ts.isMatVector(); - Mat errs = Mat_(NUM_CAMERAS, NUM_FRAMES); - auto * errs_ptr = (double *) errs.data; - for (int c = 0; c < NUM_CAMERAS; c++) { - const Mat rvec = r_mat_vec ? Rs.getMat(c) : Rs.getMat().row(c).t(); - const Mat tvec = t_mat_vec ? Ts.getMat(c) : Ts.getMat().row(c).t(); - for (int f = 0; f < NUM_FRAMES; f++) { - if (detection_mask_mat[c][f]) { - const Mat rvec0 = rvecs_mat_vec ? rvecs0.getMat(f) : rvecs0_.row(f).t(); - const Mat tvec0 = tvecs_mat_vec ? tvecs0.getMat(f) : tvecs0_.row(f).t(); - const double err2 = multiview::computeReprojectionMSE(objPoints.getMat(f), imagePoints[c][f], Ks[c], - distortions[c], rvec0, tvec0, rvec, tvec, is_fisheye_vec[c]); - (*errs_ptr++) = sqrt(err2); - sum_errors += err2; - cnt_errors += 1; - } else (*errs_ptr++) = -1.0; - } - } errs.copyTo(perFrameErrors); } diff --git a/modules/calib/test/test_cameracalibration.cpp b/modules/calib/test/test_cameracalibration.cpp index 1a06e318ef..1b10159ac7 100644 --- a/modules/calib/test/test_cameracalibration.cpp +++ b/modules/calib/test/test_cameracalibration.cpp @@ -2482,10 +2482,10 @@ double CV_MultiviewCalibrationTest_CPP::calibrateStereoCamera( const vector image_sizes (2, imageSize); Mat visibility_mat = Mat_::ones(2, numImgs); - std::vector is_fisheye(2, false); + std::vector models(2, cv::CALIB_MODEL_PINHOLE); std::vector all_flags(2, flags); - double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, - Rs, Ts, Ks, distortions, rvecs, tvecs, is_fisheye, errors_mat, noArray(), false, all_flags); + double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models, + Rs, Ts, Ks, distortions, 0, all_flags, rvecs, tvecs, errors_mat); if (perViewErrors1.size() != (size_t)numImgs) { diff --git a/modules/calib/test/test_fisheye.cpp b/modules/calib/test/test_fisheye.cpp index 8ed2310f26..e5d528cd22 100644 --- a/modules/calib/test/test_fisheye.cpp +++ b/modules/calib/test/test_fisheye.cpp @@ -615,9 +615,9 @@ TEST_F(fisheyeTest, multiview_calibration) right_pts.copyTo(image_points_all[1][i]); } std::vector image_sizes(2, imageSize); - cv::Mat visibility_mat = cv::Mat_::ones(2, (int)leftPoints.size()), errors_mat, output_pairs; + cv::Mat visibility_mat = cv::Mat_::ones(2, (int)leftPoints.size()); std::vector Rs, Ts, Ks, distortions, rvecs0, tvecs0; - std::vector is_fisheye(2, true); + std::vector models(2, cv::CALIB_MODEL_FISHEYE); int flag = 0; flag |= cv::CALIB_RECOMPUTE_EXTRINSIC; flag |= cv::CALIB_CHECK_COND; @@ -625,8 +625,8 @@ TEST_F(fisheyeTest, multiview_calibration) std::vector all_flags(2, flag); - calibrateMultiview (objectPoints, image_points_all, image_sizes, visibility_mat, - Rs, Ts, Ks, distortions, rvecs0, tvecs0, is_fisheye, errors_mat, output_pairs, false, all_flags); + calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models, + Rs, Ts, Ks, distortions, 0, all_flags, rvecs0, tvecs0); cv::Matx33d R_correct( 0.9975587205950972, 0.06953016383322372, 0.006492709911733523, -0.06956823121068059, 0.9975601387249519, 0.005833595226966235, -0.006071257768382089, -0.006271040135405457, 0.9999619062167968); diff --git a/modules/calib/test/test_multiview_calib.cpp b/modules/calib/test/test_multiview_calib.cpp index 9ff324c0df..361d6c4a58 100644 --- a/modules/calib/test/test_multiview_calib.cpp +++ b/modules/calib/test/test_multiview_calib.cpp @@ -27,7 +27,7 @@ TEST(multiview_calibration, accuracy) { board_pattern[j*board_size.width+i] = cv::Vec3f ((float)i, (float)j, 0)*board_len; } } - std::vector is_fisheye(num_cameras, false); + std::vector models(num_cameras, cv::CALIB_MODEL_PINHOLE); std::vector image_sizes(num_cameras); std::vector Ks_gt, distortions_gt, Rs_gt, Ts_gt; for (int c = 0; c < num_cameras; c++) { @@ -87,7 +87,7 @@ TEST(multiview_calibration, accuracy) { int num_visible_patterns = 0; for (int c = 0; c < num_cameras; c++) { cv::Mat img_pts; - if (is_fisheye[c]) { + if (models[c] == cv::CALIB_MODEL_FISHEYE) { cv::fisheye::projectPoints(pattern_new, img_pts, Rs_gt[c], Ts_gt[c], Ks_gt[c], distortions_gt[c]); } else { cv::projectPoints(pattern_new, Rs_gt[c], Ts_gt[c], Ks_gt[c], distortions_gt[c], img_pts); @@ -132,9 +132,8 @@ TEST(multiview_calibration, accuracy) { } std::vector Ks, distortions, Rs, Ts; - cv::Mat errors_mat, output_pairs, rvecs0, tvecs0; - calibrateMultiview (objPoints, image_points_all, image_sizes, visibility_mat, - Rs, Ts, Ks, distortions, rvecs0, tvecs0, is_fisheye, errors_mat, output_pairs, false); + calibrateMultiview(objPoints, image_points_all, image_sizes, visibility_mat, + models, Rs, Ts, Ks, distortions); const double K_err_tol = 1e1, dist_tol = 5e-2, R_tol = 1e-2, T_tol = 1e-2; for (int c = 0; c < num_cameras; c++) { @@ -197,6 +196,40 @@ struct MultiViewTest : public ::testing::Test } } + double calibrateMono(const std::vector& board_pattern, + const std::vector& image_points, + const cv::Size& image_size, + cv::CameraModel model, + int flags, + Mat& K, + Mat& dist) + { + std::vector filtered_image_points; + for(size_t i = 0; i < image_points.size(); i++) + { + if(!image_points[i].empty()) + filtered_image_points.push_back(image_points[i]); + } + std::vector> objPoints(filtered_image_points.size(), board_pattern); + + std::vector rvec, tvec; + cv::Mat K1, dist1; + if(model == cv::CALIB_MODEL_PINHOLE) + { + return cv::calibrateCamera(objPoints, filtered_image_points, image_size, K, dist, rvec, tvec, flags); + } + else if(model == cv::CALIB_MODEL_FISHEYE) + { + return cv::fisheye::calibrate(objPoints, filtered_image_points, image_size, K, dist, rvec, tvec, flags); + } + else + { + CV_Error(Error::StsBadArg, "Unsupported camera model!"); + } + + return FLT_MAX; + } + void validateCameraPose(const Mat& R, Mat T, const Mat& R_gt, const Mat& T_gt, double angle_tol = 1.*M_PI/180., double pos_tol = 0.01) { @@ -235,7 +268,7 @@ TEST_F(MultiViewTest, OneLine) const string root = cvtest::TS::ptr()->get_data_path() + "cv/cameracalibration/multiview/3cams-one-line/"; const std::vector cam_names = {"cam_0", "cam_1", "cam_3"}; const std::vector image_sizes = {{1920, 1080}, {1920, 1080}, {1920, 1080} }; - std::vector is_fisheye(3, false); + std::vector models(3, cv::CALIB_MODEL_PINHOLE); double rs_1_gt_data[9] = { 0.9996914489704484, -0.01160060078752197, -0.02196435559568884, @@ -279,10 +312,113 @@ TEST_F(MultiViewTest, OneLine) std::vector flagsForIntrinsics(3, CALIB_RATIONAL_MODEL); std::vector Ks, distortions, Rs, Rs_rvec, Ts; - cv::Mat errors_mat, output_pairs, rvecs0, tvecs0; - double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, - Rs_rvec, Ts, Ks, distortions, rvecs0, tvecs0, is_fisheye, errors_mat, output_pairs, - false, flagsForIntrinsics); + double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, + Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); + CV_LOG_INFO(NULL, "RMS: " << rms); + + EXPECT_LE(rms, .3); + + Rs.resize(Rs_rvec.size()); + for(int c = 0; c < 3; c++) + { + cv::Rodrigues(Rs_rvec[c], Rs[c]); + CV_LOG_INFO(NULL, "R" << c << ":" << Rs[c]); + CV_LOG_INFO(NULL, "T" << c << ":" << Ts[c]); + } + + validateAllPoses(Rs_gt, Ts_gt, Rs, Ts); +} + +TEST_F(MultiViewTest, OneLineInitialGuess) +{ + const string root = cvtest::TS::ptr()->get_data_path() + "cv/cameracalibration/multiview/3cams-one-line/"; + const std::vector cam_names = {"cam_0", "cam_1", "cam_3"}; + const std::vector image_sizes = {{1920, 1080}, {1920, 1080}, {1920, 1080} }; + std::vector models(3, cv::CALIB_MODEL_PINHOLE); + + double rs_1_gt_data[9] = { + 0.9996914489704484, -0.01160060078752197, -0.02196435559568884, + 0.012283315339906, 0.9994374509454836, 0.03120739995344806, + 0.02158997497973892, -0.03146756598408248, 0.9992715673286274 + }; + double rs_2_gt_data[9] = { + 0.9988848194142131, -0.0255827884561986, -0.03968171466355882, + 0.0261796234191418, 0.999550713317242, 0.0145944792515729, + 0.03929051872229011, -0.0156170561181697, 0.9991057815350362 + }; + + double ts_1_gt_data[3] = {0.5078811293323259, 0.002753469433719865, 0.02413521839310227}; + double ts_2_gt_data[3] = {1.007213763725429, 0.01645068247976361, 0.05394643957910365}; + + std::vector Rs_gt = { + cv::Mat::eye(3, 3, CV_64FC1), + cv::Mat(3, 3, CV_64FC1, rs_1_gt_data), + cv::Mat(3, 3, CV_64FC1, rs_2_gt_data) + }; + + std::vector Ts_gt = { + cv::Mat::zeros(3, 1, CV_64FC1), + cv::Mat(3, 1, CV_64FC1, ts_1_gt_data), + cv::Mat(3, 1, CV_64FC1, ts_2_gt_data) + }; + + const int num_frames = 96; + std::vector> image_points_all; + cv::Mat visibility; + loadImagePoints(root, cam_names, num_frames, image_points_all, visibility); + EXPECT_EQ(cam_names.size(), image_points_all.size()); + for(size_t i = 0; i < cam_names.size(); i++) + { + EXPECT_TRUE(!image_points_all[i].empty()); + } + + std::vector board_pattern = genAsymmetricObjectPoints(); + std::vector> objPoints(num_frames, board_pattern); + + std::vector flagsForIntrinsics(3, CALIB_RATIONAL_MODEL); + + std::vector Ks, distortions; + std::vector Rs(3); + std::vector Ts(3); + std::vector Rs_rvec(3); + for(int c = 0; c < 3; c++) + { + Mat K, dist; + double mono_rms = calibrateMono(board_pattern, image_points_all[c], image_sizes[c], + cv::CALIB_MODEL_PINHOLE, cv::CALIB_RATIONAL_MODEL, + K, dist); + + CV_LOG_INFO(NULL, "K:" << K); + CV_LOG_INFO(NULL, "dist:" << dist); + Ks.push_back(K); + distortions.push_back(dist); + CV_LOG_INFO(NULL, "Calibrate mono RMS #" << c << ": " << mono_rms); + EXPECT_LE(mono_rms, .3); + } + + const auto euler2rot = [] (double x, double y, double z) { + cv::Matx33d R_x(1, 0, 0, 0, cos(x), -sin(x), 0, sin(x), cos(x)); + cv::Matx33d R_y(cos(y), 0, sin(y), 0, 1, 0, -sin(y), 0, cos(y)); + cv::Matx33d R_z(cos(z), -sin(z), 0, sin(z), cos(z), 0, 0, 0, 1); + return cv::Mat(R_z * R_y * R_x); + }; + + // Introduce small noise by rotating ground truth camera pose a bit + Rs[0] = Rs_gt[0].clone(); + Ts[0] = Ts_gt[0].clone(); + double sign = 1.; + for (int c = 1; c < 3; c++) + { + Mat noise = euler2rot(0., sign*M_PI/180., 0.); + sign *= -1.; + Rs[c] = noise*Rs_gt[c]; + Ts[c] = Ts_gt[c].clone(); + cv::Rodrigues(Rs[c], Rs_rvec[c]); + } + + int flags = cv::CALIB_USE_EXTRINSIC_GUESS | cv::CALIB_USE_INTRINSIC_GUESS; + double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, + Rs_rvec, Ts, Ks, distortions, flags, flagsForIntrinsics); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, .3); @@ -303,7 +439,7 @@ TEST_F(MultiViewTest, CamsToFloor) const string root = cvtest::TS::ptr()->get_data_path() + "cv/cameracalibration/multiview/3cams-to-floor/"; const std::vector cam_names = {"cam_0", "cam_1", "cam_2"}; std::vector image_sizes = {{1920, 1080}, {1920, 1080}, {1280, 720}}; - std::vector is_fisheye(3, false); + std::vector models(3, cv::CALIB_MODEL_PINHOLE); double rs_1_gt_data[9] = { -0.05217184989559624, 0.6470741242690249, -0.7606399777686852, @@ -347,10 +483,8 @@ TEST_F(MultiViewTest, CamsToFloor) std::vector flagsForIntrinsics(3, cv::CALIB_RATIONAL_MODEL); std::vector Ks, distortions, Rs, Rs_rvec, Ts; - cv::Mat errors_mat, output_pairs, rvecs0, tvecs0; - double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, - Rs_rvec, Ts, Ks, distortions, rvecs0, tvecs0, is_fisheye, errors_mat, output_pairs, - false, flagsForIntrinsics); + double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, + Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, 1.); @@ -368,40 +502,6 @@ TEST_F(MultiViewTest, CamsToFloor) struct RegisterCamerasTest: public MultiViewTest { - double calibrateMono(const std::vector& board_pattern, - const std::vector& image_points, - const cv::Size& image_size, - cv::CameraModel model, - int flags, - Mat& K, - Mat& dist) - { - std::vector filtered_image_points; - for(size_t i = 0; i < image_points.size(); i++) - { - if(!image_points[i].empty()) - filtered_image_points.push_back(image_points[i]); - } - std::vector> objPoints(filtered_image_points.size(), board_pattern); - - std::vector rvec, tvec; - cv::Mat K1, dist1; - if(model == cv::CALIB_MODEL_PINHOLE) - { - return cv::calibrateCamera(objPoints, filtered_image_points, image_size, K, dist, rvec, tvec, flags); - } - else if(model == cv::CALIB_MODEL_FISHEYE) - { - return cv::fisheye::calibrate(objPoints, filtered_image_points, image_size, K, dist, rvec, tvec, flags); - } - else - { - CV_Error(Error::StsBadArg, "Unsupported camera model!"); - } - - return FLT_MAX; - } - void filterPoints(const std::vector>& image_points_all, std::vector& visible_image_points1, std::vector& visible_image_points2) diff --git a/samples/cpp/multiview_calibration_sample.cpp b/samples/cpp/multiview_calibration_sample.cpp index 3db10169e3..24d47bb0b7 100644 --- a/samples/cpp/multiview_calibration_sample.cpp +++ b/samples/cpp/multiview_calibration_sample.cpp @@ -13,17 +13,16 @@ // ! [detectPointsAndCalibrate_signature] static void detectPointsAndCalibrate (cv::Size pattern_size, float pattern_distance, const std::string &pattern_type, - const std::vector &is_fisheye, const std::vector &filenames, + const std::vector &models, const std::vector &filenames, const cv::String* dict_path=nullptr) // ! [detectPointsAndCalibrate_signature] { // ! [calib_init] std::vector board (pattern_size.area()); - const int num_cameras = (int)is_fisheye.size(); + const int num_cameras = (int)models.size(); std::vector> image_points_all; std::vector image_sizes; std::vector Ks, distortions, Ts, Rs; - cv::Mat rvecs0, tvecs0, errors_mat, output_pairs; if (pattern_type == "checkerboard" || pattern_type == "charuco") { for (int i = 0; i < pattern_size.height; i++) { for (int j = 0; j < pattern_size.width; j++) { @@ -162,9 +161,9 @@ static void detectPointsAndCalibrate (cv::Size pattern_size, float pattern_dista std::vector> objPoints(num_frames, board); // ! [multiview_calib] + const double rmse = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, - Rs, Ts, Ks, distortions, cv::noArray(), cv::noArray(), - is_fisheye, errors_mat, output_pairs); + models, Rs, Ts, Ks, distortions); // ! [multiview_calib] std::cout << "average RMSE over detection mask " << rmse << "\n"; for (int c = 0; c < (int)Rs.size(); c++) { @@ -219,13 +218,13 @@ int main (int argc, char **argv) { CV_Assert(pattern_size_count == 1); pattern_size.height = std::stoi(temp_str); - std::vector is_fisheye; + std::vector models; const cv::String is_fisheye_str = parser.get("is_fisheye"); for (char i : is_fisheye_str) { if (i == '0') { - is_fisheye.push_back(false); + models.push_back(cv::CALIB_MODEL_PINHOLE); } else if (i == '1') { - is_fisheye.push_back(true); + models.push_back(cv::CALIB_MODEL_FISHEYE); } } const cv::String filenames_str = parser.get("filenames"); @@ -240,13 +239,13 @@ int main (int argc, char **argv) { } } filenames.emplace_back(temp_str); - CV_CheckEQ(filenames.size(), is_fisheye.size(), "filenames size must be equal to number of cameras!"); + CV_CheckEQ(filenames.size(), models.size(), "filenames size must be equal to number of cameras!"); if (parser.has("board_dict_path")) { cv::String board_dict_path = parser.get("board_dict_path"); - detectPointsAndCalibrate (pattern_size, parser.get("pattern_distance"), parser.get("pattern_type"), is_fisheye, filenames, &board_dict_path); + detectPointsAndCalibrate (pattern_size, parser.get("pattern_distance"), parser.get("pattern_type"), models, filenames, &board_dict_path); } else { - detectPointsAndCalibrate (pattern_size, parser.get("pattern_distance"), parser.get("pattern_type"), is_fisheye, filenames); + detectPointsAndCalibrate (pattern_size, parser.get("pattern_distance"), parser.get("pattern_type"), models, filenames); } return 0; } diff --git a/samples/python/multiview_calibration.py b/samples/python/multiview_calibration.py index f79dd138a6..d7453a1dca 100644 --- a/samples/python/multiview_calibration.py +++ b/samples/python/multiview_calibration.py @@ -275,11 +275,11 @@ def showUndistorted(image_points, Ks, distortions, image_names): def plotProjection(points_2d, pattern_points, rvec0, tvec0, rvec1, tvec1, - K, dist_coeff, is_fisheye, cam_idx, frame_idx, per_acc, + K, dist_coeff, model, cam_idx, frame_idx, per_acc, image=None): rvec2, tvec2 = cv.composeRT(rvec0, tvec0, rvec1, tvec1)[:2] - if is_fisheye: + if model == cv.CALIB_MODEL_FISHEYE: points_2d_est = cv.fisheye.projectPoints( pattern_points[:, None], rvec2, tvec2, K, dist_coeff.flatten() )[0].reshape(-1, 2) @@ -361,7 +361,7 @@ def calibrateFromPoints( pattern_points, image_points, image_sizes, - is_fisheye, + models, image_names=None, find_intrinsics_in_python=False, Ks=None, @@ -370,7 +370,7 @@ def calibrateFromPoints( """ pattern_points: NUM_POINTS x 3 (numpy array) image_points: NUM_CAMERAS x NUM_FRAMES x NUM_POINTS x 2 - is_fisheye: NUM_CAMERAS (bool) + models: NUM_CAMERAS (cv.CALIB_MODEL_PINHOLE | cv.CALIB_MODEL_FISHEYE) image_sizes: NUM_CAMERAS x [width, height] """ num_cameras = len(image_points) @@ -380,7 +380,7 @@ def calibrateFromPoints( with np.printoptions(threshold=np.inf): # type: ignore print("detection mask Matrix:\n", str(detection_mask).replace('0\n ', '0').replace('1\n ', '1')) - pinhole_flag = cv.CALIB_ZERO_TANGENT_DIST + pinhole_flag = cv.CALIB_RATIONAL_MODEL fisheye_flag = cv.CALIB_RECOMPUTE_EXTRINSIC+cv.CALIB_FIX_SKEW if Ks is not None and distortions is not None: USE_INTRINSICS_GUESS = True @@ -389,7 +389,7 @@ def calibrateFromPoints( if find_intrinsics_in_python: Ks, distortions = [], [] for c in range(num_cameras): - if is_fisheye[c]: + if models[c] == cv.CALIB_MODEL_FISHEYE: image_points_c = [ image_points[c][f][:, None] for f in range(num_frames) if len(image_points[c][f]) > 0 ] @@ -428,13 +428,13 @@ def calibrateFromPoints( imagePoints=image_points, imageSize=image_sizes, detectionMask=detection_mask, + models=np.array(models, dtype=np.uint8), Rs=None, Ts=None, Ks=Ks, distortions=distortions, - isFisheye=np.array(is_fisheye, dtype=np.uint8), - useIntrinsicsGuess=USE_INTRINSICS_GUESS, - flagsForIntrinsics=np.array([pinhole_flag if not is_fisheye[x] else fisheye_flag for x in range(num_cameras)], dtype=int), + flagsForIntrinsics=np.array([pinhole_flag if models[x] == cv.CALIB_MODEL_PINHOLE else fisheye_flag for x in range(num_cameras)], dtype=int), + flags = cv.CALIB_USE_INTRINSIC_GUESS if USE_INTRINSICS_GUESS else 0 ) # [multiview_calib] # except Exception as e: @@ -461,7 +461,7 @@ def calibrateFromPoints( 'errors_per_frame': errors_per_frame, 'output_pairs': output_pairs, 'image_points': image_points, - 'is_fisheye': is_fisheye, + 'models': models, 'image_sizes': image_sizes, 'pattern_points': pattern_points, 'detection_mask': detection_mask, @@ -469,7 +469,7 @@ def calibrateFromPoints( } -def visualizeResults(detection_mask, Rs, Ts, Ks, distortions, is_fisheye, +def visualizeResults(detection_mask, Rs, Ts, Ks, distortions, models, image_points, errors_per_frame, rvecs0, tvecs0, pattern_points, image_sizes, output_pairs, image_names, cam_ids): rvecs = [cv.Rodrigues(R)[0] for R in Rs] @@ -509,7 +509,7 @@ def visualizeResults(detection_mask, Rs, Ts, Ks, distortions, is_fisheye, Ts[cam_idx], Ks[cam_idx], distortions[cam_idx], - is_fisheye[cam_idx], + models[cam_idx], cam_idx, frame_idx, (errors_per_frame[cam_idx, frame_idx] < errors).sum() * 100 / len(errors), @@ -527,7 +527,7 @@ def visualizeFromFile(file): assert file_read.isOpened(), file read_keys = [ 'Rs', 'distortions', 'Ks', 'Ts', 'rvecs0', 'tvecs0', - 'errors_per_frame', 'output_pairs', 'image_points', 'is_fisheye', + 'errors_per_frame', 'output_pairs', 'image_points', 'models', 'image_sizes', 'pattern_points', 'detection_mask', 'cam_ids', ] input = {} @@ -549,7 +549,7 @@ def saveToFile(path_to_save, **kwargs): path_to_save = datetime.now().strftime("%d-%b-%Y (%H:%M:%S.%f)")+'.yaml' save_file = cv.FileStorage(path_to_save, cv.FileStorage_WRITE) - kwargs['is_fisheye'] = np.array(kwargs['is_fisheye'], dtype=int) + kwargs['models'] = np.array(kwargs['models'], dtype=int) image_points = kwargs['image_points'] for i in range(len(image_points)): @@ -574,7 +574,7 @@ def saveToFile(path_to_save, **kwargs): save_file.release() -def compareGT(gt_file, detection_mask, Rs, Ts, Ks, distortions, is_fisheye, +def compareGT(gt_file, detection_mask, Rs, Ts, Ks, distortions, models, image_points, errors_per_frame, rvecs0, tvecs0, pattern_points, image_sizes, output_pairs, image_names, cam_ids): @@ -610,7 +610,7 @@ def compareGT(gt_file, detection_mask, Rs, Ts, Ks, distortions, is_fisheye, points = np.concatenate([X[:,:,None], Y[:,:,None]], axis=2).reshape([-1, 1, 2]) # Undistort the image points with the estimated distortions - if is_fisheye[cam]: + if models[cam] == cv.CALIB_MODEL_FISHEYE: points_undist = cv.fisheye.undistortPoints(points, Ks[cam],distortions[cam]) else: points_undist = cv.undistortPoints(points, Ks[cam], distortions[cam]) @@ -618,7 +618,7 @@ def compareGT(gt_file, detection_mask, Rs, Ts, Ks, distortions, is_fisheye, pt_norm = np.concatenate([points_undist, np.ones([points_undist.shape[0], 1, 1])], axis=2) # Distort the image points with the ground truth distortions - if is_fisheye[cam]: + if models[cam] == cv.CALIB_MODEL_FISHEYE: projected = cv.fisheye.projectPoints(pt_norm, np.zeros([3, 1]), np.zeros([3, 1]), Ks_gt[cam], distortions_gt[cam])[0] else: projected = cv.projectPoints(pt_norm, np.zeros([3, 1]), np.zeros([3, 1]), Ks_gt[cam], distortions_gt[cam])[0] @@ -772,7 +772,7 @@ def detect(cam_idx, frame_idx, img_name, pattern_type, return cam_idx, frame_idx, img_size, np.array([], dtype=np.float32) -def calibrateFromImages(files_with_images, grid_size, pattern_type, is_fisheye, +def calibrateFromImages(files_with_images, grid_size, pattern_type, models, dist_m, winsize, points_json_file, debug_corners, RESIZE_IMAGE, find_intrinsics_in_python, is_parallel_detection=True, cam_ids=None, intrinsics_dir='', board_dict_path=None): @@ -780,7 +780,7 @@ def calibrateFromImages(files_with_images, grid_size, pattern_type, is_fisheye, files_with_images: NUM_CAMERAS - path to file containing image names (NUM_FRAMES) grid_size: [width, height] -- size of grid pattern dist_m: length of a grid cell - is_fisheye: NUM_CAMERAS (bool) + models: NUM_CAMERAS (cv.CALIB_MODEL_PINHOLE | cv.CALIB_MODEL_FISHEYE) """ # [calib_init] if pattern_type.lower() == 'checkerboard' or pattern_type.lower() == 'charuco': @@ -795,10 +795,12 @@ def calibrateFromImages(files_with_images, grid_size, pattern_type, is_fisheye, if pattern_type.lower() == 'charuco': assert (board_dict_path is not None) and os.path.exists(board_dict_path) board_dict = json.load(open(board_dict_path, 'r')) + else: + board_dict = None # [calib_init] - assert len(files_with_images) == len(is_fisheye) and len(grid_size) == 2 + assert len(files_with_images) == len(models) and len(grid_size) == 2 if cam_ids is None: cam_ids = list(range(len(files_with_images))) @@ -887,7 +889,7 @@ def calibrateFromImages(files_with_images, grid_size, pattern_type, is_fisheye, 'object_points': pattern.tolist(), 'image_points': image_points_cameras_list, 'image_sizes': image_sizes, - 'is_fisheye': is_fisheye, + 'model': models, }, wf) Ks = None @@ -909,7 +911,7 @@ def calibrateFromImages(files_with_images, grid_size, pattern_type, is_fisheye, pattern, image_points_cameras, image_sizes, - is_fisheye, + models, all_images_names, find_intrinsics_in_python, Ks=Ks, @@ -933,7 +935,7 @@ def calibrateFromJSON(json_file, find_intrinsics_in_python): np.array(data['object_points'], dtype=np.float32).T, data['image_points'], data['image_sizes'], - data['is_fisheye'], + data['models'], images_names, find_intrinsics_in_python, Ks, @@ -992,7 +994,7 @@ if __name__ == '__main__': files_with_images=[x.strip() for x in params.filenames.split(',')], grid_size=[int(v) for v in params.pattern_size.split(',')], pattern_type=params.pattern_type, - is_fisheye=[bool(int(v)) for v in params.is_fisheye.split(',')], + models=[int(v) for v in params.is_fisheye.split(',')], dist_m=params.pattern_distance, winsize=tuple([int(v) for v in params.winsize.split(',')]), points_json_file=params.points_json_file, @@ -1009,7 +1011,9 @@ if __name__ == '__main__': if params.gt_file is not None: assert os.path.exists(params.gt_file), f'Path to gt file does not exist: {params.gt_file}' compareGT(params.gt_file, **output) - visualizeResults(**output) + + if params.visualize: + visualizeResults(**output) print('Saving:', params.path_to_save) saveToFile(params.path_to_save, **output)