From 706a065d141b9d258d21ce9c3540dd18ce114d83 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Sat, 21 May 2011 13:27:57 +0000 Subject: [PATCH] memory optimizations in opencv_stitching --- modules/stitching/main.cpp | 139 ++++++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 41 deletions(-) diff --git a/modules/stitching/main.cpp b/modules/stitching/main.cpp index d99a779264..161740a355 100644 --- a/modules/stitching/main.cpp +++ b/modules/stitching/main.cpp @@ -198,6 +198,7 @@ int main(int argc, char* argv[]) LOGLN("Reading images and finding features..."); t = getTickCount(); + vector images(num_images); vector features(num_images); SurfFeaturesFinder finder(trygpu); Mat full_img, img; @@ -220,8 +221,11 @@ int main(int argc, char* argv[]) } resize(full_img, img, Size(), work_scale, work_scale); } + images[i] = img.clone(); finder(img, features[i]); } + full_img.release(); + img.release(); LOGLN("Reading images and finding features, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); LOGLN("Pairwise matching... "); @@ -234,9 +238,14 @@ int main(int argc, char* argv[]) LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); vector indices = leaveBiggestComponent(features, pairwise_matches, conf_thresh); + vector img_subset; vector img_names_subset; for (size_t i = 0; i < indices.size(); ++i) + { + img_subset.push_back(images[indices[i]]); img_names_subset.push_back(img_names[indices[i]]); + } + images = img_subset; img_names = img_names_subset; num_images = static_cast(img_names.size()); @@ -290,41 +299,21 @@ int main(int argc, char* argv[]) nth_element(focals.begin(), focals.end(), focals.begin() + focals.size() / 2); float camera_focal = static_cast(focals[focals.size() / 2]); - vector images(num_images); - - LOGLN("Compose scaling..."); + LOGLN("Warping images (auxiliary)... "); t = getTickCount(); - for (int i = 0; i < num_images; ++i) - { - Mat full_img = imread(img_names[i]); - if (!is_compose_scale_set) - { - compose_scale = min(1.0, sqrt(compose_megapix * 1e6 / full_img.size().area())); - is_compose_scale_set = true; - } - Mat img; - resize(full_img, img, Size(), compose_scale, compose_scale); - images[i] = img; - cameras[i].focal *= compose_scale / work_scale; - } - camera_focal *= static_cast(compose_scale / work_scale); - LOGLN("Compose scaling, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + vector corners(num_images); + vector masks_warped(num_images); + vector images_warped(num_images); + vector sizes(num_images); vector masks(num_images); + for (int i = 0; i < num_images; ++i) { masks[i].create(images[i].size(), CV_8U); masks[i].setTo(Scalar::all(255)); } - vector corners(num_images); - vector sizes(num_images); - vector masks_warped(num_images); - vector images_warped(num_images); - - LOGLN("Warping images... "); - t = getTickCount(); - Ptr warper = Warper::createByCameraFocal(camera_focal, warp_type); for (int i = 0; i < num_images; ++i) { @@ -335,38 +324,105 @@ int main(int argc, char* argv[]) INTER_NEAREST, BORDER_CONSTANT); } - vector images_f(num_images); + vector images_warped_f(num_images); for (int i = 0; i < num_images; ++i) - images_warped[i].convertTo(images_f[i], CV_32F); + images_warped[i].convertTo(images_warped_f[i], CV_32F); LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); LOGLN("Finding seams..."); t = getTickCount(); + Ptr seam_finder = SeamFinder::createDefault(seam_find_type); - seam_finder->find(images_f, corners, masks_warped); + seam_finder->find(images_warped_f, corners, masks_warped); + LOGLN("Finding seams, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); - LOGLN("Blending images..."); + // Release unused memory + images.clear(); + images_warped.clear(); + images_warped_f.clear(); + masks.clear(); + + LOGLN("Compositing..."); t = getTickCount(); - Ptr blender = Blender::createDefault(blend_type); + Mat img_warped, img_warped_f; + Mat dilated_mask, seam_mask, mask, mask_warped; + Ptr blender; + double compose_aspect = 1; - if (blend_type == Blender::MULTI_BAND) + for (int img_idx = 0; img_idx < num_images; ++img_idx) { - // Ensure last pyramid layer area is about 1 pix - MultiBandBlender* mb = dynamic_cast((Blender*)(blender)); - mb->setNumBands(static_cast(ceil(log(static_cast(images_f[0].size().area())) / log(4.0)))); - LOGLN("Multi-band blending num. bands: " << mb->numBands()); - } + LOGLN("Compositing image #" << img_idx); - blender->prepare(corners, sizes); - for (int i = 0; i < num_images; ++i) - blender->feed(images_f[i], masks_warped[i], corners[i]); + // Read image and resize it if necessary + full_img = imread(img_names[img_idx]); + if (!is_compose_scale_set) + { + if (compose_megapix > 0) + compose_scale = min(1.0, sqrt(compose_megapix * 1e6 / full_img.size().area())); + is_compose_scale_set = true; + compose_aspect = compose_scale / work_scale; + camera_focal *= static_cast(compose_aspect); + warper = Warper::createByCameraFocal(camera_focal, warp_type); + } + if (abs(compose_scale - 1) > 1e-1) + resize(full_img, img, Size(), compose_scale, compose_scale); + else + img = full_img; + full_img.release(); + + // Update cameras paramters + cameras[img_idx].focal *= compose_aspect; + + // Warp the current image + warper->warp(img, static_cast(cameras[img_idx].focal), cameras[img_idx].R, + img_warped); + img_warped.convertTo(img_warped_f, CV_32F); + img_warped.release(); + + // Warp current image mask + mask.create(img.size(), CV_8U); + mask.setTo(Scalar::all(255)); + warper->warp(mask, static_cast(cameras[img_idx].focal), cameras[img_idx].R, mask_warped, + INTER_NEAREST, BORDER_CONSTANT); + mask.release(); + dilate(masks_warped[img_idx], dilated_mask, Mat()); + resize(dilated_mask, seam_mask, mask_warped.size()); + mask_warped = seam_mask & mask_warped; + + if (static_cast(blender) == 0) + { + // Create blender + + blender = Blender::createDefault(blend_type); + if (blend_type == Blender::MULTI_BAND) + { + // Ensure last pyramid layer area is about 1 pix + MultiBandBlender* mb = dynamic_cast(static_cast(blender)); + mb->setNumBands(static_cast(ceil(log(static_cast(img_warped_f.size().area())) / log(4.0)))); + } + + // Determine the final image size + Rect dst_roi = resultRoi(corners, sizes); + for (int i = 0; i < num_images; ++i) + { + corners[i] = dst_roi.tl() + (corners[i] - dst_roi.tl()) * compose_aspect; + sizes[i] = Size(static_cast((sizes[i].width + 1) * compose_aspect), + static_cast((sizes[i].height + 1) * compose_aspect)); + } + blender->prepare(corners, sizes); + } + + // Blend the current image + blender->feed(img_warped_f, mask_warped, corners[img_idx]); + } + Mat result, result_mask; blender->blend(result, result_mask); - LOGLN("Blending images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); imwrite(result_name, result); @@ -374,3 +430,4 @@ int main(int argc, char* argv[]) return 0; } +