From 8e0c0dc347f7fdbfd8dc405c2706e911c8ba7a4a Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Tue, 9 Sep 2025 14:21:18 +0300 Subject: [PATCH] Merge pull request #27755 from dkurt:ffmpeg/sws_scale_frame Optimize FFmpeg VideoCapture with swscale threads option #27755 ### Pull Request Readiness Checklist resolves https://github.com/opencv/opencv/issues/21969 * Switch to `sws_scale_from` for `libswscale >= 6.4.100` (FFmpeg >= 5.0) * Use new context init API with threads option (`libswscale >= 8.12.100`: https://github.com/FFmpeg/FFmpeg/commit/2a091d4f2ee1e367d05a6bbbe96b204257cbda87) * Replicate `sws_getCachedContext` with threads option for `libswscale < 8.12.100` 1 hour mp4 video every frame reading | HW | sws_scale | sws_scale_frame + 16 threads | sws_scale_frame + 24 threads (#cpus) | |---|---|---|---| | Intel Core i9-12900 CPU | 45.1 sec | 25.4 sec (x1.77) | 30 sec (x1.50) | | NVIDIA GPU 4090 | 232 sec | 89.4 sec (x2.59) | 77 sec (x3.01) | ``` import time import numpy as np import os import cv2 as cv # os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] = "hwaccel;cuvid|video_codec;h264_cuvid|vsync;0" start = time.time() video = "test.mp4" cap = cv.VideoCapture(video, cv.CAP_FFMPEG) while True: has_frame, frame = cap.read() if not has_frame: break print(time.time() - start) ``` See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/videoio/src/cap_ffmpeg_impl.hpp | 71 ++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index cda32be873..ed1d99bdb0 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -607,6 +607,7 @@ struct CvCapture_FFMPEG int hw_device; int use_opencl; int extraDataIdx; + int requestedThreads; }; void CvCapture_FFMPEG::init() @@ -658,6 +659,7 @@ void CvCapture_FFMPEG::init() hw_device = -1; use_opencl = 0; extraDataIdx = 1; + requestedThreads = cv::getNumberOfCPUs(); } @@ -993,13 +995,7 @@ inline void fill_codec_context(AVCodecContext * enc, AVDictionary * dict) if (!enc->thread_count) { int nCpus = cv::getNumberOfCPUs(); - int requestedThreads = std::min(nCpus, 16); // [OPENCV:FFMPEG:24] Application has requested XX threads. Using a thread count greater than 16 is not recommended. - std::string threads_option = utils::getConfigurationParameterString("OPENCV_FFMPEG_THREADS"); - if (!threads_option.empty()) - { - requestedThreads = atoi(threads_option.c_str()); - } - enc->thread_count = requestedThreads; + enc->thread_count = std::min(nCpus, 16); // [OPENCV:FFMPEG:24] Application has requested XX threads. Using a thread count greater than 16 is not recommended. } AVDictionaryEntry* avdiscard_entry = av_dict_get(dict, "avdiscard", NULL, 0); @@ -1120,7 +1116,7 @@ bool CvCapture_FFMPEG::open(const char* _filename, const Ptr& str #endif if (params.has(CAP_PROP_N_THREADS)) { - nThreads = params.get(CAP_PROP_N_THREADS); + nThreads = requestedThreads = params.get(CAP_PROP_N_THREADS); } if (params.warnUnusedParameters()) { @@ -1129,6 +1125,15 @@ bool CvCapture_FFMPEG::open(const char* _filename, const Ptr& str } } + if (params.empty() || !params.has(CAP_PROP_N_THREADS)) + { + std::string threads_option = utils::getConfigurationParameterString("OPENCV_FFMPEG_THREADS"); + if (!threads_option.empty()) + { + nThreads = requestedThreads = atoi(threads_option.c_str()); + } + } + #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ interrupt_metadata.timeout_after_ms = open_timeout; @@ -1748,6 +1753,51 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, frame.height != video_st->CV_FFMPEG_CODEC_FIELD->height || frame.data == NULL ) { +#if LIBSWSCALE_BUILD >= CALC_FFMPEG_VERSION(6, 4, 100) + int buffer_width = video_st->CV_FFMPEG_CODEC_FIELD->width; + int buffer_height = video_st->CV_FFMPEG_CODEC_FIELD->height; + + // Reproduce sws_getCachedContext but with threads option + int64_t src_h_chr_pos = -513, dst_h_chr_pos = -513, + src_v_chr_pos = -513, dst_v_chr_pos = -513; + if (img_convert_ctx) + { + av_opt_get_int(img_convert_ctx, "src_h_chr_pos", 0, &src_h_chr_pos); + av_opt_get_int(img_convert_ctx, "src_v_chr_pos", 0, &src_v_chr_pos); + av_opt_get_int(img_convert_ctx, "dst_h_chr_pos", 0, &dst_h_chr_pos); + av_opt_get_int(img_convert_ctx, "dst_v_chr_pos", 0, &dst_v_chr_pos); + sws_freeContext(img_convert_ctx); + img_convert_ctx = NULL; + } + + img_convert_ctx = sws_alloc_context(); + if (img_convert_ctx == NULL) + return false;//CV_Error(0, "Cannot initialize the conversion context!"); + + av_opt_set_int(img_convert_ctx, "sws_flags", SWS_BICUBIC, 0); + av_opt_set_int(img_convert_ctx, "threads", requestedThreads, 0); + + if (swscale_version() < CALC_FFMPEG_VERSION(8, 12, 100)) + { + av_opt_set_int(img_convert_ctx, "src_h_chr_pos", src_h_chr_pos, 0); + av_opt_set_int(img_convert_ctx, "src_v_chr_pos", src_v_chr_pos, 0); + av_opt_set_int(img_convert_ctx, "dst_h_chr_pos", dst_h_chr_pos, 0); + av_opt_set_int(img_convert_ctx, "dst_v_chr_pos", dst_v_chr_pos, 0); + av_opt_set_int(img_convert_ctx, "srcw", buffer_width, 0); + av_opt_set_int(img_convert_ctx, "srch", buffer_height, 0); + av_opt_set_int(img_convert_ctx, "dstw", buffer_width, 0); + av_opt_set_int(img_convert_ctx, "dsth", buffer_height, 0); + av_opt_set_pixel_fmt(img_convert_ctx, "src_format", (AVPixelFormat)sw_picture->format, 0); + av_opt_set_pixel_fmt(img_convert_ctx, "dst_format", result_format, 0); + av_opt_set_double(img_convert_ctx, "param0", SWS_PARAM_DEFAULT, 0); + av_opt_set_double(img_convert_ctx, "param1", SWS_PARAM_DEFAULT, 0); + + if (sws_init_context(img_convert_ctx, NULL, NULL) < 0) { + sws_freeContext(img_convert_ctx); + img_convert_ctx = NULL; + } + } +#else // Some sws_scale optimizations have some assumptions about alignment of data/step/width/height // Also we use coded_width/height to workaround problem with legacy ffmpeg versions (like n0.8) int buffer_width = context->coded_width, buffer_height = context->coded_height; @@ -1761,6 +1811,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, SWS_BICUBIC, NULL, NULL, NULL ); +#endif if (img_convert_ctx == NULL) return false;//CV_Error(0, "Cannot initialize the conversion context!"); @@ -1790,6 +1841,9 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, frame.step = rgb_picture.linesize[0]; } +#if LIBSWSCALE_BUILD >= CALC_FFMPEG_VERSION(6, 4, 100) + sws_scale_frame(img_convert_ctx, &rgb_picture, sw_picture); +#else sws_scale( img_convert_ctx, sw_picture->data, @@ -1798,6 +1852,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, rgb_picture.data, rgb_picture.linesize ); +#endif *data = frame.data; *step = frame.step;