From b1e0c4d119650b7065e794101857db9828ae6448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Csaba=20Kert=C3=A9sz?= Date: Wed, 8 Nov 2023 11:41:50 +0100 Subject: [PATCH] Merge pull request #24243 from kecsap:4.x Fix gstreamer backend with manual pipelines #24243 - Fix broken seeking in audio/video playback - Fix broken audio playback - Fix unreliable seeking - Estimate frame count if it is not available directly - Return -1 for frame count and fps if it is not available. - Return 0 for fps if the video has variable frame rate - Enable and fix tests ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [-] There is a reference to the original bug report and related work => Reproducible test provided - [-] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [X] The feature is well documented and sample code can be built with the project CMake 1. Download two test videos: ```bash wget https://github.com/ietf-wg-cellar/matroska-test-files/raw/master/test_files/test1.mkv wget https://test-videos.co.uk/vids/jellyfish/mkv/360/Jellyfish_360_10s_5MB.mkv ``` 2. I modified a OpenCV videoio sample to demonstrate the problem, here it is the patch: http://dpaste.com//C9MAT2K6W 3. Build the sample, on Ubuntu: ```bash g++ -g videocapture_audio_combination.cpp -I/usr/include/opencv4 `pkg-config --libs --cflags opencv4` -o videocapture_audio_combination ``` 4. Play an audio stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"[ERROR:0@0.009] global cap.cpp:164 open VIDEOIO(GSTREAMER): raised OpenCV exception: OpenCV(4.8.0-dev) ./modules/videoio/src/cap_gstreamer.cpp:153: error: (-215:Assertion failed) ptr in function 'get' [ WARN:0@0.009] global cap.cpp:204 open VIDEOIO(GSTREAMER): backend is generally available but can't be used to capture by name ERROR! Can't to open file: filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink ``` 5. Play a video stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.034] global cap_gstreamer.cpp:1728 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1898 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 [ WARN:0@0.034] global cap_gstreamer.cpp:1817 getProperty OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: https://github.com/opencv/opencv/issues/19025 Timestamp: 0.6218 Timestamp: 33.1085 Timestamp: 67.1274 Timestamp: 100.1182 Timestamp: 133.1204 Timestamp: 167.1195 Timestamp: 200.1161 Timestamp: 233.1147 Timestamp: 267.1194 Timestamp: 300.1202 [ WARN:0@0.338] global cap_gstreamer.cpp:1949 setProperty OpenCV | GStreamer warning: GStreamer: unable to seek 0:00:00.338215907 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop: error: Internal data stream error. 0:00:00.338235884 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop: error: streaming stopped, reason not-linked (-1) 0:00:00.338264287 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event: error: Internal data stream error. 0:00:00.338270329 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event: error: streaming stopped, reason not-linked (-1) [ WARN:0@0.339] global cap_gstreamer.cpp:2784 handleMessage OpenCV | GStreamer warning: Embedded video playback halted; module filesrc0 reported: Internal data stream error. [ WARN:0@0.339] global cap_gstreamer.cpp:1199 startPipeline OpenCV | GStreamer warning: unable to start pipeline Number of audio samples: 0 Number of video frames: 10 [ WARN:0@0.339] global cap_gstreamer.cpp:1164 isPipelinePlaying OpenCV | GStreamer warning: GStreamer: pipeline have not been created ``` 6. Play an audio stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 48000 CAP_PROP_AUDIO_TOTAL_CHANNELS: 2 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.025] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 24.0000 Timestamp: 48.0000 Timestamp: 72.0000 Timestamp: 96.0000 Timestamp: 120.0000 Timestamp: 144.0000 Timestamp: 168.0000 Timestamp: 192.0000 Timestamp: 216.0000 Timestamp: 3500.0000 Timestamp: 3504.0000 Timestamp: 3528.0000 Timestamp: 3552.0000 Timestamp: 3576.0000 Timestamp: 3600.0000 Timestamp: 3624.0000 Timestamp: 3648.0000 Timestamp: 3672.0000 Timestamp: 3696.0000 Timestamp: 3720.0000 Timestamp: 3744.0000 Timestamp: 3768.0000 Timestamp: 3792.0000 Timestamp: 3816.0000 Timestamp: 3840.0000 Timestamp: 3864.0000 Timestamp: 3888.0000 Timestamp: 3912.0000 Timestamp: 3936.0000 ``` 7. Play a video stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.033] global cap_gstreamer.cpp:1746 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 33.0000 Timestamp: 67.0000 Timestamp: 100.0000 Timestamp: 133.0000 Timestamp: 167.0000 Timestamp: 200.0000 Timestamp: 233.0000 Timestamp: 267.0000 Timestamp: 300.0000 0:00:00.335931693 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip: Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335952823 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip: Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335988029 3893501 0x55bbe76ad920 WARN basesrc gstbasesrc.c:1742:gst_base_src_perform_seek: duplicate event found 184 Timestamp: 3467.0000 Timestamp: 3500.0000 Timestamp: 3533.0000 Timestamp: 3567.0000 Timestamp: 3600.0000 Timestamp: 3633.0000 Timestamp: 3667.0000 Timestamp: 3700.0000 Timestamp: 3733.0000 Timestamp: 3767.0000 Timestamp: 3800.0000 Timestamp: 3833.0000 Timestamp: 3867.0000 Timestamp: 3900.0000 Timestamp: 3933.0000 Timestamp: 3967.0000 Timestamp: 4000.0000 Timestamp: 4033.0000 Timestamp: 4067.0000 Timestamp: 4100.0000 ``` --- modules/videoio/src/cap_gstreamer.cpp | 144 ++++++++++++++++-------- modules/videoio/test/test_audio.cpp | 3 +- modules/videoio/test/test_gstreamer.cpp | 43 +++++++ modules/videoio/test/test_video_io.cpp | 21 ++-- 4 files changed, 158 insertions(+), 53 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 305d527ce9..cdaccabe45 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -114,6 +114,7 @@ template<> inline void GSafePtr_release(GstBuffer** pPtr) { if (pPtr) template<> inline void GSafePtr_release(GstSample** pPtr) { if (pPtr) { gst_sample_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstBus** pPtr) { if (pPtr) { gst_object_unref(G_OBJECT(*pPtr)); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstMessage** pPtr) { if (pPtr) { gst_message_unref(*pPtr); *pPtr = NULL; } } +template<> inline void GSafePtr_release(GstQuery** pPtr) { if (pPtr) { gst_query_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GMainLoop** pPtr) { if (pPtr) { g_main_loop_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstEncodingVideoProfile** pPtr) { if (pPtr) { gst_encoding_profile_unref(*pPtr); *pPtr = NULL; } } @@ -367,6 +368,7 @@ private: gint audioBitPerFrame; gint audioSampleSize; std::string audioFormat; + guint64 timestamp; Mat audioFrame; std::deque bufferAudioData; @@ -433,7 +435,8 @@ GStreamerCapture::GStreamerCapture() : audioSamplesPerSecond(44100), audioBitPerFrame(0), audioSampleSize(0), - audioFormat("S16LE") + audioFormat("S16LE"), + timestamp(0) , va_type(VIDEO_ACCELERATION_NONE) , hw_device(-1) {} @@ -680,6 +683,11 @@ bool GStreamerCapture::grabVideoFrame() stopFlag = true; emulatedFrameNumber++; } + if (usedVideoSample) + { + auto *buffer = gst_sample_get_buffer((GstSample*)usedVideoSample); + timestamp = GST_BUFFER_PTS(buffer); + } returnFlag = true; } } @@ -792,6 +800,7 @@ bool GStreamerCapture::grabAudioFrame() CV_LOG_ERROR(NULL, "GStreamer: Failed. Buffer is empty"); return false; } + timestamp = GST_BUFFER_PTS(buf); if (!gst_buffer_map(buf, &map_info, GST_MAP_READ)) { CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory"); @@ -1389,6 +1398,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr uri; GSafePtr bus; + GSafePtr queue; GSafePtr uridecodebin; GSafePtr color; GSafePtr convert; @@ -1493,6 +1503,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (strstr(name, "opencvsink") != NULL || strstr(name, "appsink") != NULL) { sink.attach(GST_ELEMENT(gst_object_ref(element))); + audiosink.attach(GST_ELEMENT(gst_object_ref(element))); } else if (strstr(name, COLOR_ELEM_NAME) != NULL) { @@ -1534,6 +1545,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (videoStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); sink.reset(gst_element_factory_make("appsink", NULL)); CV_Assert(sink); // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) @@ -1541,7 +1554,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); CV_Assert(color); - gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); + gst_bin_add_many(GST_BIN(pipeline.get()), queue.get(), uridecodebin.get(), color.get(), sink.get(), NULL); if (element_from_uri) { @@ -1566,6 +1579,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (audioStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); convert.reset(gst_element_factory_make("audioconvert", NULL)); resample.reset(gst_element_factory_make("audioresample", NULL)); audiosink.reset(gst_element_factory_make("appsink", NULL)); @@ -1573,7 +1588,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam CV_Assert(resample); CV_Assert(audiosink); - gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); + gst_bin_add_many (GST_BIN (pipeline.get()), queue.get(), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); if (!gst_element_link_many (convert.get(), resample.get(), audiosink.get(), NULL)) { CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink"); @@ -1646,14 +1661,17 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (manualpipeline) { - GSafePtr peer_caps; - GSafePtr sink_pad; - sink_pad.attach(gst_element_get_static_pad(sink, "sink")); - peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); - if (!gst_caps_can_intersect(caps, peer_caps)) + if (videoStream >= 0) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); - CV_Assert(caps); + GSafePtr peer_caps; + GSafePtr sink_pad; + sink_pad.attach(gst_element_get_static_pad(sink, "sink")); + peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); + if (!gst_caps_can_intersect(caps, peer_caps)) + { + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); + CV_Assert(caps); + } } } if (videoStream >= 0) @@ -1661,6 +1679,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam gst_app_sink_set_caps(GST_APP_SINK(sink.get()), caps); caps.release(); } + { GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-init"); @@ -1688,18 +1707,6 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr buffer_caps; buffer_caps.attach(gst_pad_get_current_caps(pad)); - GstFormat format; - - format = GST_FORMAT_DEFAULT; - if(!gst_element_query_duration(sink, format, &duration)) - { - handleMessage(pipeline); - CV_WARN("unable to query duration of stream"); - duration = -1; - } - - handleMessage(pipeline); - const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer if (!gst_structure_get_int (structure, "width", &width) || !gst_structure_get_int (structure, "height", &height)) @@ -1708,13 +1715,55 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } gint num = 0, denom=1; + bool fps_query_success = true; + if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) { CV_WARN("cannot query video fps"); + fps_query_success = false; } fps = (double)num/(double)denom; + // If num == 0 and denom == 1 -> variable frame rate video. + if (fps_query_success && !(num == 0 && denom == 1)) + { + GSafePtr query; + query.attach(gst_query_new_duration(GST_FORMAT_DEFAULT)); + + gboolean res = gst_element_query(pipeline.get(), query); + + if (res) + { + gst_query_parse_duration(query, NULL, &duration); + } + else if (fps != 0) + { + GSafePtr query2; + query2.attach(gst_query_new_duration(GST_FORMAT_TIME)); + gboolean res2 = gst_element_query(pipeline.get(), query2); + + if (res2) + { + gst_query_parse_duration(query2, NULL, &duration); + duration = static_cast((float)duration / GST_SECOND * fps); + CV_WARN("frame count is estimated by duration and fps"); + } + else + { + CV_WARN("unable to query duration of stream"); + duration = -1; + } + } + else + { + CV_WARN("unable to query frame count of stream and fps are not available to estimate it"); + duration = -1; + } + } + + handleMessage(pipeline); + { GstFormat format_; gint64 value_ = -1; @@ -1814,20 +1863,7 @@ double GStreamerCapture::getProperty(int propId) const switch(propId) { case CV_CAP_PROP_POS_MSEC: - CV_LOG_ONCE_WARNING(NULL, "OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: " - "https://github.com/opencv/opencv/issues/19025"); - if (audioStream != -1) - { - return usedVideoSampleTimeNS * 1e-6; - } - format = GST_FORMAT_TIME; - status = gst_element_query_position(sink.get(), CV_GST_FORMAT(format), &value); - if(!status) { - handleMessage(pipeline); - CV_WARN("GStreamer: unable to query position of stream"); - return 0; - } - return value * 1e-6; // nano seconds to milli seconds + return double(timestamp) / GST_MSECOND; case CV_CAP_PROP_POS_FRAMES: if (!isPosFramesSupported) { @@ -1859,7 +1895,7 @@ double GStreamerCapture::getProperty(int propId) const case CV_CAP_PROP_FPS: return fps; case CV_CAP_PROP_FRAME_COUNT: - return duration; + return (double)duration; case CV_CAP_PROP_BRIGHTNESS: case CV_CAP_PROP_CONTRAST: case CV_CAP_PROP_SATURATION: @@ -1936,13 +1972,15 @@ bool GStreamerCapture::setProperty(int propId, double value) return false; } - bool wasPlaying = this->isPipelinePlaying(); - if (wasPlaying) + bool needRestart = this->isPipelinePlaying() && (propId == CV_CAP_PROP_FRAME_WIDTH || propId == CV_CAP_PROP_FRAME_HEIGHT || propId == CV_CAP_PROP_FPS); + if (needRestart) { this->stopPipeline(); + } switch(propId) { case CV_CAP_PROP_POS_MSEC: + { if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_TIME, flags, (gint64) (value * GST_MSECOND))) { handleMessage(pipeline); @@ -1950,6 +1988,9 @@ bool GStreamerCapture::setProperty(int propId, double value) } else { + // Optimistically caching the target timestamp before reading the first frame from the new position since + // the timestamp in GStreamer can be reliable extracted from the read frames. + timestamp = (gint64)value; if (isPosFramesEmulated) { if (value == 0) @@ -1963,7 +2004,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_POS_FRAMES: { if (!isPosFramesSupported) @@ -1977,24 +2019,34 @@ bool GStreamerCapture::setProperty(int propId, double value) return true; } } - return false; CV_WARN("unable to seek"); + return false; } + // Certain mov and mp4 files seek incorrectly if the pipeline is not stopped before. + if (this->isPipelinePlaying()) { + this->stopPipeline(); + } + if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_DEFAULT, flags, (gint64) value)) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); - break; + return false; } // wait for status update gst_element_get_state(pipeline, NULL, NULL, GST_CLOCK_TIME_NONE); return true; } case CV_CAP_PROP_POS_AVI_RATIO: + { + // https://stackoverflow.com/questions/31290315 + // GStreamer docs: GST_FORMAT_PERCENT (5) – percentage of stream (few, if any, elements implement this as of May 2009) + CV_WARN("GStreamer: seeking by file percent are not supported by most GStreamer elements"); if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_PERCENT, flags, (gint64) (value * GST_FORMAT_PERCENT_MAX))) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); + return false; } else { @@ -2011,7 +2063,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_FRAME_WIDTH: if(value > 0) setFilter("width", G_TYPE_INT, (int) value, 0); @@ -2099,8 +2152,9 @@ bool GStreamerCapture::setProperty(int propId, double value) CV_WARN("GStreamer: unhandled property"); } - if (wasPlaying) + if (needRestart) { this->startPipeline(); + } return false; } @@ -2572,7 +2626,7 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc, if (stateret == GST_STATE_CHANGE_FAILURE) { handleMessage(pipeline); - CV_WARN("GStreamer: cannot put pipeline to play\n"); + CV_WARN("GStreamer: cannot put pipeline to play"); pipeline.release(); return false; } diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index ca46bfc4ca..bf2b4bad68 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -186,6 +186,7 @@ public: double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC); double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3; audio0_timestamp = video0_timestamp + audio_shift * 1e-9; + std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl; } ASSERT_TRUE(cap.retrieve(videoFrame)); @@ -228,7 +229,7 @@ public: EXPECT_NEAR( cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp, cap.get(CAP_PROP_POS_MSEC) * 1e-3, - (1.0 / fps) * 0.3) + (1.0 / fps) * 0.6) << "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC); } if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples) diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index a8c24be438..c9f94bf7ee 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -178,4 +178,47 @@ TEST(videoio_gstreamer, timeout_property) } } +//============================================================================== +// Seeking test with manual GStreamer pipeline +typedef testing::TestWithParam gstreamer_bunny; + +TEST_P(gstreamer_bunny, manual_seek) +{ + if (!videoio_registry::hasBackend(CAP_GSTREAMER)) + throw SkipTestException("GStreamer backend was not found"); + + const string video_file = BunnyParameters::getFilename("." + GetParam()); + const string pipeline = "filesrc location=" + video_file + " ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1"; + const double target_pos = 3000.0; + const float ms_per_frame = 1000 / BunnyParameters::getFps(); + VideoCapture cap; + cap.open(pipeline, CAP_GSTREAMER); + ASSERT_TRUE(cap.isOpened()); + Mat img; + for (int i = 0; i < 10; i++) + { + cap >> img; + } + EXPECT_FALSE(img.empty()); + cap.set(CAP_PROP_POS_MSEC, target_pos); + cap >> img; + EXPECT_FALSE(img.empty()); + double actual_pos = cap.get(CAP_PROP_POS_MSEC); + EXPECT_NEAR(actual_pos, target_pos, ms_per_frame); +} + +static const string bunny_params[] = { + // string("wmv"), + string("mov"), + string("mp4"), + // string("mpg"), + string("avi"), + // string("h264"), + // string("h265"), + string("mjpg.avi") +}; + +INSTANTIATE_TEST_CASE_P(videoio, gstreamer_bunny, testing::ValuesIn(bunny_params)); + + }} // namespace diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp index 7d7944f5eb..9c9cdddbff 100644 --- a/modules/videoio/test/test_video_io.cpp +++ b/modules/videoio/test/test_video_io.cpp @@ -67,6 +67,11 @@ public: std::cout << "CAP_PROP_FRAME_COUNT is not supported by backend. Assume 50 frames." << std::endl; n_frames = 50; } + // GStreamer can't read frame count of big_buck_bunny.wmv + if (apiPref == CAP_GSTREAMER && ext == "wmv") + { + n_frames = 125; + } { SCOPED_TRACE("consecutive read"); @@ -166,7 +171,7 @@ public: EXPECT_NO_THROW(count_prop = (int)cap.get(CAP_PROP_FRAME_COUNT)); // mpg file reports 5.08 sec * 24 fps => property returns 122 frames // but actual number of frames returned is 125 - if (ext != "mpg") + if (ext != "mpg" && !(apiPref == CAP_GSTREAMER && ext == "wmv")) { if (count_prop > 0) { @@ -200,12 +205,11 @@ public: if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends())) throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref)); - // GStreamer: https://github.com/opencv/opencv/issues/19025 - if (apiPref == CAP_GSTREAMER) + if (((apiPref == CAP_GSTREAMER) && (ext == "avi"))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + - cv::String(" does not return reliable values for CAP_PROP_POS_MSEC property")); + cv::String(" does not support CAP_PROP_POS_MSEC option")); - if (((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265")))) + if (((apiPref == CAP_FFMPEG || apiPref == CAP_GSTREAMER) && ((ext == "h264") || (ext == "h265")))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + cv::String(" does not support CAP_PROP_POS_MSEC option")); @@ -221,8 +225,8 @@ public: // mpg file reports 5.08 sec * 24 fps => property returns 122 frames,but actual number of frames returned is 125 // HACK: CAP_PROP_FRAME_COUNT is not supported for vmw + MSMF. Just force check for all 125 frames if (ext == "mpg") - EXPECT_GT(frame_count, 121); - else if ((ext == "wmv") && (apiPref == CAP_MSMF)) + EXPECT_GE(frame_count, 114); + else if ((ext == "wmv") && (apiPref == CAP_MSMF || apiPref == CAP_GSTREAMER)) frame_count = 125; else EXPECT_EQ(frame_count, 125); @@ -240,6 +244,9 @@ public: if (cvtest::debugLevel > 0) std::cout << "i = " << i << ": timestamp = " << timestamp << std::endl; const double frame_period = 1000.f/bunny_param.getFps(); + // big_buck_bunny.mpg starts at 0.500 msec + if ((ext == "mpg") && (apiPref == CAP_GSTREAMER)) + timestamp -= 500.0; // NOTE: eps == frame_period, because videoCapture returns frame beginning timestamp or frame end // timestamp depending on codec and back-end. So the first frame has timestamp 0 or frame_period. EXPECT_NEAR(timestamp, i*frame_period, frame_period) << "i=" << i;