diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 305d527ce9..cdaccabe45 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -114,6 +114,7 @@ template<> inline void GSafePtr_release(GstBuffer** pPtr) { if (pPtr) template<> inline void GSafePtr_release(GstSample** pPtr) { if (pPtr) { gst_sample_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstBus** pPtr) { if (pPtr) { gst_object_unref(G_OBJECT(*pPtr)); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstMessage** pPtr) { if (pPtr) { gst_message_unref(*pPtr); *pPtr = NULL; } } +template<> inline void GSafePtr_release(GstQuery** pPtr) { if (pPtr) { gst_query_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GMainLoop** pPtr) { if (pPtr) { g_main_loop_unref(*pPtr); *pPtr = NULL; } } template<> inline void GSafePtr_release(GstEncodingVideoProfile** pPtr) { if (pPtr) { gst_encoding_profile_unref(*pPtr); *pPtr = NULL; } } @@ -367,6 +368,7 @@ private: gint audioBitPerFrame; gint audioSampleSize; std::string audioFormat; + guint64 timestamp; Mat audioFrame; std::deque bufferAudioData; @@ -433,7 +435,8 @@ GStreamerCapture::GStreamerCapture() : audioSamplesPerSecond(44100), audioBitPerFrame(0), audioSampleSize(0), - audioFormat("S16LE") + audioFormat("S16LE"), + timestamp(0) , va_type(VIDEO_ACCELERATION_NONE) , hw_device(-1) {} @@ -680,6 +683,11 @@ bool GStreamerCapture::grabVideoFrame() stopFlag = true; emulatedFrameNumber++; } + if (usedVideoSample) + { + auto *buffer = gst_sample_get_buffer((GstSample*)usedVideoSample); + timestamp = GST_BUFFER_PTS(buffer); + } returnFlag = true; } } @@ -792,6 +800,7 @@ bool GStreamerCapture::grabAudioFrame() CV_LOG_ERROR(NULL, "GStreamer: Failed. Buffer is empty"); return false; } + timestamp = GST_BUFFER_PTS(buf); if (!gst_buffer_map(buf, &map_info, GST_MAP_READ)) { CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory"); @@ -1389,6 +1398,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr uri; GSafePtr bus; + GSafePtr queue; GSafePtr uridecodebin; GSafePtr color; GSafePtr convert; @@ -1493,6 +1503,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (strstr(name, "opencvsink") != NULL || strstr(name, "appsink") != NULL) { sink.attach(GST_ELEMENT(gst_object_ref(element))); + audiosink.attach(GST_ELEMENT(gst_object_ref(element))); } else if (strstr(name, COLOR_ELEM_NAME) != NULL) { @@ -1534,6 +1545,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam if (videoStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); sink.reset(gst_element_factory_make("appsink", NULL)); CV_Assert(sink); // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) @@ -1541,7 +1554,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); CV_Assert(color); - gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); + gst_bin_add_many(GST_BIN(pipeline.get()), queue.get(), uridecodebin.get(), color.get(), sink.get(), NULL); if (element_from_uri) { @@ -1566,6 +1579,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (audioStream >= 0) { + queue.reset(gst_element_factory_make("queue", NULL)); + CV_Assert(queue); convert.reset(gst_element_factory_make("audioconvert", NULL)); resample.reset(gst_element_factory_make("audioresample", NULL)); audiosink.reset(gst_element_factory_make("appsink", NULL)); @@ -1573,7 +1588,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam CV_Assert(resample); CV_Assert(audiosink); - gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); + gst_bin_add_many (GST_BIN (pipeline.get()), queue.get(), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL); if (!gst_element_link_many (convert.get(), resample.get(), audiosink.get(), NULL)) { CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink"); @@ -1646,14 +1661,17 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } if (manualpipeline) { - GSafePtr peer_caps; - GSafePtr sink_pad; - sink_pad.attach(gst_element_get_static_pad(sink, "sink")); - peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); - if (!gst_caps_can_intersect(caps, peer_caps)) + if (videoStream >= 0) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); - CV_Assert(caps); + GSafePtr peer_caps; + GSafePtr sink_pad; + sink_pad.attach(gst_element_get_static_pad(sink, "sink")); + peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); + if (!gst_caps_can_intersect(caps, peer_caps)) + { + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); + CV_Assert(caps); + } } } if (videoStream >= 0) @@ -1661,6 +1679,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam gst_app_sink_set_caps(GST_APP_SINK(sink.get()), caps); caps.release(); } + { GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-init"); @@ -1688,18 +1707,6 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr buffer_caps; buffer_caps.attach(gst_pad_get_current_caps(pad)); - GstFormat format; - - format = GST_FORMAT_DEFAULT; - if(!gst_element_query_duration(sink, format, &duration)) - { - handleMessage(pipeline); - CV_WARN("unable to query duration of stream"); - duration = -1; - } - - handleMessage(pipeline); - const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer if (!gst_structure_get_int (structure, "width", &width) || !gst_structure_get_int (structure, "height", &height)) @@ -1708,13 +1715,55 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } gint num = 0, denom=1; + bool fps_query_success = true; + if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) { CV_WARN("cannot query video fps"); + fps_query_success = false; } fps = (double)num/(double)denom; + // If num == 0 and denom == 1 -> variable frame rate video. + if (fps_query_success && !(num == 0 && denom == 1)) + { + GSafePtr query; + query.attach(gst_query_new_duration(GST_FORMAT_DEFAULT)); + + gboolean res = gst_element_query(pipeline.get(), query); + + if (res) + { + gst_query_parse_duration(query, NULL, &duration); + } + else if (fps != 0) + { + GSafePtr query2; + query2.attach(gst_query_new_duration(GST_FORMAT_TIME)); + gboolean res2 = gst_element_query(pipeline.get(), query2); + + if (res2) + { + gst_query_parse_duration(query2, NULL, &duration); + duration = static_cast((float)duration / GST_SECOND * fps); + CV_WARN("frame count is estimated by duration and fps"); + } + else + { + CV_WARN("unable to query duration of stream"); + duration = -1; + } + } + else + { + CV_WARN("unable to query frame count of stream and fps are not available to estimate it"); + duration = -1; + } + } + + handleMessage(pipeline); + { GstFormat format_; gint64 value_ = -1; @@ -1814,20 +1863,7 @@ double GStreamerCapture::getProperty(int propId) const switch(propId) { case CV_CAP_PROP_POS_MSEC: - CV_LOG_ONCE_WARNING(NULL, "OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: " - "https://github.com/opencv/opencv/issues/19025"); - if (audioStream != -1) - { - return usedVideoSampleTimeNS * 1e-6; - } - format = GST_FORMAT_TIME; - status = gst_element_query_position(sink.get(), CV_GST_FORMAT(format), &value); - if(!status) { - handleMessage(pipeline); - CV_WARN("GStreamer: unable to query position of stream"); - return 0; - } - return value * 1e-6; // nano seconds to milli seconds + return double(timestamp) / GST_MSECOND; case CV_CAP_PROP_POS_FRAMES: if (!isPosFramesSupported) { @@ -1859,7 +1895,7 @@ double GStreamerCapture::getProperty(int propId) const case CV_CAP_PROP_FPS: return fps; case CV_CAP_PROP_FRAME_COUNT: - return duration; + return (double)duration; case CV_CAP_PROP_BRIGHTNESS: case CV_CAP_PROP_CONTRAST: case CV_CAP_PROP_SATURATION: @@ -1936,13 +1972,15 @@ bool GStreamerCapture::setProperty(int propId, double value) return false; } - bool wasPlaying = this->isPipelinePlaying(); - if (wasPlaying) + bool needRestart = this->isPipelinePlaying() && (propId == CV_CAP_PROP_FRAME_WIDTH || propId == CV_CAP_PROP_FRAME_HEIGHT || propId == CV_CAP_PROP_FPS); + if (needRestart) { this->stopPipeline(); + } switch(propId) { case CV_CAP_PROP_POS_MSEC: + { if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_TIME, flags, (gint64) (value * GST_MSECOND))) { handleMessage(pipeline); @@ -1950,6 +1988,9 @@ bool GStreamerCapture::setProperty(int propId, double value) } else { + // Optimistically caching the target timestamp before reading the first frame from the new position since + // the timestamp in GStreamer can be reliable extracted from the read frames. + timestamp = (gint64)value; if (isPosFramesEmulated) { if (value == 0) @@ -1963,7 +2004,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_POS_FRAMES: { if (!isPosFramesSupported) @@ -1977,24 +2019,34 @@ bool GStreamerCapture::setProperty(int propId, double value) return true; } } - return false; CV_WARN("unable to seek"); + return false; } + // Certain mov and mp4 files seek incorrectly if the pipeline is not stopped before. + if (this->isPipelinePlaying()) { + this->stopPipeline(); + } + if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_DEFAULT, flags, (gint64) value)) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); - break; + return false; } // wait for status update gst_element_get_state(pipeline, NULL, NULL, GST_CLOCK_TIME_NONE); return true; } case CV_CAP_PROP_POS_AVI_RATIO: + { + // https://stackoverflow.com/questions/31290315 + // GStreamer docs: GST_FORMAT_PERCENT (5) – percentage of stream (few, if any, elements implement this as of May 2009) + CV_WARN("GStreamer: seeking by file percent are not supported by most GStreamer elements"); if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_PERCENT, flags, (gint64) (value * GST_FORMAT_PERCENT_MAX))) { handleMessage(pipeline); CV_WARN("GStreamer: unable to seek"); + return false; } else { @@ -2011,7 +2063,8 @@ bool GStreamerCapture::setProperty(int propId, double value) } } } - break; + return true; + } case CV_CAP_PROP_FRAME_WIDTH: if(value > 0) setFilter("width", G_TYPE_INT, (int) value, 0); @@ -2099,8 +2152,9 @@ bool GStreamerCapture::setProperty(int propId, double value) CV_WARN("GStreamer: unhandled property"); } - if (wasPlaying) + if (needRestart) { this->startPipeline(); + } return false; } @@ -2572,7 +2626,7 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc, if (stateret == GST_STATE_CHANGE_FAILURE) { handleMessage(pipeline); - CV_WARN("GStreamer: cannot put pipeline to play\n"); + CV_WARN("GStreamer: cannot put pipeline to play"); pipeline.release(); return false; } diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index ca46bfc4ca..bf2b4bad68 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -186,6 +186,7 @@ public: double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC); double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3; audio0_timestamp = video0_timestamp + audio_shift * 1e-9; + std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl; } ASSERT_TRUE(cap.retrieve(videoFrame)); @@ -228,7 +229,7 @@ public: EXPECT_NEAR( cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp, cap.get(CAP_PROP_POS_MSEC) * 1e-3, - (1.0 / fps) * 0.3) + (1.0 / fps) * 0.6) << "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC); } if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples) diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index a8c24be438..c9f94bf7ee 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -178,4 +178,47 @@ TEST(videoio_gstreamer, timeout_property) } } +//============================================================================== +// Seeking test with manual GStreamer pipeline +typedef testing::TestWithParam gstreamer_bunny; + +TEST_P(gstreamer_bunny, manual_seek) +{ + if (!videoio_registry::hasBackend(CAP_GSTREAMER)) + throw SkipTestException("GStreamer backend was not found"); + + const string video_file = BunnyParameters::getFilename("." + GetParam()); + const string pipeline = "filesrc location=" + video_file + " ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1"; + const double target_pos = 3000.0; + const float ms_per_frame = 1000 / BunnyParameters::getFps(); + VideoCapture cap; + cap.open(pipeline, CAP_GSTREAMER); + ASSERT_TRUE(cap.isOpened()); + Mat img; + for (int i = 0; i < 10; i++) + { + cap >> img; + } + EXPECT_FALSE(img.empty()); + cap.set(CAP_PROP_POS_MSEC, target_pos); + cap >> img; + EXPECT_FALSE(img.empty()); + double actual_pos = cap.get(CAP_PROP_POS_MSEC); + EXPECT_NEAR(actual_pos, target_pos, ms_per_frame); +} + +static const string bunny_params[] = { + // string("wmv"), + string("mov"), + string("mp4"), + // string("mpg"), + string("avi"), + // string("h264"), + // string("h265"), + string("mjpg.avi") +}; + +INSTANTIATE_TEST_CASE_P(videoio, gstreamer_bunny, testing::ValuesIn(bunny_params)); + + }} // namespace diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp index 7d7944f5eb..9c9cdddbff 100644 --- a/modules/videoio/test/test_video_io.cpp +++ b/modules/videoio/test/test_video_io.cpp @@ -67,6 +67,11 @@ public: std::cout << "CAP_PROP_FRAME_COUNT is not supported by backend. Assume 50 frames." << std::endl; n_frames = 50; } + // GStreamer can't read frame count of big_buck_bunny.wmv + if (apiPref == CAP_GSTREAMER && ext == "wmv") + { + n_frames = 125; + } { SCOPED_TRACE("consecutive read"); @@ -166,7 +171,7 @@ public: EXPECT_NO_THROW(count_prop = (int)cap.get(CAP_PROP_FRAME_COUNT)); // mpg file reports 5.08 sec * 24 fps => property returns 122 frames // but actual number of frames returned is 125 - if (ext != "mpg") + if (ext != "mpg" && !(apiPref == CAP_GSTREAMER && ext == "wmv")) { if (count_prop > 0) { @@ -200,12 +205,11 @@ public: if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends())) throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref)); - // GStreamer: https://github.com/opencv/opencv/issues/19025 - if (apiPref == CAP_GSTREAMER) + if (((apiPref == CAP_GSTREAMER) && (ext == "avi"))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + - cv::String(" does not return reliable values for CAP_PROP_POS_MSEC property")); + cv::String(" does not support CAP_PROP_POS_MSEC option")); - if (((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265")))) + if (((apiPref == CAP_FFMPEG || apiPref == CAP_GSTREAMER) && ((ext == "h264") || (ext == "h265")))) throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) + cv::String(" does not support CAP_PROP_POS_MSEC option")); @@ -221,8 +225,8 @@ public: // mpg file reports 5.08 sec * 24 fps => property returns 122 frames,but actual number of frames returned is 125 // HACK: CAP_PROP_FRAME_COUNT is not supported for vmw + MSMF. Just force check for all 125 frames if (ext == "mpg") - EXPECT_GT(frame_count, 121); - else if ((ext == "wmv") && (apiPref == CAP_MSMF)) + EXPECT_GE(frame_count, 114); + else if ((ext == "wmv") && (apiPref == CAP_MSMF || apiPref == CAP_GSTREAMER)) frame_count = 125; else EXPECT_EQ(frame_count, 125); @@ -240,6 +244,9 @@ public: if (cvtest::debugLevel > 0) std::cout << "i = " << i << ": timestamp = " << timestamp << std::endl; const double frame_period = 1000.f/bunny_param.getFps(); + // big_buck_bunny.mpg starts at 0.500 msec + if ((ext == "mpg") && (apiPref == CAP_GSTREAMER)) + timestamp -= 500.0; // NOTE: eps == frame_period, because videoCapture returns frame beginning timestamp or frame end // timestamp depending on codec and back-end. So the first frame has timestamp 0 or frame_period. EXPECT_NEAR(timestamp, i*frame_period, frame_period) << "i=" << i;