mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge pull request #24243 from kecsap:4.x
Fix gstreamer backend with manual pipelines #24243 - Fix broken seeking in audio/video playback - Fix broken audio playback - Fix unreliable seeking - Estimate frame count if it is not available directly - Return -1 for frame count and fps if it is not available. - Return 0 for fps if the video has variable frame rate - Enable and fix tests ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [X] I agree to contribute to the project under Apache 2 License. - [X] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [X] The PR is proposed to the proper branch - [-] There is a reference to the original bug report and related work => Reproducible test provided - [-] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [X] The feature is well documented and sample code can be built with the project CMake 1. Download two test videos: ```bash wget https://github.com/ietf-wg-cellar/matroska-test-files/raw/master/test_files/test1.mkv wget https://test-videos.co.uk/vids/jellyfish/mkv/360/Jellyfish_360_10s_5MB.mkv ``` 2. I modified a OpenCV videoio sample to demonstrate the problem, here it is the patch: http://dpaste.com//C9MAT2K6W 3. Build the sample, on Ubuntu: ```bash g++ -g videocapture_audio_combination.cpp -I/usr/include/opencv4 `pkg-config --libs --cflags opencv4` -o videocapture_audio_combination ``` 4. Play an audio stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"[ERROR:0@0.009] global cap.cpp:164 open VIDEOIO(GSTREAMER): raised OpenCV exception: OpenCV(4.8.0-dev) ./modules/videoio/src/cap_gstreamer.cpp:153: error: (-215:Assertion failed) ptr in function 'get' [ WARN:0@0.009] global cap.cpp:204 open VIDEOIO(GSTREAMER): backend is generally available but can't be used to capture by name ERROR! Can't to open file: filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink ``` 5. Play a video stream with seeking BEFORE the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.034] global cap_gstreamer.cpp:1728 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1898 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 [ WARN:0@0.034] global cap_gstreamer.cpp:1817 getProperty OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: https://github.com/opencv/opencv/issues/19025 Timestamp: 0.6218 Timestamp: 33.1085 Timestamp: 67.1274 Timestamp: 100.1182 Timestamp: 133.1204 Timestamp: 167.1195 Timestamp: 200.1161 Timestamp: 233.1147 Timestamp: 267.1194 Timestamp: 300.1202 [ WARN:0@0.338] global cap_gstreamer.cpp:1949 setProperty OpenCV | GStreamer warning: GStreamer: unable to seek 0:00:00.338215907 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop:<filesrc0> error: Internal data stream error. 0:00:00.338235884 3892572 0x5592899c7580 WARN basesrc gstbasesrc.c:3127:gst_base_src_loop:<filesrc0> error: streaming stopped, reason not-linked (-1) 0:00:00.338264287 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event:<queue0> error: Internal data stream error. 0:00:00.338270329 3892572 0x5592899c7580 WARN queue gstqueue.c:992:gst_queue_handle_sink_event:<queue0> error: streaming stopped, reason not-linked (-1) [ WARN:0@0.339] global cap_gstreamer.cpp:2784 handleMessage OpenCV | GStreamer warning: Embedded video playback halted; module filesrc0 reported: Internal data stream error. [ WARN:0@0.339] global cap_gstreamer.cpp:1199 startPipeline OpenCV | GStreamer warning: unable to start pipeline Number of audio samples: 0 Number of video frames: 10 [ WARN:0@0.339] global cap_gstreamer.cpp:1164 isPipelinePlaying OpenCV | GStreamer warning: GStreamer: pipeline have not been created ``` 6. Play an audio stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=test1.mkv ! queue ! matroskademux name=demux demux.audio_0 ! decodebin ! audioconvert ! appsink"CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 48000 CAP_PROP_AUDIO_TOTAL_CHANNELS: 2 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.025] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 24.0000 Timestamp: 48.0000 Timestamp: 72.0000 Timestamp: 96.0000 Timestamp: 120.0000 Timestamp: 144.0000 Timestamp: 168.0000 Timestamp: 192.0000 Timestamp: 216.0000 Timestamp: 3500.0000 Timestamp: 3504.0000 Timestamp: 3528.0000 Timestamp: 3552.0000 Timestamp: 3576.0000 Timestamp: 3600.0000 Timestamp: 3624.0000 Timestamp: 3648.0000 Timestamp: 3672.0000 Timestamp: 3696.0000 Timestamp: 3720.0000 Timestamp: 3744.0000 Timestamp: 3768.0000 Timestamp: 3792.0000 Timestamp: 3816.0000 Timestamp: 3840.0000 Timestamp: 3864.0000 Timestamp: 3888.0000 Timestamp: 3912.0000 Timestamp: 3936.0000 ``` 7. Play a video stream with seeking AFTER the fix: ```bash $ ./videocapture_audio_combination --audio "filesrc location=Jellyfish_360_10s_5MB.mkv ! queue ! matroskademux name=demux demux.video_0 ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1" [ WARN:0@0.033] global cap_gstreamer.cpp:1746 open OpenCV | GStreamer warning: Cannot query video position: status=1, value=22, duration=300 CAP_PROP_AUDIO_DATA_DEPTH: CV_16S CAP_PROP_AUDIO_SAMPLES_PER_SECOND: 44100 CAP_PROP_AUDIO_TOTAL_CHANNELS: 0 CAP_PROP_AUDIO_TOTAL_STREAMS: [ WARN:0@0.034] global cap_gstreamer.cpp:1903 getProperty OpenCV | GStreamer: CAP_PROP_AUDIO_TOTAL_STREAMS property is not supported 0 Timestamp: 0.0000 Timestamp: 33.0000 Timestamp: 67.0000 Timestamp: 100.0000 Timestamp: 133.0000 Timestamp: 167.0000 Timestamp: 200.0000 Timestamp: 233.0000 Timestamp: 267.0000 Timestamp: 300.0000 0:00:00.335931693 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip:<demux:sink> Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335952823 3893501 0x55bbe76ad920 WARN matroskareadcommon matroska-read-common.c:759:gst_matroska_read_common_parse_skip:<demux:sink> Unknown CueTrackPositions subelement 0xf0 - ignoring 0:00:00.335988029 3893501 0x55bbe76ad920 WARN basesrc gstbasesrc.c:1742:gst_base_src_perform_seek:<filesrc0> duplicate event found 184 Timestamp: 3467.0000 Timestamp: 3500.0000 Timestamp: 3533.0000 Timestamp: 3567.0000 Timestamp: 3600.0000 Timestamp: 3633.0000 Timestamp: 3667.0000 Timestamp: 3700.0000 Timestamp: 3733.0000 Timestamp: 3767.0000 Timestamp: 3800.0000 Timestamp: 3833.0000 Timestamp: 3867.0000 Timestamp: 3900.0000 Timestamp: 3933.0000 Timestamp: 3967.0000 Timestamp: 4000.0000 Timestamp: 4033.0000 Timestamp: 4067.0000 Timestamp: 4100.0000 ```
This commit is contained in:
@@ -114,6 +114,7 @@ template<> inline void GSafePtr_release<GstBuffer>(GstBuffer** pPtr) { if (pPtr)
|
||||
template<> inline void GSafePtr_release<GstSample>(GstSample** pPtr) { if (pPtr) { gst_sample_unref(*pPtr); *pPtr = NULL; } }
|
||||
template<> inline void GSafePtr_release<GstBus>(GstBus** pPtr) { if (pPtr) { gst_object_unref(G_OBJECT(*pPtr)); *pPtr = NULL; } }
|
||||
template<> inline void GSafePtr_release<GstMessage>(GstMessage** pPtr) { if (pPtr) { gst_message_unref(*pPtr); *pPtr = NULL; } }
|
||||
template<> inline void GSafePtr_release<GstQuery>(GstQuery** pPtr) { if (pPtr) { gst_query_unref(*pPtr); *pPtr = NULL; } }
|
||||
template<> inline void GSafePtr_release<GMainLoop>(GMainLoop** pPtr) { if (pPtr) { g_main_loop_unref(*pPtr); *pPtr = NULL; } }
|
||||
|
||||
template<> inline void GSafePtr_release<GstEncodingVideoProfile>(GstEncodingVideoProfile** pPtr) { if (pPtr) { gst_encoding_profile_unref(*pPtr); *pPtr = NULL; } }
|
||||
@@ -367,6 +368,7 @@ private:
|
||||
gint audioBitPerFrame;
|
||||
gint audioSampleSize;
|
||||
std::string audioFormat;
|
||||
guint64 timestamp;
|
||||
|
||||
Mat audioFrame;
|
||||
std::deque<uint8_t> bufferAudioData;
|
||||
@@ -433,7 +435,8 @@ GStreamerCapture::GStreamerCapture() :
|
||||
audioSamplesPerSecond(44100),
|
||||
audioBitPerFrame(0),
|
||||
audioSampleSize(0),
|
||||
audioFormat("S16LE")
|
||||
audioFormat("S16LE"),
|
||||
timestamp(0)
|
||||
, va_type(VIDEO_ACCELERATION_NONE)
|
||||
, hw_device(-1)
|
||||
{}
|
||||
@@ -680,6 +683,11 @@ bool GStreamerCapture::grabVideoFrame()
|
||||
stopFlag = true;
|
||||
emulatedFrameNumber++;
|
||||
}
|
||||
if (usedVideoSample)
|
||||
{
|
||||
auto *buffer = gst_sample_get_buffer((GstSample*)usedVideoSample);
|
||||
timestamp = GST_BUFFER_PTS(buffer);
|
||||
}
|
||||
returnFlag = true;
|
||||
}
|
||||
}
|
||||
@@ -792,6 +800,7 @@ bool GStreamerCapture::grabAudioFrame()
|
||||
CV_LOG_ERROR(NULL, "GStreamer: Failed. Buffer is empty");
|
||||
return false;
|
||||
}
|
||||
timestamp = GST_BUFFER_PTS(buf);
|
||||
if (!gst_buffer_map(buf, &map_info, GST_MAP_READ))
|
||||
{
|
||||
CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory");
|
||||
@@ -1389,6 +1398,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
GSafePtr<char> uri;
|
||||
GSafePtr<GstBus> bus;
|
||||
|
||||
GSafePtr<GstElement> queue;
|
||||
GSafePtr<GstElement> uridecodebin;
|
||||
GSafePtr<GstElement> color;
|
||||
GSafePtr<GstElement> convert;
|
||||
@@ -1493,6 +1503,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
if (strstr(name, "opencvsink") != NULL || strstr(name, "appsink") != NULL)
|
||||
{
|
||||
sink.attach(GST_ELEMENT(gst_object_ref(element)));
|
||||
audiosink.attach(GST_ELEMENT(gst_object_ref(element)));
|
||||
}
|
||||
else if (strstr(name, COLOR_ELEM_NAME) != NULL)
|
||||
{
|
||||
@@ -1534,6 +1545,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
|
||||
if (videoStream >= 0)
|
||||
{
|
||||
queue.reset(gst_element_factory_make("queue", NULL));
|
||||
CV_Assert(queue);
|
||||
sink.reset(gst_element_factory_make("appsink", NULL));
|
||||
CV_Assert(sink);
|
||||
// videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert)
|
||||
@@ -1541,7 +1554,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
color.reset(gst_element_factory_make(COLOR_ELEM, NULL));
|
||||
CV_Assert(color);
|
||||
|
||||
gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL);
|
||||
gst_bin_add_many(GST_BIN(pipeline.get()), queue.get(), uridecodebin.get(), color.get(), sink.get(), NULL);
|
||||
|
||||
if (element_from_uri)
|
||||
{
|
||||
@@ -1566,6 +1579,8 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
}
|
||||
if (audioStream >= 0)
|
||||
{
|
||||
queue.reset(gst_element_factory_make("queue", NULL));
|
||||
CV_Assert(queue);
|
||||
convert.reset(gst_element_factory_make("audioconvert", NULL));
|
||||
resample.reset(gst_element_factory_make("audioresample", NULL));
|
||||
audiosink.reset(gst_element_factory_make("appsink", NULL));
|
||||
@@ -1573,7 +1588,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
CV_Assert(resample);
|
||||
CV_Assert(audiosink);
|
||||
|
||||
gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL);
|
||||
gst_bin_add_many (GST_BIN (pipeline.get()), queue.get(), uridecodebin.get(), convert.get(), resample.get(), audiosink.get(), NULL);
|
||||
if (!gst_element_link_many (convert.get(), resample.get(), audiosink.get(), NULL))
|
||||
{
|
||||
CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink");
|
||||
@@ -1646,14 +1661,17 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
}
|
||||
if (manualpipeline)
|
||||
{
|
||||
GSafePtr<GstCaps> peer_caps;
|
||||
GSafePtr<GstPad> sink_pad;
|
||||
sink_pad.attach(gst_element_get_static_pad(sink, "sink"));
|
||||
peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL));
|
||||
if (!gst_caps_can_intersect(caps, peer_caps))
|
||||
if (videoStream >= 0)
|
||||
{
|
||||
caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}"));
|
||||
CV_Assert(caps);
|
||||
GSafePtr<GstCaps> peer_caps;
|
||||
GSafePtr<GstPad> sink_pad;
|
||||
sink_pad.attach(gst_element_get_static_pad(sink, "sink"));
|
||||
peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL));
|
||||
if (!gst_caps_can_intersect(caps, peer_caps))
|
||||
{
|
||||
caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}"));
|
||||
CV_Assert(caps);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (videoStream >= 0)
|
||||
@@ -1661,6 +1679,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
gst_app_sink_set_caps(GST_APP_SINK(sink.get()), caps);
|
||||
caps.release();
|
||||
}
|
||||
|
||||
{
|
||||
GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-init");
|
||||
|
||||
@@ -1688,18 +1707,6 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
GSafePtr<GstCaps> buffer_caps;
|
||||
buffer_caps.attach(gst_pad_get_current_caps(pad));
|
||||
|
||||
GstFormat format;
|
||||
|
||||
format = GST_FORMAT_DEFAULT;
|
||||
if(!gst_element_query_duration(sink, format, &duration))
|
||||
{
|
||||
handleMessage(pipeline);
|
||||
CV_WARN("unable to query duration of stream");
|
||||
duration = -1;
|
||||
}
|
||||
|
||||
handleMessage(pipeline);
|
||||
|
||||
const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer
|
||||
if (!gst_structure_get_int (structure, "width", &width) ||
|
||||
!gst_structure_get_int (structure, "height", &height))
|
||||
@@ -1708,13 +1715,55 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
|
||||
}
|
||||
|
||||
gint num = 0, denom=1;
|
||||
bool fps_query_success = true;
|
||||
|
||||
if (!gst_structure_get_fraction(structure, "framerate", &num, &denom))
|
||||
{
|
||||
CV_WARN("cannot query video fps");
|
||||
fps_query_success = false;
|
||||
}
|
||||
|
||||
fps = (double)num/(double)denom;
|
||||
|
||||
// If num == 0 and denom == 1 -> variable frame rate video.
|
||||
if (fps_query_success && !(num == 0 && denom == 1))
|
||||
{
|
||||
GSafePtr<GstQuery> query;
|
||||
query.attach(gst_query_new_duration(GST_FORMAT_DEFAULT));
|
||||
|
||||
gboolean res = gst_element_query(pipeline.get(), query);
|
||||
|
||||
if (res)
|
||||
{
|
||||
gst_query_parse_duration(query, NULL, &duration);
|
||||
}
|
||||
else if (fps != 0)
|
||||
{
|
||||
GSafePtr<GstQuery> query2;
|
||||
query2.attach(gst_query_new_duration(GST_FORMAT_TIME));
|
||||
gboolean res2 = gst_element_query(pipeline.get(), query2);
|
||||
|
||||
if (res2)
|
||||
{
|
||||
gst_query_parse_duration(query2, NULL, &duration);
|
||||
duration = static_cast<gint64>((float)duration / GST_SECOND * fps);
|
||||
CV_WARN("frame count is estimated by duration and fps");
|
||||
}
|
||||
else
|
||||
{
|
||||
CV_WARN("unable to query duration of stream");
|
||||
duration = -1;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
CV_WARN("unable to query frame count of stream and fps are not available to estimate it");
|
||||
duration = -1;
|
||||
}
|
||||
}
|
||||
|
||||
handleMessage(pipeline);
|
||||
|
||||
{
|
||||
GstFormat format_;
|
||||
gint64 value_ = -1;
|
||||
@@ -1814,20 +1863,7 @@ double GStreamerCapture::getProperty(int propId) const
|
||||
switch(propId)
|
||||
{
|
||||
case CV_CAP_PROP_POS_MSEC:
|
||||
CV_LOG_ONCE_WARNING(NULL, "OpenCV | GStreamer: CAP_PROP_POS_MSEC property result may be unrealiable: "
|
||||
"https://github.com/opencv/opencv/issues/19025");
|
||||
if (audioStream != -1)
|
||||
{
|
||||
return usedVideoSampleTimeNS * 1e-6;
|
||||
}
|
||||
format = GST_FORMAT_TIME;
|
||||
status = gst_element_query_position(sink.get(), CV_GST_FORMAT(format), &value);
|
||||
if(!status) {
|
||||
handleMessage(pipeline);
|
||||
CV_WARN("GStreamer: unable to query position of stream");
|
||||
return 0;
|
||||
}
|
||||
return value * 1e-6; // nano seconds to milli seconds
|
||||
return double(timestamp) / GST_MSECOND;
|
||||
case CV_CAP_PROP_POS_FRAMES:
|
||||
if (!isPosFramesSupported)
|
||||
{
|
||||
@@ -1859,7 +1895,7 @@ double GStreamerCapture::getProperty(int propId) const
|
||||
case CV_CAP_PROP_FPS:
|
||||
return fps;
|
||||
case CV_CAP_PROP_FRAME_COUNT:
|
||||
return duration;
|
||||
return (double)duration;
|
||||
case CV_CAP_PROP_BRIGHTNESS:
|
||||
case CV_CAP_PROP_CONTRAST:
|
||||
case CV_CAP_PROP_SATURATION:
|
||||
@@ -1936,13 +1972,15 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
return false;
|
||||
}
|
||||
|
||||
bool wasPlaying = this->isPipelinePlaying();
|
||||
if (wasPlaying)
|
||||
bool needRestart = this->isPipelinePlaying() && (propId == CV_CAP_PROP_FRAME_WIDTH || propId == CV_CAP_PROP_FRAME_HEIGHT || propId == CV_CAP_PROP_FPS);
|
||||
if (needRestart) {
|
||||
this->stopPipeline();
|
||||
}
|
||||
|
||||
switch(propId)
|
||||
{
|
||||
case CV_CAP_PROP_POS_MSEC:
|
||||
{
|
||||
if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_TIME,
|
||||
flags, (gint64) (value * GST_MSECOND))) {
|
||||
handleMessage(pipeline);
|
||||
@@ -1950,6 +1988,9 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
}
|
||||
else
|
||||
{
|
||||
// Optimistically caching the target timestamp before reading the first frame from the new position since
|
||||
// the timestamp in GStreamer can be reliable extracted from the read frames.
|
||||
timestamp = (gint64)value;
|
||||
if (isPosFramesEmulated)
|
||||
{
|
||||
if (value == 0)
|
||||
@@ -1963,7 +2004,8 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
return true;
|
||||
}
|
||||
case CV_CAP_PROP_POS_FRAMES:
|
||||
{
|
||||
if (!isPosFramesSupported)
|
||||
@@ -1977,24 +2019,34 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
CV_WARN("unable to seek");
|
||||
return false;
|
||||
}
|
||||
// Certain mov and mp4 files seek incorrectly if the pipeline is not stopped before.
|
||||
if (this->isPipelinePlaying()) {
|
||||
this->stopPipeline();
|
||||
}
|
||||
|
||||
if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_DEFAULT,
|
||||
flags, (gint64) value)) {
|
||||
handleMessage(pipeline);
|
||||
CV_WARN("GStreamer: unable to seek");
|
||||
break;
|
||||
return false;
|
||||
}
|
||||
// wait for status update
|
||||
gst_element_get_state(pipeline, NULL, NULL, GST_CLOCK_TIME_NONE);
|
||||
return true;
|
||||
}
|
||||
case CV_CAP_PROP_POS_AVI_RATIO:
|
||||
{
|
||||
// https://stackoverflow.com/questions/31290315
|
||||
// GStreamer docs: GST_FORMAT_PERCENT (5) – percentage of stream (few, if any, elements implement this as of May 2009)
|
||||
CV_WARN("GStreamer: seeking by file percent are not supported by most GStreamer elements");
|
||||
if(!gst_element_seek_simple(GST_ELEMENT(pipeline.get()), GST_FORMAT_PERCENT,
|
||||
flags, (gint64) (value * GST_FORMAT_PERCENT_MAX))) {
|
||||
handleMessage(pipeline);
|
||||
CV_WARN("GStreamer: unable to seek");
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -2011,7 +2063,8 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
return true;
|
||||
}
|
||||
case CV_CAP_PROP_FRAME_WIDTH:
|
||||
if(value > 0)
|
||||
setFilter("width", G_TYPE_INT, (int) value, 0);
|
||||
@@ -2099,8 +2152,9 @@ bool GStreamerCapture::setProperty(int propId, double value)
|
||||
CV_WARN("GStreamer: unhandled property");
|
||||
}
|
||||
|
||||
if (wasPlaying)
|
||||
if (needRestart) {
|
||||
this->startPipeline();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -2572,7 +2626,7 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
|
||||
if (stateret == GST_STATE_CHANGE_FAILURE)
|
||||
{
|
||||
handleMessage(pipeline);
|
||||
CV_WARN("GStreamer: cannot put pipeline to play\n");
|
||||
CV_WARN("GStreamer: cannot put pipeline to play");
|
||||
pipeline.release();
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -186,6 +186,7 @@ public:
|
||||
double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC);
|
||||
double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3;
|
||||
audio0_timestamp = video0_timestamp + audio_shift * 1e-9;
|
||||
|
||||
std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl;
|
||||
}
|
||||
ASSERT_TRUE(cap.retrieve(videoFrame));
|
||||
@@ -228,7 +229,7 @@ public:
|
||||
EXPECT_NEAR(
|
||||
cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp,
|
||||
cap.get(CAP_PROP_POS_MSEC) * 1e-3,
|
||||
(1.0 / fps) * 0.3)
|
||||
(1.0 / fps) * 0.6)
|
||||
<< "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC);
|
||||
}
|
||||
if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples)
|
||||
|
||||
@@ -178,4 +178,47 @@ TEST(videoio_gstreamer, timeout_property)
|
||||
}
|
||||
}
|
||||
|
||||
//==============================================================================
|
||||
// Seeking test with manual GStreamer pipeline
|
||||
typedef testing::TestWithParam<string> gstreamer_bunny;
|
||||
|
||||
TEST_P(gstreamer_bunny, manual_seek)
|
||||
{
|
||||
if (!videoio_registry::hasBackend(CAP_GSTREAMER))
|
||||
throw SkipTestException("GStreamer backend was not found");
|
||||
|
||||
const string video_file = BunnyParameters::getFilename("." + GetParam());
|
||||
const string pipeline = "filesrc location=" + video_file + " ! decodebin ! videoconvert ! video/x-raw, format=BGR ! appsink drop=1";
|
||||
const double target_pos = 3000.0;
|
||||
const float ms_per_frame = 1000 / BunnyParameters::getFps();
|
||||
VideoCapture cap;
|
||||
cap.open(pipeline, CAP_GSTREAMER);
|
||||
ASSERT_TRUE(cap.isOpened());
|
||||
Mat img;
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
cap >> img;
|
||||
}
|
||||
EXPECT_FALSE(img.empty());
|
||||
cap.set(CAP_PROP_POS_MSEC, target_pos);
|
||||
cap >> img;
|
||||
EXPECT_FALSE(img.empty());
|
||||
double actual_pos = cap.get(CAP_PROP_POS_MSEC);
|
||||
EXPECT_NEAR(actual_pos, target_pos, ms_per_frame);
|
||||
}
|
||||
|
||||
static const string bunny_params[] = {
|
||||
// string("wmv"),
|
||||
string("mov"),
|
||||
string("mp4"),
|
||||
// string("mpg"),
|
||||
string("avi"),
|
||||
// string("h264"),
|
||||
// string("h265"),
|
||||
string("mjpg.avi")
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(videoio, gstreamer_bunny, testing::ValuesIn(bunny_params));
|
||||
|
||||
|
||||
}} // namespace
|
||||
|
||||
@@ -67,6 +67,11 @@ public:
|
||||
std::cout << "CAP_PROP_FRAME_COUNT is not supported by backend. Assume 50 frames." << std::endl;
|
||||
n_frames = 50;
|
||||
}
|
||||
// GStreamer can't read frame count of big_buck_bunny.wmv
|
||||
if (apiPref == CAP_GSTREAMER && ext == "wmv")
|
||||
{
|
||||
n_frames = 125;
|
||||
}
|
||||
|
||||
{
|
||||
SCOPED_TRACE("consecutive read");
|
||||
@@ -166,7 +171,7 @@ public:
|
||||
EXPECT_NO_THROW(count_prop = (int)cap.get(CAP_PROP_FRAME_COUNT));
|
||||
// mpg file reports 5.08 sec * 24 fps => property returns 122 frames
|
||||
// but actual number of frames returned is 125
|
||||
if (ext != "mpg")
|
||||
if (ext != "mpg" && !(apiPref == CAP_GSTREAMER && ext == "wmv"))
|
||||
{
|
||||
if (count_prop > 0)
|
||||
{
|
||||
@@ -200,12 +205,11 @@ public:
|
||||
if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends()))
|
||||
throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref));
|
||||
|
||||
// GStreamer: https://github.com/opencv/opencv/issues/19025
|
||||
if (apiPref == CAP_GSTREAMER)
|
||||
if (((apiPref == CAP_GSTREAMER) && (ext == "avi")))
|
||||
throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) +
|
||||
cv::String(" does not return reliable values for CAP_PROP_POS_MSEC property"));
|
||||
cv::String(" does not support CAP_PROP_POS_MSEC option"));
|
||||
|
||||
if (((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265"))))
|
||||
if (((apiPref == CAP_FFMPEG || apiPref == CAP_GSTREAMER) && ((ext == "h264") || (ext == "h265"))))
|
||||
throw SkipTestException(cv::String("Backend ") + cv::videoio_registry::getBackendName(apiPref) +
|
||||
cv::String(" does not support CAP_PROP_POS_MSEC option"));
|
||||
|
||||
@@ -221,8 +225,8 @@ public:
|
||||
// mpg file reports 5.08 sec * 24 fps => property returns 122 frames,but actual number of frames returned is 125
|
||||
// HACK: CAP_PROP_FRAME_COUNT is not supported for vmw + MSMF. Just force check for all 125 frames
|
||||
if (ext == "mpg")
|
||||
EXPECT_GT(frame_count, 121);
|
||||
else if ((ext == "wmv") && (apiPref == CAP_MSMF))
|
||||
EXPECT_GE(frame_count, 114);
|
||||
else if ((ext == "wmv") && (apiPref == CAP_MSMF || apiPref == CAP_GSTREAMER))
|
||||
frame_count = 125;
|
||||
else
|
||||
EXPECT_EQ(frame_count, 125);
|
||||
@@ -240,6 +244,9 @@ public:
|
||||
if (cvtest::debugLevel > 0)
|
||||
std::cout << "i = " << i << ": timestamp = " << timestamp << std::endl;
|
||||
const double frame_period = 1000.f/bunny_param.getFps();
|
||||
// big_buck_bunny.mpg starts at 0.500 msec
|
||||
if ((ext == "mpg") && (apiPref == CAP_GSTREAMER))
|
||||
timestamp -= 500.0;
|
||||
// NOTE: eps == frame_period, because videoCapture returns frame beginning timestamp or frame end
|
||||
// timestamp depending on codec and back-end. So the first frame has timestamp 0 or frame_period.
|
||||
EXPECT_NEAR(timestamp, i*frame_period, frame_period) << "i=" << i;
|
||||
|
||||
Reference in New Issue
Block a user