potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-21 19:33:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #24363 from cudawarped:videoio_ffmpeg_add_stream_encapsulation

Browse Source 
videoio: Add raw encoded video stream muxing to cv::VideoWriter with CAP_FFMPEG #24363

Allow raw encoded video streams (e.g. h264[5]) to be encapsulated by `cv::VideoWriter` to video containers (e.g. mp4/mkv).

Operates in a similar way to https://github.com/opencv/opencv/pull/15290 where encapsulation is enabled by setting the `VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO` flag when constructing `cv::VideoWriter` e.g.
```
VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1 });
```
and each raw encoded frame is passed as single row of a `CV_8U` `cv::Mat`.

The main reason for this PR is to allow `cudacodec::VideoWriter` to output its encoded streams to a suitable container, see https://github.com/opencv/opencv_contrib/pull/3569.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
cudawarped
2023-10-25 13:21:01 +03:00
committed by GitHub
parent a3b3a589f9
commit 38bc519e4a
4 changed files with 358 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/videoio/test/test_ffmpeg.cpp
+200
View File
@@ -235,6 +235,206 @@ const videoio_container_params_t videoio_container_params[] =
INSTANTIATE_TEST_CASE_P(/**/, videoio_container, testing::ValuesIn(videoio_container_params));
typedef tuple<string, string, int, int> videoio_encapsulate_params_t;
typedef testing::TestWithParam< videoio_encapsulate_params_t > videoio_encapsulate;
TEST_P(videoio_encapsulate, write)
{
#ifdef _WIN32
throw SkipTestException("Test disabled until PR for raw video encapsulation is merged and windows dll is updated");
#else
const VideoCaptureAPIs api = CAP_FFMPEG;
if (!videoio_registry::hasBackend(api))
throw SkipTestException("FFmpeg backend was not found");
const string fileName = findDataFile(get<0>(GetParam()));
const string ext = get<1>(GetParam());
const int idrPeriod = get<2>(GetParam());
const int nFrames = get<3>(GetParam());
const string fileNameOut = tempfile(cv::format("test_encapsulated_stream.%s", ext.c_str()).c_str());
// Use VideoWriter to encapsulate encoded video read with VideoReader
{
VideoCapture capRaw(fileName, api, { CAP_PROP_FORMAT, -1 });
ASSERT_TRUE(capRaw.isOpened());
const int width = static_cast<int>(capRaw.get(CAP_PROP_FRAME_WIDTH));
const int height = static_cast<int>(capRaw.get(CAP_PROP_FRAME_HEIGHT));
const double fps = capRaw.get(CAP_PROP_FPS);
const int codecExtradataIndex = static_cast<int>(capRaw.get(CAP_PROP_CODEC_EXTRADATA_INDEX));
Mat extraData;
capRaw.retrieve(extraData, codecExtradataIndex);
const int fourcc = static_cast<int>(capRaw.get(CAP_PROP_FOURCC));
const bool mpeg4 = (fourcc == fourccFromString("FMP4"));
VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1, VideoWriterProperties::VIDEOWRITER_PROP_KEY_INTERVAL, idrPeriod });
ASSERT_TRUE(container.isOpened());
Mat rawFrame;
for (int i = 0; i < nFrames; i++) {
ASSERT_TRUE(capRaw.read(rawFrame));
ASSERT_FALSE(rawFrame.empty());
if (i == 0 && mpeg4) {
Mat tmp = rawFrame.clone();
const size_t newSzt = tmp.total() + extraData.total();
const int newSz = static_cast<int>(newSzt);
ASSERT_TRUE(newSzt == static_cast<size_t>(newSz));
rawFrame = Mat(1, newSz, CV_8UC1);
memcpy(rawFrame.data, extraData.data, extraData.total());
memcpy(rawFrame.data + extraData.total(), tmp.data, tmp.total());
}
container.write(rawFrame);
}
container.release();
}
std::cout << "Checking encapsulated video container: " << fileNameOut << std::endl;
// Check encapsulated video container is "identical" to the original
{
VideoCapture capReference(fileName), capActual(fileNameOut), capActualRaw(fileNameOut, api, { CAP_PROP_FORMAT, -1 });
ASSERT_TRUE(capReference.isOpened());
ASSERT_TRUE(capActual.isOpened());
ASSERT_TRUE(capActualRaw.isOpened());
const double fpsReference = capReference.get(CAP_PROP_FPS);
const double fpsActual = capActual.get(CAP_PROP_FPS);
ASSERT_EQ(fpsReference, fpsActual);
const int nFramesActual = static_cast<int>(capActual.get(CAP_PROP_FRAME_COUNT));
ASSERT_EQ(nFrames, nFramesActual);
Mat reference, actual;
for (int i = 0; i < nFrames; i++) {
ASSERT_TRUE(capReference.read(reference));
ASSERT_FALSE(reference.empty());
ASSERT_TRUE(capActual.read(actual));
ASSERT_FALSE(actual.empty());
ASSERT_EQ(0, cvtest::norm(reference, actual, NORM_INF));
ASSERT_TRUE(capActualRaw.grab());
const bool keyFrameActual = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.;
const bool keyFrameReference = idrPeriod ? i % idrPeriod == 0 : 1;
ASSERT_EQ(keyFrameReference, keyFrameActual);
}
}
ASSERT_EQ(0, remove(fileNameOut.c_str()));
#endif
}
const videoio_encapsulate_params_t videoio_encapsulate_params[] =
{
videoio_encapsulate_params_t("video/big_buck_bunny.h264", "avi", 125, 125),
videoio_encapsulate_params_t("video/big_buck_bunny.h265", "mp4", 125, 125),
videoio_encapsulate_params_t("video/big_buck_bunny.wmv", "wmv", 12, 13),
videoio_encapsulate_params_t("video/big_buck_bunny.mp4", "mp4", 12, 13),
videoio_encapsulate_params_t("video/big_buck_bunny.mjpg.avi", "mp4", 0, 4),
videoio_encapsulate_params_t("video/big_buck_bunny.mov", "mp4", 12, 13),
videoio_encapsulate_params_t("video/big_buck_bunny.avi", "mp4", 125, 125),
videoio_encapsulate_params_t("video/big_buck_bunny.mpg", "mp4", 12, 13),
videoio_encapsulate_params_t("video/VID00003-20100701-2204.wmv", "wmv", 12, 13),
videoio_encapsulate_params_t("video/VID00003-20100701-2204.mpg", "mp4", 12,13),
videoio_encapsulate_params_t("video/VID00003-20100701-2204.avi", "mp4", 12, 13),
videoio_encapsulate_params_t("video/VID00003-20100701-2204.3GP", "mp4", 51, 52),
videoio_encapsulate_params_t("video/sample_sorenson.avi", "mp4", 12, 13),
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "mp4", 3, 4),
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mpeg2video.mp4", "mp4", 12, 13),
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mjpeg.mp4", "mp4", 0, 5),
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "avi", 15, 15),
videoio_encapsulate_params_t("../cv/tracking/faceocc2/data/faceocc2.webm", "webm", 128, 129),
videoio_encapsulate_params_t("../cv/video/1920x1080.avi", "mp4", 12, 13),
videoio_encapsulate_params_t("../cv/video/768x576.avi", "avi", 15, 16)
// Not supported by with FFmpeg:
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "mp4", 15, 15),
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "mp4", 15, 15),
};
INSTANTIATE_TEST_CASE_P(/**/, videoio_encapsulate, testing::ValuesIn(videoio_encapsulate_params));
TEST(videoio_encapsulate_set_idr, write)
{
#ifdef _WIN32
throw SkipTestException("Test disabled until PR for raw video encapsulation is merged and windows dll is updated");
#else
const VideoCaptureAPIs api = CAP_FFMPEG;
if (!videoio_registry::hasBackend(api))
throw SkipTestException("FFmpeg backend was not found");
const string fileName = findDataFile("video/big_buck_bunny.mp4");
const string ext = "mp4";
const string fileNameOut = tempfile(cv::format("test_encapsulated_stream_set_idr.%s", ext.c_str()).c_str());
// Use VideoWriter to encapsulate encoded video read with VideoReader
{
VideoCapture capRaw(fileName, api, { CAP_PROP_FORMAT, -1 });
ASSERT_TRUE(capRaw.isOpened());
const int width = static_cast<int>(capRaw.get(CAP_PROP_FRAME_WIDTH));
const int height = static_cast<int>(capRaw.get(CAP_PROP_FRAME_HEIGHT));
const double fps = capRaw.get(CAP_PROP_FPS);
const int codecExtradataIndex = static_cast<int>(capRaw.get(CAP_PROP_CODEC_EXTRADATA_INDEX));
Mat extraData;
capRaw.retrieve(extraData, codecExtradataIndex);
const int fourcc = static_cast<int>(capRaw.get(CAP_PROP_FOURCC));
const bool mpeg4 = (fourcc == fourccFromString("FMP4"));
VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1 });
ASSERT_TRUE(container.isOpened());
Mat rawFrame;
int i = 0;
while (capRaw.read(rawFrame)) {
ASSERT_FALSE(rawFrame.empty());
if (i == 0 && mpeg4) {
Mat tmp = rawFrame.clone();
const size_t newSzt = tmp.total() + extraData.total();
const int newSz = static_cast<int>(newSzt);
ASSERT_TRUE(newSzt == static_cast<size_t>(newSz));
rawFrame = Mat(1, newSz, CV_8UC1);
memcpy(rawFrame.data, extraData.data, extraData.total());
memcpy(rawFrame.data + extraData.total(), tmp.data, tmp.total());
}
if (static_cast<bool>(capRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME)))
container.set(VideoWriterProperties::VIDEOWRITER_PROP_KEY_FLAG, 1);
else
container.set(VideoWriterProperties::VIDEOWRITER_PROP_KEY_FLAG, 0);
container.write(rawFrame);
i++;
}
container.release();
}
std::cout << "Checking encapsulated video container: " << fileNameOut << std::endl;
// Check encapsulated video container is "identical" to the original
{
VideoCapture capReference(fileName), capReferenceRaw(fileName, api, { CAP_PROP_FORMAT, -1 }), capActual(fileNameOut), capActualRaw(fileNameOut, api, { CAP_PROP_FORMAT, -1 });
ASSERT_TRUE(capReference.isOpened());
ASSERT_TRUE(capActual.isOpened());
ASSERT_TRUE(capReferenceRaw.isOpened());
ASSERT_TRUE(capActualRaw.isOpened());
const double fpsReference = capReference.get(CAP_PROP_FPS);
const double fpsActual = capActual.get(CAP_PROP_FPS);
ASSERT_EQ(fpsReference, fpsActual);
const int nFramesReference = static_cast<int>(capReference.get(CAP_PROP_FRAME_COUNT));
const int nFramesActual = static_cast<int>(capActual.get(CAP_PROP_FRAME_COUNT));
ASSERT_EQ(nFramesReference, nFramesActual);
Mat reference, actual;
for (int i = 0; i < nFramesReference; i++) {
ASSERT_TRUE(capReference.read(reference));
ASSERT_FALSE(reference.empty());
ASSERT_TRUE(capActual.read(actual));
ASSERT_FALSE(actual.empty());
ASSERT_EQ(0, cvtest::norm(reference, actual, NORM_INF));
ASSERT_TRUE(capReferenceRaw.grab());
ASSERT_TRUE(capActualRaw.grab());
const bool keyFrameReference = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.;
const bool keyFrameActual = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.;
ASSERT_EQ(keyFrameReference, keyFrameActual);
}
}
ASSERT_EQ(0, remove(fileNameOut.c_str()));
#endif
}
typedef tuple<string, string, int> videoio_skip_params_t;
typedef testing::TestWithParam< videoio_skip_params_t > videoio_skip;