mirror of
https://github.com/opencv/opencv.git
synced 2026-07-30 15:53:03 +04:00
Merge pull request #19460 from mikhail-nikolskiy:videoio-hw
videoio: HW decode/encode in FFMPEG backend; new properties with support in FFMPEG/GST/MSMF * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * .avi for GStreamer test * revert changes around seek() * cv_writer_open_with_params * params.warnUnusedParameters * VideoCaptureParameters in GStreamer * open_with_params * params->getUnused * Reduce PSNR threshold 33->32 (other tests use 30) * require FFMPEG 4.0+; PSNR 30 as in other tests * GStreamer AVI-demux plugin not installed in Ubuntu test environment? * fix build on very old ffmpeg * fix build on very old ffmpeg * fix build issues * fix build issues (static_cast) * FFMPEG built on Windows without H264 encoder? * fix for write_nothing test on VAAPI * fix warnings * fix cv_writer_get_prop in plugins * use avcodec_get_hw_frames_parameters; more robust fallback to SW codecs * internal function hw_check_device() for device check/logging * two separate tests for HW read and write * image size 640x480 in encode test * WITH_VA=ON (only .h headers used in OpenCV, no linkage dependency) * exception on VP9 SW encoder? * rebase master; refine info message * videoio: fix FFmpeg standalone plugin build * videoio(ffmpeg): eliminate MSVC build warnings * address review comments * videoio(hw): update videocapture_acceleration.read test - remove parallel decoding by SW code path - check PSNR against the original generated image * videoio: minor fixes * videoio(test): disable unsupported MSMF cases (SW and HW) * videoio(test): update PSNR thresholds for HW acceleration read * videoio(test): update debug messages * "hw_acceleration" whitelisting parameter * little optimization in test * D3D11VA supports decoders, doesn't support encoders * videoio(test): adjust PSNR threshold in write_read_position tests * videoio(ffmpeg): fix rejecting on acceleration device name mismatch * videoio(ffmpeg): fix compilation USE_AV_HW_CODECS=0, add more debug logging * videoio: rework VideoAccelerationType behavior - enum is not a bitset - default value is backend specific - only '_NONE' and '_ANY' may fallback on software processing - specific H/W acceleration doesn't fallback on software processing. It fails if there is no support for specified H/W acceleration. * videoio(test): fix for current FFmpeg wrapper Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
This commit is contained in:
Mikhail Nikolskii
committed by
GitHub
GitHub
parent
f70e80a6ba
commit
7bcb51eded
@@ -15,11 +15,35 @@
|
||||
|
||||
namespace cv {
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &out, const VideoCaptureAPIs& api)
|
||||
static inline
|
||||
std::ostream& operator<<(std::ostream& out, const VideoCaptureAPIs& api)
|
||||
{
|
||||
out << cv::videoio_registry::getBackendName(api); return out;
|
||||
}
|
||||
|
||||
static inline
|
||||
std::ostream& operator<<(std::ostream& out, const VideoAccelerationType& va_type)
|
||||
{
|
||||
struct {
|
||||
VideoAccelerationType va_type;
|
||||
const char* str;
|
||||
} va_types[] = {
|
||||
{VIDEO_ACCELERATION_ANY, "ANY"},
|
||||
{VIDEO_ACCELERATION_NONE, "NONE"},
|
||||
{VIDEO_ACCELERATION_D3D11, "D3D11"},
|
||||
{VIDEO_ACCELERATION_VAAPI, "VAAPI"},
|
||||
{VIDEO_ACCELERATION_MFX, "MFX"},
|
||||
};
|
||||
for (const auto& va : va_types) {
|
||||
if (va_type == va.va_type) {
|
||||
out << va.str;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
out << cv::format("UNKNOWN(0x%ux)", static_cast<unsigned int>(va_type));
|
||||
return out;
|
||||
}
|
||||
|
||||
static inline void PrintTo(const cv::VideoCaptureAPIs& api, std::ostream* os)
|
||||
{
|
||||
*os << cv::videoio_registry::getBackendName(api);
|
||||
|
||||
@@ -431,11 +431,11 @@ static Ext_Fourcc_PSNR synthetic_params[] = {
|
||||
{"mkv", "MPEG", 30.f, CAP_FFMPEG},
|
||||
{"mkv", "MJPG", 30.f, CAP_FFMPEG},
|
||||
|
||||
{"avi", "MPEG", 30.f, CAP_GSTREAMER},
|
||||
{"avi", "MPEG", 28.f, CAP_GSTREAMER},
|
||||
{"avi", "MJPG", 30.f, CAP_GSTREAMER},
|
||||
{"avi", "H264", 30.f, CAP_GSTREAMER},
|
||||
|
||||
{"mkv", "MPEG", 30.f, CAP_GSTREAMER},
|
||||
{"mkv", "MPEG", 28.f, CAP_GSTREAMER},
|
||||
{"mkv", "MJPG", 30.f, CAP_GSTREAMER},
|
||||
{"mkv", "H264", 30.f, CAP_GSTREAMER},
|
||||
|
||||
@@ -649,4 +649,332 @@ TEST_P(safe_capture, frames_independency)
|
||||
static VideoCaptureAPIs safe_apis[] = {CAP_FFMPEG, CAP_GSTREAMER, CAP_MSMF,CAP_AVFOUNDATION};
|
||||
INSTANTIATE_TEST_CASE_P(videoio, safe_capture, testing::ValuesIn(safe_apis));
|
||||
|
||||
//==================================================================================================
|
||||
// TEST_P(videocapture_acceleration, ...)
|
||||
|
||||
struct VideoCaptureAccelerationInput
|
||||
{
|
||||
const char* filename;
|
||||
double psnr_threshold;
|
||||
};
|
||||
|
||||
static inline
|
||||
std::ostream& operator<<(std::ostream& out, const VideoCaptureAccelerationInput& p)
|
||||
{
|
||||
out << p.filename;
|
||||
return out;
|
||||
}
|
||||
|
||||
typedef testing::TestWithParam<tuple<VideoCaptureAccelerationInput, VideoCaptureAPIs, VideoAccelerationType, bool>> videocapture_acceleration;
|
||||
|
||||
TEST_P(videocapture_acceleration, read)
|
||||
{
|
||||
auto param = GetParam();
|
||||
std::string filename = get<0>(param).filename;
|
||||
double psnr_threshold = get<0>(param).psnr_threshold;
|
||||
VideoCaptureAPIs backend = get<1>(param);
|
||||
VideoAccelerationType va_type = get<2>(param);
|
||||
bool use_umat = get<3>(param);
|
||||
int device_idx = -1;
|
||||
const int frameNum = 15;
|
||||
|
||||
std::string filepath = cvtest::findDataFile("video/" + filename);
|
||||
|
||||
if (backend == CAP_MSMF && (
|
||||
filename == "sample_322x242_15frames.yuv420p.mjpeg.mp4" ||
|
||||
filename == "sample_322x242_15frames.yuv420p.libx265.mp4" ||
|
||||
filename == "sample_322x242_15frames.yuv420p.libaom-av1.mp4" ||
|
||||
filename == "sample_322x242_15frames.yuv420p.mpeg2video.mp4"
|
||||
))
|
||||
throw SkipTestException("Format/codec is not supported");
|
||||
|
||||
|
||||
std::string backend_name = cv::videoio_registry::getBackendName(backend);
|
||||
if (!videoio_registry::hasBackend(backend))
|
||||
throw SkipTestException(cv::String("Backend is not available/disabled: ") + backend_name);
|
||||
|
||||
|
||||
// HW reader
|
||||
VideoCapture hw_reader(filepath, backend, {
|
||||
CAP_PROP_HW_ACCELERATION, static_cast<int>(va_type),
|
||||
CAP_PROP_HW_DEVICE, device_idx
|
||||
});
|
||||
if (!hw_reader.isOpened())
|
||||
{
|
||||
if (va_type == VIDEO_ACCELERATION_ANY || va_type == VIDEO_ACCELERATION_NONE)
|
||||
{
|
||||
// ANY HW acceleration should have fallback to SW codecs
|
||||
VideoCapture sw_reader(filepath, backend, {
|
||||
CAP_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE
|
||||
});
|
||||
if (!sw_reader.isOpened())
|
||||
throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported, skipping");
|
||||
|
||||
ASSERT_TRUE(hw_reader.isOpened()) << "ANY HW acceleration should have fallback to SW codecs";
|
||||
}
|
||||
else
|
||||
{
|
||||
throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported with HW acceleration, skipping");
|
||||
}
|
||||
}
|
||||
|
||||
VideoAccelerationType actual_va = static_cast<VideoAccelerationType>(static_cast<int>(hw_reader.get(CAP_PROP_HW_ACCELERATION)));
|
||||
if (va_type != VIDEO_ACCELERATION_ANY && va_type != VIDEO_ACCELERATION_NONE)
|
||||
{
|
||||
#ifdef _WIN32 // FIXIT FFmpeg wrapper upgrade is required
|
||||
if (actual_va == static_cast<VideoAccelerationType>(0))
|
||||
throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported with HW acceleration (legacy FFmpeg wrapper), skipping");
|
||||
#endif
|
||||
ASSERT_EQ((int)actual_va, (int)va_type) << "actual_va=" << actual_va << ", va_type=" << va_type;
|
||||
}
|
||||
std::cout << "VideoCapture " << backend_name << ":" << actual_va << std::endl << std::flush;
|
||||
|
||||
double min_psnr_original = 1000;
|
||||
for (int i = 0; i < frameNum; i++)
|
||||
{
|
||||
SCOPED_TRACE(cv::format("frame=%d", i));
|
||||
Mat frame;
|
||||
if (use_umat)
|
||||
{
|
||||
UMat umat;
|
||||
EXPECT_TRUE(hw_reader.read(umat));
|
||||
ASSERT_FALSE(umat.empty());
|
||||
umat.copyTo(frame);
|
||||
}
|
||||
else
|
||||
{
|
||||
EXPECT_TRUE(hw_reader.read(frame));
|
||||
}
|
||||
ASSERT_FALSE(frame.empty());
|
||||
|
||||
if (cvtest::debugLevel > 0)
|
||||
{
|
||||
imwrite(cv::format("test_frame%03d.png", i), frame);
|
||||
}
|
||||
|
||||
Mat original(frame.size(), CV_8UC3, Scalar::all(0));
|
||||
generateFrame(i, frameNum, original);
|
||||
double psnr = cvtest::PSNR(frame, original);
|
||||
if (psnr < min_psnr_original)
|
||||
min_psnr_original = psnr;
|
||||
}
|
||||
|
||||
std::ostringstream ss; ss << actual_va;
|
||||
std::string actual_va_str = ss.str();
|
||||
std::cout << "VideoCapture with acceleration = " << cv::format("%-6s @ %-10s", actual_va_str.c_str(), backend_name.c_str())
|
||||
<< " on " << filename
|
||||
<< " with PSNR-original = " << min_psnr_original
|
||||
<< std::endl << std::flush;
|
||||
EXPECT_GE(min_psnr_original, psnr_threshold);
|
||||
}
|
||||
|
||||
static const VideoCaptureAccelerationInput hw_filename[] = {
|
||||
{ "sample_322x242_15frames.yuv420p.libxvid.mp4", 28.0 },
|
||||
{ "sample_322x242_15frames.yuv420p.mjpeg.mp4", 20.0 },
|
||||
{ "sample_322x242_15frames.yuv420p.mpeg2video.mp4", 24.0 }, // GSTREAMER on Ubuntu 18.04
|
||||
{ "sample_322x242_15frames.yuv420p.libx264.mp4", 24.0 }, // GSTREAMER on Ubuntu 18.04
|
||||
{ "sample_322x242_15frames.yuv420p.libx265.mp4", 30.0 },
|
||||
{ "sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", 30.0 },
|
||||
{ "sample_322x242_15frames.yuv420p.libaom-av1.mp4", 30.0 }
|
||||
};
|
||||
|
||||
static const VideoCaptureAPIs hw_backends[] = {
|
||||
CAP_FFMPEG,
|
||||
CAP_GSTREAMER,
|
||||
#ifdef _WIN32
|
||||
CAP_MSMF,
|
||||
#endif
|
||||
};
|
||||
|
||||
static const VideoAccelerationType hw_types[] = {
|
||||
VIDEO_ACCELERATION_NONE,
|
||||
VIDEO_ACCELERATION_ANY,
|
||||
VIDEO_ACCELERATION_MFX,
|
||||
#ifdef _WIN32
|
||||
VIDEO_ACCELERATION_D3D11,
|
||||
#else
|
||||
VIDEO_ACCELERATION_VAAPI,
|
||||
#endif
|
||||
};
|
||||
|
||||
static bool hw_use_umat[] = {
|
||||
false,
|
||||
//true
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(videoio, videocapture_acceleration, testing::Combine(
|
||||
testing::ValuesIn(hw_filename),
|
||||
testing::ValuesIn(hw_backends),
|
||||
testing::ValuesIn(hw_types),
|
||||
testing::ValuesIn(hw_use_umat)
|
||||
));
|
||||
|
||||
////////////////////////////////////////// TEST_P(video_acceleration, write_read)
|
||||
|
||||
typedef tuple<Ext_Fourcc_PSNR, VideoAccelerationType, bool> VATestParams;
|
||||
|
||||
typedef testing::TestWithParam<VATestParams> videowriter_acceleration;
|
||||
|
||||
TEST_P(videowriter_acceleration, write)
|
||||
{
|
||||
auto param = GetParam();
|
||||
VideoCaptureAPIs backend = get<0>(param).api;
|
||||
std::string codecid = get<0>(param).fourcc;
|
||||
std::string extension = get<0>(param).ext;
|
||||
double psnr_threshold = get<0>(param).PSNR;
|
||||
VideoAccelerationType va_type = get<1>(param);
|
||||
int device_idx = -1;
|
||||
bool use_umat = get<2>(param);
|
||||
std::string backend_name = cv::videoio_registry::getBackendName(backend);
|
||||
if (!videoio_registry::hasBackend(backend))
|
||||
throw SkipTestException(cv::String("Backend is not available/disabled: ") + backend_name);
|
||||
|
||||
const Size sz(640, 480);
|
||||
const int frameNum = 15;
|
||||
const double fps = 25;
|
||||
|
||||
std::string filename = tempfile("videowriter_acceleration.") + extension;
|
||||
|
||||
// Write video
|
||||
VideoAccelerationType actual_va;
|
||||
{
|
||||
VideoWriter hw_writer(
|
||||
filename,
|
||||
backend,
|
||||
VideoWriter::fourcc(codecid[0], codecid[1], codecid[2], codecid[3]),
|
||||
fps,
|
||||
sz,
|
||||
{
|
||||
VIDEOWRITER_PROP_HW_ACCELERATION, static_cast<int>(va_type),
|
||||
VIDEOWRITER_PROP_HW_DEVICE, device_idx
|
||||
}
|
||||
);
|
||||
|
||||
if (!hw_writer.isOpened()) {
|
||||
if (va_type == VIDEO_ACCELERATION_ANY || va_type == VIDEO_ACCELERATION_NONE)
|
||||
{
|
||||
// ANY HW acceleration should have fallback to SW codecs
|
||||
{
|
||||
VideoWriter sw_writer(
|
||||
filename,
|
||||
backend,
|
||||
VideoWriter::fourcc(codecid[0], codecid[1], codecid[2], codecid[3]),
|
||||
fps,
|
||||
sz,
|
||||
{
|
||||
VIDEOWRITER_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE,
|
||||
}
|
||||
);
|
||||
if (!sw_writer.isOpened()) {
|
||||
remove(filename.c_str());
|
||||
throw SkipTestException(backend_name + " VideoWriter on codec " + codecid + " not supported, skipping");
|
||||
}
|
||||
}
|
||||
remove(filename.c_str());
|
||||
ASSERT_TRUE(hw_writer.isOpened()) << "ANY HW acceleration should have fallback to SW codecs";
|
||||
} else {
|
||||
throw SkipTestException(backend_name + " VideoWriter on " + filename + " not supported with HW acceleration, skipping");
|
||||
}
|
||||
}
|
||||
|
||||
actual_va = static_cast<VideoAccelerationType>(static_cast<int>(hw_writer.get(VIDEOWRITER_PROP_HW_ACCELERATION)));
|
||||
if (va_type != VIDEO_ACCELERATION_ANY && va_type != VIDEO_ACCELERATION_NONE)
|
||||
{
|
||||
#ifdef _WIN32 // FIXIT FFmpeg wrapper upgrade is required
|
||||
if (actual_va == static_cast<VideoAccelerationType>(-1))
|
||||
throw SkipTestException(backend_name + " VideoWriter on " + filename + " not supported with HW acceleration (legacy FFmpeg wrapper), skipping");
|
||||
#endif
|
||||
ASSERT_EQ((int)actual_va, (int)va_type) << "actual_va=" << actual_va << ", va_type=" << va_type;
|
||||
}
|
||||
std::cout << "VideoWriter " << backend_name << ":" << actual_va << std::endl << std::flush;
|
||||
|
||||
Mat frame(sz, CV_8UC3);
|
||||
for (int i = 0; i < frameNum; ++i) {
|
||||
generateFrame(i, frameNum, frame);
|
||||
if (use_umat) {
|
||||
UMat umat;
|
||||
frame.copyTo(umat);
|
||||
hw_writer.write(umat);
|
||||
}
|
||||
else {
|
||||
hw_writer.write(frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::ifstream ofile(filename, std::ios::binary);
|
||||
ofile.seekg(0, std::ios::end);
|
||||
int64 fileSize = (int64)ofile.tellg();
|
||||
ASSERT_GT(fileSize, 0);
|
||||
std::cout << "File size: " << fileSize << std::endl;
|
||||
|
||||
// Read video and check PSNR on every frame
|
||||
{
|
||||
VideoCapture reader(
|
||||
filename,
|
||||
CAP_ANY /*backend*/,
|
||||
{ CAP_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE }
|
||||
);
|
||||
ASSERT_TRUE(reader.isOpened());
|
||||
double min_psnr = 1000;
|
||||
Mat reference(sz, CV_8UC3);
|
||||
for (int i = 0; i < frameNum; ++i) {
|
||||
Mat actual;
|
||||
if (use_umat) {
|
||||
UMat umat;
|
||||
EXPECT_TRUE(reader.read(umat));
|
||||
umat.copyTo(actual);
|
||||
}
|
||||
else {
|
||||
EXPECT_TRUE(reader.read(actual));
|
||||
}
|
||||
EXPECT_FALSE(actual.empty());
|
||||
generateFrame(i, frameNum, reference);
|
||||
EXPECT_EQ(reference.size(), actual.size());
|
||||
EXPECT_EQ(reference.depth(), actual.depth());
|
||||
EXPECT_EQ(reference.channels(), actual.channels());
|
||||
double psnr = cvtest::PSNR(actual, reference);
|
||||
EXPECT_GE(psnr, psnr_threshold) << " frame " << i;
|
||||
if (psnr < min_psnr)
|
||||
min_psnr = psnr;
|
||||
}
|
||||
Mat actual;
|
||||
EXPECT_FALSE(reader.read(actual));
|
||||
{
|
||||
std::ostringstream ss; ss << actual_va;
|
||||
std::string actual_va_str = ss.str();
|
||||
std::cout << "VideoWriter with acceleration = " << cv::format("%-6s @ %-10s", actual_va_str.c_str(), backend_name.c_str())
|
||||
<< " on codec=" << codecid << " (." << extension << ")"
|
||||
<< ", bitrate = " << fileSize / (frameNum / fps)
|
||||
<< ", with PSNR-original = " << min_psnr
|
||||
<< std::endl << std::flush;
|
||||
}
|
||||
remove(filename.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
static Ext_Fourcc_PSNR hw_codecs[] = {
|
||||
{"mp4", "MPEG", 29.f, CAP_FFMPEG},
|
||||
{"mp4", "H264", 29.f, CAP_FFMPEG},
|
||||
{"mp4", "HEVC", 29.f, CAP_FFMPEG},
|
||||
{"avi", "MJPG", 29.f, CAP_FFMPEG},
|
||||
{"avi", "XVID", 29.f, CAP_FFMPEG},
|
||||
//{"webm", "VP8", 29.f, CAP_FFMPEG},
|
||||
//{"webm", "VP9", 29.f, CAP_FFMPEG},
|
||||
|
||||
{"mkv", "MPEG", 29.f, CAP_GSTREAMER},
|
||||
{"mkv", "H264", 29.f, CAP_GSTREAMER},
|
||||
|
||||
#ifdef _WIN32
|
||||
{"mp4", "MPEG", 29.f, CAP_MSMF},
|
||||
{"mp4", "H264", 29.f, CAP_MSMF},
|
||||
#endif
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(videoio, videowriter_acceleration, testing::Combine(
|
||||
testing::ValuesIn(hw_codecs),
|
||||
testing::ValuesIn(hw_types),
|
||||
testing::ValuesIn(hw_use_umat)
|
||||
));
|
||||
|
||||
} // namespace
|
||||
|
||||
Reference in New Issue
Block a user