Merge pull request #19755 from mikhail-nikolskiy:ffmpeg-umat

cv::UMat output/input in VideoCapture/VideoWriter (data stays in GPU memory) * FFMPEG with UMat input/output * OpenCL_D3D* context * fix Linux build * cosmetic changes * fix build if USE_AV_HW_CODECS=0 * simplify how child context pointer stored in parent context * QSV interop with OpenCL on Windows * detect_msdk.cmake via pkg-config * fix av_buffer_ref() usage * revert windows-decode-mfx whitelisting; remove debug msg * address review comments * rename property to HW_ACCELERATION_USE_OPENCL * fix issue with "cl_khr_d3d11_sharing" extension not reported by OpenCL GPU+CPU platform * core(ocl): add OpenCL stubs for configurations without OpenCL * videoio(ffmpeg): update #if guards * Put OpenCL related code under HAVE_OPENCL; simplify reuse of media context from OpenCL context * videoio(test): skip unsupported tests - plugins don't support OpenCL/UMat yet - change handling of *_USE_OPENCL flag * videoio(ffmpeg): OpenCL dependency * videoio(ffmpeg): MediaSDK/oneVPL dependency * cleanup, logging * cmake: fix handling of 3rdparty interface targets Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
2026-07-31 08:13:04 +04:00 · 2021-05-14 19:48:50 +03:00
parent bb92eb5a93
commit a604d44d06
17 changed files with 1347 additions and 783 deletions
@@ -92,8 +92,17 @@ public:
        unsigned char* data = 0;
        int step=0, width=0, height=0, cn=0;

-        if (!ffmpegCapture ||
-           !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn))
+        if (!ffmpegCapture)
+            return false;
+
+        // if UMat, try GPU to GPU copy using OpenCL extensions
+        if (frame.isUMat()) {
+            if (ffmpegCapture->retrieveHWFrame(frame)) {
+                return true;
+            }
+        }
+
+        if (!icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn))
            return false;

        cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step);
@@ -176,6 +185,13 @@ public:
            return;
        CV_Assert(image.depth() == CV_8U);

+        // if UMat, try GPU to GPU copy using OpenCL extensions
+        if (image.isUMat()) {
+            if (ffmpegWriter->writeHWFrame(image)) {
+                return;
+            }
+        }
+
        icvWriteFrame_FFMPEG_p(ffmpegWriter, (const uchar*)image.getMat().ptr(), (int)image.step(), image.cols(), image.rows(), image.channels(), 0);
    }
    virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params )
@@ -5,7 +5,10 @@
 // Copyright (C) 2020-2021 Intel Corporation

 #include "opencv2/videoio.hpp"
-#if defined(__OPENCV_BUILD) || defined(OPENCV_HAVE_CVCONFIG_H)  // TODO Properly detect and add D3D11 / LIBVA dependencies for standalone plugins
+#ifdef HAVE_OPENCL
+#include "opencv2/core/ocl.hpp"
+#endif
+#if defined(__OPENCV_BUILD) && !defined(BUILD_PLUGIN)  // TODO Properly detect and add D3D11 / LIBVA dependencies for standalone plugins
 #include "cvconfig.h"
 #endif
 #include <sstream>
@@ -14,16 +17,31 @@
 #define D3D11_NO_HELPERS
 #include <d3d11.h>
 #include <codecvt>
+#include "opencv2/core/directx.hpp"
+#ifdef HAVE_OPENCL
+#include <CL/cl_d3d11.h>
 #endif
+#endif // HAVE_D3D11

 #ifdef HAVE_VA
 #include <va/va_backend.h>
+#ifdef HAVE_VA_INTEL
+#include "opencv2/core/va_intel.hpp"
+#ifndef CL_TARGET_OPENCL_VERSION
+#define CL_TARGET_OPENCL_VERSION 120
 #endif
+#ifdef HAVE_VA_INTEL_OLD_HEADER
+#include <CL/va_ext.h>
+#else
+#include <CL/cl_va_api_media_sharing_intel.h>
+#endif
+#endif
+#endif // HAVE_VA

+// FFMPEG "C" headers
 extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libavutil/avutil.h>
-
 #include <libavutil/hwcontext.h>
 #ifdef HAVE_D3D11
 #include <libavutil/hwcontext_d3d11va.h>
@@ -31,8 +49,23 @@ extern "C" {
 #ifdef HAVE_VA
 #include <libavutil/hwcontext_vaapi.h>
 #endif
+#ifdef HAVE_MFX // dependency only on MFX header files, no linkage dependency
+#include <libavutil/hwcontext_qsv.h>
+#endif
 }

+#define HW_DEFAULT_POOL_SIZE    32
+#define HW_DEFAULT_SW_FORMAT    AV_PIX_FMT_NV12
+
+using namespace cv;
+
+static AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *),
+                              const char *disabled_codecs, AVPixelFormat *hw_pix_fmt);
+static AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname, bool use_opencl);
+static AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format);
+static AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt);
+static VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type);
+
 static
 const char* getVideoAccelerationName(VideoAccelerationType va_type)
 {
@@ -70,7 +103,7 @@ std::string getDecoderConfiguration(VideoAccelerationType va_type, AVDictionary
    case VIDEO_ACCELERATION_ANY: return "d3d11va";
    case VIDEO_ACCELERATION_D3D11: return "d3d11va";
    case VIDEO_ACCELERATION_VAAPI: return "";
-    case VIDEO_ACCELERATION_MFX: return "";
+    case VIDEO_ACCELERATION_MFX: return ""; // "qsv" fails if non-Intel D3D11 device
    }
    return "";
 #else
@@ -80,7 +113,7 @@ std::string getDecoderConfiguration(VideoAccelerationType va_type, AVDictionary
    case VIDEO_ACCELERATION_ANY: return "vaapi.iHD";
    case VIDEO_ACCELERATION_D3D11: return "";
    case VIDEO_ACCELERATION_VAAPI: return "vaapi.iHD";
-    case VIDEO_ACCELERATION_MFX: return "";
+    case VIDEO_ACCELERATION_MFX: return "qsv.iHD";
    }
    return "";
 #endif
@@ -125,7 +158,6 @@ std::string getEncoderConfiguration(VideoAccelerationType va_type, AVDictionary
 #endif
 }

-
 static
 std::string getDecoderDisabledCodecs(AVDictionary *dict)
 {
@@ -170,19 +202,6 @@ std::string getEncoderDisabledCodecs(AVDictionary *dict)
 #endif
 }

-
-#define HW_DEFAULT_POOL_SIZE    32
-#define HW_DEFAULT_SW_FORMAT    AV_PIX_FMT_NV12
-
-using namespace cv;
-
-static AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *),
-                              const char *disabled_codecs, AVPixelFormat *hw_pix_fmt);
-static AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname);
-static AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format);
-static AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt);
-static VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type);
-
 static
 bool hw_check_device(AVBufferRef* ctx, AVHWDeviceType hw_type, const std::string& device_subname) {
    if (!ctx)
@@ -259,75 +278,343 @@ bool hw_check_device(AVBufferRef* ctx, AVHWDeviceType hw_type, const std::string
 }

 static
-AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname) {
-    if (AV_HWDEVICE_TYPE_NONE == hw_type)
-        return NULL;
-
-    AVHWDeviceType child_type = hw_type;
-    if (hw_type == AV_HWDEVICE_TYPE_QSV) {
-#ifdef _WIN32
-        child_type = AV_HWDEVICE_TYPE_DXVA2;
-#else
-        child_type = AV_HWDEVICE_TYPE_VAAPI;
-#endif
-    }
-
-    AVBufferRef* hw_device_ctx = NULL;
-    char device[128] = "";
-    char* pdevice = NULL;
-    if (hw_device >= 0 && hw_device < 100000) {
-        if (child_type == AV_HWDEVICE_TYPE_VAAPI) {
-            snprintf(device, sizeof(device), "/dev/dri/renderD%d", 128 + hw_device);
-        } else {
-            snprintf(device, sizeof(device), "%d", hw_device);
-        }
-        pdevice = device;
-    }
-    const char *hw_child_name = av_hwdevice_get_type_name(child_type);
-    const char *device_name = pdevice ? pdevice : "'default'";
-    int err = av_hwdevice_ctx_create(&hw_device_ctx, child_type, pdevice, NULL, 0);
-    if (hw_device_ctx && err >= 0)
+AVBufferRef* hw_create_derived_context(AVHWDeviceType hw_type, AVBufferRef* hw_device_ctx) {
+    AVBufferRef* derived_ctx = NULL;
+    const char* hw_name = av_hwdevice_get_type_name(hw_type);
+    int err = av_hwdevice_ctx_create_derived(&derived_ctx, hw_type, hw_device_ctx, 0);
+    if (!derived_ctx || err < 0)
    {
-        CV_LOG_DEBUG(NULL, "FFMPEG: Created video acceleration context (av_hwdevice_ctx_create) for " << hw_child_name << " on device " << device_name);
-        if (!hw_check_device(hw_device_ctx, hw_type, device_subname)) {
-            av_buffer_unref(&hw_device_ctx);
-            return NULL;
-        }
-        if (hw_type != child_type) {
-            AVBufferRef *derived_ctx = NULL;
-            const char *hw_name = av_hwdevice_get_type_name(hw_type);
-            err = av_hwdevice_ctx_create_derived(&derived_ctx, hw_type, hw_device_ctx, 0);
-            if (!derived_ctx || err < 0)
-            {
-                if (derived_ctx)
-                    av_buffer_unref(&derived_ctx);
-                CV_LOG_INFO(NULL, "FFMPEG: Failed to create derived video acceleration (av_hwdevice_ctx_create_derived) for " << hw_name << ". Error=" << err);
-            }
-            else
-            {
-                CV_LOG_DEBUG(NULL, "FFMPEG: Created derived video acceleration context (av_hwdevice_ctx_create_derived) for " << hw_name);
-            }
-            av_buffer_unref(&hw_device_ctx);
-            return derived_ctx;
-        } else {
-            return hw_device_ctx;
-        }
+        if (derived_ctx)
+            av_buffer_unref(&derived_ctx);
+        CV_LOG_INFO(NULL, "FFMPEG: Failed to create derived video acceleration (av_hwdevice_ctx_create_derived) for " << hw_name << ". Error=" << err);
+        return NULL;
    }
    else
    {
-        const char *hw_name = hw_child_name;
-        CV_LOG_INFO(NULL, "FFMPEG: Failed to create " << hw_name << " video acceleration (av_hwdevice_ctx_create) on device " << device_name);
+        // Store child context in 'user_opaque' field of parent context.
+        struct FreeChildContext {
+            static void free(struct AVHWDeviceContext* ctx) {
+                AVBufferRef* child_ctx = (AVBufferRef*)ctx->user_opaque;
+                if (child_ctx)
+                    av_buffer_unref(&child_ctx);
+            }
+        };
+        AVHWDeviceContext* ctx = (AVHWDeviceContext*)derived_ctx->data;
+        ctx->user_opaque = av_buffer_ref(hw_device_ctx);
+        ctx->free = FreeChildContext::free;
+        CV_LOG_INFO(NULL, "FFMPEG: Created derived video acceleration context (av_hwdevice_ctx_create_derived) for " << hw_name);
+        return derived_ctx;
+    }
+}
+
+#ifdef HAVE_OPENCL // GPU buffer interop with cv::UMat
+
+// FFmpeg context attached to OpenCL context
+class OpenCL_FFMPEG_Context : public ocl::Context::UserContext {
+public:
+    OpenCL_FFMPEG_Context(AVBufferRef* ctx) {
+        ctx_ = av_buffer_ref(ctx);
+    }
+    virtual ~OpenCL_FFMPEG_Context() {
+        av_buffer_unref(&ctx_);
+    }
+    AVBufferRef* GetAVHWDevice() {
+        return ctx_;
+    }
+private:
+    AVBufferRef* ctx_;
+};
+
+#ifdef HAVE_MFX
+static
+int hw_find_qsv_surface_index(AVFrame* hw_frame)
+{
+    if (AV_PIX_FMT_QSV != hw_frame->format)
+        return -1;
+    mfxFrameSurface1* surface = (mfxFrameSurface1*)hw_frame->data[3]; // As defined by AV_PIX_FMT_QSV
+    AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frame->hw_frames_ctx->data;
+    AVQSVFramesContext* qsv_ctx = (AVQSVFramesContext*)frames_ctx->hwctx;
+    for (int i = 0; i < qsv_ctx->nb_surfaces; i++) {
+        if (surface == qsv_ctx->surfaces + i) {
+            return i;
+        }
+    }
+    return -1;
+}
+#endif
+
+#ifdef HAVE_VA
+static
+VADisplay hw_get_va_display(AVHWDeviceContext* hw_device_ctx)
+{
+    if (hw_device_ctx->type == AV_HWDEVICE_TYPE_QSV) { // we stored pointer to child context in 'user_opaque' field
+        AVBufferRef* ctx = (AVBufferRef*)hw_device_ctx->user_opaque;
+        hw_device_ctx = (AVHWDeviceContext*)ctx->data;
+    }
+    if (hw_device_ctx && hw_device_ctx->type == AV_HWDEVICE_TYPE_VAAPI) {
+        return ((AVVAAPIDeviceContext*)hw_device_ctx->hwctx)->display;
+    }
+    return NULL;
+}
+#endif // HAVE_VA
+
+#ifdef HAVE_VA_INTEL
+static
+VASurfaceID hw_get_va_surface(AVFrame* hw_frame) {
+    if (AV_PIX_FMT_VAAPI == hw_frame->format) {
+        return (VASurfaceID)(size_t)hw_frame->data[3]; // As defined by AV_PIX_FMT_VAAPI
+    }
+#ifdef HAVE_MFX
+    else if (AV_PIX_FMT_QSV == hw_frame->format) {
+        int frame_idx = hw_find_qsv_surface_index(hw_frame);
+        if (frame_idx >= 0) { // frame index is same in parent (QSV) and child (VAAPI) frame context
+            AVHWFramesContext *frames_ctx = (AVHWFramesContext *) hw_frame->hw_frames_ctx->data;
+            AVHWFramesContext *child_ctx = (AVHWFramesContext *) frames_ctx->user_opaque;
+            if (child_ctx && AV_HWDEVICE_TYPE_VAAPI == child_ctx->device_ctx->type) {
+                AVVAAPIFramesContext *vaapi_ctx = (AVVAAPIFramesContext *) child_ctx->hwctx;
+                CV_Assert(frame_idx < vaapi_ctx->nb_surfaces);
+                return vaapi_ctx->surface_ids[frame_idx];
+            }
+        }
+    }
+#endif // HAVE_MFX
+    return VA_INVALID_SURFACE;
+}
+#endif // HAVE_VA_INTEL
+
+#ifdef HAVE_D3D11
+static
+AVD3D11VADeviceContext* hw_get_d3d11_device_ctx(AVHWDeviceContext* hw_device_ctx) {
+    if (AV_HWDEVICE_TYPE_QSV == hw_device_ctx->type) { // we stored pointer to child context in 'user_opaque' field
+        AVBufferRef* ctx = (AVBufferRef*)hw_device_ctx->user_opaque;
+        hw_device_ctx = (AVHWDeviceContext*)ctx->data;
+    }
+    if (AV_HWDEVICE_TYPE_D3D11VA == hw_device_ctx->type) {
+        return (AVD3D11VADeviceContext*)hw_device_ctx->hwctx;
+    }
+    return NULL;
+}
+
+ID3D11Texture2D* hw_get_d3d11_texture(AVFrame* hw_frame, int* subresource) {
+    ID3D11Texture2D* texture = NULL;
+    if (AV_PIX_FMT_D3D11 == hw_frame->format) {
+        texture = (ID3D11Texture2D*)hw_frame->data[0]; // As defined by AV_PIX_FMT_D3D11
+        *subresource = (intptr_t)hw_frame->data[1]; // As defined by AV_PIX_FMT_D3D11
+    }
+#ifdef HAVE_MFX
+    else if (AV_PIX_FMT_QSV == hw_frame->format) {
+        AVHWFramesContext *frames_ctx = (AVHWFramesContext *) hw_frame->hw_frames_ctx->data;
+        AVHWFramesContext *child_ctx = (AVHWFramesContext *) frames_ctx->user_opaque;
+        if (child_ctx && AV_HWDEVICE_TYPE_D3D11VA == child_ctx->device_ctx->type) {
+            texture = ((AVD3D11VAFramesContext*)child_ctx->hwctx)->texture;
+        }
+        *subresource = hw_find_qsv_surface_index(hw_frame);
+        CV_Assert(*subresource >= 0);
+    }
+#endif
+    return texture;
+}
+
+// In D3D11 case we allocate additional texture as single texture (not texture array) because
+// OpenCL interop with D3D11 doesn't support/work with NV12 sub-texture of texture array.
+ID3D11Texture2D* hw_get_d3d11_single_texture(AVFrame* hw_frame, AVD3D11VADeviceContext* d3d11_device_ctx, ID3D11Texture2D* texture) {
+    AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frame->hw_frames_ctx->data;
+    if (AV_HWDEVICE_TYPE_QSV == frames_ctx->device_ctx->type) {
+        frames_ctx = (AVHWFramesContext*)frames_ctx->user_opaque; // we stored pointer to child context in 'user_opaque' field
+    }
+    if (!frames_ctx || AV_HWDEVICE_TYPE_D3D11VA != frames_ctx->device_ctx->type) {
        return NULL;
    }
+    ID3D11Texture2D* singleTexture = (ID3D11Texture2D*)frames_ctx->user_opaque;
+    if (!singleTexture && d3d11_device_ctx && texture) {
+        D3D11_TEXTURE2D_DESC desc = {};
+        texture->GetDesc(&desc);
+        desc.ArraySize = 1;
+        desc.BindFlags |= D3D11_BIND_SHADER_RESOURCE;
+        desc.MiscFlags |= D3D11_RESOURCE_MISC_SHARED;
+        if (SUCCEEDED(d3d11_device_ctx->device->CreateTexture2D(&desc, NULL, &singleTexture))) {
+            frames_ctx->user_opaque = singleTexture;
+        }
+    }
+    return singleTexture;
+}
+#endif // HAVE_D3D11
+
+static
+AVHWDeviceType hw_check_opencl_context(AVHWDeviceContext* ctx) {
+    ocl::OpenCLExecutionContext& ocl_context = ocl::OpenCLExecutionContext::getCurrentRef();
+    if (!ctx || ocl_context.empty())
+        return AV_HWDEVICE_TYPE_NONE;
+#ifdef HAVE_VA_INTEL
+    VADisplay vadisplay_ocl = ocl_context.getContext().getOpenCLContextProperty(CL_CONTEXT_VA_API_DISPLAY_INTEL);
+    VADisplay vadisplay_ctx = hw_get_va_display(ctx);
+    if (vadisplay_ocl && vadisplay_ocl == vadisplay_ctx)
+        return AV_HWDEVICE_TYPE_VAAPI;
+#endif
+#ifdef HAVE_D3D11
+    ID3D11Device* d3d11device_ocl = (ID3D11Device*)ocl_context.getContext().getOpenCLContextProperty(CL_CONTEXT_D3D11_DEVICE_KHR);
+    AVD3D11VADeviceContext* d3d11_device_ctx = hw_get_d3d11_device_ctx(ctx);
+    if (d3d11_device_ctx && d3d11device_ocl && d3d11_device_ctx->device == d3d11device_ocl)
+        return AV_HWDEVICE_TYPE_D3D11VA;
+#endif
+    return AV_HWDEVICE_TYPE_NONE;
+}
+
+static
+void hw_init_opencl(AVBufferRef* ctx) {
+    if (!ctx)
+        return;
+    AVHWDeviceContext* hw_device_ctx = (AVHWDeviceContext*)ctx->data;
+    if (!hw_device_ctx)
+        return;
+#ifdef HAVE_VA_INTEL
+    VADisplay va_display = hw_get_va_display(hw_device_ctx);
+    if (va_display) {
+        va_intel::ocl::initializeContextFromVA(va_display);
+    }
+#endif
+#ifdef HAVE_D3D11
+    AVD3D11VADeviceContext* d3d11_device_ctx = hw_get_d3d11_device_ctx(hw_device_ctx);
+    if (d3d11_device_ctx) {
+        directx::ocl::initializeContextFromD3D11Device(d3d11_device_ctx->device);
+    }
+#endif
+    if (hw_check_opencl_context(hw_device_ctx) != AV_HWDEVICE_TYPE_NONE) {
+        // Attach AVHWDeviceContext to OpenCL context
+        ocl::Context &ocl_context = ocl::OpenCLExecutionContext::getCurrent().getContext();
+        ocl_context.setUserContext(std::make_shared<OpenCL_FFMPEG_Context>(ctx));
+    }
 }

 static
-AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format)
-{
-    AVBufferRef *hw_frames_ref = nullptr;
+AVBufferRef* hw_create_context_from_opencl(ocl::OpenCLExecutionContext& ocl_context, AVHWDeviceType hw_type) {
+    if (ocl_context.empty())
+        return NULL;
+    auto ocl_ffmpeg_context = ocl_context.getContext().getUserContext<OpenCL_FFMPEG_Context>();
+    if (!ocl_ffmpeg_context)
+        return NULL;
+    AVBufferRef* ctx = ocl_ffmpeg_context->GetAVHWDevice();
+    if (hw_type != ((AVHWDeviceContext*)ctx->data)->type) {
+        ctx = hw_create_derived_context(hw_type, ctx);
+    }
+    else {
+        ctx = av_buffer_ref(ctx);
+    }
    if (ctx)
+        CV_LOG_INFO(NULL, "FFMPEG: Using " << av_hwdevice_get_type_name(hw_type) << " video acceleration context attached to OpenCL context");
+    return ctx;
+}
+
+#endif // HAVE_OPENCL
+
+static
+AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname, bool use_opencl) {
+    AVBufferRef* hw_device_ctx = NULL;
+    if (AV_HWDEVICE_TYPE_NONE == hw_type)
+        return NULL;
+
+#ifdef HAVE_OPENCL
+    // Check if OpenCL context has AVHWDeviceContext attached to it
+    ocl::OpenCLExecutionContext& ocl_context = ocl::OpenCLExecutionContext::getCurrentRef();
+    try {
+        hw_device_ctx = hw_create_context_from_opencl(ocl_context, hw_type);
+        if (hw_device_ctx) {
+            if (hw_device >= 0)
+                CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: ignoring property HW_DEVICE as device context already created and attached to OpenCL context");
+            return hw_device_ctx;
+        }
+    }
+    catch (...) {
+        CV_LOG_INFO(NULL, "FFMPEG: Exception creating Video Acceleration context using current OpenCL context");
+    }
+#endif
+
+    // Create new media context. In QSV case, first create 'child' context.
+    std::vector<AVHWDeviceType> child_types = { hw_type };
+    if (hw_type == AV_HWDEVICE_TYPE_QSV) {
+#ifdef _WIN32
+        child_types = { AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2 };
+#else
+        child_types = { AV_HWDEVICE_TYPE_VAAPI };
+#endif
+    }
+    for (AVHWDeviceType child_type : child_types) {
+        char device[128] = "";
+        char* pdevice = NULL;
+        if (hw_device >= 0 && hw_device < 100000) {
+            if (child_type == AV_HWDEVICE_TYPE_VAAPI) {
+                snprintf(device, sizeof(device), "/dev/dri/renderD%d", 128 + hw_device);
+            }
+            else {
+                snprintf(device, sizeof(device), "%d", hw_device);
+            }
+            pdevice = device;
+        }
+        const char* hw_child_name = av_hwdevice_get_type_name(child_type);
+        const char* device_name = pdevice ? pdevice : "'default'";
+        int err = av_hwdevice_ctx_create(&hw_device_ctx, child_type, pdevice, NULL, 0);
+        if (hw_device_ctx && err >= 0)
+        {
+            if (!hw_check_device(hw_device_ctx, hw_type, device_subname)) {
+                av_buffer_unref(&hw_device_ctx);
+                continue;
+            }
+            CV_LOG_INFO(NULL, "FFMPEG: Created video acceleration context (av_hwdevice_ctx_create) for " << hw_child_name << " on device " << device_name);
+#ifdef HAVE_OPENCL
+            // if OpenCL context not created yet or property HW_ACCELERATION_USE_OPENCL set, create OpenCL context with binding to video acceleration context
+            if (ocl::haveOpenCL()) {
+                if (ocl_context.empty() || use_opencl) {
+                    try {
+                        hw_init_opencl(hw_device_ctx);
+                        ocl_context = ocl::OpenCLExecutionContext::getCurrentRef();
+                        if (!ocl_context.empty()) {
+                            CV_LOG_INFO(NULL, "FFMPEG: Created OpenCL context with " << hw_child_name <<
+                                " video acceleration on OpenCL device: " << ocl_context.getDevice().name());
+                        }
+                    } catch (...) {
+                        CV_LOG_INFO(NULL, "FFMPEG: Exception creating OpenCL context with " << hw_child_name << " video acceleration");
+                    }
+                }
+                else {
+                    CV_LOG_INFO(NULL, "FFMPEG: Can't bind " << hw_child_name << " video acceleration context to already created OpenCL context");
+                }
+            }
+#else
+            CV_UNUSED(use_opencl);
+#endif
+            if (hw_type != child_type) {
+                AVBufferRef* derived_ctx = hw_create_derived_context(hw_type, hw_device_ctx);
+                av_buffer_unref(&hw_device_ctx);
+                return derived_ctx;
+            } else {
+                return hw_device_ctx;
+            }
+        }
+        else
+        {
+            const char* hw_name = hw_child_name;
+            CV_LOG_INFO(NULL, "FFMPEG: Failed to create " << hw_name << " video acceleration (av_hwdevice_ctx_create) on device " << device_name);
+        }
+    }
+    return NULL;
+}
+
+static
+AVBufferRef* hw_create_frames(struct AVCodecContext* codec_ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format)
+{
+    AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)hw_device_ctx->data;
+    AVBufferRef* child_ctx = hw_device_ctx;
+    // In QSV case we first allocate child D3D11/VAAPI frames (except DXVA2 as no OpenCL interop), then derive to parent QSV frames
+    if (AV_HWDEVICE_TYPE_QSV == device_ctx->type) {
+        AVBufferRef *ctx = (AVBufferRef *) device_ctx->user_opaque; // child context stored during creation of derived context
+        if (ctx && AV_HWDEVICE_TYPE_DXVA2 != ((AVHWDeviceContext *) ctx->data)->type) {
+            child_ctx = ctx;
+        }
+    }
+    AVBufferRef *hw_frames_ref = nullptr;
+    if (codec_ctx)
    {
-        int res = avcodec_get_hw_frames_parameters(ctx, hw_device_ctx, hw_format, &hw_frames_ref);
+        int res = avcodec_get_hw_frames_parameters(codec_ctx, child_ctx, hw_format, &hw_frames_ref);
        if (res < 0)
        {
            CV_LOG_DEBUG(NULL, "FFMPEG: avcodec_get_hw_frames_parameters() call failed: " << res)
@@ -335,7 +622,7 @@ AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device
    }
    if (!hw_frames_ref)
    {
-        hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx);
+        hw_frames_ref = av_hwframe_ctx_alloc(child_ctx);
    }
    if (!hw_frames_ref)
    {
@@ -345,12 +632,41 @@ AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device
    AVHWFramesContext *frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
    frames_ctx->width = width;
    frames_ctx->height = height;
-    if (frames_ctx->format == AV_PIX_FMT_NONE)
-        frames_ctx->format = hw_format;
+    if (frames_ctx->format == AV_PIX_FMT_NONE) {
+        if (child_ctx == hw_device_ctx) {
+            frames_ctx->format = hw_format;
+        }
+        else {
+            AVHWFramesConstraints* constraints = av_hwdevice_get_hwframe_constraints(child_ctx, NULL);
+            if (constraints) {
+                frames_ctx->format = constraints->valid_hw_formats[0];
+                av_hwframe_constraints_free(&constraints);
+            }
+        }
+    }
    if (frames_ctx->sw_format == AV_PIX_FMT_NONE)
        frames_ctx->sw_format = HW_DEFAULT_SW_FORMAT;
    if (frames_ctx->initial_pool_size == 0)
        frames_ctx->initial_pool_size = HW_DEFAULT_POOL_SIZE;
+
+#ifdef HAVE_D3D11
+    if (frames_ctx->device_ctx && AV_HWDEVICE_TYPE_D3D11VA == frames_ctx->device_ctx->type) {
+        // BindFlags
+        AVD3D11VAFramesContext* frames_hwctx = (AVD3D11VAFramesContext*)frames_ctx->hwctx;
+        frames_hwctx->BindFlags |= D3D11_BIND_DECODER | D3D11_BIND_VIDEO_ENCODER;
+        // See function hw_get_d3d11_single_texture(), it allocates additional ID3D11Texture2D texture and
+        // attaches it as 'user_opaque' field. We have to set free() callback before av_hwframe_ctx_init() call.
+        struct D3D11SingleTexture {
+            static void free(struct AVHWFramesContext* ctx) {
+                ID3D11Texture2D* singleTexture = (ID3D11Texture2D*)ctx->user_opaque;
+                if (ctx->user_opaque)
+                    singleTexture->Release();
+            }
+        };
+        frames_ctx->free = D3D11SingleTexture::free;
+    }
+#endif
+
    int res = av_hwframe_ctx_init(hw_frames_ref);
    if (res < 0)
    {
@@ -358,7 +674,25 @@ AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device
        av_buffer_unref(&hw_frames_ref);
        return NULL;
    }
-    return hw_frames_ref;
+
+    if (child_ctx != hw_device_ctx) {
+        AVBufferRef* derived_frame_ctx = NULL;
+        int flags = AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_WRITE;
+        res = av_hwframe_ctx_create_derived(&derived_frame_ctx, hw_format, hw_device_ctx, hw_frames_ref, flags);
+        av_buffer_unref(&hw_frames_ref);
+        if (res < 0)
+        {
+            CV_LOG_INFO(NULL, "FFMPEG: Failed to create derived HW frame context (av_hwframe_ctx_create_derived): " << res);
+            return NULL;
+        }
+        else {
+            ((AVHWFramesContext*)derived_frame_ctx->data)->user_opaque = frames_ctx;
+            return derived_frame_ctx;
+        }
+    }
+    else {
+        return hw_frames_ref;
+    }
 }

 static
@@ -455,6 +789,110 @@ AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPi
    return fmt[0];
 }

+// GPU color conversion NV12->BGRA via OpenCL extensions
+static bool
+hw_copy_frame_to_umat(AVBufferRef* ctx, AVFrame* hw_frame, cv::OutputArray output) {
+    CV_UNUSED(hw_frame);
+    CV_UNUSED(output);
+    if (!ctx)
+        return false;
+
+#ifdef HAVE_OPENCL
+    try {
+        // check that current OpenCL context initilized with binding to same VAAPI/D3D11 context
+        AVHWDeviceContext *hw_device_ctx = (AVHWDeviceContext *) ctx->data;
+        AVHWDeviceType child_type = hw_check_opencl_context(hw_device_ctx);
+        if (child_type == AV_HWDEVICE_TYPE_NONE)
+            return false;
+
+#ifdef HAVE_VA_INTEL
+        if (child_type == AV_HWDEVICE_TYPE_VAAPI) {
+            VADisplay va_display = hw_get_va_display(hw_device_ctx);
+            VASurfaceID va_surface = hw_get_va_surface(hw_frame);
+            if (va_display && va_surface != VA_INVALID_SURFACE) {
+                va_intel::convertFromVASurface(va_display, va_surface, {hw_frame->width, hw_frame->height}, output);
+                return true;
+            }
+        }
+#endif
+
+#ifdef HAVE_D3D11
+        if (child_type == AV_HWDEVICE_TYPE_D3D11VA) {
+            AVD3D11VADeviceContext* d3d11_device_ctx = hw_get_d3d11_device_ctx(hw_device_ctx);
+            int subresource = 0;
+            ID3D11Texture2D* texture = hw_get_d3d11_texture(hw_frame, &subresource);
+            ID3D11Texture2D* singleTexture = hw_get_d3d11_single_texture(hw_frame, d3d11_device_ctx, texture);
+            if (texture && singleTexture) {
+                // Copy D3D11 sub-texture to D3D11 single texture
+                d3d11_device_ctx->device_context->CopySubresourceRegion(singleTexture, 0, 0, 0, 0, texture, subresource, NULL);
+                // Copy D3D11 single texture to cv::UMat
+                directx::convertFromD3D11Texture2D(singleTexture, output);
+                return true;
+            }
+        }
+#endif
+    }
+    catch (...)
+    {
+        return false;
+    }
+#endif // HAVE_OPENCL
+
+    return false;
+}
+
+// GPU color conversion BGRA->NV12 via OpenCL extensions
+static bool
+hw_copy_umat_to_frame(AVBufferRef* ctx, cv::InputArray input, AVFrame* hw_frame) {
+    CV_UNUSED(input);
+    CV_UNUSED(hw_frame);
+    if (!ctx)
+        return false;
+
+#ifdef HAVE_OPENCL
+    try {
+        // check that current OpenCL context initilized with binding to same VAAPI/D3D11 context
+        AVHWDeviceContext *hw_device_ctx = (AVHWDeviceContext *) ctx->data;
+        AVHWDeviceType child_type = hw_check_opencl_context(hw_device_ctx);
+        if (child_type == AV_HWDEVICE_TYPE_NONE)
+            return false;
+
+#ifdef HAVE_VA_INTEL
+        if (child_type == AV_HWDEVICE_TYPE_VAAPI) {
+            VADisplay va_display = hw_get_va_display(hw_device_ctx);
+            VASurfaceID va_surface = hw_get_va_surface(hw_frame);
+            if (va_display != NULL && va_surface != VA_INVALID_SURFACE) {
+                va_intel::convertToVASurface(va_display, input, va_surface, {hw_frame->width, hw_frame->height});
+                return true;
+            }
+        }
+#endif
+
+#ifdef HAVE_D3D11
+        if (child_type == AV_HWDEVICE_TYPE_D3D11VA) {
+            AVD3D11VADeviceContext* d3d11_device_ctx = hw_get_d3d11_device_ctx(hw_device_ctx);
+            int subresource = 0;
+            ID3D11Texture2D* texture = hw_get_d3d11_texture(hw_frame, &subresource);
+            ID3D11Texture2D* singleTexture = hw_get_d3d11_single_texture(hw_frame, d3d11_device_ctx, texture);
+            if (texture && singleTexture) {
+                // Copy cv::UMat to D3D11 single texture
+                directx::convertToD3D11Texture2D(input, singleTexture);
+                // Copy D3D11 single texture to D3D11 sub-texture
+                d3d11_device_ctx->device_context->CopySubresourceRegion(texture, subresource, 0, 0, 0, singleTexture, 0, NULL);
+                return true;
+            }
+        }
+#endif
+    }
+    catch (...)
+    {
+        return false;
+    }
+#endif // HAVE_OPENCL
+
+    return false;
+}
+
 static
 VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type) {
    struct HWTypeFFMPEG {
@@ -476,6 +476,7 @@ struct CvCapture_FFMPEG
    bool setProperty(int, double);
    bool grabFrame();
    bool retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn);
+    bool retrieveHWFrame(cv::OutputArray output);
    void rotateFrame(cv::Mat &mat) const;

    void init();
@@ -537,6 +538,7 @@ struct CvCapture_FFMPEG
 #endif
    VideoAccelerationType va_type;
    int hw_device;
+    int use_opencl;
 };

 void CvCapture_FFMPEG::init()
@@ -574,6 +576,7 @@ void CvCapture_FFMPEG::init()
    bsfc = NULL;
    va_type = cv::VIDEO_ACCELERATION_NONE;  // TODO OpenCV 5.0: change to _ANY?
    hw_device = -1;
+    use_opencl = 0;
 }


@@ -922,6 +925,9 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters&
                return false;
            }
        }
+        if (params.has(CAP_PROP_HW_ACCELERATION_USE_OPENCL)) {
+            use_opencl = params.get<int>(CAP_PROP_HW_ACCELERATION_USE_OPENCL);
+        }
        if (params.warnUnusedParameters())
        {
            CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details. Bailout");
@@ -1051,7 +1057,7 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters&
                    if (codec) {
                        if (hw_pix_fmt != AV_PIX_FMT_NONE)
                            enc->get_format = hw_get_format_callback; // set callback to select HW pixel format, not SW format
-                        enc->hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname());
+                        enc->hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0);
                        if (!enc->hw_device_ctx)
                        {
                            CV_LOG_DEBUG(NULL, "FFMPEG: ... can't create H/W device: '" << accel_iter.hw_type_device_string() << "'");
@@ -1476,6 +1482,22 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int*
    return true;
 }

+bool CvCapture_FFMPEG::retrieveHWFrame(cv::OutputArray output)
+{
+#if USE_AV_HW_CODECS
+    // check that we have HW frame in GPU memory
+    if (!picture || !picture->hw_frames_ctx) {
+        return false;
+    }
+
+    // GPU color conversion NV12->BGRA, from GPU media buffer to GPU OpenCL buffer
+    return hw_copy_frame_to_umat(video_st->codec->hw_device_ctx, picture, output);
+#else
+    CV_UNUSED(output);
+    return false;
+#endif
+}
+
 double CvCapture_FFMPEG::getProperty( int property_id ) const
 {
    if( !video_st ) return 0;
@@ -1549,6 +1571,8 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
        return static_cast<double>(va_type);
    case CAP_PROP_HW_DEVICE:
        return static_cast<double>(hw_device);
+    case CAP_PROP_HW_ACCELERATION_USE_OPENCL:
+        return static_cast<double>(use_opencl);
 #endif  // USE_AV_HW_CODECS
    default:
        break;
@@ -1752,6 +1776,7 @@ struct CvVideoWriter_FFMPEG
               double fps, int width, int height, const VideoWriterParameters& params );
    void close();
    bool writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin );
+    bool writeHWFrame(cv::InputArray input);
    double getProperty(int propId) const;

    void init();
@@ -1774,6 +1799,7 @@ struct CvVideoWriter_FFMPEG
    struct SwsContext *img_convert_ctx;
    VideoAccelerationType va_type;
    int               hw_device;
+    int               use_opencl;
 };

 static const char * icvFFMPEGErrStr(int err)
@@ -1836,6 +1862,7 @@ void CvVideoWriter_FFMPEG::init()
    frame_idx = 0;
    va_type = VIDEO_ACCELERATION_NONE;
    hw_device = -1;
+    use_opencl = 0;
    ok = false;
 }

@@ -2210,6 +2237,41 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int
    return ret;
 }

+bool CvVideoWriter_FFMPEG::writeHWFrame(cv::InputArray input) {
+#if USE_AV_HW_CODECS
+    if (!video_st->codec->hw_frames_ctx)
+        return false;
+
+    // Get hardware frame from frame pool
+    AVFrame* hw_frame = av_frame_alloc();
+    if (!hw_frame) {
+        return false;
+    }
+    if (av_hwframe_get_buffer(video_st->codec->hw_frames_ctx, hw_frame, 0) < 0) {
+        av_frame_free(&hw_frame);
+        return false;
+    }
+
+    // GPU to GPU copy
+    if (!hw_copy_umat_to_frame(video_st->codec->hw_device_ctx, input, hw_frame)) {
+        av_frame_free(&hw_frame);
+        return false;
+    }
+
+    // encode
+    hw_frame->pts = frame_idx;
+    icv_av_write_frame_FFMPEG( oc, video_st, outbuf, outbuf_size, hw_frame, frame_idx);
+    frame_idx++;
+
+    av_frame_free(&hw_frame);
+
+    return true;
+#else
+    CV_UNUSED(input);
+    return false;
+#endif
+}
+
 double CvVideoWriter_FFMPEG::getProperty(int propId) const
 {
    CV_UNUSED(propId);
@@ -2222,6 +2284,10 @@ double CvVideoWriter_FFMPEG::getProperty(int propId) const
    {
        return static_cast<double>(hw_device);
    }
+    else if (propId == VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL)
+    {
+        return static_cast<double>(use_opencl);
+    }
 #endif
    return 0;
 }
@@ -2375,6 +2441,9 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
            return false;
        }
    }
+    if (params.has(VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL)) {
+        use_opencl = params.get<int>(VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL);
+    }

    if (params.warnUnusedParameters())
    {
@@ -2638,7 +2707,7 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
            if (!codec)
                continue;

-            hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname());
+            hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0);
            if (!hw_device_ctx)
                continue;
        }