mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #28650 from amd:fast_pyrDown
imgproc: Optimized OpticalFlowPyrLK (PyrDownH) #28650 - Optimized Uchar PyrDown Horizontal processing intermediate storage. - Improved horizontal processing with AVX512 vbmi ISA. - Todo: further refinement for other archs. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Madan mohan Manokar
committed by
GitHub
GitHub
parent
06574736b3
commit
6b640b424c
@@ -12,6 +12,7 @@ ocv_add_dispatched_file(smooth SSE2 SSE4_1 AVX2 AVX512_ICL)
|
||||
ocv_add_dispatched_file(sumpixels SSE2 AVX2 AVX512_SKX)
|
||||
ocv_add_dispatched_file(equalize_hist AVX512_ICL)
|
||||
ocv_add_dispatched_file(imgwarp SSE4_1 AVX2 AVX512_SKX AVX512_ICL)
|
||||
ocv_add_dispatched_file(pyramids_avx512_vbmi AVX512_ICL)
|
||||
ocv_define_module(imgproc opencv_core WRAP java objc python js)
|
||||
|
||||
if(OPENCV_CORE_EXCLUDE_C_API)
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
|
||||
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
|
||||
// Copyright (C) 2014-2015, Itseez Inc., all rights reserved.
|
||||
// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without modification,
|
||||
@@ -54,6 +55,12 @@ template<typename T, int shift> struct FixPtCast
|
||||
typedef T rtype;
|
||||
rtype operator ()(type1 arg) const { return (T)((arg + (1 << (shift-1))) >> shift); }
|
||||
};
|
||||
template<typename T, int shift> struct FixPtUcharCast
|
||||
{
|
||||
typedef ushort type1;
|
||||
typedef T rtype;
|
||||
rtype operator ()(type1 arg) const { return (T)((arg + (1 << (shift-1))) >> shift); }
|
||||
};
|
||||
|
||||
template<typename T, int shift> struct FltCast
|
||||
{
|
||||
@@ -83,6 +90,76 @@ template<typename T1, typename T2> int PyrUpVecV(T1**, T2**, int) { return 0; }
|
||||
template<typename T1, typename T2> int PyrUpVecVOneRow(T1**, T2*, int) { return 0; }
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
// Dispatched AVX-512 VBMI implementations
|
||||
int PyrDownVecH_uchar_ushort_1_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_2_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_3_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_4_dispatch(const uchar* src, ushort* row, int width);
|
||||
|
||||
// uchar -> ushort intermediate storage implementations
|
||||
template<> int PyrDownVecH<uchar, ushort, 1>(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = PyrDownVecH_uchar_ushort_1_dispatch(src, row, width);
|
||||
return x;
|
||||
}
|
||||
|
||||
template<> int PyrDownVecH<uchar, ushort, 2>(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = PyrDownVecH_uchar_ushort_2_dispatch(src, row, width);
|
||||
return x;
|
||||
}
|
||||
|
||||
template<> int PyrDownVecH<uchar, ushort, 3>(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = PyrDownVecH_uchar_ushort_3_dispatch(src, row, width);
|
||||
return x;
|
||||
}
|
||||
|
||||
template<> int PyrDownVecH<uchar, ushort, 4>(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = PyrDownVecH_uchar_ushort_4_dispatch(src, row, width);
|
||||
return x;
|
||||
|
||||
}
|
||||
|
||||
template<> int PyrDownVecV<ushort, uchar>(ushort** src, uchar* dst, int width)
|
||||
{
|
||||
int x = 0;
|
||||
const ushort *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4];
|
||||
|
||||
for( ; x <= width - VTraits<v_uint8>::vlanes(); x += VTraits<v_uint8>::vlanes() )
|
||||
{
|
||||
v_uint16 r0, r1, r2, r3, r4, t0, t1;
|
||||
r0 = vx_load(row0 + x);
|
||||
r1 = vx_load(row1 + x);
|
||||
r2 = vx_load(row2 + x);
|
||||
r3 = vx_load(row3 + x);
|
||||
r4 = vx_load(row4 + x);
|
||||
t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2)));
|
||||
r0 = vx_load(row0 + x + VTraits<v_uint16>::vlanes());
|
||||
r1 = vx_load(row1 + x + VTraits<v_uint16>::vlanes());
|
||||
r2 = vx_load(row2 + x + VTraits<v_uint16>::vlanes());
|
||||
r3 = vx_load(row3 + x + VTraits<v_uint16>::vlanes());
|
||||
r4 = vx_load(row4 + x + VTraits<v_uint16>::vlanes());
|
||||
t1 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2)));
|
||||
v_store(dst + x, v_rshr_pack<8>(t0, t1));
|
||||
}
|
||||
if (x <= width - VTraits<v_uint16>::vlanes())
|
||||
{
|
||||
v_uint16 r0, r1, r2, r3, r4, t0;
|
||||
r0 = vx_load(row0 + x);
|
||||
r1 = vx_load(row1 + x);
|
||||
r2 = vx_load(row2 + x);
|
||||
r3 = vx_load(row3 + x);
|
||||
r4 = vx_load(row4 + x);
|
||||
t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2)));
|
||||
v_rshr_pack_store<8>(dst + x, t0);
|
||||
x += VTraits<v_uint16>::vlanes();
|
||||
}
|
||||
vx_cleanup();
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
template<> int PyrDownVecH<uchar, int, 1>(const uchar* src, int* row, int width)
|
||||
{
|
||||
@@ -1291,10 +1368,23 @@ void cv::pyrDown( InputArray _src, OutputArray _dst, const Size& _dsz, int borde
|
||||
{
|
||||
CALL_HAL(pyrDown, cv_hal_pyrdown, src.data, src.step, src.cols, src.rows, dst.data, dst.step, dst.cols, dst.rows, depth, src.channels(), borderType);
|
||||
}
|
||||
bool use_avx512vbmi = checkHardwareSupport(CPU_AVX_512VBMI);
|
||||
|
||||
PyrFunc func = 0;
|
||||
if( depth == CV_8U )
|
||||
func = pyrDown_< FixPtCast<uchar, 8> >;
|
||||
{
|
||||
if(use_avx512vbmi)
|
||||
{
|
||||
// intermediate storage in 16bit only improves when used along with AVX512_VBMI ISA.
|
||||
// AVX2: Usage with AVX2 has negligible measurable uplift in performance.
|
||||
// ARM: Using 16bit intermediate storage has shown 5 to 10% degradation on ARM platform.
|
||||
func = pyrDown_< FixPtUcharCast<uchar, 8> >;
|
||||
}
|
||||
else
|
||||
{
|
||||
func = pyrDown_< FixPtCast<uchar, 8> >;
|
||||
}
|
||||
}
|
||||
else if( depth == CV_16S )
|
||||
func = pyrDown_< FixPtCast<short, 8> >;
|
||||
else if( depth == CV_16U )
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
#include "pyramids_avx512_vbmi.simd.hpp"
|
||||
#include "pyramids_avx512_vbmi.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX512_ICL,...,BASELINE
|
||||
|
||||
namespace cv {
|
||||
|
||||
// declared in pyramids.cpp
|
||||
int PyrDownVecH_uchar_ushort_1_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_2_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_3_dispatch(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_4_dispatch(const uchar* src, ushort* row, int width);
|
||||
|
||||
int PyrDownVecH_uchar_ushort_1_dispatch(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
CV_CPU_DISPATCH(PyrDownVecH_uchar_ushort_1_vbmi, (src, row, width),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_2_dispatch(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
CV_CPU_DISPATCH(PyrDownVecH_uchar_ushort_2_vbmi, (src, row, width),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_3_dispatch(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
CV_CPU_DISPATCH(PyrDownVecH_uchar_ushort_3_vbmi, (src, row, width),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_4_dispatch(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
CV_CPU_DISPATCH(PyrDownVecH_uchar_ushort_4_vbmi, (src, row, width),
|
||||
CV_CPU_DISPATCH_MODES_ALL);
|
||||
}
|
||||
|
||||
} // namespace cv
|
||||
@@ -0,0 +1,259 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved.
|
||||
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
namespace cv {
|
||||
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
|
||||
|
||||
// AVX-512 VBMI vpermb-based PyrDownVecH for uchar->ushort.
|
||||
int PyrDownVecH_uchar_ushort_1_vbmi(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_2_vbmi(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_3_vbmi(const uchar* src, ushort* row, int width);
|
||||
int PyrDownVecH_uchar_ushort_4_vbmi(const uchar* src, ushort* row, int width);
|
||||
|
||||
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
|
||||
|
||||
int PyrDownVecH_uchar_ushort_1_vbmi(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = 0;
|
||||
|
||||
#if !CV_AVX_512VBMI
|
||||
CV_UNUSED(src); CV_UNUSED(row); CV_UNUSED(width);
|
||||
#else
|
||||
// cn=1: 32 output pixels per iteration.
|
||||
// Source stride = 2 bytes/pixel. Tap k for pixel i: src[2*i + k], k=0..4.
|
||||
// Max source byte: 2*31 + 4 = 66. Fits in 2 zmm (128 bytes).
|
||||
__m512i vidx0 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
62,60,58,56,54,52,50,48,46,44,42,40,38,36,34,32,
|
||||
30,28,26,24,22,20,18,16,14,12,10, 8, 6, 4, 2, 0);
|
||||
__m512i vidx1 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(1));
|
||||
__m512i vidx2 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(2));
|
||||
__m512i vidx3 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(3));
|
||||
__m512i vidx4 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(4));
|
||||
__m512i v6 = _mm512_set1_epi16(6);
|
||||
for (; x <= width - 32; x += 32, src += 64, row += 32)
|
||||
{
|
||||
__m512i sA = _mm512_loadu_si512(src);
|
||||
__m512i sB = _mm512_loadu_si512(src + 64);
|
||||
|
||||
__m512i t0 = _mm512_permutex2var_epi8(sA, vidx0, sB);
|
||||
__m512i t1 = _mm512_permutex2var_epi8(sA, vidx1, sB);
|
||||
__m512i t2 = _mm512_permutex2var_epi8(sA, vidx2, sB);
|
||||
__m512i t3 = _mm512_permutex2var_epi8(sA, vidx3, sB);
|
||||
__m512i t4 = _mm512_permutex2var_epi8(sA, vidx4, sB);
|
||||
|
||||
__m512i w0 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t0));
|
||||
__m512i w1 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t1));
|
||||
__m512i w2 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t2));
|
||||
__m512i w3 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t3));
|
||||
__m512i w4 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t4));
|
||||
|
||||
__m512i res = _mm512_add_epi16(
|
||||
_mm512_add_epi16(_mm512_mullo_epi16(w2, v6),
|
||||
_mm512_slli_epi16(_mm512_add_epi16(w1, w3), 2)),
|
||||
_mm512_add_epi16(w0, w4));
|
||||
|
||||
_mm512_storeu_si512(row, res);
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
#endif
|
||||
return x;
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_2_vbmi(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = 0;
|
||||
|
||||
#if !CV_AVX_512VBMI
|
||||
CV_UNUSED(src); CV_UNUSED(row); CV_UNUSED(width);
|
||||
#else
|
||||
// cn=2: 16 output pixels x 2 channels = 32 output ushorts per iteration.
|
||||
// Source stride = 4 bytes/pixel. Tap k for pixel i, ch j: src[4*i + 2*k + j].
|
||||
// Max source byte: 4*15 + 8 + 1 = 69. Fits in 2 zmm.
|
||||
__m512i vidx0 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
61,60,57,56,53,52,49,48,45,44,41,40,37,36,33,32,
|
||||
29,28,25,24,21,20,17,16,13,12, 9, 8, 5, 4, 1, 0);
|
||||
__m512i vidx1 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(2));
|
||||
__m512i vidx2 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(4));
|
||||
__m512i vidx3 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(6));
|
||||
__m512i vidx4 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(8));
|
||||
__m512i v6 = _mm512_set1_epi16(6);
|
||||
|
||||
for (; x <= width - 32; x += 32, src += 64, row += 32)
|
||||
{
|
||||
__m512i sA = _mm512_loadu_si512(src);
|
||||
__m512i sB = _mm512_loadu_si512(src + 64);
|
||||
|
||||
__m512i t0 = _mm512_permutex2var_epi8(sA, vidx0, sB);
|
||||
__m512i t1 = _mm512_permutex2var_epi8(sA, vidx1, sB);
|
||||
__m512i t2 = _mm512_permutex2var_epi8(sA, vidx2, sB);
|
||||
__m512i t3 = _mm512_permutex2var_epi8(sA, vidx3, sB);
|
||||
__m512i t4 = _mm512_permutex2var_epi8(sA, vidx4, sB);
|
||||
|
||||
__m512i w0 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t0));
|
||||
__m512i w1 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t1));
|
||||
__m512i w2 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t2));
|
||||
__m512i w3 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t3));
|
||||
__m512i w4 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t4));
|
||||
|
||||
__m512i res = _mm512_add_epi16(
|
||||
_mm512_add_epi16(_mm512_mullo_epi16(w2, v6),
|
||||
_mm512_slli_epi16(_mm512_add_epi16(w1, w3), 2)),
|
||||
_mm512_add_epi16(w0, w4));
|
||||
|
||||
_mm512_storeu_si512(row, res);
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
#endif
|
||||
return x;
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_3_vbmi(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = 0;
|
||||
|
||||
#if !CV_AVX_512VBMI
|
||||
CV_UNUSED(src); CV_UNUSED(row); CV_UNUSED(width);
|
||||
#else
|
||||
// Each iteration: 16 output pixels × 3 channels = 48 output ushorts.
|
||||
// Source span: 16×6 + 4*3 = 108 bytes. Load 128 contiguous bytes in 2 zmm.
|
||||
__m512i vidx0 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
92,91,90, 86,85,84, 80,79,78, 74,73,72, 68,67,66, 62,
|
||||
61,60, 56,55,54, 50,49,48, 44,43,42, 38,37,36, 32,31,
|
||||
30, 26,25,24, 20,19,18, 14,13,12, 8, 7, 6, 2, 1, 0);
|
||||
__m512i vidx1 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
95,94,93, 89,88,87, 83,82,81, 77,76,75, 71,70,69, 65,
|
||||
64,63, 59,58,57, 53,52,51, 47,46,45, 41,40,39, 35,34,
|
||||
33, 29,28,27, 23,22,21, 17,16,15, 11,10, 9, 5, 4, 3);
|
||||
__m512i vidx2 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
98,97,96, 92,91,90, 86,85,84, 80,79,78, 74,73,72, 68,
|
||||
67,66, 62,61,60, 56,55,54, 50,49,48, 44,43,42, 38,37,
|
||||
36, 32,31,30, 26,25,24, 20,19,18, 14,13,12, 8, 7, 6);
|
||||
__m512i vidx3 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
101,100,99, 95,94,93, 89,88,87, 83,82,81, 77,76,75, 71,
|
||||
70,69, 65,64,63, 59,58,57, 53,52,51, 47,46,45, 41,40,
|
||||
39, 35,34,33, 29,28,27, 23,22,21, 17,16,15, 11,10, 9);
|
||||
__m512i vidx4 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
104,103,102, 98,97,96, 92,91,90, 86,85,84, 80,79,78, 74,
|
||||
73,72, 68,67,66, 62,61,60, 56,55,54, 50,49,48, 44,43,
|
||||
42, 38,37,36, 32,31,30, 26,25,24, 20,19,18, 14,13,12);
|
||||
|
||||
__m512i v6 = _mm512_set1_epi16(6);
|
||||
|
||||
//16 output pixels × 3 channels
|
||||
for (; x <= width - 48; x += 48, src += 96, row += 48)
|
||||
{
|
||||
__m512i sA = _mm512_loadu_si512(src);
|
||||
__m512i sB = _mm512_loadu_si512(src + 64);
|
||||
|
||||
__m512i t0 = _mm512_permutex2var_epi8(sA, vidx0, sB);
|
||||
__m512i t1 = _mm512_permutex2var_epi8(sA, vidx1, sB);
|
||||
__m512i t2 = _mm512_permutex2var_epi8(sA, vidx2, sB);
|
||||
__m512i t3 = _mm512_permutex2var_epi8(sA, vidx3, sB);
|
||||
__m512i t4 = _mm512_permutex2var_epi8(sA, vidx4, sB);
|
||||
|
||||
// Expand lower 32 bytes of each tap to u16 (first 32 of 48 bytes)
|
||||
__m512i t0_lo = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t0));
|
||||
__m512i t1_lo = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t1));
|
||||
__m512i t2_lo = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t2));
|
||||
__m512i t3_lo = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t3));
|
||||
__m512i t4_lo = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t4));
|
||||
|
||||
// Expand upper 16 bytes (bytes 32..47) to u16
|
||||
__m256i t0_hi = _mm256_cvtepu8_epi16(_mm512_extracti32x4_epi32(t0, 2));
|
||||
__m256i t1_hi = _mm256_cvtepu8_epi16(_mm512_extracti32x4_epi32(t1, 2));
|
||||
__m256i t2_hi = _mm256_cvtepu8_epi16(_mm512_extracti32x4_epi32(t2, 2));
|
||||
__m256i t3_hi = _mm256_cvtepu8_epi16(_mm512_extracti32x4_epi32(t3, 2));
|
||||
__m256i t4_hi = _mm256_cvtepu8_epi16(_mm512_extracti32x4_epi32(t4, 2));
|
||||
|
||||
// Compute low half (32xu16 values):
|
||||
__m512i sum13_lo = _mm512_add_epi16(t1_lo, t3_lo);
|
||||
__m512i res_lo = _mm512_add_epi16(
|
||||
_mm512_add_epi16(_mm512_mullo_epi16(t2_lo, v6),
|
||||
_mm512_slli_epi16(sum13_lo, 2)),
|
||||
_mm512_add_epi16(t0_lo, t4_lo));
|
||||
|
||||
// Compute high half (16xu16 values):
|
||||
__m256i v6_256 = _mm256_set1_epi16(6);
|
||||
__m256i sum13_hi = _mm256_add_epi16(t1_hi, t3_hi);
|
||||
__m256i res_hi = _mm256_add_epi16(
|
||||
_mm256_add_epi16(_mm256_mullo_epi16(t2_hi, v6_256),
|
||||
_mm256_slli_epi16(sum13_hi, 2)),
|
||||
_mm256_add_epi16(t0_hi, t4_hi));
|
||||
|
||||
_mm512_storeu_si512(row, res_lo); //32 ushort
|
||||
_mm256_storeu_si256((__m256i*)(row + 32), res_hi); //16 ushort
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
#endif // CV_AVX_512VBMI
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
int PyrDownVecH_uchar_ushort_4_vbmi(const uchar* src, ushort* row, int width)
|
||||
{
|
||||
int x = 0;
|
||||
|
||||
#if !CV_AVX_512VBMI
|
||||
CV_UNUSED(src); CV_UNUSED(row); CV_UNUSED(width);
|
||||
#else
|
||||
// cn=4: 8 output pixels x 4 channels = 32 output ushorts per iteration.
|
||||
// Source stride = 8 bytes/pixel. Tap k for pixel i, ch j: src[8*i + 4*k + j].
|
||||
// Max source byte: 8*7 + 4*4 + 3 = 75. Fits in 2 zmm.
|
||||
__m512i vidx0 = _v512_set_epu8(
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
59,58,57,56,51,50,49,48,43,42,41,40,35,34,33,32,
|
||||
27,26,25,24,19,18,17,16,11,10, 9, 8, 3, 2, 1, 0);
|
||||
__m512i vidx1 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(4));
|
||||
__m512i vidx2 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(8));
|
||||
__m512i vidx3 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(12));
|
||||
__m512i vidx4 = _mm512_add_epi8(vidx0, _mm512_set1_epi8(16));
|
||||
|
||||
__m512i v6 = _mm512_set1_epi16(6);
|
||||
|
||||
for (; x <= width - 32; x += 32, src += 64, row += 32)
|
||||
{
|
||||
__m512i sA = _mm512_loadu_si512(src);
|
||||
__m512i sB = _mm512_loadu_si512(src + 64);
|
||||
|
||||
__m512i t0 = _mm512_permutex2var_epi8(sA, vidx0, sB);
|
||||
__m512i t1 = _mm512_permutex2var_epi8(sA, vidx1, sB);
|
||||
__m512i t2 = _mm512_permutex2var_epi8(sA, vidx2, sB);
|
||||
__m512i t3 = _mm512_permutex2var_epi8(sA, vidx3, sB);
|
||||
__m512i t4 = _mm512_permutex2var_epi8(sA, vidx4, sB);
|
||||
|
||||
__m512i w0 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t0));
|
||||
__m512i w1 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t1));
|
||||
__m512i w2 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t2));
|
||||
__m512i w3 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t3));
|
||||
__m512i w4 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(t4));
|
||||
|
||||
__m512i res = _mm512_add_epi16(
|
||||
_mm512_add_epi16(_mm512_mullo_epi16(w2, v6),
|
||||
_mm512_slli_epi16(_mm512_add_epi16(w1, w3), 2)),
|
||||
_mm512_add_epi16(w0, w4));
|
||||
|
||||
_mm512_storeu_si512(row, res);
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
#endif
|
||||
return x;
|
||||
}
|
||||
|
||||
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
|
||||
|
||||
CV_CPU_OPTIMIZATION_NAMESPACE_END
|
||||
} // namespace cv
|
||||
Reference in New Issue
Block a user