diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/cholesky.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/cholesky.hpp index aaf0d863f3..e519a1ad4a 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/cholesky.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/cholesky.hpp @@ -4,42 +4,21 @@ #ifndef OPENCV_HAL_RVV_CHOLESKY_HPP_INCLUDED #define OPENCV_HAL_RVV_CHOLESKY_HPP_INCLUDED +#include +#include #include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace cholesky { #undef cv_hal_Cholesky32f -#define cv_hal_Cholesky32f cv::cv_hal_rvv::cholesky::Cholesky +#define cv_hal_Cholesky32f cv::cv_hal_rvv::cholesky::Cholesky #undef cv_hal_Cholesky64f -#define cv_hal_Cholesky64f cv::cv_hal_rvv::cholesky::Cholesky - -template struct rvv; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e32m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vfloat32m4_t vfmv_v_f(float a, size_t b) { return __riscv_vfmv_v_f_f32m4(a, b); } - static inline vfloat32m1_t vfmv_s_f(float a, size_t b) { return __riscv_vfmv_s_f_f32m1(a, b); } - static inline vfloat32m4_t vle(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline vfloat32m4_t vlse(const float* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_f32m4(a, b, c); } - static inline void vsse(float* a, ptrdiff_t b, vfloat32m4_t c, size_t d) { __riscv_vsse32(a, b, c, d); } -}; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e64m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e64m4(a); } - static inline vfloat64m4_t vfmv_v_f(double a, size_t b) { return __riscv_vfmv_v_f_f64m4(a, b); } - static inline vfloat64m1_t vfmv_s_f(double a, size_t b) { return __riscv_vfmv_s_f_f64m1(a, b); } - static inline vfloat64m4_t vle(const double* a, size_t b) { return __riscv_vle64_v_f64m4(a, b); } - static inline vfloat64m4_t vlse(const double* a, ptrdiff_t b, size_t c) { return __riscv_vlse64_v_f64m4(a, b, c); } - static inline void vsse(double* a, ptrdiff_t b, vfloat64m4_t c, size_t d) { __riscv_vsse64(a, b, c, d); } -}; +#define cv_hal_Cholesky64f cv::cv_hal_rvv::cholesky::Cholesky // the algorithm is copied from core/src/matrix_decomp.cpp, // in the function template static int cv::CholImpl -template +template inline int Cholesky(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n, bool* info) { int i, j, k; @@ -47,30 +26,30 @@ inline int Cholesky(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, src1_step /= sizeof(src1[0]); src2_step /= sizeof(src2[0]); - int vlmax = rvv::vsetvlmax(), vl; + int vlmax = RVV_T::setvlmax(), vl; for( i = 0; i < m; i++ ) { for( j = 0; j < i; j++ ) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < j; k += vl ) { - vl = rvv::vsetvl(j - k); - auto vec_src1 = rvv::vle(src1 + i * src1_step + k, vl); - auto vec_src2 = rvv::vle(src1 + j * src1_step + k, vl); + vl = RVV_T::setvl(j - k); + auto vec_src1 = RVV_T::vload(src1 + i * src1_step + k, vl); + auto vec_src2 = RVV_T::vload(src1 + j * src1_step + k, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - s = src1[i*src1_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + s = src1[i*src1_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV::vmv_s(0, vlmax), vlmax)); src1[i*src1_step + j] = (T)(s*src1[j*src1_step + j]); } - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < j; k += vl ) { - vl = rvv::vsetvl(j - k); - auto vec_src = rvv::vle(src1 + i * src1_step + k, vl); + vl = RVV_T::setvl(j - k); + auto vec_src = RVV_T::vload(src1 + i * src1_step + k, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src, vec_src, vl); } - s = src1[i*src1_step + i] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + s = src1[i*src1_step + i] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV::vmv_s(0, vlmax), vlmax)); if( s < std::numeric_limits::epsilon() ) { *info = false; @@ -83,10 +62,10 @@ inline int Cholesky(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, { for( i = 0; i < m; i += vl ) { - vl = rvv::vsetvl(m - i); - auto vec_src = rvv::vlse(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vl); + vl = RVV_T::setvl(m - i); + auto vec_src = RVV_T::vload_stride(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vl); vec_src = __riscv_vfrdiv(vec_src, 1, vl); - rvv::vsse(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vec_src, vl); + RVV_T::vstore_stride(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vec_src, vl); } *info = true; return CV_HAL_ERROR_OK; @@ -96,15 +75,15 @@ inline int Cholesky(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, { for( j = 0; j < n; j++ ) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < i; k += vl ) { - vl = rvv::vsetvl(i - k); - auto vec_src1 = rvv::vle(src1 + i * src1_step + k, vl); - auto vec_src2 = rvv::vlse(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); + vl = RVV_T::setvl(i - k); + auto vec_src1 = RVV_T::vload(src1 + i * src1_step + k, vl); + auto vec_src2 = RVV_T::vload_stride(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - s = src2[i*src2_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + s = src2[i*src2_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV::vmv_s(0, vlmax), vlmax)); src2[i*src2_step + j] = (T)(s*src1[i*src1_step + i]); } } @@ -113,24 +92,24 @@ inline int Cholesky(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, { for( j = 0; j < n; j++ ) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = i + 1; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vlse(src1 + k * src1_step + i, sizeof(T) * src1_step, vl); - auto vec_src2 = rvv::vlse(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload_stride(src1 + k * src1_step + i, sizeof(T) * src1_step, vl); + auto vec_src2 = RVV_T::vload_stride(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - s = src2[i*src2_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + s = src2[i*src2_step + j] - __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV::vmv_s(0, vlmax), vlmax)); src2[i*src2_step + j] = (T)(s*src1[i*src1_step + i]); } } for( i = 0; i < m; i += vl ) { - vl = rvv::vsetvl(m - i); - auto vec_src = rvv::vlse(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vl); + vl = RVV_T::setvl(m - i); + auto vec_src = RVV_T::vload_stride(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vl); vec_src = __riscv_vfrdiv(vec_src, 1, vl); - rvv::vsse(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vec_src, vl); + RVV_T::vstore_stride(src1 + i * src1_step + i, sizeof(T) * (src1_step + 1), vec_src, vl); } *info = true; diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/dxt.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/dxt.hpp index 3c763b386b..11ac133760 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/dxt.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/dxt.hpp @@ -5,6 +5,8 @@ #define OPENCV_HAL_RVV_DXT_HPP_INCLUDED #include +#include "hal_rvv_1p0/types.hpp" +#include "opencv2/core/types.hpp" namespace cv { namespace cv_hal_rvv { namespace dxt { @@ -13,29 +15,21 @@ namespace cv { namespace cv_hal_rvv { namespace dxt { template struct rvv; -template<> struct rvv +template<> struct rvv : RVV_F32MF2 { - using T = vfloat32mf2_t; - static inline size_t vsetvl_itab(size_t a) { return __riscv_vsetvl_e32m8(a); } - static inline vuint32m8_t vlse_itab(const uint* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_u32m8(a, b, c); } - static inline vfloat32m8_t vlse_itab_f(const float* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_f32m8(a, b, c); } - static inline void vsse_itab(float* a, ptrdiff_t b, vfloat32m8_t c, size_t d) { return __riscv_vsse32(a, b, c, d); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e32mf2(a); } - static inline vfloat32m1_t vfmv_s(float a, size_t b) { return __riscv_vfmv_s_f_f32m1(a, b); } + using T = RVV_F32MF2::VecType; + using TabType = RVV_U32M8; + using TabTypeF = RVV_SameLen; static inline void vlseg(const float* a, T& b, T& c, size_t d) { auto x = __riscv_vlseg2e32_v_f32mf2x2(a, d); b = __riscv_vget_v_f32mf2x2_f32mf2(x, 0), c = __riscv_vget_v_f32mf2x2_f32mf2(x, 1); } static inline void vlsseg(const float* a, ptrdiff_t b, T& c, T& d, size_t e) { auto x = __riscv_vlsseg2e32_v_f32mf2x2(a, b, e); c = __riscv_vget_v_f32mf2x2_f32mf2(x, 0), d = __riscv_vget_v_f32mf2x2_f32mf2(x, 1); } static inline void vsseg(float* a, T b, T c, size_t d) { __riscv_vsseg2e32(a, __riscv_vset_v_f32mf2_f32mf2x2(__riscv_vset_v_f32mf2_f32mf2x2(vfloat32mf2x2_t(), 0, b), 1, c), d); } }; -template<> struct rvv +template<> struct rvv : RVV_F64M1 { - using T = vfloat64m1_t; - static inline size_t vsetvl_itab(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vuint32m4_t vlse_itab(const uint* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_u32m4(a, b, c); } - static inline vfloat64m8_t vlse_itab_f(const double* a, ptrdiff_t b, size_t c) { return __riscv_vlse64_v_f64m8(a, b, c); } - static inline void vsse_itab(double* a, ptrdiff_t b, vfloat64m8_t c, size_t d) { return __riscv_vsse64(a, b, c, d); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e64m1(a); } - static inline vfloat64m1_t vfmv_s(double a, size_t b) { return __riscv_vfmv_s_f_f64m1(a, b); } + using T = RVV_F64M1::VecType; + using TabType = RVV_U32M4; + using TabTypeF = RVV_SameLen; static inline void vlseg(const double* a, T& b, T& c, size_t d) { auto x = __riscv_vlseg2e64_v_f64m1x2(a, d); b = __riscv_vget_v_f64m1x2_f64m1(x, 0), c = __riscv_vget_v_f64m1x2_f64m1(x, 1); } static inline void vlsseg(const double* a, ptrdiff_t b, T& c, T& d, size_t e) { auto x = __riscv_vlsseg2e64_v_f64m1x2(a, b, e); c = __riscv_vget_v_f64m1x2_f64m1(x, 0), d = __riscv_vget_v_f64m1x2_f64m1(x, 1); } static inline void vsseg(double* a, T b, T c, size_t d) { __riscv_vsseg2e64(a, __riscv_vset_v_f64m1_f64m1x2(__riscv_vset_v_f64m1_f64m1x2(vfloat64m1x2_t(), 0, b), 1, c), d); } @@ -43,7 +37,7 @@ template<> struct rvv // the algorithm is copied from core/src/dxt.cpp, // in the function template static void cv::DFT and cv::DFT_R2, cv::DFT_R3, cv::DFT_R5 -template +template inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T scale, int* itab, const Complex* wave, int tab_size, int len, bool isInverse, bool noPermute) { @@ -52,7 +46,9 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s int dw0 = tab_size, dw; int i, j, k; Complex t; - using VT = typename rvv::T; + using VT = typename rvv::VecType; + using TabType = typename rvv::TabType; + using TabTypeF = typename rvv::TabTypeF; int tab_step = tab_size == n ? 1 : tab_size == n*2 ? 2 : tab_size/n; int vl; @@ -64,29 +60,29 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s { for( i = 0; i < n; i += vl ) { - vl = rvv::vsetvl_itab(n - i); - auto vec_itab = rvv::vlse_itab(reinterpret_cast(itab + i * tab_step), sizeof(int) * tab_step, vl); + vl = TabType::setvl(n - i); + auto vec_itab = TabType::vload_stride(reinterpret_cast(itab + i * tab_step), sizeof(int) * tab_step, vl); vec_itab = __riscv_vmul(vec_itab, sizeof(T) * 2, vl); auto vec_src_re = __riscv_vloxei32(reinterpret_cast(src), vec_itab, vl); vec_itab = __riscv_vadd(vec_itab, sizeof(T), vl); auto vec_src_im = __riscv_vloxei32(reinterpret_cast(src), vec_itab, vl); - rvv::vsse_itab(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); - rvv::vsse_itab(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); } } else { for( i = 0; i < n; i += vl ) { - vl = rvv::vsetvl_itab(n - i); - auto vec_itab = rvv::vlse_itab(reinterpret_cast(itab + i * tab_step), sizeof(int) * tab_step, vl); + vl = TabType::setvl(n - i); + auto vec_itab = TabType::vload_stride(reinterpret_cast(itab + i * tab_step), sizeof(int) * tab_step, vl); vec_itab = __riscv_vmul(vec_itab, sizeof(T) * 2, vl); auto vec_src_re = __riscv_vloxei32(reinterpret_cast(src), vec_itab, vl); vec_itab = __riscv_vadd(vec_itab, sizeof(T), vl); auto vec_src_im = __riscv_vloxei32(reinterpret_cast(src), vec_itab, vl); vec_src_im = __riscv_vfneg(vec_src_im, vl); - rvv::vsse_itab(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); - rvv::vsse_itab(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); } } } @@ -131,10 +127,10 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s { for( i = 0; i < n; i += vl ) { - vl = rvv::vsetvl_itab(n - i); - auto vec_src_im = rvv::vlse_itab_f(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); + vl = TabType::setvl(n - i); + auto vec_src_im = TabTypeF::vload_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); vec_src_im = __riscv_vfneg(vec_src_im, vl); - rvv::vsse_itab(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); } } } @@ -177,7 +173,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( j = 1; j < nx; j += vl ) { - vl = rvv::vsetvl(nx - j); + vl = rvv::setvl(nx - j); v0 = dst + i + j; v1 = v0 + nx*2; @@ -238,7 +234,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( j = 1; j < nx; j += vl ) { - vl = rvv::vsetvl(nx - j); + vl = rvv::setvl(nx - j); v = dst + i + j; VT vec_re, vec_im, vec_w_re, vec_w_im; @@ -284,7 +280,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( j = 1; j < nx; j += vl ) { - vl = rvv::vsetvl(nx - j); + vl = rvv::setvl(nx - j); v = dst + i + j; VT vec_re, vec_im, vec_w_re, vec_w_im; @@ -323,7 +319,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s { for( j = 0; j < nx; j += vl ) { - vl = rvv::vsetvl(nx - j); + vl = rvv::setvl(nx - j); Complex* v0 = dst + i + j; Complex* v1 = v0 + nx*2; Complex* v2 = v1 + nx*2; @@ -417,7 +413,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s { for( p = 1; p <= factor2; p += vl ) { - vl = rvv::vsetvl(factor2 + 1 - p); + vl = rvv::setvl(factor2 + 1 - p); VT vec_a_re, vec_a_im, vec_b_re, vec_b_im; rvv::vlsseg(reinterpret_cast(v + nx * p), sizeof(T) * nx * 2, vec_a_re, vec_a_im, vl); @@ -427,8 +423,8 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s auto vec_i0 = __riscv_vfsub(vec_a_im, vec_b_im, vl); auto vec_i1 = __riscv_vfadd(vec_a_im, vec_b_im, vl); - vn_0.re += __riscv_vfmv_f(__riscv_vfredosum(vec_r0, rvv::vfmv_s(0, vl), vl)); - vn_0.im += __riscv_vfmv_f(__riscv_vfredosum(vec_i1, rvv::vfmv_s(0, vl), vl)); + vn_0.re += __riscv_vfmv_f(__riscv_vfredosum(vec_r0, RVV::vmv_s(0, vl), vl)); + vn_0.im += __riscv_vfmv_f(__riscv_vfredosum(vec_i1, RVV::vmv_s(0, vl), vl)); rvv::vsseg(reinterpret_cast(a + p - 1), vec_r0, vec_i0, vl); rvv::vsseg(reinterpret_cast(b + p - 1), vec_r1, vec_i1, vl); @@ -440,7 +436,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( p = 1; p <= factor2; p += vl ) { - vl = rvv::vsetvl(factor2 + 1 - p); + vl = rvv::setvl(factor2 + 1 - p); VT vec_re, vec_im, vec_w_re, vec_w_im; rvv::vlsseg(reinterpret_cast(v + nx * p), sizeof(T) * nx * 2, vec_re, vec_im, vl); @@ -458,8 +454,8 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s vec_r1 = __riscv_vfsub(vec_r2, vec_r1, vl); vec_i1 = __riscv_vfadd(vec_i2, vec_i1, vl); - vn_0.re += __riscv_vfmv_f(__riscv_vfredosum(vec_r0, rvv::vfmv_s(0, vl), vl)); - vn_0.im += __riscv_vfmv_f(__riscv_vfredosum(vec_i1, rvv::vfmv_s(0, vl), vl)); + vn_0.re += __riscv_vfmv_f(__riscv_vfredosum(vec_r0, RVV::vmv_s(0, vl), vl)); + vn_0.im += __riscv_vfmv_f(__riscv_vfredosum(vec_i1, RVV::vmv_s(0, vl), vl)); rvv::vsseg(reinterpret_cast(a + p - 1), vec_r0, vec_i0, vl); rvv::vsseg(reinterpret_cast(b + p - 1), vec_r1, vec_i1, vl); @@ -480,7 +476,7 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( q = 0; q < factor2; q += vl ) { - vl = rvv::vsetvl(factor2 - q); + vl = rvv::setvl(factor2 - q); auto vec_d = __riscv_vadd(vec_dd, (q + 1) * dd % tab_size, vl); auto vmask = __riscv_vmsgeu(vec_d, tab_size, vl); @@ -498,10 +494,10 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s auto vec_i0 = __riscv_vfmul(vec_w, vec_a_im, vl); auto vec_i1 = __riscv_vfmul(vec_w, vec_b_re, vl); - T r0 = __riscv_vfmv_f(__riscv_vfredosum(vec_r0, rvv::vfmv_s(0, vl), vl)); - T i0 = __riscv_vfmv_f(__riscv_vfredosum(vec_i0, rvv::vfmv_s(0, vl), vl)); - T r1 = __riscv_vfmv_f(__riscv_vfredosum(vec_r1, rvv::vfmv_s(0, vl), vl)); - T i1 = __riscv_vfmv_f(__riscv_vfredosum(vec_i1, rvv::vfmv_s(0, vl), vl)); + T r0 = __riscv_vfmv_f(__riscv_vfredosum(vec_r0, RVV::vmv_s(0, vl), vl)); + T i0 = __riscv_vfmv_f(__riscv_vfredosum(vec_i0, RVV::vmv_s(0, vl), vl)); + T r1 = __riscv_vfmv_f(__riscv_vfredosum(vec_r1, RVV::vmv_s(0, vl), vl)); + T i1 = __riscv_vfmv_f(__riscv_vfredosum(vec_i1, RVV::vmv_s(0, vl), vl)); s1.re += r0 + i0; s0.re += r0 - i0; s1.im += r1 - i1; s0.im += r1 + i1; @@ -523,23 +519,23 @@ inline int dft(const Complex* src, Complex* dst, int nf, int *factors, T s for( i = 0; i < len; i += vl ) { - vl = rvv::vsetvl_itab(len - i); - auto vec_src_re = rvv::vlse_itab_f(reinterpret_cast(dst + i), sizeof(T) * 2, vl); - auto vec_src_im = rvv::vlse_itab_f(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); + vl = TabType::setvl(len - i); + auto vec_src_re = TabTypeF::vload_stride(reinterpret_cast(dst + i), sizeof(T) * 2, vl); + auto vec_src_im = TabTypeF::vload_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); vec_src_re = __riscv_vfmul(vec_src_re, re_scale, vl); vec_src_im = __riscv_vfmul(vec_src_im, im_scale, vl); - rvv::vsse_itab(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); - rvv::vsse_itab(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i), sizeof(T) * 2, vec_src_re, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); } } else if( isInverse ) { for( i = 0; i < len; i += vl ) { - vl = rvv::vsetvl_itab(len - i); - auto vec_src_im = rvv::vlse_itab_f(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); + vl = TabType::setvl(len - i); + auto vec_src_im = TabTypeF::vload_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vl); vec_src_im = __riscv_vfneg(vec_src_im, vl); - rvv::vsse_itab(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); + TabTypeF::vstore_stride(reinterpret_cast(dst + i) + 1, sizeof(T) * 2, vec_src_im, vl); } } diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/flip.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/flip.hpp index fe5c5a8951..95de5793ed 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/flip.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/flip.hpp @@ -5,7 +5,7 @@ #include #include -#include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { @@ -14,62 +14,33 @@ namespace cv { namespace cv_hal_rvv { struct FlipVlen256 { - using TableElemType = uchar; - using TableType = vuint8m8_t; + using SrcType = RVV_U8M8; + using TabType = RVV_U8M8; + using TabVecType = typename TabType::VecType; - static inline size_t setvlmax() + static inline void gather(const uchar* src, TabVecType tab, uchar* dst, size_t vl) { - return __riscv_vsetvlmax_e8m8(); - } - - static inline TableType vid(size_t vl) - { - return __riscv_vid_v_u8m8(vl); - } - - static inline TableType loadTable(const TableElemType* ptr, size_t vl) - { - return __riscv_vle8_v_u8m8(ptr, vl); - } - - static inline void gather(const uchar* src, TableType tab, uchar* dst, size_t vl) - { - auto v = __riscv_vle8_v_u8m8(src, vl); - __riscv_vse8(dst, __riscv_vrgather(v, tab, vl), vl); + auto src_v = SrcType::vload(src, vl); + SrcType::vstore(dst, __riscv_vrgather(src_v, tab, vl), vl); } }; -struct FlipVlen512 +struct FlipVlen512 : RVV_U8M8 { - using TableElemType = uint16_t; - using TableType = vuint16m8_t; + using SrcType = RVV_U8M4; + using TabType = RVV_U16M8; + using TabVecType = typename TabType::VecType; - static inline size_t setvlmax() + static inline void gather(const uchar* src, TabVecType tab, uchar* dst, size_t vl) { - return __riscv_vsetvlmax_e8m4(); - } - - static inline TableType vid(size_t vl) - { - return __riscv_vid_v_u16m8(vl); - } - - static inline TableType loadTable(const TableElemType* ptr, size_t vl) - { - return __riscv_vle16_v_u16m8(ptr, vl); - } - - static inline void gather(const uchar* src, TableType tab, uchar* dst, size_t vl) - { - auto v = __riscv_vle8_v_u8m4(src, vl); - __riscv_vse8(dst, __riscv_vrgatherei16(v, tab, vl), vl); + auto src_v = SrcType::vload(src, vl); + SrcType::vstore(dst, __riscv_vrgatherei16(src_v, tab, vl), vl); } }; template -inline void flipFillBuffer(cv::AutoBuffer& _buf, size_t len, int esz) +inline void flipFillBuffer(T* buf, size_t len, int esz) { - T* buf = _buf.data(); for (int i = (int)len - esz; i >= 0; i -= esz, buf += esz) for (int j = 0; j < esz; j++) buf[j] = (T)(i + j); @@ -107,7 +78,9 @@ inline void flipX(int esz, } } -template +template inline void flipY(int esz, const uchar* src_data, size_t src_step, @@ -117,15 +90,16 @@ inline void flipY(int esz, size_t dst_step) { size_t w = (size_t)src_width * esz; - size_t vl = std::min(FlipVlen::setvlmax() / esz * esz, w); - typename FlipVlen::TableType tab_v; + size_t vl = std::min(SrcType::setvlmax() / esz * esz, w); + typename TabType::VecType tab_v; if (esz == 1) - tab_v = __riscv_vrsub(FlipVlen::vid(vl), vl - 1, vl); + tab_v = __riscv_vrsub(TabType::vid(vl), vl - 1, vl); else { - cv::AutoBuffer buf(vl); + // max vlen supported is 1024 (vlmax of u8m4 for vlen 1024 is 512) + typename TabType::ElemType buf[512]; flipFillBuffer(buf, vl, esz); - tab_v = FlipVlen::loadTable(buf.data(), vl); + tab_v = TabType::vload(buf, vl); } if (vl == w) for (; src_height; src_height--, src_data += src_step, dst_data += dst_step) @@ -143,7 +117,9 @@ inline void flipY(int esz, } } -template +template inline void flipXY(int esz, const uchar* src_data, size_t src_step, @@ -153,15 +129,16 @@ inline void flipXY(int esz, size_t dst_step) { size_t w = (size_t)src_width * esz; - size_t vl = std::min(FlipVlen::setvlmax() / esz * esz, w); - typename FlipVlen::TableType tab_v; + size_t vl = std::min(SrcType::setvlmax() / esz * esz, w); + typename TabType::VecType tab_v; if (esz == 1) - tab_v = __riscv_vrsub(FlipVlen::vid(vl), vl - 1, vl); + tab_v = __riscv_vrsub(TabType::vid(vl), vl - 1, vl); else { - cv::AutoBuffer buf(vl); + // max vlen supported is 1024 (vlmax of u8m4 for vlen 1024 is 512) + typename TabType::ElemType buf[512]; flipFillBuffer(buf, vl, esz); - tab_v = FlipVlen::loadTable(buf.data(), vl); + tab_v = TabType::vload(buf, vl); } auto src0 = src_data, src1 = src_data + src_step * (src_height - 1); auto dst0 = dst_data, dst1 = dst_data + dst_step * (src_height - 1); diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/lu.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/lu.hpp index d9988972da..b90f6de53d 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/lu.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/lu.hpp @@ -4,42 +4,22 @@ #ifndef OPENCV_HAL_RVV_LU_HPP_INCLUDED #define OPENCV_HAL_RVV_LU_HPP_INCLUDED +#include +#include +#include #include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace lu { #undef cv_hal_LU32f -#define cv_hal_LU32f cv::cv_hal_rvv::lu::LU +#define cv_hal_LU32f cv::cv_hal_rvv::lu::LU #undef cv_hal_LU64f -#define cv_hal_LU64f cv::cv_hal_rvv::lu::LU - -template struct rvv; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e32m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vfloat32m4_t vfmv_v_f(float a, size_t b) { return __riscv_vfmv_v_f_f32m4(a, b); } - static inline vfloat32m1_t vfmv_s_f(float a, size_t b) { return __riscv_vfmv_s_f_f32m1(a, b); } - static inline vfloat32m4_t vle(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline vfloat32m4_t vlse(const float* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_f32m4(a, b, c); } - static inline void vse(float* a, vfloat32m4_t b, size_t c) { __riscv_vse32(a, b, c); } -}; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e64m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e64m4(a); } - static inline vfloat64m4_t vfmv_v_f(double a, size_t b) { return __riscv_vfmv_v_f_f64m4(a, b); } - static inline vfloat64m1_t vfmv_s_f(double a, size_t b) { return __riscv_vfmv_s_f_f64m1(a, b); } - static inline vfloat64m4_t vle(const double* a, size_t b) { return __riscv_vle64_v_f64m4(a, b); } - static inline vfloat64m4_t vlse(const double* a, ptrdiff_t b, size_t c) { return __riscv_vlse64_v_f64m4(a, b, c); } - static inline void vse(double* a, vfloat64m4_t b, size_t c) { __riscv_vse64(a, b, c); } -}; +#define cv_hal_LU64f cv::cv_hal_rvv::lu::LU // the algorithm is copied from core/src/matrix_decomp.cpp, // in the function template static int cv::LUImpl -template +template inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n, int* info) { T eps; @@ -54,7 +34,7 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n src1_step /= sizeof(src1[0]); src2_step /= sizeof(src2[0]); - int vlmax = rvv::vsetvlmax(), vl; + int vlmax = RVV_T::setvlmax(), vl; if( src2 ) { for( i = 0; i < m; i++ ) @@ -75,19 +55,19 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n { for( j = i; j < m; j += vl ) { - vl = rvv::vsetvl(m - j); - auto vec_src1 = rvv::vle(src1 + i * src1_step + j, vl); - auto vec_src2 = rvv::vle(src1 + k * src1_step + j, vl); - rvv::vse(src1 + k * src1_step + j, vec_src1, vl); - rvv::vse(src1 + i * src1_step + j, vec_src2, vl); + vl = RVV_T::setvl(m - j); + auto vec_src1 = RVV_T::vload(src1 + i * src1_step + j, vl); + auto vec_src2 = RVV_T::vload(src1 + k * src1_step + j, vl); + RVV_T::vstore(src1 + k * src1_step + j, vec_src1, vl); + RVV_T::vstore(src1 + i * src1_step + j, vec_src2, vl); } for( j = 0; j < n; j += vl ) { - vl = rvv::vsetvl(n - j); - auto vec_src1 = rvv::vle(src2 + i * src2_step + j, vl); - auto vec_src2 = rvv::vle(src2 + k * src2_step + j, vl); - rvv::vse(src2 + k * src2_step + j, vec_src1, vl); - rvv::vse(src2 + i * src2_step + j, vec_src2, vl); + vl = RVV_T::setvl(n - j); + auto vec_src1 = RVV_T::vload(src2 + i * src2_step + j, vl); + auto vec_src2 = RVV_T::vload(src2 + k * src2_step + j, vl); + RVV_T::vstore(src2 + k * src2_step + j, vec_src1, vl); + RVV_T::vstore(src2 + i * src2_step + j, vec_src2, vl); } p = -p; } @@ -100,19 +80,19 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n for( k = i+1; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src = rvv::vle(src1 + i * src1_step + k, vl); - auto vec_dst = rvv::vle(src1 + j * src1_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src = RVV_T::vload(src1 + i * src1_step + k, vl); + auto vec_dst = RVV_T::vload(src1 + j * src1_step + k, vl); vec_dst = __riscv_vfmacc(vec_dst, alpha, vec_src, vl); - rvv::vse(src1 + j * src1_step + k, vec_dst, vl); + RVV_T::vstore(src1 + j * src1_step + k, vec_dst, vl); } for( k = 0; k < n; k += vl ) { - vl = rvv::vsetvl(n - k); - auto vec_src = rvv::vle(src2 + i * src2_step + k, vl); - auto vec_dst = rvv::vle(src2 + j * src2_step + k, vl); + vl = RVV_T::setvl(n - k); + auto vec_src = RVV_T::vload(src2 + i * src2_step + k, vl); + auto vec_dst = RVV_T::vload(src2 + j * src2_step + k, vl); vec_dst = __riscv_vfmacc(vec_dst, alpha, vec_src, vl); - rvv::vse(src2 + j * src2_step + k, vec_dst, vl); + RVV_T::vstore(src2 + j * src2_step + k, vec_dst, vl); } } } @@ -121,15 +101,15 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n for( j = 0; j < n; j++ ) { T s = src2[i*src2_step + j]; - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = i+1; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(src1 + i * src1_step + k, vl); - auto vec_src2 = rvv::vlse(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(src1 + i * src1_step + k, vl); + auto vec_src2 = RVV_T::vload_stride(src2 + k * src2_step + j, sizeof(T) * src2_step, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - s -= __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + s -= __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); src2[i*src2_step + j] = s/src1[i*src1_step + i]; } } @@ -153,11 +133,11 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n { for( j = i; j < m; j += vl ) { - vl = rvv::vsetvl(m - j); - auto vec_src1 = rvv::vle(src1 + i * src1_step + j, vl); - auto vec_src2 = rvv::vle(src1 + k * src1_step + j, vl); - rvv::vse(src1 + k * src1_step + j, vec_src1, vl); - rvv::vse(src1 + i * src1_step + j, vec_src2, vl); + vl = RVV_T::setvl(m - j); + auto vec_src1 = RVV_T::vload(src1 + i * src1_step + j, vl); + auto vec_src2 = RVV_T::vload(src1 + k * src1_step + j, vl); + RVV_T::vstore(src1 + k * src1_step + j, vec_src1, vl); + RVV_T::vstore(src1 + i * src1_step + j, vec_src2, vl); } p = -p; } @@ -170,11 +150,11 @@ inline int LU(T* src1, size_t src1_step, int m, T* src2, size_t src2_step, int n for( k = i+1; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src = rvv::vle(src1 + i * src1_step + k, vl); - auto vec_dst = rvv::vle(src1 + j * src1_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src = RVV_T::vload(src1 + i * src1_step + k, vl); + auto vec_dst = RVV_T::vload(src1 + j * src1_step + k, vl); vec_dst = __riscv_vfmacc(vec_dst, alpha, vec_src, vl); - rvv::vse(src1 + j * src1_step + k, vec_dst, vl); + RVV_T::vstore(src1 + j * src1_step + k, vec_dst, vl); } } } diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/lut.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/lut.hpp index b5c4008186..e869731ce5 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/lut.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/lut.hpp @@ -7,95 +7,61 @@ #include #include +#include "hal_rvv_1p0/types.hpp" + namespace cv { namespace cv_hal_rvv { #undef cv_hal_lut #define cv_hal_lut cv::cv_hal_rvv::lut // need vlen >= 256 -struct LUTCacheU8 +struct LUTCacheU8 : RVV_U8M8 { - using ElemType = uchar; + using TabType = RVV_U8M8; + using SrcType = RVV_SameLen; + using IdxType = RVV_SameLen; + + using ElemType = typename TabType::ElemType; constexpr static size_t elem_size = sizeof(ElemType); - static inline vuint8m8_t loadTable(const ElemType* lut_data) + static inline typename TabType::VecType + gather(typename IdxType::VecType src_v, typename TabType::VecType lut_v, size_t vl) { - return __riscv_vle8_v_u8m8(lut_data, 256); - } - - static inline size_t setvl(size_t len) - { - return __riscv_vsetvl_e8m8(len); - } - - static inline size_t setvlmax() - { - return __riscv_vsetvlmax_e8m8(); - } - - static inline void lut(const uchar* src, vuint8m8_t lut_v, ElemType* dst, size_t vl) - { - auto src_v = __riscv_vle8_v_u8m8(src, vl); - auto dst_v = __riscv_vrgather(lut_v, src_v, vl); - __riscv_vse8(dst, dst_v, vl); + return __riscv_vrgather(lut_v, src_v, vl); } }; // need vlen >= 512 -struct LUTCacheU16 +struct LUTCacheU16 : RVV_U16M8 { - using ElemType = uint16_t; + using TabType = RVV_U16M8; + using SrcType = RVV_SameLen; + using IdxType = RVV_SameLen; + + using ElemType = typename TabType::ElemType; constexpr static size_t elem_size = sizeof(ElemType); - static inline vuint16m8_t loadTable(const ElemType* lut_data) + static inline typename TabType::VecType + gather(typename IdxType::VecType src_v, typename TabType::VecType lut_v, size_t vl) { - return __riscv_vle16_v_u16m8(lut_data, 256); - } - - static inline size_t setvl(size_t len) - { - return __riscv_vsetvl_e16m8(len); - } - - static inline size_t setvlmax() - { - return __riscv_vsetvlmax_e16m8(); - } - - static inline void lut(const uchar* src, vuint16m8_t lut_v, ElemType* dst, size_t vl) - { - auto src_v = __riscv_vzext_vf2(__riscv_vle8_v_u8m4(src, vl), vl); - auto dst_v = __riscv_vrgather(lut_v, src_v, vl); - __riscv_vse16(dst, dst_v, vl); + return __riscv_vrgather(lut_v, src_v, vl); } }; // need vlen >= 1024 -struct LUTCacheU32 +struct LUTCacheU32 : RVV_U32M8 { - using ElemType = uint32_t; + using TabType = RVV_U32M8; + using SrcType = RVV_SameLen; + using IdxType = RVV_SameLen; + + using ElemType = typename TabType::ElemType; constexpr static size_t elem_size = sizeof(ElemType); - static inline vuint32m8_t loadTable(const ElemType* lut_data) + static inline typename TabType::VecType + gather(typename IdxType::VecType src_v, typename TabType::VecType lut_v, size_t vl) { - return __riscv_vle32_v_u32m8(lut_data, 256); - } - - static inline size_t setvl(size_t len) - { - return __riscv_vsetvl_e32m8(len); - } - - static inline size_t setvlmax() - { - return __riscv_vsetvlmax_e32m8(); - } - - static inline void lut(const uchar* src, vuint32m8_t lut_v, ElemType* dst, size_t vl) - { - auto src_v = __riscv_vzext_vf2(__riscv_vle8_v_u8m2(src, vl), vl); - auto dst_v = __riscv_vrgatherei16(lut_v, src_v, vl); - __riscv_vse32(dst, dst_v, vl); + return __riscv_vrgatherei16(lut_v, src_v, vl); } }; @@ -134,7 +100,7 @@ public: w = w * h; h = 1; } - auto lut = LUT_TYPE::loadTable((ElemType*)lut_data); + auto lut = LUT_TYPE::vload((ElemType*)lut_data, 256); size_t maxlv = LUT_TYPE::setvlmax(); for (; h; h--, src += src_step, dst += dst_step) { @@ -144,12 +110,18 @@ public: auto d = (ElemType*)dst; for (; l >= vl; l -= vl, s += vl, d += vl) { - LUT_TYPE::lut(s, lut, d, vl); + auto src_v = LUT_TYPE::SrcType::vload(s, vl); + auto idx_v = LUT_TYPE::IdxType::cast(src_v, vl); + auto dst_v = LUT_TYPE::gather(idx_v, lut, vl); + LUT_TYPE::vstore(d, dst_v, vl); } for (; l > 0; l -= vl, s += vl, d += vl) { vl = LUT_TYPE::setvl(l); - LUT_TYPE::lut(s, lut, d, vl); + auto src_v = LUT_TYPE::SrcType::vload(s, vl); + auto idx_v = LUT_TYPE::IdxType::cast(src_v, vl); + auto dst_v = LUT_TYPE::gather(idx_v, lut, vl); + LUT_TYPE::vstore(d, dst_v, vl); } } } diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/minmax.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/minmax.hpp index aa8cf2937f..2e03a3b338 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/minmax.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/minmax.hpp @@ -5,6 +5,8 @@ #define OPENCV_HAL_RVV_MINMAX_HPP_INCLUDED #include +#include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace minmax { @@ -13,61 +15,13 @@ namespace cv { namespace cv_hal_rvv { namespace minmax { #undef cv_hal_minMaxIdxMaskStep #define cv_hal_minMaxIdxMaskStep cv::cv_hal_rvv::minmax::minMaxIdx -template struct rvv; - -#define HAL_RVV_GENERATOR(T, EEW, TYPE, IS_U, EMUL, M_EMUL, B_LEN) \ -template<> struct rvv \ -{ \ - using vec_t = v##IS_U##int##EEW##EMUL##_t; \ - using bool_t = vbool##B_LEN##_t; \ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e##EEW##EMUL(); } \ - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e##EEW##EMUL(a); } \ - static inline vec_t vmv_v_x(T a, size_t b) { return __riscv_vmv_v_x_##TYPE##EMUL(a, b); } \ - static inline vec_t vle(const T* a, size_t b) { return __riscv_vle##EEW##_v_##TYPE##EMUL(a, b); } \ - static inline vuint8##M_EMUL##_t vle_mask(const uchar* a, size_t b) { return __riscv_vle8_v_u8##M_EMUL(a, b); } \ - static inline vec_t vmin_tu(vec_t a, vec_t b, vec_t c, size_t d) { return __riscv_vmin##IS_U##_tu(a, b, c, d); } \ - static inline vec_t vmax_tu(vec_t a, vec_t b, vec_t c, size_t d) { return __riscv_vmax##IS_U##_tu(a, b, c, d); } \ - static inline vec_t vmin_tumu(bool_t a, vec_t b, vec_t c, vec_t d, size_t e) { return __riscv_vmin##IS_U##_tumu(a, b, c, d, e); } \ - static inline vec_t vmax_tumu(bool_t a, vec_t b, vec_t c, vec_t d, size_t e) { return __riscv_vmax##IS_U##_tumu(a, b, c, d, e); } \ - static inline vec_t vredmin(vec_t a, vec_t b, size_t c) { return __riscv_vredmin##IS_U(a, b, c); } \ - static inline vec_t vredmax(vec_t a, vec_t b, size_t c) { return __riscv_vredmax##IS_U(a, b, c); } \ -}; -HAL_RVV_GENERATOR(uchar , 8 , u8 , u, m1, m1 , 8 ) -HAL_RVV_GENERATOR(schar , 8 , i8 , , m1, m1 , 8 ) -HAL_RVV_GENERATOR(ushort, 16, u16, u, m1, mf2, 16) -HAL_RVV_GENERATOR(short , 16, i16, , m1, mf2, 16) -#undef HAL_RVV_GENERATOR - -#define HAL_RVV_GENERATOR(T, NAME, EEW, TYPE, IS_F, F_OR_S, F_OR_X, EMUL, M_EMUL, P_EMUL, B_LEN) \ -template<> struct rvv \ -{ \ - using vec_t = v##NAME##EEW##EMUL##_t; \ - using bool_t = vbool##B_LEN##_t; \ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e##EEW##EMUL(); } \ - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e##EEW##EMUL(a); } \ - static inline vec_t vmv_v_x(T a, size_t b) { return __riscv_v##IS_F##mv_v_##F_OR_X##_##TYPE##EMUL(a, b); } \ - static inline vuint32##P_EMUL##_t vid(size_t a) { return __riscv_vid_v_u32##P_EMUL(a); } \ - static inline vuint32##P_EMUL##_t vundefined() { return __riscv_vundefined_u32##P_EMUL(); } \ - static inline vec_t vle(const T* a, size_t b) { return __riscv_vle##EEW##_v_##TYPE##EMUL(a, b); } \ - static inline vuint8##M_EMUL##_t vle_mask(const uchar* a, size_t b) { return __riscv_vle8_v_u8##M_EMUL(a, b); } \ - static inline bool_t vmlt(vec_t a, vec_t b, size_t c) { return __riscv_vm##F_OR_S##lt(a, b, c); } \ - static inline bool_t vmgt(vec_t a, vec_t b, size_t c) { return __riscv_vm##F_OR_S##gt(a, b, c); } \ - static inline bool_t vmlt_mu(bool_t a, bool_t b, vec_t c, vec_t d, size_t e) { return __riscv_vm##F_OR_S##lt##_mu(a, b, c, d, e); } \ - static inline bool_t vmgt_mu(bool_t a, bool_t b, vec_t c, vec_t d, size_t e) { return __riscv_vm##F_OR_S##gt##_mu(a, b, c, d, e); } \ - static inline T vmv_x_s(vec_t a) { return __riscv_v##IS_F##mv_##F_OR_X(a); } \ -}; -HAL_RVV_GENERATOR(int , int , 32, i32, , s, x, m4, m1 , m4, 8 ) -HAL_RVV_GENERATOR(float , float, 32, f32, f, f, f, m4, m1 , m4, 8 ) -HAL_RVV_GENERATOR(double, float, 64, f64, f, f, f, m4, mf2, m2, 16) -#undef HAL_RVV_GENERATOR - -template +template inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, int height, double* minVal, double* maxVal, int* minIdx, int* maxIdx, uchar* mask, size_t mask_step) { - int vlmax = rvv::vsetvlmax(); - auto vec_min = rvv::vmv_v_x(std::numeric_limits::max(), vlmax); - auto vec_max = rvv::vmv_v_x(std::numeric_limits::lowest(), vlmax); + int vlmax = VEC_T::setvlmax(); + auto vec_min = VEC_T::vmv(std::numeric_limits::max(), vlmax); + auto vec_max = VEC_T::vmv(std::numeric_limits::lowest(), vlmax); T val_min, val_max; if (mask) @@ -79,19 +33,19 @@ inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width; j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); - auto vec_mask = rvv::vle_mask(mask_row + j, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); + auto vec_mask = BOOL_T::vload(mask_row + j, vl); auto bool_mask = __riscv_vmsne(vec_mask, 0, vl); - vec_min = rvv::vmin_tumu(bool_mask, vec_min, vec_min, vec_src, vl); - vec_max = rvv::vmax_tumu(bool_mask, vec_max, vec_max, vec_src, vl); + vec_min = VEC_T::vmin_tumu(bool_mask, vec_min, vec_min, vec_src, vl); + vec_max = VEC_T::vmax_tumu(bool_mask, vec_max, vec_max, vec_src, vl); } } - auto sc_minval = rvv::vmv_v_x(std::numeric_limits::max(), vlmax); - auto sc_maxval = rvv::vmv_v_x(std::numeric_limits::lowest(), vlmax); - sc_minval = rvv::vredmin(vec_min, sc_minval, vlmax); - sc_maxval = rvv::vredmax(vec_max, sc_maxval, vlmax); + auto sc_minval = VEC_T::vmv(std::numeric_limits::max(), vlmax); + auto sc_maxval = VEC_T::vmv(std::numeric_limits::lowest(), vlmax); + sc_minval = VEC_T::vredmin(vec_min, sc_minval, vlmax); + sc_maxval = VEC_T::vredmax(vec_max, sc_maxval, vlmax); val_min = __riscv_vmv_x(sc_minval); val_max = __riscv_vmv_x(sc_maxval); @@ -103,9 +57,9 @@ inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width && (!found_min || !found_max); j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); - auto vec_mask = rvv::vle_mask(mask_row + j, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); + auto vec_mask = BOOL_T::vload(mask_row + j, vl); auto bool_mask = __riscv_vmsne(vec_mask, 0, vl); auto bool_zero = __riscv_vmxor(bool_mask, bool_mask, vl); if (!found_min) @@ -141,17 +95,17 @@ inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width; j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); - vec_min = rvv::vmin_tu(vec_min, vec_min, vec_src, vl); - vec_max = rvv::vmax_tu(vec_max, vec_max, vec_src, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); + vec_min = VEC_T::vmin_tu(vec_min, vec_min, vec_src, vl); + vec_max = VEC_T::vmax_tu(vec_max, vec_max, vec_src, vl); } } - auto sc_minval = rvv::vmv_v_x(std::numeric_limits::max(), vlmax); - auto sc_maxval = rvv::vmv_v_x(std::numeric_limits::lowest(), vlmax); - sc_minval = rvv::vredmin(vec_min, sc_minval, vlmax); - sc_maxval = rvv::vredmax(vec_max, sc_maxval, vlmax); + auto sc_minval = VEC_T::vmv(std::numeric_limits::max(), vlmax); + auto sc_maxval = VEC_T::vmv(std::numeric_limits::lowest(), vlmax); + sc_minval = VEC_T::vredmin(vec_min, sc_minval, vlmax); + sc_maxval = VEC_T::vredmax(vec_max, sc_maxval, vlmax); val_min = __riscv_vmv_x(sc_minval); val_max = __riscv_vmv_x(sc_maxval); @@ -162,8 +116,8 @@ inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width && (!found_min || !found_max); j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); if (!found_min) { auto bool_minpos = __riscv_vmseq(vec_src, val_min, vl); @@ -201,15 +155,15 @@ inline int minMaxIdxReadTwice(const uchar* src_data, size_t src_step, int width, return CV_HAL_ERROR_OK; } -template +template inline int minMaxIdxReadOnce(const uchar* src_data, size_t src_step, int width, int height, double* minVal, double* maxVal, int* minIdx, int* maxIdx, uchar* mask, size_t mask_step) { - int vlmax = rvv::vsetvlmax(); - auto vec_min = rvv::vmv_v_x(std::numeric_limits::max(), vlmax); - auto vec_max = rvv::vmv_v_x(std::numeric_limits::lowest(), vlmax); - auto vec_pos = rvv::vid(vlmax); - auto vec_minpos = rvv::vundefined(), vec_maxpos = rvv::vundefined(); + int vlmax = VEC_T::setvlmax(); + auto vec_min = VEC_T::vmv(std::numeric_limits::max(), vlmax); + auto vec_max = VEC_T::vmv(std::numeric_limits::lowest(), vlmax); + auto vec_pos = IDX_T::vid(vlmax); + auto vec_minpos = IDX_T::vundefined(), vec_maxpos = IDX_T::vundefined(); T val_min, val_max; if (mask) @@ -221,14 +175,14 @@ inline int minMaxIdxReadOnce(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width; j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); - auto vec_mask = rvv::vle_mask(mask_row + j, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); + auto vec_mask = BOOL_T::vload(mask_row + j, vl); auto bool_mask = __riscv_vmsne(vec_mask, 0, vl); auto bool_zero = __riscv_vmxor(bool_mask, bool_mask, vl); - auto bool_minpos = rvv::vmlt_mu(bool_mask, bool_zero, vec_src, vec_min, vl); - auto bool_maxpos = rvv::vmgt_mu(bool_mask, bool_zero, vec_src, vec_max, vl); + auto bool_minpos = VEC_T::vmlt_mu(bool_mask, bool_zero, vec_src, vec_min, vl); + auto bool_maxpos = VEC_T::vmgt_mu(bool_mask, bool_zero, vec_src, vec_max, vl); vec_minpos = __riscv_vmerge_tu(vec_minpos, vec_minpos, vec_pos, bool_minpos, vl); vec_maxpos = __riscv_vmerge_tu(vec_maxpos, vec_maxpos, vec_pos, bool_maxpos, vl); @@ -246,11 +200,11 @@ inline int minMaxIdxReadOnce(const uchar* src_data, size_t src_step, int width, int vl; for (int j = 0; j < width; j += vl) { - vl = rvv::vsetvl(width - j); - auto vec_src = rvv::vle(src_row + j, vl); + vl = VEC_T::setvl(width - j); + auto vec_src = VEC_T::vload(src_row + j, vl); - auto bool_minpos = rvv::vmlt(vec_src, vec_min, vl); - auto bool_maxpos = rvv::vmgt(vec_src, vec_max, vl); + auto bool_minpos = VEC_T::vmlt(vec_src, vec_min, vl); + auto bool_maxpos = VEC_T::vmgt(vec_src, vec_max, vl); vec_minpos = __riscv_vmerge_tu(vec_minpos, vec_minpos, vec_pos, bool_minpos, vl); vec_maxpos = __riscv_vmerge_tu(vec_maxpos, vec_maxpos, vec_pos, bool_maxpos, vl); @@ -265,18 +219,18 @@ inline int minMaxIdxReadOnce(const uchar* src_data, size_t src_step, int width, val_max = std::numeric_limits::lowest(); for (int i = 0; i < vlmax; i++) { - if (val_min > rvv::vmv_x_s(vec_min)) + if (val_min > VEC_T::vmv_x(vec_min)) { - val_min = rvv::vmv_x_s(vec_min); + val_min = VEC_T::vmv_x(vec_min); if (minIdx) { minIdx[0] = __riscv_vmv_x(vec_minpos) / width; minIdx[1] = __riscv_vmv_x(vec_minpos) % width; } } - if (val_max < rvv::vmv_x_s(vec_max)) + if (val_max < VEC_T::vmv_x(vec_max)) { - val_max = rvv::vmv_x_s(vec_max); + val_max = VEC_T::vmv_x(vec_max); if (maxIdx) { maxIdx[0] = __riscv_vmv_x(vec_maxpos) / width; @@ -309,19 +263,19 @@ inline int minMaxIdx(const uchar* src_data, size_t src_step, int width, int heig switch (depth) { case CV_8UC1: - return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_8SC1: - return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_16UC1: - return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_16SC1: - return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadTwice(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_32SC1: - return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_32FC1: - return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); case CV_64FC1: - return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); + return minMaxIdxReadOnce(src_data, src_step, width, height, minVal, maxVal, minIdx, maxIdx, mask, mask_step); } return CV_HAL_ERROR_NOT_IMPLEMENTED; diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/pyramids.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/pyramids.hpp index d1a543b2cb..76f040fec8 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/pyramids.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/pyramids.hpp @@ -5,6 +5,7 @@ #define OPENCV_HAL_RVV_PYRAMIDS_HPP_INCLUDED #include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace pyramids { @@ -17,43 +18,96 @@ template struct rvv; template<> struct rvv { - static inline size_t vsetvl_WT(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vuint8m1_t vle_T(const uchar* a, size_t b) { return __riscv_vle8_v_u8m1(a, b); } - static inline vint32m4_t vle_WT(const int* a, size_t b) { return __riscv_vle32_v_i32m4(a, b); } - static inline vuint32m4_t vle_M(const uint* a, size_t b) { return __riscv_vle32_v_u32m4(a, b); } - static inline vuint8m1_t vlse_T(const uchar* a, ptrdiff_t b, size_t c) { return __riscv_vlse8_v_u8m1(a, b, c); } - static inline vuint8m1_t vloxei_T(const uchar* a, vuint32m4_t b, size_t c) { return __riscv_vloxei32_v_u8m1(a, b, c); } - static inline void vse_T(uchar* a, vuint8m1_t b, size_t c) { return __riscv_vse8(a, b, c); } - static inline vint32m4_t vcvt_T_WT(vuint8m1_t a, size_t b) { return __riscv_vreinterpret_v_u32m4_i32m4(__riscv_vzext_vf4(a, b)); } - static inline vuint8m1_t vcvt_WT_T(vint32m4_t a, int b, size_t c) { return __riscv_vncvt_x(__riscv_vncvt_x(__riscv_vreinterpret_v_i32m4_u32m4(__riscv_vsra(__riscv_vadd(a, 1 << (b - 1), c), b, c)), c), c); } + using T = RVV_U8M1; + using WT = RVV_SameLen; + using MT = RVV_SameLen; + + static inline WT::VecType vcvt_T_WT(T::VecType a, size_t b) { return WT::cast(MT::cast(a, b), b); } + static inline T::VecType vcvt_WT_T(WT::VecType a, int b, size_t c) { return T::cast(MT::cast(__riscv_vsra(__riscv_vadd(a, 1 << (b - 1), c), b, c), c), c); } + static inline WT::VecType down0(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), + __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl); + } + static inline WT::VecType down1(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl); + } + static inline WT::VecType up00(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vadd(__riscv_vadd(vec_src0, vec_src2, vl), __riscv_vadd(__riscv_vsll(vec_src1, 2, vl), __riscv_vsll(vec_src1, 1, vl), vl), vl); + } + static inline WT::VecType up01(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vsll(__riscv_vadd(vec_src1, vec_src2, vl), 2, vl); + } + static inline WT::VecType up10(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return up00(vec_src0, vec_src1, vec_src2, vl); + } + static inline WT::VecType up11(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return up01(vec_src1, vec_src2, vl); + } }; template<> struct rvv { - static inline size_t vsetvl_WT(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vint16m2_t vle_T(const short* a, size_t b) { return __riscv_vle16_v_i16m2(a, b); } - static inline vint32m4_t vle_WT(const int* a, size_t b) { return __riscv_vle32_v_i32m4(a, b); } - static inline vuint32m4_t vle_M(const uint* a, size_t b) { return __riscv_vle32_v_u32m4(a, b); } - static inline vint16m2_t vlse_T(const short* a, ptrdiff_t b, size_t c) { return __riscv_vlse16_v_i16m2(a, b, c); } - static inline vint16m2_t vloxei_T(const short* a, vuint32m4_t b, size_t c) { return __riscv_vloxei32_v_i16m2(a, b, c); } - static inline void vse_T(short* a, vint16m2_t b, size_t c) { return __riscv_vse16(a, b, c); } - static inline vint32m4_t vcvt_T_WT(vint16m2_t a, size_t b) { return __riscv_vsext_vf2(a, b); } - static inline vint16m2_t vcvt_WT_T(vint32m4_t a, int b, size_t c) { return __riscv_vncvt_x(__riscv_vsra(__riscv_vadd(a, 1 << (b - 1), c), b, c), c); } + using T = RVV_I16M2; + using WT = RVV_SameLen; + using MT = RVV_SameLen; + + static inline WT::VecType vcvt_T_WT(T::VecType a, size_t b) { return WT::cast(a, b); } + static inline T::VecType vcvt_WT_T(WT::VecType a, int b, size_t c) { return T::cast(__riscv_vsra(__riscv_vadd(a, 1 << (b - 1), c), b, c), c); } + static inline WT::VecType down0(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), + __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl); + } + static inline WT::VecType down1(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl); + } + static inline WT::VecType up00(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vadd(__riscv_vadd(vec_src0, vec_src2, vl), __riscv_vadd(__riscv_vsll(vec_src1, 2, vl), __riscv_vsll(vec_src1, 1, vl), vl), vl); + } + static inline WT::VecType up01(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vsll(__riscv_vadd(vec_src1, vec_src2, vl), 2, vl); + } + static inline WT::VecType up10(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return up00(vec_src0, vec_src1, vec_src2, vl); + } + static inline WT::VecType up11(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return up01(vec_src1, vec_src2, vl); + } }; template<> struct rvv { - static inline size_t vsetvl_WT(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vfloat32m4_t vle_T(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline vfloat32m4_t vle_WT(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline vuint32m4_t vle_M(const uint* a, size_t b) { return __riscv_vle32_v_u32m4(a, b); } - static inline vfloat32m4_t vlse_T(const float* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_f32m4(a, b, c); } - static inline vfloat32m4_t vloxei_T(const float* a, vuint32m4_t b, size_t c) { return __riscv_vloxei32_v_f32m4(a, b, c); } - static inline void vse_T(float* a, vfloat32m4_t b, size_t c) { return __riscv_vse32(a, b, c); } + using T = RVV_F32M4; + using WT = RVV_SameLen; + using MT = RVV_SameLen; + + static inline WT::VecType vcvt_T_WT(T::VecType a, size_t b) { return WT::cast(a, b); } + static inline T::VecType vcvt_WT_T(WT::VecType a, [[maybe_unused]] int b, size_t c) { return T::cast(a, c); } + static inline WT::VecType down0(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl); + } + static inline WT::VecType down1(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, WT::VecType vec_src3, WT::VecType vec_src4, size_t vl) { + return __riscv_vfmul(down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), 1.f / 256.f, vl); + } + static inline WT::VecType up00(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vfadd(__riscv_vfmadd(vec_src1, 6, vec_src0, vl), vec_src2, vl); + } + static inline WT::VecType up01(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vfmul(__riscv_vfadd(vec_src1, vec_src2, vl), 4, vl); + } + static inline WT::VecType up10(WT::VecType vec_src0, WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vfmul(__riscv_vfadd(__riscv_vfmadd(vec_src1, 6, vec_src0, vl), vec_src2, vl), 1.f / 64.f, vl); + } + static inline WT::VecType up11(WT::VecType vec_src1, WT::VecType vec_src2, size_t vl) { + return __riscv_vfmul(__riscv_vfadd(vec_src1, vec_src2, vl), 1.f / 16.f, vl); + } }; -template struct pyrDownVec0 +template +struct pyrDownVec0 { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; + void operator()(const T* src, WT* row, const uint* tabM, int start, int end) { int vl; @@ -62,308 +116,160 @@ template struct pyrDownVec0 case 1: for( int x = start; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + x * 2 - 2, 2 * sizeof(T), vl), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + x * 2 - 1, 2 * sizeof(T), vl), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + x * 2, 2 * sizeof(T), vl), vl); - auto vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + x * 2 + 1, 2 * sizeof(T), vl), vl); - auto vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + x * 2 + 2, 2 * sizeof(T), vl), vl); - __riscv_vse32(row + x, __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); + vl = RVV::WT::setvl(end - x); + auto vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + x * 2 - 2, 2 * sizeof(T), vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + x * 2 - 1, 2 * sizeof(T), vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + x * 2, 2 * sizeof(T), vl), vl); + auto vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + x * 2 + 1, 2 * sizeof(T), vl), vl); + auto vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + x * 2 + 2, 2 * sizeof(T), vl), vl); + __riscv_vse32(row + x, RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); } break; case 2: for( int x = start / 2; x < end / 2; x += vl ) { - vl = rvv::vsetvl_WT(end / 2 - x); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 2, 4 * sizeof(T), vl), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 2, 4 * sizeof(T), vl), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 2, 4 * sizeof(T), vl), vl); - auto vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 2, 4 * sizeof(T), vl), vl); - auto vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 2, 4 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 2, 2 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 2 + 1, 4 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 2 + 1, 4 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 2 + 1, 4 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 2 + 1, 4 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 2 + 1, 4 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 2 + 1, 2 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); + vl = RVV::WT::setvl(end / 2 - x); + auto vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 2, 4 * sizeof(T), vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 2, 4 * sizeof(T), vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 2, 4 * sizeof(T), vl), vl); + auto vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 2, 4 * sizeof(T), vl), vl); + auto vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 2, 4 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 2, 2 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 2 + 1, 4 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 2 + 1, 4 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 2 + 1, 4 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 2 + 1, 4 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 2 + 1, 4 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 2 + 1, 2 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); } break; case 3: for( int x = start / 3; x < end / 3; x += vl ) { - vl = rvv::vsetvl_WT(end / 3 - x); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 3, 6 * sizeof(T), vl), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 3, 6 * sizeof(T), vl), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 3, 6 * sizeof(T), vl), vl); - auto vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 3, 6 * sizeof(T), vl), vl); - auto vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 3, 6 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 3, 3 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 3 + 1, 6 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 3 + 1, 6 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 3 + 1, 6 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 3 + 1, 6 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 3 + 1, 6 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 3 + 1, 3 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 3 + 2, 6 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 3 + 2, 6 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 3 + 2, 6 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 3 + 2, 6 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 3 + 2, 6 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 3 + 2, 3 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); + vl = RVV::WT::setvl(end / 3 - x); + auto vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 3, 6 * sizeof(T), vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 3, 6 * sizeof(T), vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 3, 6 * sizeof(T), vl), vl); + auto vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 3, 6 * sizeof(T), vl), vl); + auto vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 3, 6 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 3, 3 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 3 + 1, 6 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 3 + 1, 6 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 3 + 1, 6 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 3 + 1, 6 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 3 + 1, 6 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 3 + 1, 3 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 3 + 2, 6 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 3 + 2, 6 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 3 + 2, 6 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 3 + 2, 6 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 3 + 2, 6 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 3 + 2, 3 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); } break; case 4: for( int x = start / 4; x < end / 4; x += vl ) { - vl = rvv::vsetvl_WT(end / 4 - x); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 4, 8 * sizeof(T), vl), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 4, 8 * sizeof(T), vl), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 4, 8 * sizeof(T), vl), vl); - auto vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 4, 8 * sizeof(T), vl), vl); - auto vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 4, 8 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 4, 4 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 4 + 1, 8 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 4 + 1, 8 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 4 + 1, 8 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 4 + 1, 8 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 4 + 1, 8 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 4 + 1, 4 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 4 + 2, 8 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 4 + 2, 8 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 4 + 2, 8 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 4 + 2, 8 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 4 + 2, 8 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 4 + 2, 4 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - vec_src0 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 2) * 4 + 3, 8 * sizeof(T), vl), vl); - vec_src1 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 - 1) * 4 + 3, 8 * sizeof(T), vl), vl); - vec_src2 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2) * 4 + 3, 8 * sizeof(T), vl), vl); - vec_src3 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 1) * 4 + 3, 8 * sizeof(T), vl), vl); - vec_src4 = rvv::vcvt_T_WT(rvv::vlse_T(src + (x * 2 + 2) * 4 + 3, 8 * sizeof(T), vl), vl); - __riscv_vsse32(row + x * 4 + 3, 4 * sizeof(WT), __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); + vl = RVV::WT::setvl(end / 4 - x); + auto vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 4, 8 * sizeof(T), vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 4, 8 * sizeof(T), vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 4, 8 * sizeof(T), vl), vl); + auto vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 4, 8 * sizeof(T), vl), vl); + auto vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 4, 8 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 4, 4 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 4 + 1, 8 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 4 + 1, 8 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 4 + 1, 8 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 4 + 1, 8 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 4 + 1, 8 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 4 + 1, 4 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 4 + 2, 8 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 4 + 2, 8 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 4 + 2, 8 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 4 + 2, 8 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 4 + 2, 8 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 4 + 2, 4 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); + vec_src0 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 2) * 4 + 3, 8 * sizeof(T), vl), vl); + vec_src1 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 - 1) * 4 + 3, 8 * sizeof(T), vl), vl); + vec_src2 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2) * 4 + 3, 8 * sizeof(T), vl), vl); + vec_src3 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 1) * 4 + 3, 8 * sizeof(T), vl), vl); + vec_src4 = RVV::vcvt_T_WT(RVV::T::vload_stride(src + (x * 2 + 2) * 4 + 3, 8 * sizeof(T), vl), vl); + __riscv_vsse32(row + x * 4 + 3, 4 * sizeof(WT), RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); } break; default: for( int x = start; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_tabM = rvv::vle_M(tabM + x, vl); + vl = RVV::WT::setvl(end - x); + auto vec_tabM = RVV::MT::vload(tabM + x, vl); vec_tabM = __riscv_vmul(__riscv_vsub(vec_tabM, start * 2, vl), sizeof(T), vl); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vloxei_T(src, vec_tabM, vl), vl); + auto vec_src0 = RVV::vcvt_T_WT(__riscv_vloxei32(src, vec_tabM, vl), vl); vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(T), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vloxei_T(src, vec_tabM, vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(__riscv_vloxei32(src, vec_tabM, vl), vl); vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(T), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vloxei_T(src, vec_tabM, vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(__riscv_vloxei32(src, vec_tabM, vl), vl); vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(T), vl); - auto vec_src3 = rvv::vcvt_T_WT(rvv::vloxei_T(src, vec_tabM, vl), vl); + auto vec_src3 = RVV::vcvt_T_WT(__riscv_vloxei32(src, vec_tabM, vl), vl); vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(T), vl); - auto vec_src4 = rvv::vcvt_T_WT(rvv::vloxei_T(src, vec_tabM, vl), vl); - __riscv_vse32(row + x, __riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), vl); - } - } - } -}; -template<> struct pyrDownVec0 -{ - void operator()(const float* src, float* row, const uint* tabM, int start, int end) - { - int vl; - switch (start) - { - case 1: - for( int x = start; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vlse_T(src + x * 2 - 2, 2 * sizeof(float), vl); - auto vec_src1 = rvv::vlse_T(src + x * 2 - 1, 2 * sizeof(float), vl); - auto vec_src2 = rvv::vlse_T(src + x * 2, 2 * sizeof(float), vl); - auto vec_src3 = rvv::vlse_T(src + x * 2 + 1, 2 * sizeof(float), vl); - auto vec_src4 = rvv::vlse_T(src + x * 2 + 2, 2 * sizeof(float), vl); - __riscv_vse32(row + x, __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - } - break; - case 2: - for( int x = start / 2; x < end / 2; x += vl ) - { - vl = rvv::vsetvl_WT(end / 2 - x); - auto vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 2, 4 * sizeof(float), vl); - auto vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 2, 4 * sizeof(float), vl); - auto vec_src2 = rvv::vlse_T(src + (x * 2) * 2, 4 * sizeof(float), vl); - auto vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 2, 4 * sizeof(float), vl); - auto vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 2, 4 * sizeof(float), vl); - __riscv_vsse32(row + x * 2, 2 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 2 + 1, 4 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 2 + 1, 4 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 2 + 1, 4 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 2 + 1, 4 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 2 + 1, 4 * sizeof(float), vl); - __riscv_vsse32(row + x * 2 + 1, 2 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - } - break; - case 3: - for( int x = start / 3; x < end / 3; x += vl ) - { - vl = rvv::vsetvl_WT(end / 3 - x); - auto vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 3, 6 * sizeof(float), vl); - auto vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 3, 6 * sizeof(float), vl); - auto vec_src2 = rvv::vlse_T(src + (x * 2) * 3, 6 * sizeof(float), vl); - auto vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 3, 6 * sizeof(float), vl); - auto vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 3, 6 * sizeof(float), vl); - __riscv_vsse32(row + x * 3, 3 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 3 + 1, 6 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 3 + 1, 6 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 3 + 1, 6 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 3 + 1, 6 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 3 + 1, 6 * sizeof(float), vl); - __riscv_vsse32(row + x * 3 + 1, 3 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 3 + 2, 6 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 3 + 2, 6 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 3 + 2, 6 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 3 + 2, 6 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 3 + 2, 6 * sizeof(float), vl); - __riscv_vsse32(row + x * 3 + 2, 3 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - } - break; - case 4: - for( int x = start / 4; x < end / 4; x += vl ) - { - vl = rvv::vsetvl_WT(end / 4 - x); - auto vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 4, 8 * sizeof(float), vl); - auto vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 4, 8 * sizeof(float), vl); - auto vec_src2 = rvv::vlse_T(src + (x * 2) * 4, 8 * sizeof(float), vl); - auto vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 4, 8 * sizeof(float), vl); - auto vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 4, 8 * sizeof(float), vl); - __riscv_vsse32(row + x * 4, 4 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 4 + 1, 8 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 4 + 1, 8 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 4 + 1, 8 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 4 + 1, 8 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 4 + 1, 8 * sizeof(float), vl); - __riscv_vsse32(row + x * 4 + 1, 4 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 4 + 2, 8 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 4 + 2, 8 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 4 + 2, 8 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 4 + 2, 8 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 4 + 2, 8 * sizeof(float), vl); - __riscv_vsse32(row + x * 4 + 2, 4 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - vec_src0 = rvv::vlse_T(src + (x * 2 - 2) * 4 + 3, 8 * sizeof(float), vl); - vec_src1 = rvv::vlse_T(src + (x * 2 - 1) * 4 + 3, 8 * sizeof(float), vl); - vec_src2 = rvv::vlse_T(src + (x * 2) * 4 + 3, 8 * sizeof(float), vl); - vec_src3 = rvv::vlse_T(src + (x * 2 + 1) * 4 + 3, 8 * sizeof(float), vl); - vec_src4 = rvv::vlse_T(src + (x * 2 + 2) * 4 + 3, 8 * sizeof(float), vl); - __riscv_vsse32(row + x * 4 + 3, 4 * sizeof(float), __riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), vl); - } - break; - default: - for( int x = start; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_tabM = rvv::vle_M(tabM + x, vl); - vec_tabM = __riscv_vmul(__riscv_vsub(vec_tabM, start * 2, vl), sizeof(float), vl); - auto vec_src0 = rvv::vloxei_T(src, vec_tabM, vl); - vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(float), vl); - auto vec_src1 = rvv::vloxei_T(src, vec_tabM, vl); - vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(float), vl); - auto vec_src2 = rvv::vloxei_T(src, vec_tabM, vl); - vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(float), vl); - auto vec_src3 = rvv::vloxei_T(src, vec_tabM, vl); - vec_tabM = __riscv_vadd(vec_tabM, start * sizeof(float), vl); - auto vec_src4 = rvv::vloxei_T(src, vec_tabM, vl); - __riscv_vse32(row + x, __riscv_vfmadd(__riscv_vfadd(__riscv_vfadd(vec_src1, vec_src2, vl), vec_src3, vl), 4, - __riscv_vfadd(__riscv_vfadd(vec_src0, vec_src4, vl), __riscv_vfadd(vec_src2, vec_src2, vl), vl), vl), vl); + auto vec_src4 = RVV::vcvt_T_WT(__riscv_vloxei32(src, vec_tabM, vl), vl); + __riscv_vse32(row + x, RVV::down0(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), vl); } } } }; -template struct pyrDownVec1 +template +struct pyrDownVec1 { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; + void operator()(WT* row0, WT* row1, WT* row2, WT* row3, WT* row4, T* dst, int end) { int vl; for( int x = 0 ; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - auto vec_src3 = rvv::vle_WT(row3 + x, vl); - auto vec_src4 = rvv::vle_WT(row4 + x, vl); - rvv::vse_T(dst + x, rvv::vcvt_WT_T(__riscv_vadd(__riscv_vadd(__riscv_vadd(vec_src0, vec_src4, vl), __riscv_vadd(vec_src2, vec_src2, vl), vl), - __riscv_vsll(__riscv_vadd(__riscv_vadd(vec_src1, vec_src2, vl), vec_src3, vl), 2, vl), vl), 8, vl), vl); - } - } -}; -template<> struct pyrDownVec1 -{ - void operator()(float* row0, float* row1, float* row2, float* row3, float* row4, float* dst, int end) - { - int vl; - for( int x = 0 ; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - auto vec_src3 = rvv::vle_WT(row3 + x, vl); - auto vec_src4 = rvv::vle_WT(row4 + x, vl); - rvv::vse_T(dst + x, __riscv_vfmul(__riscv_vfmadd(vec_src2, 6, __riscv_vfmadd(__riscv_vfadd(vec_src1, vec_src3, vl), 4, __riscv_vfadd(vec_src0, vec_src4, vl), vl), vl), 1.f / 256.f, vl), vl); + vl = RVV::WT::setvl(end - x); + auto vec_src0 = RVV::WT::vload(row0 + x, vl); + auto vec_src1 = RVV::WT::vload(row1 + x, vl); + auto vec_src2 = RVV::WT::vload(row2 + x, vl); + auto vec_src3 = RVV::WT::vload(row3 + x, vl); + auto vec_src4 = RVV::WT::vload(row4 + x, vl); + RVV::T::vstore(dst + x, RVV::vcvt_WT_T(RVV::down1(vec_src0, vec_src1, vec_src2, vec_src3, vec_src4, vl), 8, vl), vl); } } }; -template struct pyrUpVec0 +template +struct pyrUpVec0 { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; + void operator()(const T* src, WT* row, const uint* dtab, int start, int end) { int vl; for( int x = start; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vcvt_T_WT(rvv::vle_T(src + x - start, vl), vl); - auto vec_src1 = rvv::vcvt_T_WT(rvv::vle_T(src + x, vl), vl); - auto vec_src2 = rvv::vcvt_T_WT(rvv::vle_T(src + x + start, vl), vl); + vl = RVV::WT::setvl(end - x); + auto vec_src0 = RVV::vcvt_T_WT(RVV::T::vload(src + x - start, vl), vl); + auto vec_src1 = RVV::vcvt_T_WT(RVV::T::vload(src + x, vl), vl); + auto vec_src2 = RVV::vcvt_T_WT(RVV::T::vload(src + x + start, vl), vl); - auto vec_dtab = rvv::vle_M(dtab + x, vl); + auto vec_dtab = RVV::MT::vload(dtab + x, vl); vec_dtab = __riscv_vmul(vec_dtab, sizeof(WT), vl); - __riscv_vsoxei32(row, vec_dtab, __riscv_vadd(__riscv_vadd(vec_src0, vec_src2, vl), __riscv_vadd(__riscv_vsll(vec_src1, 2, vl), __riscv_vsll(vec_src1, 1, vl), vl), vl), vl); - __riscv_vsoxei32(row, __riscv_vadd(vec_dtab, start * sizeof(WT), vl), __riscv_vsll(__riscv_vadd(vec_src1, vec_src2, vl), 2, vl), vl); - } - } -}; -template<> struct pyrUpVec0 -{ - void operator()(const float* src, float* row, const uint* dtab, int start, int end) - { - int vl; - for( int x = start; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_T(src + x - start, vl); - auto vec_src1 = rvv::vle_T(src + x, vl); - auto vec_src2 = rvv::vle_T(src + x + start, vl); - - auto vec_dtab = rvv::vle_M(dtab + x, vl); - vec_dtab = __riscv_vmul(vec_dtab, sizeof(float), vl); - __riscv_vsoxei32(row, vec_dtab, __riscv_vfadd(__riscv_vfmadd(vec_src1, 6, vec_src0, vl), vec_src2, vl), vl); - __riscv_vsoxei32(row, __riscv_vadd(vec_dtab, start * sizeof(float), vl), __riscv_vfmul(__riscv_vfadd(vec_src1, vec_src2, vl), 4, vl), vl); + __riscv_vsoxei32(row, vec_dtab, RVV::up00(vec_src0, vec_src1, vec_src2, vl), vl); + __riscv_vsoxei32(row, __riscv_vadd(vec_dtab, start * sizeof(WT), vl), RVV::up01(vec_src1, vec_src2, vl), vl); } } }; -template struct pyrUpVec1 +template +struct pyrUpVec1 { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; + void operator()(WT* row0, WT* row1, WT* row2, T* dst0, T* dst1, int end) { int vl; @@ -371,59 +277,29 @@ template struct pyrUpVec1 { for( int x = 0 ; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - rvv::vse_T(dst0 + x, rvv::vcvt_WT_T(__riscv_vadd(__riscv_vadd(vec_src0, vec_src2, vl), __riscv_vadd(__riscv_vsll(vec_src1, 2, vl), __riscv_vsll(vec_src1, 1, vl), vl), vl), 6, vl), vl); - rvv::vse_T(dst1 + x, rvv::vcvt_WT_T(__riscv_vsll(__riscv_vadd(vec_src1, vec_src2, vl), 2, vl), 6, vl), vl); + vl = RVV::WT::setvl(end - x); + auto vec_src0 = RVV::WT::vload(row0 + x, vl); + auto vec_src1 = RVV::WT::vload(row1 + x, vl); + auto vec_src2 = RVV::WT::vload(row2 + x, vl); + RVV::T::vstore(dst0 + x, RVV::vcvt_WT_T(RVV::up10(vec_src0, vec_src1, vec_src2, vl), 6, vl), vl); + RVV::T::vstore(dst1 + x, RVV::vcvt_WT_T(RVV::up11(vec_src1, vec_src2, vl), 6, vl), vl); } } else { for( int x = 0 ; x < end; x += vl ) { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - rvv::vse_T(dst0 + x, rvv::vcvt_WT_T(__riscv_vadd(__riscv_vadd(vec_src0, vec_src2, vl), __riscv_vadd(__riscv_vsll(vec_src1, 2, vl), __riscv_vsll(vec_src1, 1, vl), vl), vl), 6, vl), vl); - } - } - } -}; -template<> struct pyrUpVec1 -{ - void operator()(float* row0, float* row1, float* row2, float* dst0, float* dst1, int end) - { - int vl; - if (dst0 != dst1) - { - for( int x = 0 ; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - rvv::vse_T(dst0 + x, __riscv_vfmul(__riscv_vfadd(__riscv_vfmadd(vec_src1, 6, vec_src0, vl), vec_src2, vl), 1.f / 64.f, vl), vl); - rvv::vse_T(dst1 + x, __riscv_vfmul(__riscv_vfadd(vec_src1, vec_src2, vl), 1.f / 16.f, vl), vl); - } - } - else - { - for( int x = 0 ; x < end; x += vl ) - { - vl = rvv::vsetvl_WT(end - x); - auto vec_src0 = rvv::vle_WT(row0 + x, vl); - auto vec_src1 = rvv::vle_WT(row1 + x, vl); - auto vec_src2 = rvv::vle_WT(row2 + x, vl); - rvv::vse_T(dst0 + x, __riscv_vfmul(__riscv_vfadd(__riscv_vfmadd(vec_src1, 6, vec_src0, vl), vec_src2, vl), 1.f / 64.f, vl), vl); + vl = RVV::WT::setvl(end - x); + auto vec_src0 = RVV::WT::vload(row0 + x, vl); + auto vec_src1 = RVV::WT::vload(row1 + x, vl); + auto vec_src2 = RVV::WT::vload(row2 + x, vl); + RVV::T::vstore(dst0 + x, RVV::vcvt_WT_T(RVV::up10(vec_src0, vec_src1, vec_src2, vl), 6, vl), vl); } } } }; -template +template struct PyrDownInvoker : ParallelLoopBody { PyrDownInvoker(const uchar* _src_data, size_t _src_step, int _src_width, int _src_height, uchar* _dst_data, size_t _dst_step, int _dst_width, int _dst_height, int _cn, int _borderType, int* _tabR, int* _tabM, int* _tabL) @@ -494,7 +370,8 @@ static inline int borderInterpolate( int p, int len, int borderType ) // the algorithm is copied from imgproc/src/pyramids.cpp, // in the function template void cv::pyrDown_ -template + +template inline int pyrDown(const uchar* src_data, size_t src_step, int src_width, int src_height, uchar* dst_data, size_t dst_step, int dst_width, int dst_height, int cn, int borderType) { const int PD_SZ = 5; @@ -524,13 +401,15 @@ inline int pyrDown(const uchar* src_data, size_t src_step, int src_width, int sr for (int x = 0; x < dst_width*cn; x++) tabM[x] = (x/cn)*2*cn + x % cn; - cv::parallel_for_(Range(0,dst_height), PyrDownInvoker(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, borderType, tabR, tabM, tabL), cv::getNumThreads()); + cv::parallel_for_(Range(0,dst_height), PyrDownInvoker(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, borderType, tabR, tabM, tabL), cv::getNumThreads()); return CV_HAL_ERROR_OK; } -template -void PyrDownInvoker::operator()(const Range& range) const +template +void PyrDownInvoker::operator()(const Range& range) const { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; const int PD_SZ = 5; int bufstep = (dst_width*cn + 15) & -16; @@ -567,7 +446,7 @@ void PyrDownInvoker::operator()(const Range& range) const if( x == _dst_width ) break; - pyrDownVec0()(src, row, reinterpret_cast(tabM), cn, width0); + pyrDownVec0()(src, row, reinterpret_cast(tabM), cn, width0); x = width0; // tabR @@ -584,15 +463,17 @@ void PyrDownInvoker::operator()(const Range& range) const rows[k] = buf + ((y*2 - PD_SZ/2 + k - sy0) % PD_SZ)*bufstep; row0 = rows[0]; row1 = rows[1]; row2 = rows[2]; row3 = rows[3]; row4 = rows[4]; - pyrDownVec1()(row0, row1, row2, row3, row4, dst, _dst_width); + pyrDownVec1()(row0, row1, row2, row3, row4, dst, _dst_width); } } // the algorithm is copied from imgproc/src/pyramids.cpp, // in the function template void cv::pyrUp_ -template +template inline int pyrUp(const uchar* src_data, size_t src_step, int src_width, int src_height, uchar* dst_data, size_t dst_step, int dst_width, int dst_height, int cn) { + using T = typename RVV::T::ElemType; + using WT = typename RVV::WT::ElemType; const int PU_SZ = 3; int bufstep = ((dst_width+1)*cn + 15) & -16; @@ -653,7 +534,7 @@ inline int pyrUp(const uchar* src_data, size_t src_step, int src_width, int src_ } } - pyrUpVec0()(src, row, reinterpret_cast(dtab), cn, src_width - cn); + pyrUpVec0()(src, row, reinterpret_cast(dtab), cn, src_width - cn); } // do vertical convolution and decimation and write the result to the destination image @@ -661,7 +542,7 @@ inline int pyrUp(const uchar* src_data, size_t src_step, int src_width, int src_ rows[k] = buf + ((y - PU_SZ/2 + k - sy0) % PU_SZ)*bufstep; row0 = rows[0]; row1 = rows[1]; row2 = rows[2]; - pyrUpVec1()(row0, row1, row2, dst0, dst1, dst_width); + pyrUpVec1()(row0, row1, row2, dst0, dst1, dst_width); } if (dst_height > src_height*2) @@ -686,11 +567,11 @@ inline int pyrDown(const uchar* src_data, size_t src_step, int src_width, int sr switch (depth) { case CV_8U: - return pyrDown(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); + return pyrDown>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); case CV_16S: - return pyrDown(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); + return pyrDown>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); case CV_32F: - return pyrDown(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); + return pyrDown>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn, border_type); } return CV_HAL_ERROR_NOT_IMPLEMENTED; @@ -704,11 +585,11 @@ inline int pyrUp(const uchar* src_data, size_t src_step, int src_width, int src_ switch (depth) { case CV_8U: - return pyrUp(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); + return pyrUp>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); case CV_16S: - return pyrUp(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); + return pyrUp>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); case CV_32F: - return pyrUp(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); + return pyrUp>(src_data, src_step, src_width, src_height, dst_data, dst_step, dst_width, dst_height, cn); } return CV_HAL_ERROR_NOT_IMPLEMENTED; diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/qr.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/qr.hpp index 37aa0dd8b3..58b6f770fc 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/qr.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/qr.hpp @@ -4,44 +4,23 @@ #ifndef OPENCV_HAL_RVV_QR_HPP_INCLUDED #define OPENCV_HAL_RVV_QR_HPP_INCLUDED +#include +#include +#include +#include #include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace qr { #undef cv_hal_QR32f -#define cv_hal_QR32f cv::cv_hal_rvv::qr::QR +#define cv_hal_QR32f cv::cv_hal_rvv::qr::QR #undef cv_hal_QR64f -#define cv_hal_QR64f cv::cv_hal_rvv::qr::QR - -template struct rvv; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e32m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vfloat32m4_t vfmv_v_f(float a, size_t b) { return __riscv_vfmv_v_f_f32m4(a, b); } - static inline vfloat32m1_t vfmv_s_f(float a, size_t b) { return __riscv_vfmv_s_f_f32m1(a, b); } - static inline vfloat32m4_t vle(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline vfloat32m4_t vlse(const float* a, ptrdiff_t b, size_t c) { return __riscv_vlse32_v_f32m4(a, b, c); } - static inline void vse(float* a, vfloat32m4_t b, size_t c) { __riscv_vse32(a, b, c); } - static inline void vsse(float* a, ptrdiff_t b, vfloat32m4_t c, size_t d) { __riscv_vsse32(a, b, c, d); } -}; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e64m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e64m4(a); } - static inline vfloat64m4_t vfmv_v_f(double a, size_t b) { return __riscv_vfmv_v_f_f64m4(a, b); } - static inline vfloat64m1_t vfmv_s_f(double a, size_t b) { return __riscv_vfmv_s_f_f64m1(a, b); } - static inline vfloat64m4_t vle(const double* a, size_t b) { return __riscv_vle64_v_f64m4(a, b); } - static inline vfloat64m4_t vlse(const double* a, ptrdiff_t b, size_t c) { return __riscv_vlse64_v_f64m4(a, b, c); } - static inline void vse(double* a, vfloat64m4_t b, size_t c) { __riscv_vse64(a, b, c); } - static inline void vsse(double* a, ptrdiff_t b, vfloat64m4_t c, size_t d) { __riscv_vsse64(a, b, c, d); } -}; +#define cv_hal_QR64f cv::cv_hal_rvv::qr::QR // the algorithm is copied from core/src/matrix_decomp.cpp, // in the function template static int cv::QRImpl -template +template inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t src2_step, T* dst, int* info) { T eps; @@ -61,50 +40,50 @@ inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t sr if (dst == NULL) dst = val + m; - int vlmax = rvv::vsetvlmax(), vl; + int vlmax = RVV_T::setvlmax(), vl; for (int l = 0; l < n; l++) { //generate val int vlSize = m - l; - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for (int i = 0; i < vlSize; i += vl) { - vl = rvv::vsetvl(vlSize - i); - auto vec_src = rvv::vlse(src1 + (l + i) * src1_step + l, sizeof(T) * src1_step, vl); - rvv::vse(val + i, vec_src, vl); + vl = RVV_T::setvl(vlSize - i); + auto vec_src = RVV_T::vload_stride(src1 + (l + i) * src1_step + l, sizeof(T) * src1_step, vl); + RVV_T::vstore(val + i, vec_src, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src, vec_src, vl); } - T vlNorm = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + T vlNorm = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); T tmpV = val[0]; val[0] = val[0] + (val[0] >= 0 ? 1 : -1) * std::sqrt(vlNorm); vlNorm = std::sqrt(vlNorm + val[0] * val[0] - tmpV*tmpV); for (int i = 0; i < vlSize; i += vl) { - vl = rvv::vsetvl(vlSize - i); - auto vec_src = rvv::vle(val + i, vl); + vl = RVV_T::setvl(vlSize - i); + auto vec_src = RVV_T::vload(val + i, vl); vec_src = __riscv_vfdiv(vec_src, vlNorm, vl); - rvv::vse(val + i, vec_src, vl); + RVV_T::vstore(val + i, vec_src, vl); } //multiply A_l*val for (int j = l; j < n; j++) { - vec_sum = rvv::vfmv_v_f(0, vlmax); + vec_sum = RVV_T::vmv(0, vlmax); for (int i = l; i < m; i += vl) { - vl = rvv::vsetvl(m - i); - auto vec_src1 = rvv::vle(val + i - l, vl); - auto vec_src2 = rvv::vlse(src1 + i * src1_step + j, sizeof(T) * src1_step, vl); + vl = RVV_T::setvl(m - i); + auto vec_src1 = RVV_T::vload(val + i - l, vl); + auto vec_src2 = RVV_T::vload_stride(src1 + i * src1_step + j, sizeof(T) * src1_step, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - T v_lA = 2 * __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + T v_lA = 2 * __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); for (int i = l; i < m; i += vl) { - vl = rvv::vsetvl(m - i); - auto vec_src1 = rvv::vle(val + i - l, vl); - auto vec_src2 = rvv::vlse(src1 + i * src1_step + j, sizeof(T) * src1_step, vl); + vl = RVV_T::setvl(m - i); + auto vec_src1 = RVV_T::vload(val + i - l, vl); + auto vec_src2 = RVV_T::vload_stride(src1 + i * src1_step + j, sizeof(T) * src1_step, vl); vec_src2 = __riscv_vfnmsac(vec_src2, v_lA, vec_src1, vl); - rvv::vsse(src1 + i * src1_step + j, sizeof(T) * src1_step, vec_src2, vl); + RVV_T::vstore_stride(src1 + i * src1_step + j, sizeof(T) * src1_step, vec_src2, vl); } } @@ -112,10 +91,10 @@ inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t sr dst[l] = val[0] * val[0]; for (int i = 1; i < vlSize; i += vl) { - vl = rvv::vsetvl(vlSize - i); - auto vec_src = rvv::vle(val + i, vl); + vl = RVV_T::setvl(vlSize - i); + auto vec_src = RVV_T::vload(val + i, vl); vec_src = __riscv_vfdiv(vec_src, val[0], vl); - rvv::vsse(src1 + (l + i) * src1_step + l, sizeof(T) * src1_step, vec_src, vl); + RVV_T::vstore_stride(src1 + (l + i) * src1_step + l, sizeof(T) * src1_step, vec_src, vl); } } @@ -128,31 +107,31 @@ inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t sr val[0] = (T)1; for (int j = 1; j < m - l; j += vl) { - vl = rvv::vsetvl(m - l - j); - auto vec_src = rvv::vlse(src1 + (j + l) * src1_step + l, sizeof(T) * src1_step, vl); - rvv::vse(val + j, vec_src, vl); + vl = RVV_T::setvl(m - l - j); + auto vec_src = RVV_T::vload_stride(src1 + (j + l) * src1_step + l, sizeof(T) * src1_step, vl); + RVV_T::vstore(val + j, vec_src, vl); } //h_l*x for (int j = 0; j < k; j++) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for (int i = l; i < m; i += vl) { - vl = rvv::vsetvl(m - i); - auto vec_src1 = rvv::vle(val + i - l, vl); - auto vec_src2 = rvv::vlse(src2 + i * src2_step + j, sizeof(T) * src2_step, vl); + vl = RVV_T::setvl(m - i); + auto vec_src1 = RVV_T::vload(val + i - l, vl); + auto vec_src2 = RVV_T::vload_stride(src2 + i * src2_step + j, sizeof(T) * src2_step, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - T v_lB = 2 * dst[l] * __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + T v_lB = 2 * dst[l] * __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); for (int i = l; i < m; i += vl) { - vl = rvv::vsetvl(m - i); - auto vec_src1 = rvv::vle(val + i - l, vl); - auto vec_src2 = rvv::vlse(src2 + i * src2_step + j, sizeof(T) * src2_step, vl); + vl = RVV_T::setvl(m - i); + auto vec_src1 = RVV_T::vload(val + i - l, vl); + auto vec_src2 = RVV_T::vload_stride(src2 + i * src2_step + j, sizeof(T) * src2_step, vl); vec_src2 = __riscv_vfnmsac(vec_src2, v_lB, vec_src1, vl); - rvv::vsse(src2 + i * src2_step + j, sizeof(T) * src2_step, vec_src2, vl); + RVV_T::vstore_stride(src2 + i * src2_step + j, sizeof(T) * src2_step, vec_src2, vl); } } } @@ -163,11 +142,11 @@ inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t sr { for (int p = 0; p < k; p += vl) { - vl = rvv::vsetvl(k - p); - auto vec_src1 = rvv::vle(src2 + i * src2_step + p, vl); - auto vec_src2 = rvv::vle(src2 + j * src2_step + p, vl); + vl = RVV_T::setvl(k - p); + auto vec_src1 = RVV_T::vload(src2 + i * src2_step + p, vl); + auto vec_src2 = RVV_T::vload(src2 + j * src2_step + p, vl); vec_src1 = __riscv_vfnmsac(vec_src1, src1[i*src1_step + j], vec_src2, vl); - rvv::vse(src2 + i * src2_step + p, vec_src1, vl); + RVV_T::vstore(src2 + i * src2_step + p, vec_src1, vl); } } if (std::abs(src1[i*src1_step + i]) < eps) @@ -177,10 +156,10 @@ inline int QR(T* src1, size_t src1_step, int m, int n, int k, T* src2, size_t sr } for (int p = 0; p < k; p += vl) { - vl = rvv::vsetvl(k - p); - auto vec_src = rvv::vle(src2 + i * src2_step + p, vl); + vl = RVV_T::setvl(k - p); + auto vec_src = RVV_T::vload(src2 + i * src2_step + p, vl); vec_src = __riscv_vfdiv(vec_src, src1[i*src1_step + i], vl); - rvv::vse(src2 + i * src2_step + p, vec_src, vl); + RVV_T::vstore(src2 + i * src2_step + p, vec_src, vl); } } } diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/svd.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/svd.hpp index c2d861e02b..5d22d73227 100644 --- a/3rdparty/hal_rvv/hal_rvv_1p0/svd.hpp +++ b/3rdparty/hal_rvv/hal_rvv_1p0/svd.hpp @@ -4,40 +4,23 @@ #ifndef OPENCV_HAL_RVV_SVD_HPP_INCLUDED #define OPENCV_HAL_RVV_SVD_HPP_INCLUDED +#include +#include +#include +#include #include +#include "hal_rvv_1p0/types.hpp" namespace cv { namespace cv_hal_rvv { namespace svd { #undef cv_hal_SVD32f -#define cv_hal_SVD32f cv::cv_hal_rvv::svd::SVD +#define cv_hal_SVD32f cv::cv_hal_rvv::svd::SVD #undef cv_hal_SVD64f -#define cv_hal_SVD64f cv::cv_hal_rvv::svd::SVD - -template struct rvv; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e32m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e32m4(a); } - static inline vfloat32m4_t vfmv_v_f(float a, size_t b) { return __riscv_vfmv_v_f_f32m4(a, b); } - static inline vfloat32m1_t vfmv_s_f(float a, size_t b) { return __riscv_vfmv_s_f_f32m1(a, b); } - static inline vfloat32m4_t vle(const float* a, size_t b) { return __riscv_vle32_v_f32m4(a, b); } - static inline void vse(float* a, vfloat32m4_t b, size_t c) { __riscv_vse32(a, b, c); } -}; - -template<> struct rvv -{ - static inline size_t vsetvlmax() { return __riscv_vsetvlmax_e64m4(); } - static inline size_t vsetvl(size_t a) { return __riscv_vsetvl_e64m4(a); } - static inline vfloat64m4_t vfmv_v_f(double a, size_t b) { return __riscv_vfmv_v_f_f64m4(a, b); } - static inline vfloat64m1_t vfmv_s_f(double a, size_t b) { return __riscv_vfmv_s_f_f64m1(a, b); } - static inline vfloat64m4_t vle(const double* a, size_t b) { return __riscv_vle64_v_f64m4(a, b); } - static inline void vse(double* a, vfloat64m4_t b, size_t c) { __riscv_vse64(a, b, c); } -}; +#define cv_hal_SVD64f cv::cv_hal_rvv::svd::SVD // the algorithm is copied from core/src/lapack.cpp, // in the function template static void cv::JacobiSVDImpl_ -template +template inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, int m, int n, int flags) { T minval, eps; @@ -72,7 +55,7 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, src_step /= sizeof(src[0]); vt_step /= sizeof(vt[0]); - int vlmax = rvv::vsetvlmax(), vl; + int vlmax = RVV_T::setvlmax(), vl; for( i = 0; i < n; i++ ) { for( k = 0, sd = 0; k < m; k++ ) @@ -100,15 +83,15 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, T *Ai = src + i*src_step, *Aj = src + j*src_step; double a = W[i], p = 0, b = W[j]; - auto vec_sum1 = rvv::vfmv_v_f(0, vlmax); + auto vec_sum1 = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(Ai + k, vl); - auto vec_src2 = rvv::vle(Aj + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(Ai + k, vl); + auto vec_src2 = RVV_T::vload(Aj + k, vl); vec_sum1 = __riscv_vfmacc_tu(vec_sum1, vec_src1, vec_src2, vl); } - p = __riscv_vfmv_f(__riscv_vfredosum(vec_sum1, rvv::vfmv_s_f(0, vlmax), vlmax)); + p = __riscv_vfmv_f(__riscv_vfredosum(vec_sum1, RVV_BaseType::vmv_s(0, vlmax), vlmax)); if( std::abs(p) <= eps*std::sqrt((double)a*b) ) continue; @@ -127,22 +110,22 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, s = (T)(p/(gamma*c*2)); } - vec_sum1 = rvv::vfmv_v_f(0, vlmax); - auto vec_sum2 = rvv::vfmv_v_f(0, vlmax); + vec_sum1 = RVV_T::vmv(0, vlmax); + auto vec_sum2 = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(Ai + k, vl); - auto vec_src2 = rvv::vle(Aj + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(Ai + k, vl); + auto vec_src2 = RVV_T::vload(Aj + k, vl); auto vec_t0 = __riscv_vfadd(__riscv_vfmul(vec_src1, c, vl), __riscv_vfmul(vec_src2, s, vl), vl); auto vec_t1 = __riscv_vfsub(__riscv_vfmul(vec_src2, c, vl), __riscv_vfmul(vec_src1, s, vl), vl); - rvv::vse(Ai + k, vec_t0, vl); - rvv::vse(Aj + k, vec_t1, vl); + RVV_T::vstore(Ai + k, vec_t0, vl); + RVV_T::vstore(Aj + k, vec_t1, vl); vec_sum1 = __riscv_vfmacc_tu(vec_sum1, vec_t0, vec_t0, vl); vec_sum2 = __riscv_vfmacc_tu(vec_sum2, vec_t1, vec_t1, vl); } - W[i] = __riscv_vfmv_f(__riscv_vfredosum(vec_sum1, rvv::vfmv_s_f(0, vlmax), vlmax)); - W[j] = __riscv_vfmv_f(__riscv_vfredosum(vec_sum2, rvv::vfmv_s_f(0, vlmax), vlmax)); + W[i] = __riscv_vfmv_f(__riscv_vfredosum(vec_sum1, RVV_BaseType::vmv_s(0, vlmax), vlmax)); + W[j] = __riscv_vfmv_f(__riscv_vfredosum(vec_sum2, RVV_BaseType::vmv_s(0, vlmax), vlmax)); changed = true; @@ -151,13 +134,13 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, T *Vi = vt + i*vt_step, *Vj = vt + j*vt_step; for( k = 0; k < n; k += vl ) { - vl = rvv::vsetvl(n - k); - auto vec_src1 = rvv::vle(Vi + k, vl); - auto vec_src2 = rvv::vle(Vj + k, vl); + vl = RVV_T::setvl(n - k); + auto vec_src1 = RVV_T::vload(Vi + k, vl); + auto vec_src2 = RVV_T::vload(Vj + k, vl); auto vec_t0 = __riscv_vfadd(__riscv_vfmul(vec_src1, c, vl), __riscv_vfmul(vec_src2, s, vl), vl); auto vec_t1 = __riscv_vfsub(__riscv_vfmul(vec_src2, c, vl), __riscv_vfmul(vec_src1, s, vl), vl); - rvv::vse(Vi + k, vec_t0, vl); - rvv::vse(Vj + k, vec_t1, vl); + RVV_T::vstore(Vi + k, vec_t0, vl); + RVV_T::vstore(Vj + k, vec_t1, vl); } } } @@ -167,14 +150,14 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, for( i = 0; i < n; i++ ) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src = rvv::vle(src + i * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src = RVV_T::vload(src + i * src_step + k, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src, vec_src, vl); } - W[i] = std::sqrt(__riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax))); + W[i] = std::sqrt(__riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax))); } for( i = 0; i < n-1; i++ ) @@ -224,56 +207,56 @@ inline int SVD(T* src, size_t src_step, T* w, T*, size_t, T* vt, size_t vt_step, { for( j = 0; j < i; j++ ) { - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(src + i * src_step + k, vl); - auto vec_src2 = rvv::vle(src + j * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(src + i * src_step + k, vl); + auto vec_src2 = RVV_T::vload(src + j * src_step + k, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - sd = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + sd = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); - vec_sum = rvv::vfmv_v_f(0, vlmax); + vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(src + i * src_step + k, vl); - auto vec_src2 = rvv::vle(src + j * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(src + i * src_step + k, vl); + auto vec_src2 = RVV_T::vload(src + j * src_step + k, vl); vec_src1 = __riscv_vfnmsac(vec_src1, sd, vec_src2, vl); - rvv::vse(src + i * src_step + k, vec_src1, vl); + RVV_T::vstore(src + i * src_step + k, vec_src1, vl); vec_sum = __riscv_vfadd_tu(vec_sum, vec_sum, __riscv_vfabs(vec_src1, vl), vl); } - T asum = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax)); + T asum = __riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax)); asum = asum > eps*100 ? 1/asum : 0; for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src = rvv::vle(src + i * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src = RVV_T::vload(src + i * src_step + k, vl); vec_src = __riscv_vfmul(vec_src, asum, vl); - rvv::vse(src + i * src_step + k, vec_src, vl); + RVV_T::vstore(src + i * src_step + k, vec_src, vl); } } } - auto vec_sum = rvv::vfmv_v_f(0, vlmax); + auto vec_sum = RVV_T::vmv(0, vlmax); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src1 = rvv::vle(src + i * src_step + k, vl); - auto vec_src2 = rvv::vle(src + j * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src1 = RVV_T::vload(src + i * src_step + k, vl); + auto vec_src2 = RVV_T::vload(src + j * src_step + k, vl); vec_sum = __riscv_vfmacc_tu(vec_sum, vec_src1, vec_src2, vl); } - sd = std::sqrt(__riscv_vfmv_f(__riscv_vfredosum(vec_sum, rvv::vfmv_s_f(0, vlmax), vlmax))); + sd = std::sqrt(__riscv_vfmv_f(__riscv_vfredosum(vec_sum, RVV_BaseType::vmv_s(0, vlmax), vlmax))); } s = (T)(sd > minval ? 1/sd : 0.); for( k = 0; k < m; k += vl ) { - vl = rvv::vsetvl(m - k); - auto vec_src = rvv::vle(src + i * src_step + k, vl); + vl = RVV_T::setvl(m - k); + auto vec_src = RVV_T::vload(src + i * src_step + k, vl); vec_src = __riscv_vfmul(vec_src, s, vl); - rvv::vse(src + i * src_step + k, vec_src, vl); + RVV_T::vstore(src + i * src_step + k, vec_src, vl); } } diff --git a/3rdparty/hal_rvv/hal_rvv_1p0/types.hpp b/3rdparty/hal_rvv/hal_rvv_1p0/types.hpp new file mode 100644 index 0000000000..9416e8cd6e --- /dev/null +++ b/3rdparty/hal_rvv/hal_rvv_1p0/types.hpp @@ -0,0 +1,485 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level +// directory of this distribution and at http://opencv.org/license.html. +#pragma once + +#include +#include + +namespace cv { namespace cv_hal_rvv { + +enum RVV_LMUL +{ + LMUL_1 = 1, + LMUL_2 = 2, + LMUL_4 = 4, + LMUL_8 = 8, + LMUL_f2, + LMUL_f4, + LMUL_f8, +}; + +template +struct RVV; + +// -------------------------------Supported types-------------------------------- + +using RVV_U8M1 = struct RVV; +using RVV_U8M2 = struct RVV; +using RVV_U8M4 = struct RVV; +using RVV_U8M8 = struct RVV; +using RVV_U8MF2 = struct RVV; +using RVV_U8MF4 = struct RVV; +using RVV_U8MF8 = struct RVV; + +using RVV_I8M1 = struct RVV; +using RVV_I8M2 = struct RVV; +using RVV_I8M4 = struct RVV; +using RVV_I8M8 = struct RVV; +using RVV_I8MF2 = struct RVV; +using RVV_I8MF4 = struct RVV; +using RVV_I8MF8 = struct RVV; + +using RVV_U16M1 = struct RVV; +using RVV_U16M2 = struct RVV; +using RVV_U16M4 = struct RVV; +using RVV_U16M8 = struct RVV; +using RVV_U16MF2 = struct RVV; +using RVV_U16MF4 = struct RVV; + +using RVV_I16M1 = struct RVV; +using RVV_I16M2 = struct RVV; +using RVV_I16M4 = struct RVV; +using RVV_I16M8 = struct RVV; +using RVV_I16MF2 = struct RVV; +using RVV_I16MF4 = struct RVV; + +using RVV_U32M1 = struct RVV; +using RVV_U32M2 = struct RVV; +using RVV_U32M4 = struct RVV; +using RVV_U32M8 = struct RVV; +using RVV_U32MF2 = struct RVV; + +using RVV_I32M1 = struct RVV; +using RVV_I32M2 = struct RVV; +using RVV_I32M4 = struct RVV; +using RVV_I32M8 = struct RVV; +using RVV_I32MF2 = struct RVV; + +using RVV_U64M1 = struct RVV; +using RVV_U64M2 = struct RVV; +using RVV_U64M4 = struct RVV; +using RVV_U64M8 = struct RVV; + +using RVV_I64M1 = struct RVV; +using RVV_I64M2 = struct RVV; +using RVV_I64M4 = struct RVV; +using RVV_I64M8 = struct RVV; + +using RVV_F32M1 = struct RVV; +using RVV_F32M2 = struct RVV; +using RVV_F32M4 = struct RVV; +using RVV_F32M8 = struct RVV; +using RVV_F32MF2 = struct RVV; + +using RVV_F64M1 = struct RVV; +using RVV_F64M2 = struct RVV; +using RVV_F64M4 = struct RVV; +using RVV_F64M8 = struct RVV; + +// Only for dst type lmul >= 1 +template +using RVV_SameLen = + RVV; + +template +using RVV_BaseType = RVV; + +// -------------------------------Supported operations-------------------------------- + +#define HAL_RVV_SIZE_RELATED(EEW, TYPE, LMUL, S_OR_F, X_OR_F, IS_U, IS_F) \ +static inline size_t setvlmax() { return __riscv_vsetvlmax_e##EEW##LMUL(); } \ +static inline size_t setvl(size_t vl) { return __riscv_vsetvl_e##EEW##LMUL(vl); } \ +static inline VecType vload(const ElemType* ptr, size_t vl) { \ + return __riscv_vle##EEW##_v_##TYPE##LMUL(ptr, vl); \ +} \ +static inline VecType vload(BoolType vm, const ElemType* ptr, size_t vl) { \ + return __riscv_vle##EEW(vm, ptr, vl); \ +} \ +static inline VecType vload_stride(const ElemType* ptr, ptrdiff_t unit, size_t vl) { \ + return __riscv_vlse##EEW##_v_##TYPE##LMUL(ptr, unit, vl); \ +} \ +static inline VecType vload_stride(BoolType vm, const ElemType* ptr, ptrdiff_t unit, size_t vl) { \ + return __riscv_vlse##EEW(vm, ptr, unit, vl); \ +} \ +static inline void vstore(ElemType* ptr, VecType v, size_t vl) { \ + __riscv_vse##EEW(ptr, v, vl); \ +} \ +static inline void vstore(BoolType vm, ElemType* ptr, VecType v, size_t vl) { \ + __riscv_vse##EEW(vm, ptr, v, vl); \ +} \ +static inline void vstore_stride(ElemType* ptr, ptrdiff_t unit, VecType v, size_t vl) { \ + __riscv_vsse##EEW(ptr, unit, v, vl); \ +} \ +static inline void vstore_stride(BoolType vm, ElemType* ptr, ptrdiff_t unit, VecType v, size_t vl) { \ + __riscv_vsse##EEW(vm, ptr, unit, v, vl); \ +} \ +static inline VecType vundefined() { return __riscv_vundefined_##TYPE##LMUL(); } \ +static inline VecType vmv(ElemType a, size_t vl) { \ + return __riscv_v##IS_F##mv_v_##X_OR_F##_##TYPE##LMUL(a, vl); \ +} \ +static inline VecType vmv_s(ElemType a, size_t vl) { \ + return __riscv_v##IS_F##mv_s_##X_OR_F##_##TYPE##LMUL(a, vl); \ +} \ +HAL_RVV_SIZE_RELATED_CUSTOM(EEW, TYPE, LMUL) + +#define HAL_RVV_SIZE_UNRELATED(S_OR_F, X_OR_F, IS_U, IS_F) \ +static inline ElemType vmv_x(VecType vs2) { return __riscv_v##IS_F##mv_##X_OR_F(vs2); } \ + \ +static inline BoolType vmlt(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##lt##IS_U(vs2, vs1, vl); \ +} \ +static inline BoolType vmle(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##le##IS_U(vs2, vs1, vl); \ +} \ +static inline BoolType vmgt(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##gt##IS_U(vs2, vs1, vl); \ +} \ +static inline BoolType vmge(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##ge##IS_U(vs2, vs1, vl); \ +} \ +static inline BoolType vmlt_mu(BoolType vm, BoolType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##lt##IS_U##_mu(vm, vd, vs2, vs1, vl); \ +} \ +static inline BoolType vmle_mu(BoolType vm, BoolType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##le##IS_U##_mu(vm, vd, vs2, vs1, vl); \ +} \ +static inline BoolType vmgt_mu(BoolType vm, BoolType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##gt##IS_U##_mu(vm, vd, vs2, vs1, vl); \ +} \ +static inline BoolType vmge_mu(BoolType vm, BoolType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_vm##S_OR_F##ge##IS_U##_mu(vm, vd, vs2, vs1, vl); \ +} \ + \ +static inline VecType vmin(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##min##IS_U(vs2, vs1, vl); \ +} \ +static inline VecType vmax(VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##max##IS_U(vs2, vs1, vl); \ +} \ +static inline VecType vmin_tu(VecType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##min##IS_U##_tu(vd, vs2, vs1, vl); \ +} \ +static inline VecType vmax_tu(VecType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##max##IS_U##_tu(vd, vs2, vs1, vl); \ +} \ +static inline VecType vmin_tumu(BoolType vm, VecType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##min##IS_U##_tumu(vm, vd, vs2, vs1, vl); \ +} \ +static inline VecType vmax_tumu(BoolType vm, VecType vd, VecType vs2, VecType vs1, size_t vl) { \ + return __riscv_v##IS_F##max##IS_U##_tumu(vm, vd, vs2, vs1, vl); \ +} \ + \ +static inline BaseType vredmin(VecType vs2, BaseType vs1, size_t vl) { \ + return __riscv_v##IS_F##redmin##IS_U(vs2, vs1, vl); \ +} \ +static inline BaseType vredmax(VecType vs2, BaseType vs1, size_t vl) { \ + return __riscv_v##IS_F##redmax##IS_U(vs2, vs1, vl); \ +} + +#define HAL_RVV_BOOL_TYPE(S_OR_F, X_OR_F, IS_U, IS_F) \ + decltype(__riscv_vm##S_OR_F##eq(std::declval(), std::declval(), 0)) + +#define HAL_RVV_DEFINE_ONE(ELEM_TYPE, VEC_TYPE, LMUL_TYPE, \ + EEW, TYPE, LMUL, ...) \ + template <> struct RVV \ + { \ + using ElemType = ELEM_TYPE; \ + using VecType = v##VEC_TYPE##LMUL##_t; \ + using BoolType = HAL_RVV_BOOL_TYPE(__VA_ARGS__); \ + using BaseType = v##VEC_TYPE##m1_t; \ + \ + static constexpr size_t lmul = LMUL_TYPE; \ + \ + HAL_RVV_SIZE_RELATED(EEW, TYPE, LMUL, __VA_ARGS__) \ + HAL_RVV_SIZE_UNRELATED(__VA_ARGS__) \ + \ + template \ + inline static VecType cast(FROM v, size_t vl); \ + }; \ + \ + template <> \ + inline RVV::VecType \ + RVV::cast( \ + RVV::VecType v, \ + [[maybe_unused]] size_t vl \ + ) \ + { \ + return v; \ + } + +// -------------------------------Define all types-------------------------------- + +#define HAL_RVV_DEFINE_ALL(ELEM_TYPE, VEC_TYPE, \ + EEW, TYPE, ...) \ + HAL_RVV_DEFINE_ONE(ELEM_TYPE, VEC_TYPE, LMUL_1, \ + EEW, TYPE, m1, __VA_ARGS__) \ + HAL_RVV_DEFINE_ONE(ELEM_TYPE, VEC_TYPE, LMUL_2, \ + EEW, TYPE, m2, __VA_ARGS__) \ + HAL_RVV_DEFINE_ONE(ELEM_TYPE, VEC_TYPE, LMUL_4, \ + EEW, TYPE, m4, __VA_ARGS__) \ + HAL_RVV_DEFINE_ONE(ELEM_TYPE, VEC_TYPE, LMUL_8, \ + EEW, TYPE, m8, __VA_ARGS__) + +#define HAL_RVV_SIGNED_PARAM s,x, , +#define HAL_RVV_UNSIGNED_PARAM s,x,u, +#define HAL_RVV_FLOAT_PARAM f,f, ,f + +// -------------------------------Define Unsigned Integer-------------------------------- + +#define HAL_RVV_SIZE_RELATED_CUSTOM(EEW, TYPE, LMUL) \ + static inline VecType vid(size_t vl) { return __riscv_vid_v_##TYPE##LMUL(vl); } + +// LMUL = 1, 2, 4, 8 +HAL_RVV_DEFINE_ALL( uint8_t, uint8, 8, u8, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ALL(uint16_t, uint16, 16, u16, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ALL(uint32_t, uint32, 32, u32, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ALL(uint64_t, uint64, 64, u64, HAL_RVV_UNSIGNED_PARAM) + +// LMUL = f2 +HAL_RVV_DEFINE_ONE( uint8_t, uint8, LMUL_f2, 8, u8, mf2, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ONE(uint16_t, uint16, LMUL_f2, 16, u16, mf2, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ONE(uint32_t, uint32, LMUL_f2, 32, u32, mf2, HAL_RVV_UNSIGNED_PARAM) + +// LMUL = f4 +HAL_RVV_DEFINE_ONE( uint8_t, uint8, LMUL_f4, 8, u8, mf4, HAL_RVV_UNSIGNED_PARAM) +HAL_RVV_DEFINE_ONE(uint16_t, uint16, LMUL_f4, 16, u16, mf4, HAL_RVV_UNSIGNED_PARAM) + +// LMUL = f8 +HAL_RVV_DEFINE_ONE( uint8_t, uint8, LMUL_f8, 8, u8, mf8, HAL_RVV_UNSIGNED_PARAM) + +#undef HAL_RVV_SIZE_RELATED_CUSTOM + +// -------------------------------Define Signed Integer-------------------------------- + +#define HAL_RVV_SIZE_RELATED_CUSTOM(EEW, TYPE, LMUL) + +// LMUL = 1, 2, 4, 8 +HAL_RVV_DEFINE_ALL( int8_t, int8, 8, i8, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ALL(int16_t, int16, 16, i16, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ALL(int32_t, int32, 32, i32, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ALL(int64_t, int64, 64, i64, HAL_RVV_SIGNED_PARAM) + +// LMUL = f2 +HAL_RVV_DEFINE_ONE( int8_t, int8, LMUL_f2, 8, i8, mf2, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ONE(int16_t, int16, LMUL_f2, 16, i16, mf2, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ONE(int32_t, int32, LMUL_f2, 32, i32, mf2, HAL_RVV_SIGNED_PARAM) + +// LMUL = f4 +HAL_RVV_DEFINE_ONE( int8_t, int8, LMUL_f4, 8, i8, mf4, HAL_RVV_SIGNED_PARAM) +HAL_RVV_DEFINE_ONE(int16_t, int16, LMUL_f4, 16, i16, mf4, HAL_RVV_SIGNED_PARAM) + +// LMUL = f8 +HAL_RVV_DEFINE_ONE( int8_t, int8, LMUL_f8, 8, i8, mf8, HAL_RVV_SIGNED_PARAM) + +// -------------------------------Define Floating Point-------------------------------- + +// LMUL = 1, 2, 4, 8 +HAL_RVV_DEFINE_ALL( float, float32, 32, f32, HAL_RVV_FLOAT_PARAM) +HAL_RVV_DEFINE_ALL(double, float64, 64, f64, HAL_RVV_FLOAT_PARAM) + +// LMUL = f2 +HAL_RVV_DEFINE_ONE( float, float32, LMUL_f2, 32, f32, mf2, HAL_RVV_FLOAT_PARAM) + +#undef HAL_RVV_SIZE_RELATED_CUSTOM +#undef HAL_RVV_DEFINE_ALL +#undef HAL_RVV_DEFINE_ONE +#undef HAL_RVV_BOOL_TYPE +#undef HAL_RVV_SIZE_UNRELATED +#undef HAL_RVV_SIZE_RELATED + +// -------------------------------Define cast-------------------------------- + +#define HAL_RVV_CVT(ONE, TWO) \ + template <> \ + inline ONE::VecType ONE::cast(TWO::VecType v, size_t vl) { return __riscv_vncvt_x(v, vl); } \ + template <> \ + inline TWO::VecType TWO::cast(ONE::VecType v, size_t vl) { return __riscv_vwcvt_x(v, vl); } + +HAL_RVV_CVT(RVV_I8M4, RVV_I16M8) +HAL_RVV_CVT(RVV_I8M2, RVV_I16M4) +HAL_RVV_CVT(RVV_I8M1, RVV_I16M2) +HAL_RVV_CVT(RVV_I8MF2, RVV_I16M1) +HAL_RVV_CVT(RVV_I8MF4, RVV_I16MF2) +HAL_RVV_CVT(RVV_I8MF8, RVV_I16MF4) + +HAL_RVV_CVT(RVV_I16M4, RVV_I32M8) +HAL_RVV_CVT(RVV_I16M2, RVV_I32M4) +HAL_RVV_CVT(RVV_I16M1, RVV_I32M2) +HAL_RVV_CVT(RVV_I16MF2, RVV_I32M1) +HAL_RVV_CVT(RVV_I16MF4, RVV_I32MF2) + +HAL_RVV_CVT(RVV_I32M4, RVV_I64M8) +HAL_RVV_CVT(RVV_I32M2, RVV_I64M4) +HAL_RVV_CVT(RVV_I32M1, RVV_I64M2) +HAL_RVV_CVT(RVV_I32MF2, RVV_I64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(ONE, TWO) \ + template <> \ + inline ONE::VecType ONE::cast(TWO::VecType v, size_t vl) { return __riscv_vncvt_x(v, vl); } \ + template <> \ + inline TWO::VecType TWO::cast(ONE::VecType v, size_t vl) { return __riscv_vwcvtu_x(v, vl); } + +HAL_RVV_CVT(RVV_U8M4, RVV_U16M8) +HAL_RVV_CVT(RVV_U8M2, RVV_U16M4) +HAL_RVV_CVT(RVV_U8M1, RVV_U16M2) +HAL_RVV_CVT(RVV_U8MF2, RVV_U16M1) +HAL_RVV_CVT(RVV_U8MF4, RVV_U16MF2) +HAL_RVV_CVT(RVV_U8MF8, RVV_U16MF4) + +HAL_RVV_CVT(RVV_U16M4, RVV_U32M8) +HAL_RVV_CVT(RVV_U16M2, RVV_U32M4) +HAL_RVV_CVT(RVV_U16M1, RVV_U32M2) +HAL_RVV_CVT(RVV_U16MF2, RVV_U32M1) +HAL_RVV_CVT(RVV_U16MF4, RVV_U32MF2) + +HAL_RVV_CVT(RVV_U32M4, RVV_U64M8) +HAL_RVV_CVT(RVV_U32M2, RVV_U64M4) +HAL_RVV_CVT(RVV_U32M1, RVV_U64M2) +HAL_RVV_CVT(RVV_U32MF2, RVV_U64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(ONE, FOUR) \ + template <> \ + inline FOUR::VecType FOUR::cast(ONE::VecType v, size_t vl) { return __riscv_vsext_vf4(v, vl); } \ + template <> \ + inline ONE::VecType ONE::cast(FOUR::VecType v, size_t vl) { \ + return __riscv_vncvt_x(__riscv_vncvt_x(v, vl), vl); \ + } + +HAL_RVV_CVT(RVV_I8M2, RVV_I32M8) +HAL_RVV_CVT(RVV_I8M1, RVV_I32M4) +HAL_RVV_CVT(RVV_I8MF2, RVV_I32M2) +HAL_RVV_CVT(RVV_I8MF4, RVV_I32M1) +HAL_RVV_CVT(RVV_I8MF8, RVV_I32MF2) + +HAL_RVV_CVT(RVV_I16M2, RVV_I64M8) +HAL_RVV_CVT(RVV_I16M1, RVV_I64M4) +HAL_RVV_CVT(RVV_I16MF2, RVV_I64M2) +HAL_RVV_CVT(RVV_I16MF4, RVV_I64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(ONE, FOUR) \ + template <> \ + inline FOUR::VecType FOUR::cast(ONE::VecType v, size_t vl) { return __riscv_vzext_vf4(v, vl); } \ + template <> \ + inline ONE::VecType ONE::cast(FOUR::VecType v, size_t vl) { \ + return __riscv_vncvt_x(__riscv_vncvt_x(v, vl), vl); \ + } + +HAL_RVV_CVT(RVV_U8M2, RVV_U32M8) +HAL_RVV_CVT(RVV_U8M1, RVV_U32M4) +HAL_RVV_CVT(RVV_U8MF2, RVV_U32M2) +HAL_RVV_CVT(RVV_U8MF4, RVV_U32M1) +HAL_RVV_CVT(RVV_U8MF8, RVV_U32MF2) + +HAL_RVV_CVT(RVV_U16M2, RVV_U64M8) +HAL_RVV_CVT(RVV_U16M1, RVV_U64M4) +HAL_RVV_CVT(RVV_U16MF2, RVV_U64M2) +HAL_RVV_CVT(RVV_U16MF4, RVV_U64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(ONE, EIGHT) \ + template <> \ + inline EIGHT::VecType EIGHT::cast(ONE::VecType v, size_t vl) { return __riscv_vsext_vf8(v, vl); } \ + template <> \ + inline ONE::VecType ONE::cast(EIGHT::VecType v, size_t vl) { \ + return __riscv_vncvt_x(__riscv_vncvt_x(__riscv_vncvt_x(v ,vl), vl), vl); \ + } + +HAL_RVV_CVT(RVV_I8M1, RVV_I64M8) +HAL_RVV_CVT(RVV_I8MF2, RVV_I64M4) +HAL_RVV_CVT(RVV_I8MF4, RVV_I64M2) +HAL_RVV_CVT(RVV_I8MF8, RVV_I64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(ONE, EIGHT) \ + template <> \ + inline EIGHT::VecType EIGHT::cast(ONE::VecType v, size_t vl) { return __riscv_vzext_vf8(v, vl); } \ + template <> \ + inline ONE::VecType ONE::cast(EIGHT::VecType v, size_t vl) { \ + return __riscv_vncvt_x(__riscv_vncvt_x(__riscv_vncvt_x(v ,vl), vl), vl); \ + } + +HAL_RVV_CVT(RVV_U8M1, RVV_U64M8) +HAL_RVV_CVT(RVV_U8MF2, RVV_U64M4) +HAL_RVV_CVT(RVV_U8MF4, RVV_U64M2) +HAL_RVV_CVT(RVV_U8MF8, RVV_U64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(F32, F64) \ + template <> \ + inline F32::VecType F32::cast(F64::VecType v, size_t vl) { return __riscv_vfncvt_f(v, vl); } \ + template <> \ + inline F64::VecType F64::cast(F32::VecType v, size_t vl) { return __riscv_vfwcvt_f(v, vl); } + +HAL_RVV_CVT(RVV_F32M4, RVV_F64M8) +HAL_RVV_CVT(RVV_F32M2, RVV_F64M4) +HAL_RVV_CVT(RVV_F32M1, RVV_F64M2) +HAL_RVV_CVT(RVV_F32MF2, RVV_F64M1) + +#undef HAL_RVV_CVT + +#define HAL_RVV_CVT(A, B, A_TYPE, B_TYPE, LMUL_TYPE, LMUL) \ + template <> \ + inline RVV::VecType RVV::cast( \ + RVV::VecType v, [[maybe_unused]] size_t vl \ + ) { \ + return __riscv_vreinterpret_##A_TYPE##LMUL(v); \ + } \ + template <> \ + inline RVV::VecType RVV::cast( \ + RVV::VecType v, [[maybe_unused]] size_t vl \ + ) { \ + return __riscv_vreinterpret_##B_TYPE##LMUL(v); \ + } + +#define HAL_RVV_CVT2(A, B, A_TYPE, B_TYPE) \ + HAL_RVV_CVT(A, B, A_TYPE, B_TYPE, LMUL_1, m1) \ + HAL_RVV_CVT(A, B, A_TYPE, B_TYPE, LMUL_2, m2) \ + HAL_RVV_CVT(A, B, A_TYPE, B_TYPE, LMUL_4, m4) \ + HAL_RVV_CVT(A, B, A_TYPE, B_TYPE, LMUL_8, m8) + +HAL_RVV_CVT2( uint8_t, int8_t, u8, i8) +HAL_RVV_CVT2(uint16_t, int16_t, u16, i16) +HAL_RVV_CVT2(uint32_t, int32_t, u32, i32) +HAL_RVV_CVT2(uint64_t, int64_t, u64, i64) + +HAL_RVV_CVT2( int32_t, float, i32, f32) +HAL_RVV_CVT2(uint32_t, float, u32, f32) +HAL_RVV_CVT2( int64_t, double, i64, f64) +HAL_RVV_CVT2(uint64_t, double, u64, f64) + +#undef HAL_RVV_CVT2 + +HAL_RVV_CVT( uint8_t, int8_t, u8, i8, LMUL_f2, mf2) +HAL_RVV_CVT(uint16_t, int16_t, u16, i16, LMUL_f2, mf2) +HAL_RVV_CVT(uint32_t, int32_t, u32, i32, LMUL_f2, mf2) +HAL_RVV_CVT( int32_t, float, i32, f32, LMUL_f2, mf2) +HAL_RVV_CVT(uint32_t, float, u32, f32, LMUL_f2, mf2) + +HAL_RVV_CVT( uint8_t, int8_t, u8, i8, LMUL_f4, mf4) +HAL_RVV_CVT(uint16_t, int16_t, u16, i16, LMUL_f4, mf4) + +HAL_RVV_CVT( uint8_t, int8_t, u8, i8, LMUL_f8, mf8) + +#undef HAL_RVV_CVT + +}} // namespace cv::cv_hal_rvv