From 358e3065abe9caa5ab7b86e004c85b90709f8ecf Mon Sep 17 00:00:00 2001 From: cudawarped <12133430+cudawarped@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:21:11 +0300 Subject: [PATCH 01/34] cuda: add default ptx when CUDA_ARCH_BIN is missing and allow CUDA_ARCH_PTX to be passed in isolation --- cmake/OpenCVDetectCUDA.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index 4a562bdaf9..752ae0edcf 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -228,7 +228,7 @@ if(CUDA_FOUND) endif() endmacro() - set(__cuda_arch_ptx "") + set(__cuda_arch_ptx ${CUDA_ARCH_PTX}) if(CUDA_GENERATION STREQUAL "Fermi") set(__cuda_arch_bin ${_arch_fermi}) elseif(CUDA_GENERATION STREQUAL "Kepler") @@ -259,7 +259,7 @@ if(CUDA_FOUND) set(__cuda_arch_bin ${CUDA_ARCH_BIN}) endif() - if(NOT DEFINED __cuda_arch_bin) + if(NOT DEFINED __cuda_arch_bin AND NOT DEFINED __cuda_arch_ptx) if(ARM) set(__cuda_arch_bin "3.2") set(__cuda_arch_ptx "") @@ -295,6 +295,7 @@ if(CUDA_FOUND) ${_arch_lovelace} ${_arch_hopper} ) + list(GET __cuda_arch_bin -1 __cuda_arch_ptx) endif() endif() From 5c9f58e124bc1bc208b47d5745c58291b1a17b16 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Thu, 7 Sep 2023 13:03:41 +0300 Subject: [PATCH 02/34] More strict test for MSMF FOURCC (camera). --- modules/videoio/test/test_camera.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index 8b0f0efe83..4466919f50 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -130,7 +130,12 @@ TEST(DISABLED_videoio_camera, msmf_read_yuyv) std::cout << "Capturing FPS: " << capture.get(CAP_PROP_FPS) << std::endl; int fourcc = (int)capture.get(CAP_PROP_FOURCC); std::cout << "FOURCC code: " << cv::format("0x%8x", fourcc) << std::endl; - test_readFrames(capture); + cv::Mat frame; + for (int i = 0; i < 10; i++) + { + capture >> frame; + EXPECT_EQ(2, frame.channels()); + } capture.release(); } From ae1d1b6d5517d1beb1c30221e5091762f63bbff5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 12 Sep 2023 12:17:57 +0300 Subject: [PATCH 03/34] fix drawDetectedCornersCharuco, drawDetectedMarkers, drawDetectedDiamonds added tests --- .../misc/python/test/test_objdetect_aruco.py | 64 +++++++++++++++++++ .../objdetect/src/aruco/aruco_detector.cpp | 18 +++--- .../objdetect/src/aruco/charuco_detector.cpp | 39 ++++++----- .../objdetect/test/test_charucodetection.cpp | 52 +++++++++++++++ 4 files changed, 150 insertions(+), 23 deletions(-) diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index dda58b6460..8dd407d32f 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -394,5 +394,69 @@ class aruco_objdetect_test(NewOpenCVTests): self.assertEqual(2, img_points.shape[2]) np.testing.assert_array_equal(chessboard_corners, obj_points[:, :, :2].reshape(-1, 2)) + def test_draw_detected_markers(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx4 Mat with 2 channels + points1 = np.array(detected_points).reshape(-1, 4, 1, 2) + img = cv.aruco.drawDetectedMarkers(img, points1, borderColor=255) + + # check that the marker borders are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 1) + self.assertEqual(img[10, 10], 255) + self.assertEqual(img[50, 10], 255) + self.assertEqual(img[50, 50], 255) + self.assertEqual(img[10, 50], 255) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedMarkers(img, points2, borderColor=255) + + def test_draw_detected_charuco(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx1 Mat with 2 channels + points = np.array(detected_points).reshape(-1, 1, 2) + img = cv.aruco.drawDetectedCornersCharuco(img, points, cornerColor=255) + + # check that the 4 charuco corners are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 4) + for contour in contours: + center_x = round(np.average(contour[:, 0, 0])) + center_y = round(np.average(contour[:, 0, 1])) + center = [center_x, center_y] + self.assertTrue(center in detected_points[0]) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedCornersCharuco(img, points2, borderColor=255) + + def test_draw_detected_diamonds(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx4 Mat with 2 channels + points = np.array(detected_points).reshape(-1, 4, 1, 2) + img = cv.aruco.drawDetectedDiamonds(img, points, borderColor=255) + + # check that the diamonds borders are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 1) + self.assertEqual(img[10, 10], 255) + self.assertEqual(img[50, 10], 255) + self.assertEqual(img[50, 50], 255) + self.assertEqual(img[10, 50], 255) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedDiamonds(img, points2, borderColor=255) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 84ccc6e323..b18ddd98ba 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1315,24 +1315,26 @@ void drawDetectedMarkers(InputOutputArray _image, InputArrayOfArrays _corners, int nMarkers = (int)_corners.total(); for(int i = 0; i < nMarkers; i++) { Mat currentMarker = _corners.getMat(i); - CV_Assert(currentMarker.total() == 4 && currentMarker.type() == CV_32FC2); + CV_Assert(currentMarker.total() == 4 && currentMarker.channels() == 2); + if (currentMarker.type() != CV_32SC2) + currentMarker.convertTo(currentMarker, CV_32SC2); // draw marker sides for(int j = 0; j < 4; j++) { - Point2f p0, p1; - p0 = currentMarker.ptr(0)[j]; - p1 = currentMarker.ptr(0)[(j + 1) % 4]; + Point p0, p1; + p0 = currentMarker.ptr(0)[j]; + p1 = currentMarker.ptr(0)[(j + 1) % 4]; line(_image, p0, p1, borderColor, 1); } // draw first corner mark - rectangle(_image, currentMarker.ptr(0)[0] - Point2f(3, 3), - currentMarker.ptr(0)[0] + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, currentMarker.ptr(0)[0] - Point(3, 3), + currentMarker.ptr(0)[0] + Point(3, 3), cornerColor, 1, LINE_AA); // draw ID if(_ids.total() != 0) { - Point2f cent(0, 0); + Point cent(0, 0); for(int p = 0; p < 4; p++) - cent += currentMarker.ptr(0)[p]; + cent += currentMarker.ptr(0)[p]; cent = cent / 4.; stringstream s; s << "id=" << _ids.getMat().ptr(0)[i]; diff --git a/modules/objdetect/src/aruco/charuco_detector.cpp b/modules/objdetect/src/aruco/charuco_detector.cpp index 55a7f8d888..4605de19c5 100644 --- a/modules/objdetect/src/aruco/charuco_detector.cpp +++ b/modules/objdetect/src/aruco/charuco_detector.cpp @@ -507,20 +507,27 @@ void drawDetectedCornersCharuco(InputOutputArray _image, InputArray _charucoCorn InputArray _charucoIds, Scalar cornerColor) { CV_Assert(!_image.getMat().empty() && (_image.getMat().channels() == 1 || _image.getMat().channels() == 3)); - CV_Assert((_charucoCorners.getMat().total() == _charucoIds.getMat().total()) || - _charucoIds.getMat().total() == 0); + CV_Assert((_charucoCorners.total() == _charucoIds.total()) || + _charucoIds.total() == 0); + CV_Assert(_charucoCorners.channels() == 2); - size_t nCorners = _charucoCorners.getMat().total(); + Mat charucoCorners = _charucoCorners.getMat(); + if (charucoCorners.type() != CV_32SC2) + charucoCorners.convertTo(charucoCorners, CV_32SC2); + Mat charucoIds; + if (!_charucoIds.empty()) + charucoIds = _charucoIds.getMat(); + size_t nCorners = charucoCorners.total(); for(size_t i = 0; i < nCorners; i++) { - Point2f corner = _charucoCorners.getMat().at((int)i); + Point corner = charucoCorners.at((int)i); // draw first corner mark - rectangle(_image, corner - Point2f(3, 3), corner + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, corner - Point(3, 3), corner + Point(3, 3), cornerColor, 1, LINE_AA); // draw ID if(!_charucoIds.empty()) { - int id = _charucoIds.getMat().at((int)i); + int id = charucoIds.at((int)i); stringstream s; s << "id=" << id; - putText(_image, s.str(), corner + Point2f(5, -5), FONT_HERSHEY_SIMPLEX, 0.5, + putText(_image, s.str(), corner + Point(5, -5), FONT_HERSHEY_SIMPLEX, 0.5, cornerColor, 2); } } @@ -540,25 +547,27 @@ void drawDetectedDiamonds(InputOutputArray _image, InputArrayOfArrays _corners, int nMarkers = (int)_corners.total(); for(int i = 0; i < nMarkers; i++) { Mat currentMarker = _corners.getMat(i); - CV_Assert(currentMarker.total() == 4 && currentMarker.type() == CV_32FC2); + CV_Assert(currentMarker.total() == 4 && currentMarker.channels() == 2); + if (currentMarker.type() != CV_32SC2) + currentMarker.convertTo(currentMarker, CV_32SC2); // draw marker sides for(int j = 0; j < 4; j++) { - Point2f p0, p1; - p0 = currentMarker.at< Point2f >(j); - p1 = currentMarker.at< Point2f >((j + 1) % 4); + Point p0, p1; + p0 = currentMarker.at(j); + p1 = currentMarker.at((j + 1) % 4); line(_image, p0, p1, borderColor, 1); } // draw first corner mark - rectangle(_image, currentMarker.at< Point2f >(0) - Point2f(3, 3), - currentMarker.at< Point2f >(0) + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, currentMarker.at(0) - Point(3, 3), + currentMarker.at(0) + Point(3, 3), cornerColor, 1, LINE_AA); // draw id composed by four numbers if(_ids.total() != 0) { - Point2f cent(0, 0); + Point cent(0, 0); for(int p = 0; p < 4; p++) - cent += currentMarker.at< Point2f >(p); + cent += currentMarker.at(p); cent = cent / 4.; stringstream s; s << "id=" << _ids.getMat().at< Vec4i >(i); diff --git a/modules/objdetect/test/test_charucodetection.cpp b/modules/objdetect/test/test_charucodetection.cpp index 9e561bc40a..418826d947 100644 --- a/modules/objdetect/test/test_charucodetection.cpp +++ b/modules/objdetect/test/test_charucodetection.cpp @@ -689,6 +689,58 @@ TEST(Charuco, testmatchImagePoints) } } +typedef testing::TestWithParam CharucoDraw; +INSTANTIATE_TEST_CASE_P(/**/, CharucoDraw, testing::Values(CV_8UC2, CV_8SC2, CV_16UC2, CV_16SC2, CV_32SC2, CV_32FC2, CV_64FC2)); +TEST_P(CharucoDraw, testDrawDetected) { + vector> detected_golds = {{Point(20, 20), Point(80, 20), Point(80, 80), Point2f(20, 80)}}; + Point center_gold = (detected_golds[0][0] + detected_golds[0][1] + detected_golds[0][2] + detected_golds[0][3]) / 4; + int type = GetParam(); + vector detected(detected_golds[0].size(), Mat(4, 1, type)); + // copy detected_golds to detected with any 2 channels type + for (size_t i = 0ull; i < detected_golds[0].size(); i++) { + detected[0].row((int)i) = Scalar(detected_golds[0][i].x, detected_golds[0][i].y); + } + vector> contours; + Point detectedCenter; + Moments m; + Mat img; + + // check drawDetectedMarkers + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedMarkers(img, detected, noArray(), Scalar(255, 255, 255))); + // check that the marker borders are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 1ull); + m = moments(contours[0]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + ASSERT_EQ(detectedCenter, center_gold); + + + // check drawDetectedCornersCharuco + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedCornersCharuco(img, detected[0], noArray(), Scalar(255, 255, 255))); + // check that the 4 charuco corners are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 4ull); + for (size_t i = 0ull; i < 4ull; i++) { + m = moments(contours[i]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + // detectedCenter must be in detected_golds + ASSERT_TRUE(find(detected_golds[0].begin(), detected_golds[0].end(), detectedCenter) != detected_golds[0].end()); + } + + + // check drawDetectedDiamonds + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedDiamonds(img, detected, noArray(), Scalar(255, 255, 255))); + // check that the diamonds borders are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 1ull); + m = moments(contours[0]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + ASSERT_EQ(detectedCenter, center_gold); +} + typedef testing::TestWithParam CharucoBoard; INSTANTIATE_TEST_CASE_P(/**/, CharucoBoard, testing::Values(Size(3, 2), Size(3, 2), Size(6, 2), Size(2, 6), Size(3, 4), Size(4, 3), Size(7, 3), Size(3, 7))); From eb20bb3b23d234bf545999769cf2f0deec77fa68 Mon Sep 17 00:00:00 2001 From: Yuriy Chernyshov Date: Wed, 13 Sep 2023 12:15:34 +0300 Subject: [PATCH 04/34] Add missing sanitizer interface include --- modules/core/src/hal_internal.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/core/src/hal_internal.cpp b/modules/core/src/hal_internal.cpp index f581d05cf4..28227688c0 100644 --- a/modules/core/src/hal_internal.cpp +++ b/modules/core/src/hal_internal.cpp @@ -66,6 +66,7 @@ #if defined(__clang__) && defined(__has_feature) #if __has_feature(memory_sanitizer) +#include #define CV_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ __msan_unpoison(address, size) #endif From 347a1e2913119a2033483a743f1bc1c4e22ba8f3 Mon Sep 17 00:00:00 2001 From: FlyinTeller Date: Thu, 14 Sep 2023 09:04:41 +0200 Subject: [PATCH 05/34] Update OpenCVFindOpenBLAS.cmake to accomodate alternative lib name openBLAS windows release calls their library libopenblas which was not recognized before. see #24268 --- cmake/OpenCVFindOpenBLAS.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVFindOpenBLAS.cmake b/cmake/OpenCVFindOpenBLAS.cmake index d1db034908..4e3f0cc210 100644 --- a/cmake/OpenCVFindOpenBLAS.cmake +++ b/cmake/OpenCVFindOpenBLAS.cmake @@ -73,7 +73,7 @@ SET(Open_BLAS_LIB_SEARCH_PATHS ) FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS} NO_DEFAULT_PATH) -FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS} NO_DEFAULT_PATH) +FIND_LIBRARY(OpenBLAS_LIB NAMES openblas libopenblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS} NO_DEFAULT_PATH) SET(OpenBLAS_FOUND ON) From 1e54e5657927996e86b155d89f51c7b5a73461d2 Mon Sep 17 00:00:00 2001 From: alexlyulkov Date: Thu, 14 Sep 2023 14:25:24 +0700 Subject: [PATCH 06/34] Merge pull request #24266 from alexlyulkov:al/tf-argmax-default-dim Added default dimension value to tensorflow ArgMax and ArgMin layers #24266 Added default dimension value to tensorflow ArgMax and ArgMin layers. Added exception when accessing layer's input with out of range index. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=48452 --- modules/dnn/src/tensorflow/tf_importer.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index a23fac187b..756bdc949c 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2665,14 +2665,20 @@ void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::No connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// ArgMin or ArgMax node void TFImporter::parseArg(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); const std::string& type = layer.op(); - Mat dimension = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(dimension.total() == 1 && dimension.type() == CV_32SC1); - layerParams.set("axis", *dimension.ptr()); + if (layer.input_size() < 2) + layerParams.set("axis", 0); // default dimension is 0 + else + { + Mat dimension = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(dimension.total() == 1 && dimension.type() == CV_32SC1); + layerParams.set("axis", dimension.at(0)); + } layerParams.set("op", type == "ArgMax" ? "max" : "min"); layerParams.set("keepdims", false); //tensorflow doesn't have this atrr, the output's dims minus one(default); @@ -2866,6 +2872,7 @@ const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDe if (input_blob_index == -1) CV_Error(Error::StsError, "Const input blob for weights not found"); + CV_CheckLT(input_blob_index, layer.input_size(), "Input index is out of range"); Pin kernel_inp = parsePin(layer.input(input_blob_index)); if (const_layers.find(kernel_inp.name) == const_layers.end()) From c5edd20354021feabc984262aa6b6e285333dec5 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Thu, 14 Sep 2023 13:11:53 +0300 Subject: [PATCH 07/34] Higher threshold for FasterRCNN_vgg16 --- modules/dnn/test/test_caffe_importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 66eff49979..14c3ffc6d4 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -731,7 +731,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - double scoreDiff = 0.001, iouDiff = 0.03; + double scoreDiff = 0.0012, iouDiff = 0.03; #if defined(INF_ENGINE_RELEASE) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); From 3880d059b333f07896187015f1ebbb553d869f32 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 14 Sep 2023 14:16:28 +0200 Subject: [PATCH 08/34] Merge pull request #24260 from vrabaud:ubsan Fix undefined behavior arithmetic in copyMakeBorder and adjustROI. #24260 This is due to the undefined: negative int multiplied by size_t pointer increment. To test, compile with: ``` mkdir build cd build cmake ../ -DCMAKE_C_FLAGS_INIT="-fsanitize=undefined" -DCMAKE_CXX_FLAGS_INIT="-fsanitize=undefined" -DCMAKE_C_COMPILER="/usr/bin/clang" -DCMAKE_CXX_COMPILER="/usr/bin/clang++" -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=undefined -lubsan" ``` And run: ``` make -j opencv_test_core && ./bin/opencv_test_core --gtest_filter=*UndefinedBehavior* ``` ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/src/copy.cpp | 4 ++-- modules/core/src/matrix.cpp | 2 +- modules/core/test/test_mat.cpp | 12 ++++++++++++ modules/core/test/test_operations.cpp | 9 +++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/modules/core/src/copy.cpp b/modules/core/src/copy.cpp index aab4fbd3f0..d641c6f497 100644 --- a/modules/core/src/copy.cpp +++ b/modules/core/src/copy.cpp @@ -860,14 +860,14 @@ void copyMakeBorder_8u( const uchar* src, size_t srcstep, cv::Size srcroi, } dstroi.width *= elemSize; - dst += dststep*top; for( i = 0; i < top; i++ ) { j = cv::borderInterpolate(i - top, srcroi.height, borderType); - memcpy(dst + (i - top)*dststep, dst + j*dststep, dstroi.width); + memcpy(dst + i*dststep, dst + (top+j)*dststep, dstroi.width); } + dst += dststep*top; for( i = 0; i < bottom; i++ ) { j = cv::borderInterpolate(i + srcroi.height, srcroi.height, borderType); diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 8111dc2230..513e13ea5f 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -1128,7 +1128,7 @@ Mat& Mat::adjustROI( int dtop, int dbottom, int dleft, int dright ) if(col1 > col2) std::swap(col1, col2); - data += (row1 - ofs.y)*step + (col1 - ofs.x)*esz; + data += (row1 - ofs.y)*(std::ptrdiff_t)step + (col1 - ofs.x)*(std::ptrdiff_t)esz; rows = row2 - row1; cols = col2 - col1; size.p[0] = rows; size.p[1] = cols; updateContinuityFlag(); diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index aea0507149..f476118f65 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -1368,6 +1368,18 @@ TEST(Core_Mat, copyNx1ToVector) ASSERT_PRED_FORMAT2(cvtest::MatComparator(0, 0), ref_dst16, cv::Mat_(dst16)); } +TEST(Core_Mat, copyMakeBoderUndefinedBehavior) +{ + Mat1b src(4, 4), dst; + randu(src, Scalar(10), Scalar(100)); + // This could trigger a (signed int)*size_t operation which is undefined behavior. + cv::copyMakeBorder(src, dst, 1, 1, 1, 1, cv::BORDER_REFLECT_101); + EXPECT_EQ(0, cv::norm(src.row(1), dst(Rect(1,0,4,1)))); + EXPECT_EQ(0, cv::norm(src.row(2), dst(Rect(1,5,4,1)))); + EXPECT_EQ(0, cv::norm(src.col(1), dst(Rect(0,1,1,4)))); + EXPECT_EQ(0, cv::norm(src.col(2), dst(Rect(5,1,1,4)))); +} + TEST(Core_Matx, fromMat_) { Mat_ a = (Mat_(2,2) << 10, 11, 12, 13); diff --git a/modules/core/test/test_operations.cpp b/modules/core/test/test_operations.cpp index 934028f3ae..5158e65bda 100644 --- a/modules/core/test/test_operations.cpp +++ b/modules/core/test/test_operations.cpp @@ -1379,6 +1379,15 @@ TEST(MatTestRoi, adjustRoiOverflow) ASSERT_EQ(roi.rows, m.rows); } +TEST(MatTestRoi, adjustRoiUndefinedBehavior) +{ + Mat m(6, 6, CV_8U); + Mat roi(m, cv::Range(2, 4), cv::Range(2, 4)); + // This could trigger a (negative int)*size_t when updating data, + // which is undefined behavior. + roi.adjustROI(2, 2, 2, 2); + EXPECT_EQ(m.data, roi.data); +} CV_ENUM(SortRowCol, SORT_EVERY_COLUMN, SORT_EVERY_ROW) CV_ENUM(SortOrder, SORT_ASCENDING, SORT_DESCENDING) From 5e9191558d787b33e2f4d43833106b2e00c21cd1 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Fri, 15 Sep 2023 01:37:46 +0800 Subject: [PATCH 09/34] Merge pull request #24058 from hanliutong:rewrite-imgporc Rewrite Universal Intrinsic code by using new API: ImgProc module. #24058 The goal of this series of PRs is to modify the SIMD code blocks guarded by CV_SIMD macro in the `opencv/modules/imgproc` folder: rewrite them by using the new Universal Intrinsic API. For easier review, this PR includes a part of the rewritten code, and another part will be brought in the next PR (coming soon). I tested this patch on RVV (QEMU) and AVX devices, `opencv_test_imgproc` is passed. The patch is partially auto-generated by using the [rewriter](https://github.com/hanliutong/rewriter), related PR https://github.com/opencv/opencv/pull/23885 and https://github.com/opencv/opencv/pull/23980. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/accum.simd.hpp | 640 ++++++++++++------------ modules/imgproc/src/blend.cpp | 54 +- modules/imgproc/src/canny.cpp | 66 +-- modules/imgproc/src/color_rgb.simd.hpp | 174 ++++--- modules/imgproc/src/contours.cpp | 20 +- modules/imgproc/src/corner.cpp | 72 +-- modules/imgproc/src/histogram.cpp | 34 +- modules/imgproc/src/pyramids.cpp | 478 +++++++++--------- modules/imgproc/src/resize.cpp | 413 ++++++++------- modules/imgproc/src/smooth.simd.hpp | 329 ++++++------ modules/imgproc/src/spatialgradient.cpp | 40 +- 11 files changed, 1180 insertions(+), 1140 deletions(-) diff --git a/modules/imgproc/src/accum.simd.hpp b/modules/imgproc/src/accum.simd.hpp index 6b0e6d6fbe..7fe7aabeaf 100644 --- a/modules/imgproc/src/accum.simd.hpp +++ b/modules/imgproc/src/accum.simd.hpp @@ -139,7 +139,7 @@ void acc_general_(const T* src, AT* dst, const uchar* mask, int len, int cn, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -187,7 +187,7 @@ accSqr_general_( const T* src, AT* dst, const uchar* mask, int len, int cn, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -236,7 +236,7 @@ accProd_general_( const T* src1, const T* src2, AT* dst, const uchar* mask, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -285,16 +285,16 @@ accW_general_( const T* src, AT* dst, const uchar* mask, int len, int cn, double } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -309,10 +309,10 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -323,9 +323,9 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -333,10 +333,10 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -344,12 +344,12 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + (x * cn), v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -373,18 +373,18 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_load_deinterleave(dst + ((x + step * 2) * cn), v_dst010, v_dst110, v_dst210); v_load_deinterleave(dst + ((x + step * 3) * cn), v_dst011, v_dst111, v_dst211); - v_dst000 += v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst100 += v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst200 += v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst001 += v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst101 += v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst201 += v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst010 += v_cvt_f32(v_reinterpret_as_s32(v_src010)); - v_dst110 += v_cvt_f32(v_reinterpret_as_s32(v_src110)); - v_dst210 += v_cvt_f32(v_reinterpret_as_s32(v_src210)); - v_dst011 += v_cvt_f32(v_reinterpret_as_s32(v_src011)); - v_dst111 += v_cvt_f32(v_reinterpret_as_s32(v_src111)); - v_dst211 += v_cvt_f32(v_reinterpret_as_s32(v_src211)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst010 = v_add(v_dst010, v_cvt_f32(v_reinterpret_as_s32(v_src010))); + v_dst110 = v_add(v_dst110, v_cvt_f32(v_reinterpret_as_s32(v_src110))); + v_dst210 = v_add(v_dst210, v_cvt_f32(v_reinterpret_as_s32(v_src210))); + v_dst011 = v_add(v_dst011, v_cvt_f32(v_reinterpret_as_s32(v_src011))); + v_dst111 = v_add(v_dst111, v_cvt_f32(v_reinterpret_as_s32(v_src111))); + v_dst211 = v_add(v_dst211, v_cvt_f32(v_reinterpret_as_s32(v_src211))); v_store_interleave(dst + (x * cn), v_dst000, v_dst100, v_dst200); v_store_interleave(dst + ((x + step) * cn), v_dst001, v_dst101, v_dst201); @@ -400,9 +400,9 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -413,8 +413,8 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src0))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src1))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src0)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src1)))); } } else @@ -425,14 +425,14 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src0))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src1))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src0)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src1)))); } } else if (cn == 3) @@ -441,12 +441,12 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -456,12 +456,12 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn v_load_deinterleave(dst + x * cn, v_dst00, v_dst10, v_dst20); v_load_deinterleave(dst + (x + step) * cn, v_dst01, v_dst11, v_dst21); - v_dst00 += v_cvt_f32(v_reinterpret_as_s32(v_src00)); - v_dst01 += v_cvt_f32(v_reinterpret_as_s32(v_src01)); - v_dst10 += v_cvt_f32(v_reinterpret_as_s32(v_src10)); - v_dst11 += v_cvt_f32(v_reinterpret_as_s32(v_src11)); - v_dst20 += v_cvt_f32(v_reinterpret_as_s32(v_src20)); - v_dst21 += v_cvt_f32(v_reinterpret_as_s32(v_src21)); + v_dst00 = v_add(v_dst00, v_cvt_f32(v_reinterpret_as_s32(v_src00))); + v_dst01 = v_add(v_dst01, v_cvt_f32(v_reinterpret_as_s32(v_src01))); + v_dst10 = v_add(v_dst10, v_cvt_f32(v_reinterpret_as_s32(v_src10))); + v_dst11 = v_add(v_dst11, v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_dst20 = v_add(v_dst20, v_cvt_f32(v_reinterpret_as_s32(v_src20))); + v_dst21 = v_add(v_dst21, v_cvt_f32(v_reinterpret_as_s32(v_src21))); v_store_interleave(dst + x * cn, v_dst00, v_dst10, v_dst20); v_store_interleave(dst + (x + step) * cn, v_dst01, v_dst11, v_dst21); @@ -551,9 +551,9 @@ void acc_simd_(const float* src, float* dst, const uchar* mask, int len, int cn) void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -586,14 +586,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_float64 v_dst6 = vx_load(dst + x + step * 6); v_float64 v_dst7 = vx_load(dst + x + step * 7); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; - v_dst4 = v_dst4 + v_src4; - v_dst5 = v_dst5 + v_src5; - v_dst6 = v_dst6 + v_src6; - v_dst7 = v_dst7 + v_src7; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); + v_dst4 = v_add(v_dst4, v_src4); + v_dst5 = v_add(v_dst5, v_src5); + v_dst6 = v_add(v_dst6, v_src6); + v_dst7 = v_add(v_dst7, v_src7); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -613,9 +613,9 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_int0, v_int1; v_expand(v_src, v_int0, v_int1); @@ -641,14 +641,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_float64 v_dst6 = vx_load(dst + x + step * 6); v_float64 v_dst7 = vx_load(dst + x + step * 7); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; - v_dst4 = v_dst4 + v_src4; - v_dst5 = v_dst5 + v_src5; - v_dst6 = v_dst6 + v_src6; - v_dst7 = v_dst7 + v_src7; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); + v_dst4 = v_add(v_dst4, v_src4); + v_dst5 = v_add(v_dst5, v_src5); + v_dst6 = v_add(v_dst6, v_src6); + v_dst7 = v_add(v_dst7, v_src7); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -665,12 +665,12 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + (x * cn), v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -726,14 +726,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_load_deinterleave(dst + ((x + step * 6) * cn), v_dst0110, v_dst1110, v_dst2110); v_load_deinterleave(dst + ((x + step * 7) * cn), v_dst0111, v_dst1111, v_dst2111); - v_store_interleave(dst + (x * cn), v_dst0000 + v_src0000, v_dst1000 + v_src1000, v_dst2000 + v_src2000); - v_store_interleave(dst + ((x + step) * cn), v_dst0001 + v_src0001, v_dst1001 + v_src1001, v_dst2001 + v_src2001); - v_store_interleave(dst + ((x + step * 2) * cn), v_dst0010 + v_src0010, v_dst1010 + v_src1010, v_dst2010 + v_src2010); - v_store_interleave(dst + ((x + step * 3) * cn), v_dst0011 + v_src0011, v_dst1011 + v_src1011, v_dst2011 + v_src2011); - v_store_interleave(dst + ((x + step * 4) * cn), v_dst0100 + v_src0100, v_dst1100 + v_src1100, v_dst2100 + v_src2100); - v_store_interleave(dst + ((x + step * 5) * cn), v_dst0101 + v_src0101, v_dst1101 + v_src1101, v_dst2101 + v_src2101); - v_store_interleave(dst + ((x + step * 6) * cn), v_dst0110 + v_src0110, v_dst1110 + v_src1110, v_dst2110 + v_src2110); - v_store_interleave(dst + ((x + step * 7) * cn), v_dst0111 + v_src0111, v_dst1111 + v_src1111, v_dst2111 + v_src2111); + v_store_interleave(dst + (x * cn), v_add(v_dst0000, v_src0000), v_add(v_dst1000, v_src1000), v_add(v_dst2000, v_src2000)); + v_store_interleave(dst + ((x + step) * cn), v_add(v_dst0001, v_src0001), v_add(v_dst1001, v_src1001), v_add(v_dst2001, v_src2001)); + v_store_interleave(dst + ((x + step * 2) * cn), v_add(v_dst0010, v_src0010), v_add(v_dst1010, v_src1010), v_add(v_dst2010, v_src2010)); + v_store_interleave(dst + ((x + step * 3) * cn), v_add(v_dst0011, v_src0011), v_add(v_dst1011, v_src1011), v_add(v_dst2011, v_src2011)); + v_store_interleave(dst + ((x + step * 4) * cn), v_add(v_dst0100, v_src0100), v_add(v_dst1100, v_src1100), v_add(v_dst2100, v_src2100)); + v_store_interleave(dst + ((x + step * 5) * cn), v_add(v_dst0101, v_src0101), v_add(v_dst1101, v_src1101), v_add(v_dst2101, v_src2101)); + v_store_interleave(dst + ((x + step * 6) * cn), v_add(v_dst0110, v_src0110), v_add(v_dst1110, v_src1110), v_add(v_dst2110, v_src2110)); + v_store_interleave(dst + ((x + step * 7) * cn), v_add(v_dst0111, v_src0111), v_add(v_dst1111, v_src1111), v_add(v_dst2111, v_src2111)); } } } @@ -744,9 +744,9 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -767,10 +767,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -786,9 +786,9 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_int0, v_int1; v_expand(v_src, v_int0, v_int1); @@ -802,10 +802,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -818,12 +818,12 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); @@ -848,10 +848,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_load_deinterleave(dst + (x + step * 2) * cn, v_dst02, v_dst12, v_dst22); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst03, v_dst13, v_dst23); - v_store_interleave(dst + x * cn, v_dst00 + v_src00, v_dst10 + v_src10, v_dst20 + v_src20); - v_store_interleave(dst + (x + step) * cn, v_dst01 + v_src01, v_dst11 + v_src11, v_dst21 + v_src21); - v_store_interleave(dst + (x + step * 2) * cn, v_dst02 + v_src02, v_dst12 + v_src12, v_dst22 + v_src22); - v_store_interleave(dst + (x + step * 3) * cn, v_dst03 + v_src03, v_dst13 + v_src13, v_dst23 + v_src23); + v_store_interleave(dst + x * cn, v_add(v_dst00, v_src00), v_add(v_dst10, v_src10), v_add(v_dst20, v_src20)); + v_store_interleave(dst + (x + step) * cn, v_add(v_dst01, v_src01), v_add(v_dst11, v_src11), v_add(v_dst21, v_src21)); + v_store_interleave(dst + (x + step * 2) * cn, v_add(v_dst02, v_src02), v_add(v_dst12, v_src12), v_add(v_dst22, v_src22)); + v_store_interleave(dst + (x + step * 3) * cn, v_add(v_dst03, v_src03), v_add(v_dst13, v_src13), v_add(v_dst23, v_src23)); } } } @@ -1033,9 +1033,9 @@ void acc_simd_(const double* src, double* dst, const uchar* mask, int len, int c void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1052,10 +1052,10 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -1066,9 +1066,9 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int for ( ; x <= len - cVectorWidth ; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); v_src0 = v_mul_wrap(v_src0, v_src0); @@ -1078,10 +1078,10 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -1089,13 +1089,13 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int for ( ; x <= len - cVectorWidth ; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); @@ -1126,20 +1126,20 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_load_deinterleave(dst + (x + step * 2) * cn, v_dst010, v_dst110, v_dst210); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst011, v_dst111, v_dst211); - v_dst000 += v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst001 += v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst010 += v_cvt_f32(v_reinterpret_as_s32(v_src010)); - v_dst011 += v_cvt_f32(v_reinterpret_as_s32(v_src011)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst010 = v_add(v_dst010, v_cvt_f32(v_reinterpret_as_s32(v_src010))); + v_dst011 = v_add(v_dst011, v_cvt_f32(v_reinterpret_as_s32(v_src011))); - v_dst100 += v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst101 += v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst110 += v_cvt_f32(v_reinterpret_as_s32(v_src110)); - v_dst111 += v_cvt_f32(v_reinterpret_as_s32(v_src111)); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst110 = v_add(v_dst110, v_cvt_f32(v_reinterpret_as_s32(v_src110))); + v_dst111 = v_add(v_dst111, v_cvt_f32(v_reinterpret_as_s32(v_src111))); - v_dst200 += v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst201 += v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst210 += v_cvt_f32(v_reinterpret_as_s32(v_src210)); - v_dst211 += v_cvt_f32(v_reinterpret_as_s32(v_src211)); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst210 = v_add(v_dst210, v_cvt_f32(v_reinterpret_as_s32(v_src210))); + v_dst211 = v_add(v_dst211, v_cvt_f32(v_reinterpret_as_s32(v_src211))); v_store_interleave(dst + x * cn, v_dst000, v_dst100, v_dst200); v_store_interleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); @@ -1155,9 +1155,9 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1186,13 +1186,13 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_uint16 v_mask16 = vx_load_expand(mask + x); v_uint32 v_mask0, v_mask1; v_expand(v_mask16, v_mask0, v_mask1); - v_mask0 = ~(v_mask0 == v_0); - v_mask1 = ~(v_mask1 == v_0); + v_mask0 = v_not(v_eq(v_mask0, v_0)); + v_mask1 = v_not(v_eq(v_mask1, v_0)); v_uint16 v_src = vx_load(src + x); v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_src0 = v_src0 & v_mask0; - v_src1 = v_src1 & v_mask1; + v_src0 = v_and(v_src0, v_mask0); + v_src1 = v_and(v_src1, v_mask1); v_float32 v_float0, v_float1; v_float0 = v_cvt_f32(v_reinterpret_as_s32(v_src0)); @@ -1209,8 +1209,8 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_uint16 v_mask16 = vx_load_expand(mask + x); v_uint32 v_mask0, v_mask1; v_expand(v_mask16, v_mask0, v_mask1); - v_mask0 = ~(v_mask0 == v_0); - v_mask1 = ~(v_mask1 == v_0); + v_mask0 = v_not(v_eq(v_mask0, v_0)); + v_mask1 = v_not(v_eq(v_mask1, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); @@ -1218,12 +1218,12 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); v_expand(v_src2, v_int20, v_int21); - v_int00 = v_int00 & v_mask0; - v_int01 = v_int01 & v_mask1; - v_int10 = v_int10 & v_mask0; - v_int11 = v_int11 & v_mask1; - v_int20 = v_int20 & v_mask0; - v_int21 = v_int21 & v_mask1; + v_int00 = v_and(v_int00, v_mask0); + v_int01 = v_and(v_int01, v_mask1); + v_int10 = v_and(v_int10, v_mask0); + v_int11 = v_and(v_int11, v_mask1); + v_int20 = v_and(v_int20, v_mask0); + v_int21 = v_and(v_int21, v_mask1); v_float32 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_src00 = v_cvt_f32(v_reinterpret_as_s32(v_int00)); @@ -1347,9 +1347,9 @@ void accSqr_simd_(const float* src, float* dst, const uchar* mask, int len, int void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1390,9 +1390,9 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load_expand(src + x); - v_uint16 v_int = v_src & v_mask; + v_uint16 v_int = v_and(v_src, v_mask); v_uint32 v_int0, v_int1; v_expand(v_int, v_int0, v_int1); @@ -1430,10 +1430,10 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int v_uint16 v_int2 = v_expand_low(v_src2); v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_int0 = v_int0 & v_mask; - v_int1 = v_int1 & v_mask; - v_int2 = v_int2 & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_int0 = v_and(v_int0, v_mask); + v_int1 = v_and(v_int1, v_mask); + v_int2 = v_and(v_int2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_int0, v_int00, v_int01); @@ -1486,9 +1486,9 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1531,9 +1531,9 @@ void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, in for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_int_0, v_int_1; v_expand(v_src, v_int_0, v_int_1); @@ -1566,12 +1566,12 @@ void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, in for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); @@ -1810,9 +1810,9 @@ void accSqr_simd_(const double* src, double* dst, const uchar* mask, int len, in void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1829,10 +1829,10 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -1843,11 +1843,11 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_1src = vx_load(src1 + x); v_uint8 v_2src = vx_load(src2 + x); - v_1src = v_1src & v_mask; - v_2src = v_2src & v_mask; + v_1src = v_and(v_1src, v_mask); + v_2src = v_and(v_2src, v_mask); v_uint16 v_src0, v_src1; v_mul_expand(v_1src, v_2src, v_src0, v_src1); @@ -1856,10 +1856,10 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -1867,16 +1867,16 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_mul_expand(v_1src0, v_2src0, v_src00, v_src01); @@ -1896,18 +1896,18 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_load_deinterleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); v_load_deinterleave(dst + (x + step * 2) * cn, v_dst002, v_dst102, v_dst202); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst003, v_dst103, v_dst203); - v_dst000 = v_dst000 + v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst001 = v_dst001 + v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst002 = v_dst002 + v_cvt_f32(v_reinterpret_as_s32(v_src002)); - v_dst003 = v_dst003 + v_cvt_f32(v_reinterpret_as_s32(v_src003)); - v_dst100 = v_dst100 + v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst101 = v_dst101 + v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst102 = v_dst102 + v_cvt_f32(v_reinterpret_as_s32(v_src102)); - v_dst103 = v_dst103 + v_cvt_f32(v_reinterpret_as_s32(v_src103)); - v_dst200 = v_dst200 + v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst201 = v_dst201 + v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst202 = v_dst202 + v_cvt_f32(v_reinterpret_as_s32(v_src202)); - v_dst203 = v_dst203 + v_cvt_f32(v_reinterpret_as_s32(v_src203)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst002 = v_add(v_dst002, v_cvt_f32(v_reinterpret_as_s32(v_src002))); + v_dst003 = v_add(v_dst003, v_cvt_f32(v_reinterpret_as_s32(v_src003))); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst102 = v_add(v_dst102, v_cvt_f32(v_reinterpret_as_s32(v_src102))); + v_dst103 = v_add(v_dst103, v_cvt_f32(v_reinterpret_as_s32(v_src103))); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst202 = v_add(v_dst202, v_cvt_f32(v_reinterpret_as_s32(v_src202))); + v_dst203 = v_add(v_dst203, v_cvt_f32(v_reinterpret_as_s32(v_src203))); v_store_interleave(dst + x * cn, v_dst000, v_dst100, v_dst200); v_store_interleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); @@ -1923,9 +1923,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1956,10 +1956,10 @@ void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uch for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); - v_uint16 v_1src = vx_load(src1 + x) & v_mask; - v_uint16 v_2src = vx_load(src2 + x) & v_mask; + v_uint16 v_1src = v_and(vx_load(src1 + x), v_mask); + v_uint16 v_2src = v_and(vx_load(src2 + x), v_mask); v_uint32 v_1src0, v_1src1, v_2src0, v_2src1; v_expand(v_1src, v_1src0, v_1src1); @@ -1979,17 +1979,17 @@ void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uch for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint16 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint32 v_1src00, v_1src01, v_1src10, v_1src11, v_1src20, v_1src21, v_2src00, v_2src01, v_2src10, v_2src11, v_2src20, v_2src21; v_expand(v_1src0, v_1src00, v_1src01); @@ -2108,9 +2108,9 @@ void accProd_simd_(const float* src1, const float* src2, float* dst, const uchar void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2153,9 +2153,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_uint16 v_1int = vx_load_expand(src1 + x) & v_mask; - v_uint16 v_2int = vx_load_expand(src2 + x) & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_uint16 v_1int = v_and(vx_load_expand(src1 + x), v_mask); + v_uint16 v_2int = v_and(vx_load_expand(src2 + x), v_mask); v_uint32 v_1int_0, v_1int_1, v_2int_0, v_2int_1; v_expand(v_1int, v_1int_0, v_1int_1); @@ -2198,13 +2198,13 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha v_uint16 v_2int2 = v_expand_low(v_2src2); v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_1int0 = v_1int0 & v_mask; - v_1int1 = v_1int1 & v_mask; - v_1int2 = v_1int2 & v_mask; - v_2int0 = v_2int0 & v_mask; - v_2int1 = v_2int1 & v_mask; - v_2int2 = v_2int2 & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_1int0 = v_and(v_1int0, v_mask); + v_1int1 = v_and(v_1int1, v_mask); + v_1int2 = v_and(v_1int2, v_mask); + v_2int0 = v_and(v_2int0, v_mask); + v_2int1 = v_and(v_2int1, v_mask); + v_2int2 = v_and(v_2int2, v_mask); v_uint32 v_1int00, v_1int01, v_1int10, v_1int11, v_1int20, v_1int21; v_uint32 v_2int00, v_2int01, v_2int10, v_2int11, v_2int20, v_2int21; @@ -2248,9 +2248,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2293,11 +2293,11 @@ void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uc for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_1src = vx_load(src1 + x); v_uint16 v_2src = vx_load(src2 + x); - v_1src = v_1src & v_mask; - v_2src = v_2src & v_mask; + v_1src = v_and(v_1src, v_mask); + v_2src = v_and(v_2src, v_mask); v_uint32 v_1int_0, v_1int_1, v_2int_0, v_2int_1; v_expand(v_1src, v_1int_0, v_1int_1); @@ -2329,16 +2329,16 @@ void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uc for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint32 v_1int_00, v_1int_01, v_2int_00, v_2int_01; v_uint32 v_1int_10, v_1int_11, v_2int_10, v_2int_11; @@ -2594,11 +2594,11 @@ void accProd_simd_(const double* src1, const double* src2, double* dst, const uc void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2619,10 +2619,10 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst10 = vx_load(dst + x + step * 2); v_float32 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)); v_store(dst + x , v_dst00); v_store(dst + x + step , v_dst01); @@ -2663,15 +2663,15 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst10 = vx_load(dst + x + step * 2); v_float32 v_dst11 = vx_load(dst + x + step * 3); - v_mf00 = v_mf00 != zero; - v_mf01 = v_mf01 != zero; - v_mf10 = v_mf10 != zero; - v_mf11 = v_mf11 != zero; + v_mf00 = v_ne(v_mf00, zero); + v_mf01 = v_ne(v_mf01, zero); + v_mf10 = v_ne(v_mf10, zero); + v_mf11 = v_ne(v_mf11, zero); - v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha), v_dst00); - v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha), v_dst01); - v_dst10 = v_select(v_mf10, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha), v_dst10); - v_dst11 = v_select(v_mf11, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha), v_dst11); + v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)), v_dst00); + v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)), v_dst01); + v_dst10 = v_select(v_mf10, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)), v_dst10); + v_dst11 = v_select(v_mf11, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)), v_dst11); v_store(dst + x , v_dst00); v_store(dst + x + step , v_dst01); @@ -2719,25 +2719,25 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_mf10 = v_cvt_f32(v_reinterpret_as_s32(v_m10)); v_mf11 = v_cvt_f32(v_reinterpret_as_s32(v_m11)); - v_mf00 = v_mf00 != zero; - v_mf01 = v_mf01 != zero; - v_mf10 = v_mf10 != zero; - v_mf11 = v_mf11 != zero; + v_mf00 = v_ne(v_mf00, zero); + v_mf01 = v_ne(v_mf01, zero); + v_mf10 = v_ne(v_mf10, zero); + v_mf11 = v_ne(v_mf11, zero); - v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src000)) * v_alpha), v_dst00); - v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src001)) * v_alpha), v_dst01); - v_dst02 = v_select(v_mf10, v_fma(v_dst02, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src010)) * v_alpha), v_dst02); - v_dst03 = v_select(v_mf11, v_fma(v_dst03, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src011)) * v_alpha), v_dst03); + v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src000)), v_alpha)), v_dst00); + v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src001)), v_alpha)), v_dst01); + v_dst02 = v_select(v_mf10, v_fma(v_dst02, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src010)), v_alpha)), v_dst02); + v_dst03 = v_select(v_mf11, v_fma(v_dst03, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src011)), v_alpha)), v_dst03); - v_dst10 = v_select(v_mf00, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src100)) * v_alpha), v_dst10); - v_dst11 = v_select(v_mf01, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src101)) * v_alpha), v_dst11); - v_dst12 = v_select(v_mf10, v_fma(v_dst12, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src110)) * v_alpha), v_dst12); - v_dst13 = v_select(v_mf11, v_fma(v_dst13, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src111)) * v_alpha), v_dst13); + v_dst10 = v_select(v_mf00, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src100)), v_alpha)), v_dst10); + v_dst11 = v_select(v_mf01, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src101)), v_alpha)), v_dst11); + v_dst12 = v_select(v_mf10, v_fma(v_dst12, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src110)), v_alpha)), v_dst12); + v_dst13 = v_select(v_mf11, v_fma(v_dst13, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src111)), v_alpha)), v_dst13); - v_dst20 = v_select(v_mf00, v_fma(v_dst20, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src200)) * v_alpha), v_dst20); - v_dst21 = v_select(v_mf01, v_fma(v_dst21, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src201)) * v_alpha), v_dst21); - v_dst22 = v_select(v_mf10, v_fma(v_dst22, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src210)) * v_alpha), v_dst22); - v_dst23 = v_select(v_mf11, v_fma(v_dst23, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src211)) * v_alpha), v_dst23); + v_dst20 = v_select(v_mf00, v_fma(v_dst20, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src200)), v_alpha)), v_dst20); + v_dst21 = v_select(v_mf01, v_fma(v_dst21, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src201)), v_alpha)), v_dst21); + v_dst22 = v_select(v_mf10, v_fma(v_dst22, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src210)), v_alpha)), v_dst22); + v_dst23 = v_select(v_mf11, v_fma(v_dst23, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src211)), v_alpha)), v_dst23); v_store_interleave(dst + x * cn , v_dst00, v_dst10, v_dst20); v_store_interleave(dst + ( x + step ) * cn, v_dst01, v_dst11, v_dst21); @@ -2753,11 +2753,11 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2770,8 +2770,8 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_int0)) * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_int1)) * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_int0)), v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_int1)), v_alpha)); v_store(dst + x , v_dst0); v_store(dst + x + step, v_dst1); @@ -2799,11 +2799,11 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_mf0 = v_mf0 != zero; - v_mf1 = v_mf1 != zero; + v_mf0 = v_ne(v_mf0, zero); + v_mf1 = v_ne(v_mf1, zero); - v_dst0 = v_select(v_mf0, v_fma(v_dst0, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src0)) * v_alpha), v_dst0); - v_dst1 = v_select(v_mf1, v_fma(v_dst1, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src1)) * v_alpha), v_dst1); + v_dst0 = v_select(v_mf0, v_fma(v_dst0, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src0)), v_alpha)), v_dst0); + v_dst1 = v_select(v_mf1, v_fma(v_dst1, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src1)), v_alpha)), v_dst1); v_store(dst + x , v_dst0); v_store(dst + x + step, v_dst1); @@ -2833,16 +2833,16 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_mf0 = v_cvt_f32(v_reinterpret_as_s32(v_m0)); v_mf1 = v_cvt_f32(v_reinterpret_as_s32(v_m1)); - v_mf0 = v_mf0 != zero; - v_mf1 = v_mf1 != zero; + v_mf0 = v_ne(v_mf0, zero); + v_mf1 = v_ne(v_mf1, zero); - v_dst00 = v_select(v_mf0, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha), v_dst00); - v_dst10 = v_select(v_mf0, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha), v_dst10); - v_dst20 = v_select(v_mf0, v_fma(v_dst20, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src20)) * v_alpha), v_dst20); + v_dst00 = v_select(v_mf0, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)), v_dst00); + v_dst10 = v_select(v_mf0, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)), v_dst10); + v_dst20 = v_select(v_mf0, v_fma(v_dst20, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src20)), v_alpha)), v_dst20); - v_dst01 = v_select(v_mf1, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha), v_dst01); - v_dst11 = v_select(v_mf1, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha), v_dst11); - v_dst21 = v_select(v_mf1, v_fma(v_dst21, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src21)) * v_alpha), v_dst21); + v_dst01 = v_select(v_mf1, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)), v_dst01); + v_dst11 = v_select(v_mf1, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)), v_dst11); + v_dst21 = v_select(v_mf1, v_fma(v_dst21, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src21)), v_alpha)), v_dst21); v_store_interleave(dst + x * cn , v_dst00, v_dst10, v_dst20); v_store_interleave(dst + ( x + step ) * cn, v_dst01, v_dst11, v_dst21); @@ -2870,11 +2870,11 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn _mm256_storeu_ps(dst + x + 8, _mm256_add_ps(_mm256_mul_ps(_mm256_loadu_ps(dst + x + 8), v_beta), _mm256_mul_ps(_mm256_loadu_ps(src + x + 8), v_alpha))); } } -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2884,8 +2884,8 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, vx_load(src + x) * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, vx_load(src + x + step) * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(vx_load(src + x), v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(vx_load(src + x + step), v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -2898,11 +2898,11 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2927,10 +2927,10 @@ void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_fma(v_dst0, v_beta, v_src0 * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_src1 * v_alpha); - v_dst2 = v_fma(v_dst2, v_beta, v_src2 * v_alpha); - v_dst3 = v_fma(v_dst3, v_beta, v_src3 * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_src0, v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_src1, v_alpha)); + v_dst2 = v_fma(v_dst2, v_beta, v_mul(v_src2, v_alpha)); + v_dst3 = v_fma(v_dst3, v_beta, v_mul(v_src3, v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -2945,11 +2945,11 @@ void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int c void accW_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2973,10 +2973,10 @@ void accW_simd_(const ushort* src, double* dst, const uchar* mask, int len, int v_float64 v_dst10 = vx_load(dst + x + step * 2); v_float64 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_src00 * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_src01 * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_src10 * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_src11 * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_src00, v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_src01, v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_src10, v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_src11, v_alpha)); v_store(dst + x, v_dst00); v_store(dst + x + step, v_dst01); @@ -3014,11 +3014,11 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c _mm256_storeu_pd(dst + x + 12, _mm256_add_pd(_mm256_mul_pd(_mm256_loadu_pd(dst + x + 12), v_beta), _mm256_mul_pd(v_src11, v_alpha))); } } -#elif CV_SIMD_64F +#elif (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_float32::nlanes * 2; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes() * 2; + const int step = VTraits::vlanes(); if (!mask) { @@ -3026,7 +3026,7 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c for (; x <= size - cVectorWidth; x += cVectorWidth) { v_float32 v_src0 = vx_load(src + x); - v_float32 v_src1 = vx_load(src + x + v_float32::nlanes); + v_float32 v_src1 = vx_load(src + x + VTraits::vlanes()); v_float64 v_src00 = v_cvt_f64(v_src0); v_float64 v_src01 = v_cvt_f64_high(v_src0); v_float64 v_src10 = v_cvt_f64(v_src1); @@ -3037,10 +3037,10 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst10 = vx_load(dst + x + step * 2); v_float64 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_src00 * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_src01 * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_src10 * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_src11 * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_src00, v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_src01, v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_src10, v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_src11, v_alpha)); v_store(dst + x, v_dst00); v_store(dst + x + step, v_dst01); @@ -3072,11 +3072,11 @@ void accW_simd_(const double* src, double* dst, const uchar* mask, int len, int _mm256_storeu_pd(dst + x + 4, _mm256_add_pd(_mm256_mul_pd(_mm256_loadu_pd(dst + x + 4), v_beta), _mm256_mul_pd(v_src1, v_alpha))); } } -#elif CV_SIMD_64F +#elif (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_float64::nlanes * 2; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes() * 2; + const int step = VTraits::vlanes(); if (!mask) { @@ -3089,8 +3089,8 @@ void accW_simd_(const double* src, double* dst, const uchar* mask, int len, int v_float64 v_dst0 = vx_load(dst + x); v_float64 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, v_src0 * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_src1 * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_src0, v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_src1, v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); diff --git a/modules/imgproc/src/blend.cpp b/modules/imgproc/src/blend.cpp index 5a1296b509..accb45e7ad 100644 --- a/modules/imgproc/src/blend.cpp +++ b/modules/imgproc/src/blend.cpp @@ -48,12 +48,12 @@ #include "opencv2/core/hal/intrin.hpp" namespace cv { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline v_float32 blend(const v_float32& v_src1, const v_float32& v_src2, const v_float32& v_w1, const v_float32& v_w2) { const v_float32 v_eps = vx_setall_f32(1e-5f); - v_float32 v_denom = v_w1 + v_w2 + v_eps; - return (v_src1 * v_w1 + v_src2 * v_w2) / v_denom; + v_float32 v_denom = v_add(v_add(v_w1, v_w2), v_eps); + return v_div(v_add(v_mul(v_src1, v_w1), v_mul(v_src2, v_w2)), v_denom); } static inline v_float32 blend(const v_float32& v_src1, const v_float32& v_src2, const float* w_ptr1, const float* w_ptr2, int offset) { @@ -105,7 +105,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, switch(cn) { case 1: - for(int weight_offset = 0 ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13; v_float32 v_src20, v_src21, v_src22, v_src23; @@ -113,15 +113,15 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, load_expand_u8tof32(src2 + x, v_src20, v_src21, v_src22, v_src23); v_float32 v_dst0 = blend(v_src10, v_src20, weights1, weights2, weight_offset); - v_float32 v_dst1 = blend(v_src11, v_src21, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst2 = blend(v_src12, v_src22, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst3 = blend(v_src13, v_src23, weights1, weights2, weight_offset + 3*v_float32::nlanes); + v_float32 v_dst1 = blend(v_src11, v_src21, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst2 = blend(v_src12, v_src22, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst3 = blend(v_src13, v_src23, weights1, weights2, weight_offset + 3*VTraits::vlanes()); store_pack_f32tou8(dst + x, v_dst0, v_dst1, v_dst2, v_dst3); } break; case 2: - for(int weight_offset = 0 ; x <= width - 2*v_uint8::nlanes; x += 2*v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_uint8 v_src10, v_src11, v_src20, v_src21; v_load_deinterleave(src1 + x, v_src10, v_src11); @@ -135,12 +135,12 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, v_float32 v_dst0 = blend(v_src100, v_src200, weights1, weights2, weight_offset); v_float32 v_dst1 = blend(v_src110, v_src210, weights1, weights2, weight_offset); - v_float32 v_dst2 = blend(v_src101, v_src201, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst3 = blend(v_src111, v_src211, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst4 = blend(v_src102, v_src202, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst5 = blend(v_src112, v_src212, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst6 = blend(v_src103, v_src203, weights1, weights2, weight_offset + 3*v_float32::nlanes); - v_float32 v_dst7 = blend(v_src113, v_src213, weights1, weights2, weight_offset + 3*v_float32::nlanes); + v_float32 v_dst2 = blend(v_src101, v_src201, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst3 = blend(v_src111, v_src211, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst4 = blend(v_src102, v_src202, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst5 = blend(v_src112, v_src212, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst6 = blend(v_src103, v_src203, weights1, weights2, weight_offset + 3*VTraits::vlanes()); + v_float32 v_dst7 = blend(v_src113, v_src213, weights1, weights2, weight_offset + 3*VTraits::vlanes()); v_uint8 v_dsta = pack_f32tou8(v_dst0, v_dst2, v_dst4, v_dst6); v_uint8 v_dstb = pack_f32tou8(v_dst1, v_dst3, v_dst5, v_dst7); @@ -148,7 +148,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, } break; case 3: - for(int weight_offset = 0 ; x <= width - 3*v_uint8::nlanes; x += 3*v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - 3*VTraits::vlanes(); x += 3*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_uint8 v_src10, v_src11, v_src12, v_src20, v_src21, v_src22; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12); @@ -164,13 +164,13 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, expand_u8tof32(v_src22, v_src220, v_src221, v_src222, v_src223); v_float32 v_w10 = vx_load(weights1 + weight_offset); - v_float32 v_w11 = vx_load(weights1 + weight_offset + v_float32::nlanes); - v_float32 v_w12 = vx_load(weights1 + weight_offset + 2*v_float32::nlanes); - v_float32 v_w13 = vx_load(weights1 + weight_offset + 3*v_float32::nlanes); + v_float32 v_w11 = vx_load(weights1 + weight_offset + VTraits::vlanes()); + v_float32 v_w12 = vx_load(weights1 + weight_offset + 2*VTraits::vlanes()); + v_float32 v_w13 = vx_load(weights1 + weight_offset + 3*VTraits::vlanes()); v_float32 v_w20 = vx_load(weights2 + weight_offset); - v_float32 v_w21 = vx_load(weights2 + weight_offset + v_float32::nlanes); - v_float32 v_w22 = vx_load(weights2 + weight_offset + 2*v_float32::nlanes); - v_float32 v_w23 = vx_load(weights2 + weight_offset + 3*v_float32::nlanes); + v_float32 v_w21 = vx_load(weights2 + weight_offset + VTraits::vlanes()); + v_float32 v_w22 = vx_load(weights2 + weight_offset + 2*VTraits::vlanes()); + v_float32 v_w23 = vx_load(weights2 + weight_offset + 3*VTraits::vlanes()); v_src100 = blend(v_src100, v_src200, v_w10, v_w20); v_src110 = blend(v_src110, v_src210, v_w10, v_w20); v_src120 = blend(v_src120, v_src220, v_w10, v_w20); @@ -192,7 +192,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, } break; case 4: - for(int weight_offset = 0 ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13; v_float32 v_src20, v_src21, v_src22, v_src23; @@ -229,7 +229,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, switch(cn) { case 1: - for(int weight_offset = 0 ; x <= width - v_float32::nlanes; x += v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src1 = vx_load(src1 + x); v_float32 v_src2 = vx_load(src2 + x); @@ -242,7 +242,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 2: - for(int weight_offset = 0 ; x <= width - 2*v_float32::nlanes; x += 2*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src20, v_src21; v_load_deinterleave(src1 + x, v_src10, v_src11); @@ -257,7 +257,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 3: - for(int weight_offset = 0 ; x <= width - 3*v_float32::nlanes; x += 3*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 3*VTraits::vlanes(); x += 3*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src20, v_src21, v_src22; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12); @@ -273,7 +273,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 4: - for(int weight_offset = 0 ; x <= width - 4*v_float32::nlanes; x += 4*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 4*VTraits::vlanes(); x += 4*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12, v_src13); @@ -320,7 +320,7 @@ public: T * const dst_row = dst->ptr(y); int x = 0; - #if CV_SIMD + #if (CV_SIMD || CV_SIMD_SCALABLE) x = blendLinearSimd(src1_row, src2_row, weights1_row, weights2_row, dst_row, x, width, cn); #endif diff --git a/modules/imgproc/src/canny.cpp b/modules/imgproc/src/canny.cpp index 9c14929dc8..2fed0ba0c2 100644 --- a/modules/imgproc/src/canny.cpp +++ b/modules/imgproc/src/canny.cpp @@ -306,11 +306,11 @@ public: src(_src), src2(_src), map(_map), _borderPeaksParallel(borderPeaksParallel), low(_low), high(_high), aperture_size(_aperture_size), L2gradient(_L2gradient) { -#if CV_SIMD - for(int i = 0; i < v_int8::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(int i = 0; i < VTraits::vlanes(); ++i) { smask[i] = 0; - smask[i + v_int8::nlanes] = (schar)-1; + smask[i + VTraits::vlanes()] = (schar)-1; } if (true) _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); @@ -330,11 +330,11 @@ public: src(_dx), src2(_dy), map(_map), _borderPeaksParallel(borderPeaksParallel), low(_low), high(_high), aperture_size(0), L2gradient(_L2gradient) { -#if CV_SIMD - for(int i = 0; i < v_int8::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(int i = 0; i < VTraits::vlanes(); ++i) { smask[i] = 0; - smask[i + v_int8::nlanes] = (schar)-1; + smask[i + VTraits::vlanes()] = (schar)-1; } if (true) _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); @@ -396,7 +396,7 @@ public: } // _mag_p: previous row, _mag_a: actual row, _mag_n: next row -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) AutoBuffer buffer(3 * (mapstep * cn + CV_SIMD_WIDTH)); _mag_p = alignPtr(buffer.data() + 1, CV_SIMD_WIDTH); _mag_a = alignPtr(_mag_p + mapstep * cn, CV_SIMD_WIDTH); @@ -436,8 +436,8 @@ public: if (L2gradient) { int j = 0, width = src.cols * cn; -#if CV_SIMD - for ( ; j <= width - v_int16::nlanes; j += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for ( ; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) { v_int16 v_dx = vx_load((const short*)(_dx + j)); v_int16 v_dy = vx_load((const short*)(_dy + j)); @@ -447,8 +447,8 @@ public: v_expand(v_dx, v_dxp_low, v_dxp_high); v_expand(v_dy, v_dyp_low, v_dyp_high); - v_store_aligned((int *)(_mag_n + j), v_dxp_low*v_dxp_low+v_dyp_low*v_dyp_low); - v_store_aligned((int *)(_mag_n + j + v_int32::nlanes), v_dxp_high*v_dxp_high+v_dyp_high*v_dyp_high); + v_store_aligned((int *)(_mag_n + j), v_add(v_mul(v_dxp_low, v_dxp_low), v_mul(v_dyp_low, v_dyp_low))); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_mul(v_dxp_high, v_dxp_high), v_mul(v_dyp_high, v_dyp_high))); } #endif for ( ; j < width; ++j) @@ -457,8 +457,8 @@ public: else { int j = 0, width = src.cols * cn; -#if CV_SIMD - for(; j <= width - v_int16::nlanes; j += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) { v_int16 v_dx = vx_load((const short *)(_dx + j)); v_int16 v_dy = vx_load((const short *)(_dy + j)); @@ -470,8 +470,8 @@ public: v_expand(v_dx, v_dx_ml, v_dx_mh); v_expand(v_dy, v_dy_ml, v_dy_mh); - v_store_aligned((int *)(_mag_n + j), v_dx_ml + v_dy_ml); - v_store_aligned((int *)(_mag_n + j + v_int32::nlanes), v_dx_mh + v_dy_mh); + v_store_aligned((int *)(_mag_n + j), v_add(v_dx_ml, v_dy_ml)); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_dx_mh, v_dy_mh)); } #endif for ( ; j < width; ++j) @@ -515,7 +515,7 @@ public: // From here actual src row is (i - 1) // Set left and right border to 1 -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (true) _pmap = map.ptr(i) + CV_SIMD_WIDTH; else @@ -537,22 +537,22 @@ public: const int TG22 = 13573; int j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_int32 v_low = vx_setall_s32(low); const v_int8 v_one = vx_setall_s8(1); - for (; j <= src.cols - v_int8::nlanes; j += v_int8::nlanes) + for (; j <= src.cols - VTraits::vlanes(); j += VTraits::vlanes()) { v_store_aligned((signed char*)(_pmap + j), v_one); - v_int8 v_cmp = v_pack(v_pack(vx_load_aligned((const int*)(_mag_a + j )) > v_low, - vx_load_aligned((const int*)(_mag_a + j + v_int32::nlanes)) > v_low), - v_pack(vx_load_aligned((const int*)(_mag_a + j + 2*v_int32::nlanes)) > v_low, - vx_load_aligned((const int*)(_mag_a + j + 3*v_int32::nlanes)) > v_low)); + v_int8 v_cmp = v_pack(v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j)), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + VTraits::vlanes())), v_low)), + v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j + 2 * VTraits::vlanes())), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + 3 * VTraits::vlanes())), v_low))); while (v_check_any(v_cmp)) { int l = v_scan_forward(v_cmp); - v_cmp &= vx_load(smask + v_int8::nlanes - 1 - l); + v_cmp = v_and(v_cmp, vx_load(smask + VTraits::vlanes() - 1 - l)); int k = j + l; int m = _mag_a[k]; @@ -693,8 +693,8 @@ private: ptrdiff_t mapstep; int cn; mutable Mutex mutex; -#if CV_SIMD - schar smask[2*v_int8::nlanes]; +#if (CV_SIMD || CV_SIMD_SCALABLE) + schar smask[2*VTraits::max_nlanes]; #endif }; @@ -718,31 +718,31 @@ public: int j = 0; uchar *pdst = dst.ptr(i); const uchar *pmap = map.ptr(i + 1); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (true) pmap += CV_SIMD_WIDTH; else #endif pmap += 1; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_uint8 v_zero = vx_setzero_u8(); - const v_uint8 v_ff = ~v_zero; + const v_uint8 v_ff = v_not(v_zero); const v_uint8 v_two = vx_setall_u8(2); - for (; j <= dst.cols - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= dst.cols - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v_pmap = vx_load_aligned((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_pmap == v_two, v_ff, v_zero); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); v_store((pdst + j), v_pmap); } - if (j <= dst.cols - v_uint8::nlanes/2) + if (j <= dst.cols - VTraits::vlanes()/2) { v_uint8 v_pmap = vx_load_low((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_pmap == v_two, v_ff, v_zero); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); v_store_low((pdst + j), v_pmap); - j += v_uint8::nlanes/2; + j += VTraits::vlanes()/2; } } #endif diff --git a/modules/imgproc/src/color_rgb.simd.hpp b/modules/imgproc/src/color_rgb.simd.hpp index 6e11020197..67e2febd5b 100644 --- a/modules/imgproc/src/color_rgb.simd.hpp +++ b/modules/imgproc/src/color_rgb.simd.hpp @@ -122,8 +122,8 @@ struct RGB2RGB int i = 0; _Tp alphav = ColorChannel<_Tp>::max(); -#if CV_SIMD - const int vsize = vt::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); for(; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize*dcn) @@ -138,8 +138,13 @@ struct RGB2RGB v_load_deinterleave(src, a, b, c); d = v_set<_Tp>::set(alphav); } - if(bi == 2) + if(bi == 2) { + #if CV_SIMD_SCALABLE + auto t = a; a = c; c = t; // swap(a, c); + #else swap(a, c); + #endif + } if(dcn == 4) { @@ -185,53 +190,57 @@ struct RGB5x52RGB int dcn = dstcn, bidx = blueIdx, gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 vz = vx_setzero_u8(), vn0 = vx_setall_u8(255); for(; i <= n-vsize; i += vsize, src += vsize*sizeof(ushort), dst += vsize*dcn) { v_uint16 t0 = v_reinterpret_as_u16(vx_load(src)); v_uint16 t1 = v_reinterpret_as_u16(vx_load(src + - sizeof(ushort)*v_uint16::nlanes)); + sizeof(ushort)*VTraits::vlanes())); //TODO: shorten registers use when v_interleave is available v_uint8 r, g, b, a; - v_uint16 b0 = (t0 << 11) >> 8; - v_uint16 b1 = (t1 << 11) >> 8; + v_uint16 b0 = v_shr<8>(v_shl<11>(t0)); + v_uint16 b1 = v_shr<8>(v_shl<11>(t1)); b = v_pack(b0, b1); v_uint16 g0, g1, r0, r1, a0, a1; if( gb == 6 ) { - g0 = ((t0 >> 5) << 10) >> 8; - g1 = ((t1 >> 5) << 10) >> 8; + g0 = v_shr<8>(v_shl<10>(v_shr<5>(t0))); + g1 = v_shr<8>(v_shl<10>(v_shr<5>(t1))); - r0 = (t0 >> 11) << 3; - r1 = (t1 >> 11) << 3; + r0 = v_shl<3>(v_shr<11>(t0)); + r1 = v_shl<3>(v_shr<11>(t1)); a = vn0; } else { - g0 = ((t0 >> 5) << 11) >> 8; - g1 = ((t1 >> 5) << 11) >> 8; + g0 = v_shr<8>(v_shl<11>(v_shr<5>(t0))); + g1 = v_shr<8>(v_shl<11>(v_shr<5>(t1))); - r0 = ((t0 >> 10) << 11) >> 8; - r1 = ((t1 >> 10) << 11) >> 8; + r0 = v_shr<8>(v_shl<11>(v_shr<10>(t0))); + r1 = v_shr<8>(v_shl<11>(v_shr<10>(t1))); - a0 = t0 >> 15; - a1 = t1 >> 15; + a0 = v_shr<15>(t0); + a1 = v_shr<15>(t1); a = v_pack(a0, a1); - a = a != vz; + a = v_ne(a, vz); } g = v_pack(g0, g1); r = v_pack(r0, r1); - if(bidx == 2) + if(bidx == 2) { + #if CV_SIMD_SCALABLE + auto t = r; r = b; b = t; // swap(b, r); + #else swap(b, r); - + #endif + } if(dcn == 4) { v_store_interleave(dst, b, g, r, a); @@ -289,8 +298,8 @@ struct RGB2RGB5x5 int scn = srccn, bidx = blueIdx, gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint16 vn3 = vx_setall_u16((ushort)(~3)); v_uint16 vn7 = vx_setall_u16((ushort)(~7)); v_uint16 vz = vx_setzero_u16(); @@ -308,10 +317,15 @@ struct RGB2RGB5x5 { v_load_deinterleave(src, b, g, r, a); } - if(bidx == 2) + if(bidx == 2){ + #if CV_SIMD_SCALABLE + auto t = r; r = b; b = t; // swap(b, r); + #else swap(b, r); + #endif + } - r = r & v7; + r = v_and(r, v7); //TODO: shorten registers use when v_deinterleave is available v_uint16 r0, r1, g0, g1, b0, b1, a0, a1; @@ -322,20 +336,20 @@ struct RGB2RGB5x5 v_uint16 d0, d1; - b0 = b0 >> 3; - b1 = b1 >> 3; - a0 = (a0 != vz) << 15; - a1 = (a1 != vz) << 15; + b0 = v_shr<3>(b0); + b1 = v_shr<3>(b1); + a0 = v_shl<15>(v_ne(a0, vz)); + a1 = v_shl<15>(v_ne(a1, vz)); if(gb == 6) { - d0 = b0 | ((g0 & vn3) << 3) | (r0 << 8); - d1 = b1 | ((g1 & vn3) << 3) | (r1 << 8); + d0 = v_or(v_or(b0, v_shl<3>(v_and(g0, vn3))), v_shl<8>(r0)); + d1 = v_or(v_or(b1, v_shl<3>(v_and(g1, vn3))), v_shl<8>(r1)); } else { - d0 = b0 | ((g0 & vn7) << 2) | (r0 << 7) | a0; - d1 = b1 | ((g1 & vn7) << 2) | (r1 << 7) | a1; + d0 = v_or(v_or(v_or(b0, v_shl<2>(v_and(g0, vn7))), v_shl<7>(r0)), a0); + d1 = v_or(v_or(v_or(b1, v_shl<2>(v_and(g1, vn7))), v_shl<7>(r1)), a1); } v_store((ushort*)dst, d0); @@ -382,8 +396,8 @@ struct Gray2RGB int i = 0; _Tp alpha = ColorChannel<_Tp>::max(); -#if CV_SIMD - const int vsize = vt::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); vt valpha = v_set<_Tp>::set(alpha); for(; i <= n-vsize; i += vsize, src += vsize, dst += vsize*dcn) @@ -424,8 +438,8 @@ struct Gray2RGB5x5 { int gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint16 v3 = vx_setall_u16((ushort)(~3)); for(; i <= n-vsize; i += vsize, src += vsize, dst += vsize*sizeof(ushort)) @@ -433,16 +447,16 @@ struct Gray2RGB5x5 v_uint8 t8 = vx_load_low(src); v_uint16 t = v_expand_low(t8); - v_uint16 t3 = t >> 3; + v_uint16 t3 = v_shr<3>(t); v_uint16 d = t3; if(gb == 6) { - d |= ((t & v3) << 3) | (t3 << 11); + d = v_or(d, v_or(v_shl<3>(v_and(t, v3)), v_shl<11>(t3))); } else { - d |= (t3 << 5) | (t3 << 10); + d = v_or(d, v_or(v_shl<5>(t3), v_shl<10>(t3))); } v_store((ushort*)dst, d); @@ -488,8 +502,8 @@ struct RGB5x52Gray { int gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 bg2y; v_int16 r12y; @@ -504,17 +518,17 @@ struct RGB5x52Gray v_uint16 t = vx_load((ushort*)src); v_uint16 r, g, b; - b = (t << 11) >> 8; + b = v_shr<8>(v_shl<11>(t)); if(gb == 5) { - g = ((t >> 5) << 11) >> 8; - r = ((t >> 10) << 11) >> 8; + g = v_shr<8>(v_shl<11>(v_shr<5>(t))); + r = v_shr<8>(v_shl<11>(v_shr<10>(t))); } else { - g = ((t >> 5) << 10) >> 8; - r = (t >> 11) << 3; + g = v_shr<8>(v_shl<10>(v_shr<5>(t))); + r = v_shl<3>(v_shr<11>(t)); } v_uint8 d; @@ -530,11 +544,11 @@ struct RGB5x52Gray v_zip(sr, delta, rd0, rd1); v_uint32 d0, d1; - d0 = v_reinterpret_as_u32(v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)); - d1 = v_reinterpret_as_u32(v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)); + d0 = v_reinterpret_as_u32(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y))); + d1 = v_reinterpret_as_u32(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y))); - d0 = d0 >> shift; - d1 = d1 >> shift; + d0 = v_shr(d0); + d1 = v_shr(d1); dx = v_pack(d0, d1); // high part isn't used @@ -611,8 +625,8 @@ struct RGB2Gray int scn = srccn, i = 0; float cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 rv = vx_setall_f32(cr), gv = vx_setall_f32(cg), bv = vx_setall_f32(cb); for(; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize) @@ -627,7 +641,7 @@ struct RGB2Gray v_load_deinterleave(src, b, g, r, a); } - v_float32 d = v_fma(r, rv, v_fma(g, gv, b*bv)); + v_float32 d = v_fma(r, rv, v_fma(g, gv, v_mul(b, bv))); v_store(dst, d); } @@ -669,8 +683,8 @@ struct RGB2Gray short cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 bg2y; v_int16 r12y; v_int16 dummy; @@ -706,10 +720,10 @@ struct RGB2Gray v_zip(v_reinterpret_as_s16(r1), delta, rd10, rd11); v_uint32 y00, y01, y10, y11; - y00 = v_reinterpret_as_u32(v_dotprod(bg00, bg2y) + v_dotprod(rd00, r12y)) >> shift; - y01 = v_reinterpret_as_u32(v_dotprod(bg01, bg2y) + v_dotprod(rd01, r12y)) >> shift; - y10 = v_reinterpret_as_u32(v_dotprod(bg10, bg2y) + v_dotprod(rd10, r12y)) >> shift; - y11 = v_reinterpret_as_u32(v_dotprod(bg11, bg2y) + v_dotprod(rd11, r12y)) >> shift; + y00 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg00, bg2y), v_dotprod(rd00, r12y)))); + y01 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg01, bg2y), v_dotprod(rd01, r12y)))); + y10 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg10, bg2y), v_dotprod(rd10, r12y)))); + y11 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg11, bg2y), v_dotprod(rd11, r12y)))); v_uint16 y0, y1; y0 = v_pack(y00, y01); @@ -762,8 +776,8 @@ struct RGB2Gray short cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 b2y = vx_setall_s16(cb); v_int16 g2y = vx_setall_s16(cg); @@ -802,13 +816,13 @@ struct RGB2Gray // fixing 16bit signed multiplication v_int16 mr, mg, mb; - mr = (sr < z) & r2y; - mg = (sg < z) & g2y; - mb = (sb < z) & b2y; - v_int16 fixmul = v_add_wrap(mr, v_add_wrap(mg, mb)) << fix_shift; + mr = v_and(v_lt(sr, z), r2y); + mg = v_and(v_lt(sg, z), g2y); + mb = v_and(v_lt(sb, z), b2y); + v_int16 fixmul = v_shl(v_add_wrap(mr, v_add_wrap(mg, mb))); - v_int32 sy0 = (v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)) >> shift; - v_int32 sy1 = (v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)) >> shift; + v_int32 sy0 = v_shr(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y))); + v_int32 sy1 = v_shr(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y))); v_int16 y = v_add_wrap(v_pack(sy0, sy1), fixmul); @@ -973,8 +987,8 @@ struct mRGBA2RGBA uchar max_val = ColorChannel::max(); int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 amask = v_reinterpret_as_u8(vx_setall_u32(0xFF000000)); v_uint8 vmax = vx_setall_u8(max_val); @@ -989,9 +1003,9 @@ struct mRGBA2RGBA v_uint8 a; v_uint16 a16; v_uint32 a32; - a16 = v_reinterpret_as_u16(s & amask); - a32 = v_reinterpret_as_u32(a16 | (a16 >> 8)); - a = v_reinterpret_as_u8(a32 | (a32 >> 16)); + a16 = v_reinterpret_as_u16(v_and(s, amask)); + a32 = v_reinterpret_as_u32(v_or(a16, v_shr<8>(a16))); + a = v_reinterpret_as_u8(v_or(a32, v_shr<16>(a32))); // s *= max_val v_uint16 s0, s1; @@ -1000,7 +1014,7 @@ struct mRGBA2RGBA // s += a/2 v_uint16 ae0, ae1; v_expand(a, ae0, ae1); - s0 += ae0 >> 1; s1 += ae1 >> 1; + s0 = v_add(s0, v_shr<1>(ae0)); s1 = v_add(s1, v_shr<1>(ae1)); // s, a -> u32 -> float v_uint32 u00, u01, u10, u11; @@ -1035,10 +1049,10 @@ struct mRGBA2RGBA // float d = (float)s/(float)a v_float32 fd00, fd01, fd10, fd11; - fd00 = fs00/fa00; - fd01 = fs01/fa01; - fd10 = fs10/fa10; - fd11 = fs11/fa11; + fd00 = v_div(fs00, fa00); + fd01 = v_div(fs01, fa01); + fd10 = v_div(fs10, fa10); + fd11 = v_div(fs11, fa11); // d -> u32 -> u8 v_uint32 ud00, ud01, ud10, ud11; @@ -1054,8 +1068,8 @@ struct mRGBA2RGBA // if a == 0 then d = 0 v_uint8 am; - am = a != vx_setzero_u8(); - d = d & am; + am = v_ne(a, vx_setzero_u8()); + d = v_and(d, am); // put alpha values d = v_select(amask, a, d); diff --git a/modules/imgproc/src/contours.cpp b/modules/imgproc/src/contours.cpp index d8823206f2..3e3096e7a5 100644 --- a/modules/imgproc/src/contours.cpp +++ b/modules/imgproc/src/contours.cpp @@ -1080,7 +1080,7 @@ cvFindNextContour( CvContourScanner scanner ) } else { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if ((p = img[x]) != prev) { goto _next_contour; @@ -1088,9 +1088,9 @@ cvFindNextContour( CvContourScanner scanner ) else { v_uint8 v_prev = vx_setall_u8((uchar)prev); - for (; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(img + x)) != v_prev); + v_uint8 vmask = (v_ne(vx_load((uchar *)(img + x)), v_prev)); if (v_check_any(vmask)) { p = img[(x += v_scan_forward(vmask))]; @@ -1105,7 +1105,7 @@ cvFindNextContour( CvContourScanner scanner ) if( x >= width ) break; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) _next_contour: #endif { @@ -1353,11 +1353,11 @@ CvLinkedRunPoint; inline int findStartContourPoint(uchar *src_data, CvSize img_size, int j) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint8 v_zero = vx_setzero_u8(); - for (; j <= img_size.width - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= img_size.width - VTraits::vlanes(); j += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(src_data + j)) != v_zero); + v_uint8 vmask = (v_ne(vx_load((uchar *)(src_data + j)), v_zero)); if (v_check_any(vmask)) { j += v_scan_forward(vmask); @@ -1372,7 +1372,7 @@ inline int findStartContourPoint(uchar *src_data, CvSize img_size, int j) inline int findEndContourPoint(uchar *src_data, CvSize img_size, int j) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (j < img_size.width && !src_data[j]) { return j; @@ -1380,9 +1380,9 @@ inline int findEndContourPoint(uchar *src_data, CvSize img_size, int j) else { v_uint8 v_zero = vx_setzero_u8(); - for (; j <= img_size.width - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= img_size.width - VTraits::vlanes(); j += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(src_data + j)) == v_zero); + v_uint8 vmask = (v_eq(vx_load((uchar *)(src_data + j)), v_zero)); if (v_check_any(vmask)) { j += v_scan_forward(vmask); diff --git a/modules/imgproc/src/corner.cpp b/modules/imgproc/src/corner.cpp index f0ea0b5bb5..1d6ee1ac04 100644 --- a/modules/imgproc/src/corner.cpp +++ b/modules/imgproc/src/corner.cpp @@ -74,21 +74,21 @@ static void calcMinEigenVal( const Mat& _cov, Mat& _dst ) #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - v_float32x4 half = v_setall_f32(0.5f); - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + v_float32 half = vx_setall_f32(0.5f); + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_a, v_b, v_c, v_t; + v_float32 v_a, v_b, v_c, v_t; v_load_deinterleave(cov + j*3, v_a, v_b, v_c); - v_a *= half; - v_c *= half; - v_t = v_a - v_c; - v_t = v_muladd(v_b, v_b, (v_t * v_t)); - v_store(dst + j, (v_a + v_c) - v_sqrt(v_t)); + v_a = v_mul(v_a, half); + v_c = v_mul(v_c, half); + v_t = v_sub(v_a, v_c); + v_t = v_muladd(v_b, v_b, (v_mul(v_t, v_t))); + v_store(dst + j, v_sub(v_add(v_a, v_c), v_sqrt(v_t))); } } -#endif // CV_SIMD128 +#endif // CV_SIMD for( ; j < size.width; j++ ) { @@ -127,18 +127,18 @@ static void calcHarris( const Mat& _cov, Mat& _dst, double k ) #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - v_float32x4 v_k = v_setall_f32((float)k); + v_float32 v_k = vx_setall_f32((float)k); - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_a, v_b, v_c; + v_float32 v_a, v_b, v_c; v_load_deinterleave(cov + j * 3, v_a, v_b, v_c); - v_float32x4 v_ac_bb = v_a * v_c - v_b * v_b; - v_float32x4 v_ac = v_a + v_c; - v_float32x4 v_dst = v_ac_bb - v_k * v_ac * v_ac; + v_float32 v_ac_bb = v_sub(v_mul(v_a, v_c), v_mul(v_b, v_b)); + v_float32 v_ac = v_add(v_a, v_c); + v_float32 v_dst = v_sub(v_ac_bb, v_mul(v_mul(v_k, v_ac), v_ac)); v_store(dst + j, v_dst); } } @@ -282,22 +282,22 @@ cornerEigenValsVecs( const Mat& src, Mat& eigenv, int block_size, #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_dx = v_load(dxdata + j); - v_float32x4 v_dy = v_load(dydata + j); + v_float32 v_dx = vx_load(dxdata + j); + v_float32 v_dy = vx_load(dydata + j); - v_float32x4 v_dst0, v_dst1, v_dst2; - v_dst0 = v_dx * v_dx; - v_dst1 = v_dx * v_dy; - v_dst2 = v_dy * v_dy; + v_float32 v_dst0, v_dst1, v_dst2; + v_dst0 = v_mul(v_dx, v_dx); + v_dst1 = v_mul(v_dx, v_dy); + v_dst2 = v_mul(v_dy, v_dy); v_store_interleave(cov_data + j * 3, v_dst0, v_dst1, v_dst2); } } -#endif // CV_SIMD128 +#endif // CV_SIMD for( ; j < size.width; j++ ) { @@ -693,9 +693,9 @@ void cv::preCornerDetect( InputArray _src, OutputArray _dst, int ksize, int bord if( src.depth() == CV_8U ) factor *= 255; factor = 1./(factor * factor * factor); -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) float factor_f = (float)factor; - v_float32x4 v_factor = v_setall_f32(factor_f), v_m2 = v_setall_f32(-2.0f); + v_float32 v_factor = vx_setall_f32(factor_f), v_m2 = vx_setall_f32(-2.0f); #endif Size size = src.size(); @@ -711,18 +711,18 @@ void cv::preCornerDetect( InputArray _src, OutputArray _dst, int ksize, int bord j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_dx = v_load(dxdata + j); - v_float32x4 v_dy = v_load(dydata + j); + v_float32 v_dx = vx_load(dxdata + j); + v_float32 v_dy = vx_load(dydata + j); - v_float32x4 v_s1 = (v_dx * v_dx) * v_load(d2ydata + j); - v_float32x4 v_s2 = v_muladd((v_dy * v_dy), v_load(d2xdata + j), v_s1); - v_float32x4 v_s3 = v_muladd((v_dy * v_dx) * v_load(dxydata + j), v_m2, v_s2); + v_float32 v_s1 = v_mul(v_mul(v_dx, v_dx), vx_load(d2ydata + j)); + v_float32 v_s2 = v_muladd((v_mul(v_dy, v_dy)), vx_load(d2xdata + j), v_s1); + v_float32 v_s3 = v_muladd(v_mul(v_mul(v_dy, v_dx), vx_load(dxydata + j)), v_m2, v_s2); - v_store(dstdata + j, v_s3 * v_factor); + v_store(dstdata + j, v_mul(v_s3, v_factor)); } } #endif diff --git a/modules/imgproc/src/histogram.cpp b/modules/imgproc/src/histogram.cpp index 068dfd3a27..cbd60550e0 100644 --- a/modules/imgproc/src/histogram.cpp +++ b/modules/imgproc/src/histogram.cpp @@ -2053,13 +2053,13 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == CV_COMP_CORREL ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_s1 = vx_setzero_f64(); v_float64 v_s2 = vx_setzero_f64(); v_float64 v_s11 = vx_setzero_f64(); v_float64 v_s12 = vx_setzero_f64(); v_float64 v_s22 = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); @@ -2070,8 +2070,8 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_s12 = v_muladd(v_ad, v_bd, v_s12); v_s11 = v_muladd(v_ad, v_ad, v_s11); v_s22 = v_muladd(v_bd, v_bd, v_s22); - v_s1 += v_ad; - v_s2 += v_bd; + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); // 2-3 v_ad = v_cvt_f64_high(v_a); @@ -2079,8 +2079,8 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_s12 = v_muladd(v_ad, v_bd, v_s12); v_s11 = v_muladd(v_ad, v_ad, v_s11); v_s22 = v_muladd(v_bd, v_bd, v_s22); - v_s1 += v_ad; - v_s2 += v_bd; + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); } s12 += v_reduce_sum(v_s12); s11 += v_reduce_sum(v_s11); @@ -2124,12 +2124,12 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == CV_COMP_INTERSECT ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_result = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_src = v_min(vx_load(h1 + j), vx_load(h2 + j)); - v_result += v_cvt_f64(v_src) + v_cvt_f64_high(v_src); + v_result = v_add(v_result, v_add(v_cvt_f64(v_src), v_cvt_f64_high(v_src))); } result += v_reduce_sum(v_result); #elif CV_SIMD @@ -2146,26 +2146,26 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == CV_COMP_BHATTACHARYYA ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_s1 = vx_setzero_f64(); v_float64 v_s2 = vx_setzero_f64(); v_float64 v_result = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); v_float64 v_ad = v_cvt_f64(v_a); v_float64 v_bd = v_cvt_f64(v_b); - v_s1 += v_ad; - v_s2 += v_bd; - v_result += v_sqrt(v_ad * v_bd); + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); + v_result = v_add(v_result, v_sqrt(v_mul(v_ad, v_bd))); v_ad = v_cvt_f64_high(v_a); v_bd = v_cvt_f64_high(v_b); - v_s1 += v_ad; - v_s2 += v_bd; - v_result += v_sqrt(v_ad * v_bd); + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); + v_result = v_add(v_result, v_sqrt(v_mul(v_ad, v_bd))); } s1 += v_reduce_sum(v_s1); s2 += v_reduce_sum(v_s2); diff --git a/modules/imgproc/src/pyramids.cpp b/modules/imgproc/src/pyramids.cpp index c133544069..dae09564d3 100644 --- a/modules/imgproc/src/pyramids.cpp +++ b/modules/imgproc/src/pyramids.cpp @@ -84,7 +84,7 @@ template int PyrUpVecV(T1**, T2**, int) { return 0; } template int PyrUpVecVOneRow(T1**, T2*, int) { return 0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template<> int PyrDownVecH(const uchar* src, int* row, int width) { @@ -93,10 +93,8 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_reinterpret_as_s16(vx_load_expand(src01)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(vx_load_expand(src23)), v_6_4) + - (v_reinterpret_as_s32(vx_load_expand(src4)) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_reinterpret_as_s16(vx_load_expand(src01)), v_1_4), v_dotprod(v_reinterpret_as_s16(vx_load_expand(src23)), v_6_4)), v_shr<16>(v_reinterpret_as_s32(vx_load_expand(src4))))); vx_cleanup(); return x; @@ -108,42 +106,40 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4) + - v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4) + - (v_reinterpret_as_s32(v_interleave_pairs(vx_load_expand(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4), v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_pairs(vx_load_expand(src4)))))); vx_cleanup(); return x; } template<> int PyrDownVecH(const uchar* src, int* row, int width) { - int idx[v_int8::nlanes/2 + 4]; - for (int i = 0; i < v_int8::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int8::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int8::nlanes; x += 3*v_int8::nlanes/4, src += 6*v_int8::nlanes/4, row += 3*v_int8::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_uint16 r0l, r0h, r1l, r1h, r2l, r2h, r3l, r3h, r4l, r4h; v_expand(vx_lut_quads(src, idx ), r0l, r0h); - v_expand(vx_lut_quads(src, idx + v_int8::nlanes/4 + 2), r1l, r1h); + v_expand(vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r1l, r1h); v_expand(vx_lut_quads(src, idx + 1 ), r2l, r2h); - v_expand(vx_lut_quads(src, idx + v_int8::nlanes/4 + 3), r3l, r3h); + v_expand(vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r3l, r3h); v_expand(vx_lut_quads(src, idx + 2 ), r4l, r4h); - v_zip(r2l, r1l + r3l, r1l, r3l); - v_zip(r2h, r1h + r3h, r1h, r3h); - r0l += r4l; r0h += r4h; + v_zip(r2l, v_add(r1l, r3l), r1l, r3l); + v_zip(r2h, v_add(r1h, r3h), r1h, r3h); + r0l = v_add(r0l, r4l); r0h = v_add(r0h, r4h); - v_store(row , v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r1l), v_6_4) + v_reinterpret_as_s32(v_expand_low( r0l)))); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r3l), v_6_4) + v_reinterpret_as_s32(v_expand_high(r0l)))); - v_store(row + 6*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r1h), v_6_4) + v_reinterpret_as_s32(v_expand_low( r0h)))); - v_store(row + 9*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r3h), v_6_4) + v_reinterpret_as_s32(v_expand_high(r0h)))); + v_store(row , v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r1l), v_6_4), v_reinterpret_as_s32(v_expand_low(r0l))))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r3l), v_6_4), v_reinterpret_as_s32(v_expand_high(r0l))))); + v_store(row + 6*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r1h), v_6_4), v_reinterpret_as_s32(v_expand_low(r0h))))); + v_store(row + 9*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r3h), v_6_4), v_reinterpret_as_s32(v_expand_high(r0h))))); } vx_cleanup(); @@ -156,10 +152,8 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4) + - v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4) + - (v_reinterpret_as_s32(v_interleave_quads(vx_load_expand(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4), v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_quads(vx_load_expand(src4)))))); vx_cleanup(); return x; @@ -172,10 +166,8 @@ template<> int PyrDownVecH(const short* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(vx_load(src01), v_1_4) + - v_dotprod(vx_load(src23), v_6_4) + - (v_reinterpret_as_s32(vx_load(src4)) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(vx_load(src01), v_1_4), v_dotprod(vx_load(src23), v_6_4)), v_shr<16>(v_reinterpret_as_s32(vx_load(src4))))); vx_cleanup(); return x; @@ -187,34 +179,32 @@ template<> int PyrDownVecH(const short* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(vx_load(src01)), v_1_4) + - v_dotprod(v_interleave_pairs(vx_load(src23)), v_6_4) + - (v_reinterpret_as_s32(v_interleave_pairs(vx_load(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_pairs(vx_load(src01)), v_1_4), v_dotprod(v_interleave_pairs(vx_load(src23)), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_pairs(vx_load(src4)))))); vx_cleanup(); return x; } template<> int PyrDownVecH(const short* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int16::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int16::nlanes; x += 3*v_int16::nlanes/4, src += 6*v_int16::nlanes/4, row += 3*v_int16::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_int16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_pack_triplets(v_dotprod(r0, v_1_4) + v_dotprod(r2, v_6_4) + v_expand_low(r4))); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(r1, v_1_4) + v_dotprod(r3, v_6_4) + v_expand_high(r4))); + v_store(row, v_pack_triplets(v_add(v_add(v_dotprod(r0, v_1_4), v_dotprod(r2, v_6_4)), v_expand_low(r4)))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_add(v_dotprod(r1, v_1_4), v_dotprod(r3, v_6_4)), v_expand_high(r4)))); } vx_cleanup(); @@ -222,24 +212,24 @@ template<> int PyrDownVecH(const short* src, int* row, int width) } template<> int PyrDownVecH(const short* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_int16::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes, src += 2*v_int16::nlanes, row += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_int16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_dotprod(r0, v_1_4) + v_dotprod(r2, v_6_4) + v_expand_low(r4)); - v_store(row + v_int32::nlanes, v_dotprod(r1, v_1_4) + v_dotprod(r3, v_6_4) + v_expand_high(r4)); + v_store(row, v_add(v_add(v_dotprod(r0, v_1_4), v_dotprod(r2, v_6_4)), v_expand_low(r4))); + v_store(row + VTraits::vlanes(), v_add(v_add(v_dotprod(r1, v_1_4), v_dotprod(r3, v_6_4)), v_expand_high(r4))); } vx_cleanup(); @@ -255,10 +245,8 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half)), v_6_4) + - v_reinterpret_as_s32(v_reinterpret_as_u32(vx_load(src4)) >> 16) + v_half15); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half)), v_6_4)), v_reinterpret_as_s32(v_shr<16>(v_reinterpret_as_u32(vx_load(src4))))), v_half15)); vx_cleanup(); return x; @@ -272,21 +260,19 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half))), v_1_4) + - v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half))), v_6_4) + - v_reinterpret_as_s32(v_reinterpret_as_u32(v_interleave_pairs(vx_load(src4))) >> 16) + v_half15); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_add(v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half))), v_1_4), v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half))), v_6_4)), v_reinterpret_as_s32(v_shr<16>(v_reinterpret_as_u32(v_interleave_pairs(vx_load(src4)))))), v_half15)); vx_cleanup(); return x; } template<> int PyrDownVecH(const ushort* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int16::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; @@ -294,18 +280,14 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int16::nlanes; x += 3*v_int16::nlanes/4, src += 6*v_int16::nlanes/4, row += 3*v_int16::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_uint16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row , v_pack_triplets(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_low(r4)) + v_half15)); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_high(r4)) + v_half15)); + v_store(row , v_pack_triplets(v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_low(r4))), v_half15))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_high(r4))), v_half15))); } vx_cleanup(); @@ -313,11 +295,11 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt } template<> int PyrDownVecH(const ushort* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_int16::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; @@ -325,18 +307,14 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes, src += 2*v_int16::nlanes, row += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_uint16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row , v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_low(r4)) + v_half15); - v_store(row + v_int32::nlanes, v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_high(r4)) + v_half15); + v_store(row , v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_low(r4))), v_half15)); + v_store(row + VTraits::vlanes(), v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_high(r4))), v_half15)); } vx_cleanup(); @@ -349,13 +327,13 @@ template<> int PyrDownVecH(const float* src, float* row, int wi const float *src01 = src, *src23 = src + 2, *src4 = src + 3; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += v_float32::nlanes, src01 += 2*v_float32::nlanes, src23 += 2*v_float32::nlanes, src4 += 2*v_float32::nlanes, row+=v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += 2*VTraits::vlanes(), src23 += 2*VTraits::vlanes(), src4 += 2*VTraits::vlanes(), row+=VTraits::vlanes()) { v_float32 r0, r1, r2, r3, r4, rtmp; v_load_deinterleave(src01, r0, r1); v_load_deinterleave(src23, r2, r3); v_load_deinterleave(src4, rtmp, r4); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); @@ -367,13 +345,13 @@ template<> int PyrDownVecH(const float* src, float* row, int wi const float *src01 = src, *src23 = src + 4, *src4 = src + 6; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - 2*v_float32::nlanes; x += 2*v_float32::nlanes, src01 += 4*v_float32::nlanes, src23 += 4*v_float32::nlanes, src4 += 4*v_float32::nlanes, row += 2*v_float32::nlanes) + for (; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), src01 += 4*VTraits::vlanes(), src23 += 4*VTraits::vlanes(), src4 += 4*VTraits::vlanes(), row += 2*VTraits::vlanes()) { v_float32 r0a, r0b, r1a, r1b, r2a, r2b, r3a, r3b, r4a, r4b, rtmpa, rtmpb; v_load_deinterleave(src01, r0a, r0b, r1a, r1b); v_load_deinterleave(src23, r2a, r2b, r3a, r3b); v_load_deinterleave(src4, rtmpa, rtmpb, r4a, r4b); - v_store_interleave(row, v_muladd(r2a, _6, v_muladd(r1a + r3a, _4, r0a + r4a)), v_muladd(r2b, _6, v_muladd(r1b + r3b, _4, r0b + r4b))); + v_store_interleave(row, v_muladd(r2a, _6, v_muladd(v_add(r1a, r3a), _4, v_add(r0a, r4a))), v_muladd(r2b, _6, v_muladd(v_add(r1b, r3b), _4, v_add(r0b, r4b)))); } vx_cleanup(); @@ -381,23 +359,23 @@ template<> int PyrDownVecH(const float* src, float* row, int wi } template<> int PyrDownVecH(const float* src, float* row, int width) { - int idx[v_float32::nlanes/2 + 4]; - for (int i = 0; i < v_float32::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_float32::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += 3*v_float32::nlanes/4, src += 6*v_float32::nlanes/4, row += 3*v_float32::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_float32 r0 = vx_lut_quads(src, idx); - v_float32 r1 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 2); + v_float32 r1 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2); v_float32 r2 = vx_lut_quads(src, idx + 1); - v_float32 r3 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 3); + v_float32 r3 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3); v_float32 r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_pack_triplets(v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4)))); + v_store(row, v_pack_triplets(v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4))))); } vx_cleanup(); @@ -405,43 +383,43 @@ template<> int PyrDownVecH(const float* src, float* row, int wi } template<> int PyrDownVecH(const float* src, float* row, int width) { - int idx[v_float32::nlanes/2 + 4]; - for (int i = 0; i < v_float32::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_float32::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += v_float32::nlanes, src += 2*v_float32::nlanes, row += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_float32 r0 = vx_lut_quads(src, idx); - v_float32 r1 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 2); + v_float32 r1 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2); v_float32 r2 = vx_lut_quads(src, idx + 1); - v_float32 r3 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 3); + v_float32 r3 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3); v_float32 r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); return x; } -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) template<> int PyrDownVecH(const double* src, double* row, int width) { int x = 0; const double *src01 = src, *src23 = src + 2, *src4 = src + 3; v_float64 _4 = vx_setall_f64(4.f), _6 = vx_setall_f64(6.f); - for (; x <= width - v_float64::nlanes; x += v_float64::nlanes, src01 += 2*v_float64::nlanes, src23 += 2*v_float64::nlanes, src4 += 2*v_float64::nlanes, row += v_float64::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += 2*VTraits::vlanes(), src23 += 2*VTraits::vlanes(), src4 += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_float64 r0, r1, r2, r3, r4, rtmp; v_load_deinterleave(src01, r0, r1); v_load_deinterleave(src23, r2, r3); v_load_deinterleave(src4, rtmp, r4); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); @@ -454,35 +432,36 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes() ) { v_uint16 r0, r1, r2, r3, r4, t0, t1; - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + v_int32::nlanes))); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x + 2*v_int32::nlanes), vx_load(row0 + x + 3*v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x + 2*v_int32::nlanes), vx_load(row1 + x + 3*v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x + 2*v_int32::nlanes), vx_load(row2 + x + 3*v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x + 2*v_int32::nlanes), vx_load(row3 + x + 3*v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x + 2*v_int32::nlanes), vx_load(row4 + x + 3*v_int32::nlanes))); - t1 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + VTraits::vlanes()))); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x + 2*VTraits::vlanes()), vx_load(row0 + x + 3*VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x + 2*VTraits::vlanes()), vx_load(row1 + x + 3*VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x + 2*VTraits::vlanes()), vx_load(row2 + x + 3*VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x + 2*VTraits::vlanes()), vx_load(row3 + x + 3*VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x + 2*VTraits::vlanes()), vx_load(row4 + x + 3*VTraits::vlanes()))); + t1 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); v_store(dst + x, v_rshr_pack<8>(t0, t1)); } - if (x <= width - v_int16::nlanes) + if (x <= width - VTraits::vlanes()) { v_uint16 r0, r1, r2, r3, r4, t0; - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + v_int32::nlanes))); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + VTraits::vlanes()))); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); v_rshr_pack_store<8>(dst + x, t0); - x += v_uint16::nlanes; + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for ( ; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { @@ -492,10 +471,23 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) r2 = v_load(row2 + x); r3 = v_load(row3 + x); r4 = v_load(row4 + x); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); *((unaligned_int*) (dst + x)) = v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r0 = *(row0 + x); + int r1 = *(row1 + x); + int r2 = *(row2 + x); + int r3 = *(row3 + x); + int r4 = *(row4 + x); + int t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + // Similar to v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16()).get0() + *(dst + x) = (int)((((unsigned int)t0) + ((1 << (8 - 1)))) >> 8); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -508,7 +500,7 @@ int PyrDownVecV(float** src, float* dst, int width) const float *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; v_float32 _4 = vx_setall_f32(4.f), _scale = vx_setall_f32(1.f/256); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 r0, r1, r2, r3, r4; r0 = vx_load(row0 + x); @@ -516,7 +508,7 @@ int PyrDownVecV(float** src, float* dst, int width) r2 = vx_load(row2 + x); r3 = vx_load(row3 + x); r4 = vx_load(row4 + x); - v_store(dst + x, v_muladd(r1 + r3 + r2, _4, r0 + r4 + (r2 + r2)) * _scale); + v_store(dst + x, v_mul(v_muladd(v_add(v_add(r1, r3), r2), _4, v_add(v_add(r0, r4), v_add(r2, r2))), _scale)); } vx_cleanup(); @@ -528,30 +520,30 @@ template <> int PyrDownVecV(int** src, ushort* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), - r01 = vx_load(row0 + x + v_int32::nlanes), + r01 = vx_load(row0 + x + VTraits::vlanes()), r10 = vx_load(row1 + x), - r11 = vx_load(row1 + x + v_int32::nlanes), + r11 = vx_load(row1 + x + VTraits::vlanes()), r20 = vx_load(row2 + x), - r21 = vx_load(row2 + x + v_int32::nlanes), + r21 = vx_load(row2 + x + VTraits::vlanes()), r30 = vx_load(row3 + x), - r31 = vx_load(row3 + x + v_int32::nlanes), + r31 = vx_load(row3 + x + VTraits::vlanes()), r40 = vx_load(row4 + x), - r41 = vx_load(row4 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack_u<8>(r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2), - r01 + r41 + (r21 + r21) + ((r11 + r21 + r31) << 2))); + r41 = vx_load(row4 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack_u<8>(v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30))), + v_add(v_add(v_add(r01, r41), v_add(r21, r21)), v_shl<2>(v_add(v_add(r11, r21), r31))))); } - if (x <= width - v_int32::nlanes) + if (x <= width - VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), r10 = vx_load(row1 + x), r20 = vx_load(row2 + x), r30 = vx_load(row3 + x), r40 = vx_load(row4 + x); - v_rshr_pack_u_store<8>(dst + x, r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2)); - x += v_int32::nlanes; + v_rshr_pack_u_store<8>(dst + x, v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -563,30 +555,30 @@ template <> int PyrDownVecV(int** src, short* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), - r01 = vx_load(row0 + x + v_int32::nlanes), + r01 = vx_load(row0 + x + VTraits::vlanes()), r10 = vx_load(row1 + x), - r11 = vx_load(row1 + x + v_int32::nlanes), + r11 = vx_load(row1 + x + VTraits::vlanes()), r20 = vx_load(row2 + x), - r21 = vx_load(row2 + x + v_int32::nlanes), + r21 = vx_load(row2 + x + VTraits::vlanes()), r30 = vx_load(row3 + x), - r31 = vx_load(row3 + x + v_int32::nlanes), + r31 = vx_load(row3 + x + VTraits::vlanes()), r40 = vx_load(row4 + x), - r41 = vx_load(row4 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack<8>(r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2), - r01 + r41 + (r21 + r21) + ((r11 + r21 + r31) << 2))); + r41 = vx_load(row4 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack<8>(v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30))), + v_add(v_add(v_add(r01, r41), v_add(r21, r21)), v_shl<2>(v_add(v_add(r11, r21), r31))))); } - if (x <= width - v_int32::nlanes) + if (x <= width - VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), r10 = vx_load(row1 + x), r20 = vx_load(row2 + x), r30 = vx_load(row3 + x), r40 = vx_load(row4 + x); - v_rshr_pack_store<8>(dst + x, r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2)); - x += v_int32::nlanes; + v_rshr_pack_store<8>(dst + x, v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -599,39 +591,55 @@ template <> int PyrUpVecV(int** src, uchar** dst, int width) uchar *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r01 = v_pack(vx_load(row0 + x + 2 * v_int32::nlanes), vx_load(row0 + x + 3 * v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r11 = v_pack(vx_load(row1 + x + 2 * v_int32::nlanes), vx_load(row1 + x + 3 * v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)), - v_r21 = v_pack(vx_load(row2 + x + 2 * v_int32::nlanes), vx_load(row2 + x + 3 * v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10, v_2r11 = (v_r11 + v_r11); - v_store(dst0 + x, v_rshr_pack_u<6>(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), v_r01 + v_r21 + (v_2r11 + v_2r11 + v_2r11))); - v_store(dst1 + x, v_rshr_pack_u<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r01 = v_pack(vx_load(row0 + x + 2 * VTraits::vlanes()), vx_load(row0 + x + 3 * VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r11 = v_pack(vx_load(row1 + x + 2 * VTraits::vlanes()), vx_load(row1 + x + 3 * VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())), + v_r21 = v_pack(vx_load(row2 + x + 2 * VTraits::vlanes()), vx_load(row2 + x + 3 * VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10), v_2r11 = (v_add(v_r11, v_r11)); + v_store(dst0 + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_add(v_add(v_r01, v_r21), v_add(v_add(v_2r11, v_2r11), v_2r11)))); + v_store(dst1 + x, v_rshr_pack_u<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_uint16::nlanes) + if(x <= width - VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10; - v_rshr_pack_u_store<6>(dst0 + x, v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10)); - v_rshr_pack_u_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_uint16::nlanes; + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10); + v_rshr_pack_u_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10))); + v_rshr_pack_u_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_int32 v_2r10 = v_r10 + v_r10; - v_int16 d = v_pack(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), (v_r10 + v_r20) << 2); + v_int32 v_2r10 = v_add(v_r10, v_r10); + v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); *(unaligned_int*)(dst0 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); *(unaligned_int*)(dst1 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r00 = *(row0 + x), + r10 = *(row1 + x), + r20 = *(row2 + x); + int _2r10 = r10 + r10; + int d = r00 + r20 + (_2r10 + _2r10 + _2r10); + int d_shifted = (r10 + r20) << 2; + // Similar to v_rshr_pack_u<6>(d, vx_setzero_s16()).get0() + *(dst0 + x) = (int)((((unsigned int)d) + ((1 << (6 - 1)))) >> 6); + // Similar to v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16()).get0() + *(dst1 + x) = (int)((((unsigned int)d_shifted) + ((1 << (6 - 1)))) >> 6); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -643,25 +651,25 @@ template <> int PyrUpVecV(int** src, short** dst, int width) short *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst0 + x, v_rshr_pack<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); - v_store(dst1 + x, v_rshr_pack<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst0 + x, v_rshr_pack<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); + v_store(dst1 + x, v_rshr_pack<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_store<6>(dst0 + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - v_rshr_pack_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_int32::nlanes; + v_rshr_pack_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + v_rshr_pack_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -674,25 +682,25 @@ template <> int PyrUpVecV(int** src, ushort** dst, int width) ushort *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst0 + x, v_rshr_pack_u<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); - v_store(dst1 + x, v_rshr_pack_u<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst0 + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); + v_store(dst1 + x, v_rshr_pack_u<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_u_store<6>(dst0 + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - v_rshr_pack_u_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_int32::nlanes; + v_rshr_pack_u_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + v_rshr_pack_u_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -706,13 +714,13 @@ template <> int PyrUpVecV(float** src, float** dst, int width) float *dst0 = dst[0], *dst1 = dst[1]; v_float32 v_6 = vx_setall_f32(6.0f), v_scale = vx_setall_f32(1.f/64.f), v_scale4 = vx_setall_f32(1.f/16.f); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 v_r0 = vx_load(row0 + x), v_r1 = vx_load(row1 + x), v_r2 = vx_load(row2 + x); - v_store(dst1 + x, v_scale4 * (v_r1 + v_r2)); - v_store(dst0 + x, v_scale * (v_muladd(v_6, v_r1, v_r0) + v_r2)); + v_store(dst1 + x, v_mul(v_scale4, v_add(v_r1, v_r2))); + v_store(dst0 + x, v_mul(v_scale, v_add(v_muladd(v_6, v_r1, v_r0), v_r2))); } vx_cleanup(); @@ -724,36 +732,50 @@ template <> int PyrUpVecVOneRow(int** src, uchar* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r01 = v_pack(vx_load(row0 + x + 2 * v_int32::nlanes), vx_load(row0 + x + 3 * v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r11 = v_pack(vx_load(row1 + x + 2 * v_int32::nlanes), vx_load(row1 + x + 3 * v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)), - v_r21 = v_pack(vx_load(row2 + x + 2 * v_int32::nlanes), vx_load(row2 + x + 3 * v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10, v_2r11 = (v_r11 + v_r11); - v_store(dst + x, v_rshr_pack_u<6>(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), v_r01 + v_r21 + (v_2r11 + v_2r11 + v_2r11))); + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r01 = v_pack(vx_load(row0 + x + 2 * VTraits::vlanes()), vx_load(row0 + x + 3 * VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r11 = v_pack(vx_load(row1 + x + 2 * VTraits::vlanes()), vx_load(row1 + x + 3 * VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())), + v_r21 = v_pack(vx_load(row2 + x + 2 * VTraits::vlanes()), vx_load(row2 + x + 3 * VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10), v_2r11 = (v_add(v_r11, v_r11)); + v_store(dst + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_add(v_add(v_r01, v_r21), v_add(v_add(v_2r11, v_2r11), v_2r11)))); } - if(x <= width - v_uint16::nlanes) + if(x <= width - VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10; - v_rshr_pack_u_store<6>(dst + x, v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10)); - x += v_uint16::nlanes; + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10); + v_rshr_pack_u_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10))); + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_int32 v_2r10 = v_r10 + v_r10; - v_int16 d = v_pack(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), (v_r10 + v_r20) << 2); + v_int32 v_2r10 = v_add(v_r10, v_r10); + v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); *(unaligned_int*)(dst + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r00 = *(row0 + x), + r10 = *(row1 + x), + r20 = *(row2 + x); + int _2r10 = r10 + r10; + int d = r00 + r20 + (_2r10 + _2r10 + _2r10); + int d_shifted = (r10 + r20) << 2; + // Similar to v_rshr_pack_u<6>(d, vx_setzero_s16()).get0() + *(dst + x) = (int)((((unsigned int)d) + ((1 << (6 - 1)))) >> 6); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -764,23 +786,23 @@ template <> int PyrUpVecVOneRow(int** src, short* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_store<6>(dst + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - x += v_int32::nlanes; + v_rshr_pack_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -792,23 +814,23 @@ template <> int PyrUpVecVOneRow(int** src, ushort* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack_u<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_u_store<6>(dst + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - x += v_int32::nlanes; + v_rshr_pack_u_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -821,12 +843,12 @@ template <> int PyrUpVecVOneRow(float** src, float* dst, int width const float *row0 = src[0], *row1 = src[1], *row2 = src[2]; v_float32 v_6 = vx_setall_f32(6.0f), v_scale = vx_setall_f32(1.f/64.f); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 v_r0 = vx_load(row0 + x), v_r1 = vx_load(row1 + x), v_r2 = vx_load(row2 + x); - v_store(dst + x, v_scale * (v_muladd(v_6, v_r1, v_r0) + v_r2)); + v_store(dst + x, v_mul(v_scale, v_add(v_muladd(v_6, v_r1, v_r0), v_r2))); } vx_cleanup(); diff --git a/modules/imgproc/src/resize.cpp b/modules/imgproc/src/resize.cpp index 456cfc4af9..1ad8e8932d 100644 --- a/modules/imgproc/src/resize.cpp +++ b/modules/imgproc/src/resize.cpp @@ -346,8 +346,8 @@ void hlineResizeCn(uint8_t* src, int, int *o { int i = 0; ufixedpoint16 src_0(src[0]); -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_src_0 = vx_setall_u16(*((uint16_t*)&src_0)); for (; i <= dst_min - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -358,7 +358,7 @@ void hlineResizeCn(uint8_t* src, int, int *o { *(dst++) = src_0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - 2*VECSZ; i += 2*VECSZ, m += 4*VECSZ, dst += 2*VECSZ) { v_uint16 v_src0, v_src1; @@ -384,7 +384,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = m[0] * px[0] + m[1] * px[1]; } src_0 = (src + ofst[dst_width - 1])[0]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_src_0 = vx_setall_u16(*((uint16_t*)&src_0)); for (; i <= dst_width - VECSZ; i += VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -406,8 +406,8 @@ void hlineResizeCn(uint8_t* src, int, int *o } srccn; ((ufixedpoint16*)(srccn.w))[0] = src[0]; ((ufixedpoint16*)(srccn.w))[1] = src[1]; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_reinterpret_as_u16(vx_setall_u32(srccn.d)); for (; i <= dst_min - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -419,7 +419,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[0]; *(dst++) = ((ufixedpoint16*)(srccn.w))[1]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += VECSZ) { v_uint16 v_src0, v_src1; @@ -440,7 +440,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = m[0] * px[1] + m[1] * px[3]; } ((ufixedpoint16*)(srccn.w))[0] = (src + 2 * ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 2 * ofst[dst_width - 1])[1]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_reinterpret_as_u16(vx_setall_u32(srccn.d)); for (; i <= dst_width - VECSZ/2; i += VECSZ/2, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -465,8 +465,8 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[1] = src[1]; ((ufixedpoint16*)(srccn.w))[2] = src[2]; ((ufixedpoint16*)(srccn.w))[3] = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q))); for (; i <= dst_min - (VECSZ+2)/3; i += VECSZ/4, m += VECSZ/2, dst += 3*VECSZ/4) // Points that fall left from src image so became equal to leftmost src point { @@ -479,14 +479,14 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[1]; *(dst++) = ((ufixedpoint16*)(srccn.w))[2]; } -#if CV_SIMD - CV_DECL_ALIGNED(CV_SIMD_WIDTH) int ofst3[VECSZ/2]; +#if (CV_SIMD || CV_SIMD_SCALABLE) + CV_DECL_ALIGNED(CV_SIMD_WIDTH) int ofst3[VTraits::max_nlanes/2]; for (; i <= dst_max - (3*VECSZ/4 + (VECSZ+2)/3); i += VECSZ/2, m += VECSZ, dst += 3*VECSZ/2) { - v_store(ofst3, vx_load(ofst + i) * vx_setall_s32(3)); + v_store(ofst3, v_mul(vx_load(ofst + i), vx_setall_s32(3))); v_uint8 v_src01, v_src23; v_uint16 v_src0, v_src1, v_src2, v_src3; - v_zip(vx_lut_quads(src, ofst3), v_reinterpret_as_u8(v_reinterpret_as_u32(vx_lut_quads(src+2, ofst3)) >> 8), v_src01, v_src23); + v_zip(vx_lut_quads(src, ofst3), v_reinterpret_as_u8(v_shr<8>(v_reinterpret_as_u32(vx_lut_quads(src+2, ofst3)))), v_src01, v_src23); v_expand(v_src01, v_src0, v_src1); v_expand(v_src23, v_src2, v_src3); @@ -514,7 +514,7 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[0] = (src + 3*ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 3*ofst[dst_width - 1])[1]; ((ufixedpoint16*)(srccn.w))[2] = (src + 3*ofst[dst_width - 1])[2]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q))); for (; i <= dst_width - (VECSZ+2)/3; i += VECSZ/4, dst += 3*VECSZ/4) // Points that fall right from src image so became equal to rightmost src point { @@ -540,8 +540,8 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[1] = src[1]; ((ufixedpoint16*)(srccn.w))[2] = src[2]; ((ufixedpoint16*)(srccn.w))[3] = src[3]; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_reinterpret_as_u16(vx_setall_u64(srccn.q)); for (; i <= dst_min - VECSZ/4; i += VECSZ/4, m += VECSZ/2, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -555,7 +555,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[2]; *(dst++) = ((ufixedpoint16*)(srccn.w))[3]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += 2*VECSZ) { v_uint16 v_src0, v_src1, v_src2, v_src3; @@ -586,7 +586,7 @@ void hlineResizeCn(uint8_t* src, int, int *o } ((ufixedpoint16*)(srccn.w))[0] = (src + 4 * ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 4 * ofst[dst_width - 1])[1]; ((ufixedpoint16*)(srccn.w))[2] = (src + 4 * ofst[dst_width - 1])[2]; ((ufixedpoint16*)(srccn.w))[3] = (src + 4 * ofst[dst_width - 1])[3]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_reinterpret_as_u16(vx_setall_u64(srccn.q)); for (; i <= dst_width - VECSZ/4; i += VECSZ/4, dst += VECSZ) // Points that fall right from src image so became equal to rightmost src point { @@ -606,8 +606,8 @@ void hlineResizeCn(uint16_t* src, int, int { int i = 0; ufixedpoint32 src_0(src[0]); -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint32 v_src_0 = vx_setall_u32(*((uint32_t*)&src_0)); for (; i <= dst_min - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -618,16 +618,16 @@ void hlineResizeCn(uint16_t* src, int, int { *(dst++) = src_0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) { v_uint32 v_src0, v_src1; v_expand(vx_lut_pairs(src, ofst + i), v_src0, v_src1); - v_uint64 v_res0 = v_reinterpret_as_u64(v_src0 * vx_load((uint32_t*)m)); - v_uint64 v_res1 = v_reinterpret_as_u64(v_src1 * vx_load((uint32_t*)m + VECSZ)); - v_store((uint32_t*)dst, v_pack((v_res0 & vx_setall_u64(0xFFFFFFFF)) + (v_res0 >> 32), - (v_res1 & vx_setall_u64(0xFFFFFFFF)) + (v_res1 >> 32))); + v_uint64 v_res0 = v_reinterpret_as_u64(v_mul(v_src0, vx_load((uint32_t *)m))); + v_uint64 v_res1 = v_reinterpret_as_u64(v_mul(v_src1, vx_load((uint32_t *)m + VECSZ))); + v_store((uint32_t*)dst, v_pack(v_add(v_and(v_res0, vx_setall_u64(0xFFFFFFFF)), v_shr<32>(v_res0)), + v_add(v_and(v_res1, vx_setall_u64(0xFFFFFFFF)), v_shr<32>(v_res1)))); } #endif for (; i < dst_max; i += 1, m += 2) @@ -636,7 +636,7 @@ void hlineResizeCn(uint16_t* src, int, int *(dst++) = m[0] * px[0] + m[1] * px[1]; } src_0 = (src + ofst[dst_width - 1])[0]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_src_0 = vx_setall_u32(*((uint32_t*)&src_0)); for (; i <= dst_width - VECSZ; i += VECSZ, dst += VECSZ) { @@ -659,16 +659,16 @@ template <> void vlineSet(ufixedpoint16* src, uint8_t* dst, int dst_width) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint8::nlanes; - static const v_uint16 v_fixedRound = vx_setall_u16((uint16_t)((1U << 8) >> 1)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + const v_uint16 v_fixedRound = vx_setall_u16((uint16_t)((1U << 8) >> 1)); for (; i <= dst_width - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) { v_uint16 v_src0 = vx_load((uint16_t*)src); v_uint16 v_src1 = vx_load((uint16_t*)src + VECSZ/2); - v_uint16 v_res0 = (v_src0 + v_fixedRound) >> 8; - v_uint16 v_res1 = (v_src1 + v_fixedRound) >> 8; + v_uint16 v_res0 = v_shr<8>(v_add(v_src0, v_fixedRound)); + v_uint16 v_res1 = v_shr<8>(v_add(v_src1, v_fixedRound)); v_store(dst, v_pack(v_res0, v_res1)); } @@ -693,11 +693,11 @@ void vlineResize(ufixedpoint16* src, size_t src_step, { int i = 0; ufixedpoint16* src1 = src + src_step; -#if CV_SIMD - const int VECSZ = v_uint8::nlanes; - static const v_int32 v_fixedRound = vx_setall_s32((int32_t)((1 << 16) >> 1)); - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1<<15)); - static const v_int8 v_128_16 = v_reinterpret_as_s8 (vx_setall_u8 ((uint8_t) 1<<7)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + const v_int32 v_fixedRound = vx_setall_s32((int32_t)((1 << 16) >> 1)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1<<15)); + const v_int8 v_128_16 = v_reinterpret_as_s8 (vx_setall_u8 ((uint8_t) 1<<7)); v_int16 v_mul = v_reinterpret_as_s16(vx_setall_u32(((uint32_t*)m)[0])); for (; i <= dst_width - VECSZ; i += VECSZ, src += VECSZ, src1 += VECSZ, dst += VECSZ) @@ -716,10 +716,10 @@ void vlineResize(ufixedpoint16* src, size_t src_step, v_int32 v_res2 = v_dotprod(v_tmp0, v_mul); v_int32 v_res3 = v_dotprod(v_tmp1, v_mul); - v_int8 v_res = v_pack(v_pack((v_res0 + v_fixedRound) >> 16, - (v_res1 + v_fixedRound) >> 16), - v_pack((v_res2 + v_fixedRound) >> 16, - (v_res3 + v_fixedRound) >> 16)); + v_int8 v_res = v_pack(v_pack(v_shr<16>(v_add(v_res0, v_fixedRound)), + v_shr<16>(v_add(v_res1, v_fixedRound))), + v_pack(v_shr<16>(v_add(v_res2, v_fixedRound)), + v_shr<16>(v_add(v_res3, v_fixedRound)))); v_store(dst, v_reinterpret_as_u8(v_sub_wrap(v_res, v_128_16))); } @@ -828,7 +828,7 @@ public: hResize((ET*)(src + (src_height - 1) * src_step), cn, xoffsets, xcoeffs, endline, min_x, max_x, dst_width); for (; dy < range.end; dy++) vlineSet(endline, (ET*)(dst + dst_step * dy), dst_width*cn); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -1136,16 +1136,16 @@ public: switch( pix_size ) { case 1: -#if CV_SIMD - for( ; x <= dsize.width - v_uint8::nlanes; x += v_uint8::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store(D + x, vx_lut(S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) D[x] = S[x_ofse[x]]; break; case 2: -#if CV_SIMD - for( ; x <= dsize.width - v_uint16::nlanes; x += v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((ushort*)D + x, vx_lut((ushort*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1159,8 +1159,8 @@ public: } break; case 4: -#if CV_SIMD - for( ; x <= dsize.width - v_uint32::nlanes; x += v_uint32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((uint32_t*)D + x, vx_lut((uint32_t*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1175,8 +1175,8 @@ public: } break; case 8: -#if CV_SIMD - for( ; x <= dsize.width - v_uint64::nlanes; x += v_uint64::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((uint64_t*)D + x, vx_lut((uint64_t*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1250,7 +1250,7 @@ struct HResizeNoVec } }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) struct VResizeLinearVec_32s8u { @@ -1260,22 +1260,17 @@ struct VResizeLinearVec_32s8u int x = 0; v_int16 b0 = vx_setall_s16(beta[0]), b1 = vx_setall_s16(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) - v_store(dst + x, v_rshr_pack_u<2>(v_mul_hi(v_pack(vx_load_aligned(S0 + x ) >> 4, vx_load_aligned(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load_aligned(S1 + x ) >> 4, vx_load_aligned(S1 + x + v_int32::nlanes) >> 4), b1), - v_mul_hi(v_pack(vx_load_aligned(S0 + x + 2 * v_int32::nlanes) >> 4, vx_load_aligned(S0 + x + 3 * v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load_aligned(S1 + x + 2 * v_int32::nlanes) >> 4, vx_load_aligned(S1 + x + 3 * v_int32::nlanes) >> 4), b1))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_rshr_pack_u<2>(v_add(v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S0 + x)), v_shr<4>(vx_load_aligned(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S1 + x)), v_shr<4>(vx_load_aligned(S1 + x + VTraits::vlanes()))), b1)), + v_add(v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S0 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load_aligned(S0 + x + 3 * VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S1 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load_aligned(S1 + x + 3 * VTraits::vlanes()))), b1)))); else - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) - v_store(dst + x, v_rshr_pack_u<2>(v_mul_hi(v_pack(vx_load(S0 + x ) >> 4, vx_load(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x ) >> 4, vx_load(S1 + x + v_int32::nlanes) >> 4), b1), - v_mul_hi(v_pack(vx_load(S0 + x + 2 * v_int32::nlanes) >> 4, vx_load(S0 + x + 3 * v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x + 2 * v_int32::nlanes) >> 4, vx_load(S1 + x + 3 * v_int32::nlanes) >> 4), b1))); + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_rshr_pack_u<2>(v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x)), v_shr<4>(vx_load(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x)), v_shr<4>(vx_load(S1 + x + VTraits::vlanes()))), b1)), + v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load(S0 + x + 3 * VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load(S1 + x + 3 * VTraits::vlanes()))), b1)))); - for( ; x < width - v_int16::nlanes; x += v_int16::nlanes) - v_rshr_pack_u_store<2>(dst + x, v_mul_hi(v_pack(vx_load(S0 + x) >> 4, vx_load(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x) >> 4, vx_load(S1 + x + v_int32::nlanes) >> 4), b1)); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) + v_rshr_pack_u_store<2>(dst + x, v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x)), v_shr<4>(vx_load(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x)), v_shr<4>(vx_load(S1 + x + VTraits::vlanes()))), b1))); return x; } @@ -1290,17 +1285,17 @@ struct VResizeLinearVec_32f16u v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) - v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, vx_load_aligned(S1 + x ) * b1)), - v_round(v_muladd(vx_load_aligned(S0 + x + v_float32::nlanes), b0, vx_load_aligned(S1 + x + v_float32::nlanes) * b1)))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, v_mul(vx_load_aligned(S1 + x), b1))), + v_round(v_muladd(vx_load_aligned(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load_aligned(S1 + x + VTraits::vlanes()), b1))))); else - for (; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) - v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, vx_load(S1 + x ) * b1)), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, vx_load(S1 + x + v_float32::nlanes) * b1)))); - for( ; x < width - v_float32::nlanes; x += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_mul(vx_load(S1 + x), b1))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load(S1 + x + VTraits::vlanes()), b1))))); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); v_store_low(dst + x, v_pack_u(t0, t0)); } @@ -1317,17 +1312,17 @@ struct VResizeLinearVec_32f16s v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) - v_store(dst + x, v_pack(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, vx_load_aligned(S1 + x ) * b1)), - v_round(v_muladd(vx_load_aligned(S0 + x + v_float32::nlanes), b0, vx_load_aligned(S1 + x + v_float32::nlanes) * b1)))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, v_mul(vx_load_aligned(S1 + x), b1))), + v_round(v_muladd(vx_load_aligned(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load_aligned(S1 + x + VTraits::vlanes()), b1))))); else - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes) - v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, vx_load(S1 + x ) * b1)), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, vx_load(S1 + x + v_float32::nlanes) * b1)))); - for( ; x < width - v_float32::nlanes; x += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_mul(vx_load(S1 + x), b1))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load(S1 + x + VTraits::vlanes()), b1))))); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); v_store_low(dst + x, v_pack(t0, t0)); } @@ -1344,12 +1339,12 @@ struct VResizeLinearVec_32f v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) - v_store(dst + x, v_muladd(vx_load_aligned(S0 + x), b0, vx_load_aligned(S1 + x) * b1)); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_muladd(vx_load_aligned(S0 + x), b0, v_mul(vx_load_aligned(S1 + x), b1))); else - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) - v_store(dst + x, v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); return x; } @@ -1367,26 +1362,26 @@ struct VResizeCubicVec_32s8u v_float32 b0 = vx_setall_f32(beta[0] * scale), b1 = vx_setall_f32(beta[1] * scale), b2 = vx_setall_f32(beta[2] * scale), b3 = vx_setall_f32(beta[3] * scale); - if( (((size_t)S0|(size_t)S1|(size_t)S2|(size_t)S3)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + if( (((size_t)S0|(size_t)S1|(size_t)S2|(size_t)S3)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_pack_u_store(dst + x, v_pack(v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x )), b0, v_muladd(v_cvt_f32(vx_load_aligned(S1 + x )), b1, v_muladd(v_cvt_f32(vx_load_aligned(S2 + x )), b2, - v_cvt_f32(vx_load_aligned(S3 + x )) * b3)))), - v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x + v_float32::nlanes)), b0, - v_muladd(v_cvt_f32(vx_load_aligned(S1 + x + v_float32::nlanes)), b1, - v_muladd(v_cvt_f32(vx_load_aligned(S2 + x + v_float32::nlanes)), b2, - v_cvt_f32(vx_load_aligned(S3 + x + v_float32::nlanes)) * b3)))))); + v_mul(v_cvt_f32(vx_load_aligned(S3 + x)), b3))))), + v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x + VTraits::vlanes())), b0, + v_muladd(v_cvt_f32(vx_load_aligned(S1 + x + VTraits::vlanes())), b1, + v_muladd(v_cvt_f32(vx_load_aligned(S2 + x + VTraits::vlanes())), b2, + v_mul(v_cvt_f32(vx_load_aligned(S3 + x + VTraits::vlanes())), b3))))))); else - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_pack_u_store(dst + x, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + x )), b0, v_muladd(v_cvt_f32(vx_load(S1 + x )), b1, v_muladd(v_cvt_f32(vx_load(S2 + x )), b2, - v_cvt_f32(vx_load(S3 + x )) * b3)))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + x + v_float32::nlanes)), b0, - v_muladd(v_cvt_f32(vx_load(S1 + x + v_float32::nlanes)), b1, - v_muladd(v_cvt_f32(vx_load(S2 + x + v_float32::nlanes)), b2, - v_cvt_f32(vx_load(S3 + x + v_float32::nlanes)) * b3)))))); + v_mul(v_cvt_f32(vx_load(S3 + x)), b3))))), + v_round(v_muladd(v_cvt_f32(vx_load(S0 + x + VTraits::vlanes())), b0, + v_muladd(v_cvt_f32(vx_load(S1 + x + VTraits::vlanes())), b1, + v_muladd(v_cvt_f32(vx_load(S2 + x + VTraits::vlanes())), b2, + v_mul(v_cvt_f32(vx_load(S3 + x + VTraits::vlanes())), b3))))))); return x; } }; @@ -1400,15 +1395,15 @@ struct VResizeCubicVec_32f16u v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for (; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, - vx_load(S3 + x ) * b3)))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - vx_load(S3 + x + v_float32::nlanes) * b3)))))); + v_mul(vx_load(S3 + x), b3))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_mul(vx_load(S3 + x + VTraits::vlanes()), b3))))))); return x; } @@ -1423,15 +1418,15 @@ struct VResizeCubicVec_32f16s v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, - vx_load(S3 + x ) * b3)))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - vx_load(S3 + x + v_float32::nlanes) * b3)))))); + v_mul(vx_load(S3 + x), b3))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_mul(vx_load(S3 + x + VTraits::vlanes()), b3))))))); return x; } @@ -1446,11 +1441,11 @@ struct VResizeCubicVec_32f v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_muladd(vx_load(S1 + x), b1, v_muladd(vx_load(S2 + x), b2, - vx_load(S3 + x) * b3)))); + v_mul(vx_load(S3 + x), b3))))); return x; } @@ -1484,7 +1479,7 @@ struct VResizeLanczos4Vec_32f16u b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, @@ -1492,15 +1487,15 @@ struct VResizeLanczos4Vec_32f16u v_muladd(vx_load(S4 + x ), b4, v_muladd(vx_load(S5 + x ), b5, v_muladd(vx_load(S6 + x ), b6, - vx_load(S7 + x ) * b7)))))))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - v_muladd(vx_load(S3 + x + v_float32::nlanes), b3, - v_muladd(vx_load(S4 + x + v_float32::nlanes), b4, - v_muladd(vx_load(S5 + x + v_float32::nlanes), b5, - v_muladd(vx_load(S6 + x + v_float32::nlanes), b6, - vx_load(S7 + x + v_float32::nlanes) * b7)))))))))); + v_mul(vx_load(S7 + x ), b7))))))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_muladd(vx_load(S3 + x + VTraits::vlanes()), b3, + v_muladd(vx_load(S4 + x + VTraits::vlanes()), b4, + v_muladd(vx_load(S5 + x + VTraits::vlanes()), b5, + v_muladd(vx_load(S6 + x + VTraits::vlanes()), b6, + v_mul(vx_load(S7 + x + VTraits::vlanes()), b7))))))))))); return x; } @@ -1520,7 +1515,7 @@ struct VResizeLanczos4Vec_32f16s b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, @@ -1528,15 +1523,15 @@ struct VResizeLanczos4Vec_32f16s v_muladd(vx_load(S4 + x ), b4, v_muladd(vx_load(S5 + x ), b5, v_muladd(vx_load(S6 + x ), b6, - vx_load(S7 + x ) * b7)))))))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - v_muladd(vx_load(S3 + x + v_float32::nlanes), b3, - v_muladd(vx_load(S4 + x + v_float32::nlanes), b4, - v_muladd(vx_load(S5 + x + v_float32::nlanes), b5, - v_muladd(vx_load(S6 + x + v_float32::nlanes), b6, - vx_load(S7 + x + v_float32::nlanes) * b7)))))))))); + v_mul(vx_load(S7 + x), b7))))))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_muladd(vx_load(S3 + x + VTraits::vlanes()), b3, + v_muladd(vx_load(S4 + x + VTraits::vlanes()), b4, + v_muladd(vx_load(S5 + x + VTraits::vlanes()), b5, + v_muladd(vx_load(S6 + x + VTraits::vlanes()), b6, + v_mul(vx_load(S7 + x + VTraits::vlanes()), b7))))))))))); return x; } @@ -1555,7 +1550,7 @@ struct VResizeLanczos4Vec_32f b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_muladd(vx_load(S1 + x), b1, v_muladd(vx_load(S2 + x), b2, @@ -1563,7 +1558,7 @@ struct VResizeLanczos4Vec_32f v_muladd(vx_load(S4 + x), b4, v_muladd(vx_load(S5 + x), b5, v_muladd(vx_load(S6 + x), b6, - vx_load(S7 + x) * b7)))))))); + v_mul(vx_load(S7 + x), b7))))))))); return x; } @@ -1620,8 +1615,8 @@ struct HResizeLinearVec_X4 DVT s1(S0[sx0+cn], S0[sx1+cn], S0[sx2+cn], S0[sx3+cn]); DVT s0_u(S1[sx0], S1[sx1], S1[sx2], S1[sx3]); DVT s1_u(S1[sx0+cn], S1[sx1+cn], S1[sx2+cn], S1[sx3+cn]); - v_store(&D1[dx], s0_u * a_even + s1_u * a_odd); - v_store(&D0[dx], s0 * a_even + s1 * a_odd); + v_store(&D1[dx], v_add(v_mul(s0_u, a_even), v_mul(s1_u, a_odd))); + v_store(&D0[dx], v_add(v_mul(s0, a_even), v_mul(s1, a_odd))); } } for( ; k < count; k++ ) @@ -1640,7 +1635,7 @@ struct HResizeLinearVec_X4 v_load_deinterleave(&alpha[dx*2], a_even, a_odd); DVT s0(S[sx0], S[sx1], S[sx2], S[sx3]); DVT s1(S[sx0+cn], S[sx1+cn], S[sx2+cn], S[sx3+cn]); - v_store(&D[dx], s0 * a_even + s1 * a_odd); + v_store(&D[dx], v_add(v_mul(s0, a_even), v_mul(s1, a_odd))); } } return dx; @@ -1752,8 +1747,8 @@ struct HResizeLinearVecU8_X4 for( dx = 0; (xofs[dx] + cn) < smax; dx += cn ) { v_int16x8 a = v_load(alpha+dx*2); - v_store(&D0[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S0+xofs[dx]) | (v_load_expand_q(S0+xofs[dx]+cn)<<16)), a)); - v_store(&D1[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S1+xofs[dx]) | (v_load_expand_q(S1+xofs[dx]+cn)<<16)), a)); + v_store(&D0[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S0 + xofs[dx]), v_shl<16>(v_load_expand_q(S0 + xofs[dx] + cn)))), a)); + v_store(&D1[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S1 + xofs[dx]), v_shl<16>(v_load_expand_q(S1 + xofs[dx] + cn)))), a)); } } for( ; k < count; k++ ) @@ -1763,7 +1758,7 @@ struct HResizeLinearVecU8_X4 for( dx = 0; (xofs[dx] + cn) < smax; dx += cn ) { v_int16x8 a = v_load(alpha+dx*2); - v_store(&D[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S+xofs[dx]) | (v_load_expand_q(S+xofs[dx]+cn)<<16)), a)); + v_store(&D[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S + xofs[dx]), v_shl<16>(v_load_expand_q(S + xofs[dx] + cn)))), a)); } } /* Debug check to ensure truthiness that we never vector the final value. */ @@ -2452,27 +2447,27 @@ public: if (cn == 1) { v_uint16 masklow = vx_setall_u16(0x00ff); - for ( ; dx <= w - v_uint16::nlanes; dx += v_uint16::nlanes, S0 += v_uint8::nlanes, S1 += v_uint8::nlanes, D += v_uint16::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint16 r0 = v_reinterpret_as_u16(vx_load(S0)); v_uint16 r1 = v_reinterpret_as_u16(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 8) + (r0 & masklow) + (r1 >> 8) + (r1 & masklow)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<8>(r0), v_and(r0, masklow)), v_shr<8>(r1)), v_and(r1, masklow))); } } else if (cn == 3) { if (CV_SIMD_WIDTH > 64) return 0; - for ( ; dx <= w - 3*v_uint8::nlanes; dx += 3*v_uint8::nlanes, S0 += 6*v_uint8::nlanes, S1 += 6*v_uint8::nlanes, D += 3*v_uint8::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_uint16 t0, t1, t2, t3, t4, t5; v_uint16 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_uint16::nlanes) + vx_load_expand(S1 + v_uint16::nlanes); - s2 = vx_load_expand(S0 + 2*v_uint16::nlanes) + vx_load_expand(S1 + 2*v_uint16::nlanes); - s3 = vx_load_expand(S0 + 3*v_uint16::nlanes) + vx_load_expand(S1 + 3*v_uint16::nlanes); - s4 = vx_load_expand(S0 + 4*v_uint16::nlanes) + vx_load_expand(S1 + 4*v_uint16::nlanes); - s5 = vx_load_expand(S0 + 5*v_uint16::nlanes) + vx_load_expand(S1 + 5*v_uint16::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2481,18 +2476,18 @@ public: bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); - bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; + bl = v_add(s0, s3); gl = v_add(s1, s4); rl = v_add(s2, s5); #elif CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; #endif - s0 = vx_load_expand(S0 + 6*v_uint16::nlanes) + vx_load_expand(S1 + 6*v_uint16::nlanes); - s1 = vx_load_expand(S0 + 7*v_uint16::nlanes) + vx_load_expand(S1 + 7*v_uint16::nlanes); - s2 = vx_load_expand(S0 + 8*v_uint16::nlanes) + vx_load_expand(S1 + 8*v_uint16::nlanes); - s3 = vx_load_expand(S0 + 9*v_uint16::nlanes) + vx_load_expand(S1 + 9*v_uint16::nlanes); - s4 = vx_load_expand(S0 +10*v_uint16::nlanes) + vx_load_expand(S1 +10*v_uint16::nlanes); - s5 = vx_load_expand(S0 +11*v_uint16::nlanes) + vx_load_expand(S1 +11*v_uint16::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2501,7 +2496,7 @@ public: bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); - bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; + bh = v_add(s0, s3); gh = v_add(s1, s4); rh = v_add(s2, s5); #elif CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2513,7 +2508,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx <= w - v_uint8::nlanes; dx += v_uint8::nlanes, S0 += 2*v_uint8::nlanes, S1 += 2*v_uint8::nlanes, D += v_uint8::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r00, r01, r10, r11; v_load_deinterleave((uint32_t*)S0, r00, r01); @@ -2524,7 +2519,7 @@ public: v_expand(v_reinterpret_as_u8(r01), r01l, r01h); v_expand(v_reinterpret_as_u8(r10), r10l, r10h); v_expand(v_reinterpret_as_u8(r11), r11l, r11h); - v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); + v_store(D, v_rshr_pack<2>(v_add(v_add(v_add(r00l, r01l), r10l), r11l), v_add(v_add(v_add(r00h, r01h), r10h), r11h))); } } @@ -2551,11 +2546,11 @@ public: if (cn == 1) { v_uint32 masklow = vx_setall_u32(0x0000ffff); - for (; dx <= w - v_uint32::nlanes; dx += v_uint32::nlanes, S0 += v_uint16::nlanes, S1 += v_uint16::nlanes, D += v_uint32::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r0 = v_reinterpret_as_u32(vx_load(S0)); v_uint32 r1 = v_reinterpret_as_u32(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 16) + (r0 & masklow) + (r1 >> 16) + (r1 & masklow)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<16>(r0), v_and(r0, masklow)), v_shr<16>(r1)), v_and(r1, masklow))); } } else if (cn == 3) @@ -2574,38 +2569,38 @@ public: v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); #endif #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 - for ( ; dx <= w - 3*v_uint16::nlanes; dx += 3*v_uint16::nlanes, S0 += 6*v_uint16::nlanes, S1 += 6*v_uint16::nlanes, D += 3*v_uint16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_uint32 t0, t1, t2, t3, t4, t5; v_uint32 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_uint32::nlanes) + vx_load_expand(S1 + v_uint32::nlanes); - s2 = vx_load_expand(S0 + 2*v_uint32::nlanes) + vx_load_expand(S1 + 2*v_uint32::nlanes); - s3 = vx_load_expand(S0 + 3*v_uint32::nlanes) + vx_load_expand(S1 + 3*v_uint32::nlanes); - s4 = vx_load_expand(S0 + 4*v_uint32::nlanes) + vx_load_expand(S1 + 4*v_uint32::nlanes); - s5 = vx_load_expand(S0 + 5*v_uint32::nlanes) + vx_load_expand(S1 + 5*v_uint32::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_uint32 bl, gl, rl; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; + bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; #endif - s0 = vx_load_expand(S0 + 6*v_uint32::nlanes) + vx_load_expand(S1 + 6*v_uint32::nlanes); - s1 = vx_load_expand(S0 + 7*v_uint32::nlanes) + vx_load_expand(S1 + 7*v_uint32::nlanes); - s2 = vx_load_expand(S0 + 8*v_uint32::nlanes) + vx_load_expand(S1 + 8*v_uint32::nlanes); - s3 = vx_load_expand(S0 + 9*v_uint32::nlanes) + vx_load_expand(S1 + 9*v_uint32::nlanes); - s4 = vx_load_expand(S0 +10*v_uint32::nlanes) + vx_load_expand(S1 +10*v_uint32::nlanes); - s5 = vx_load_expand(S0 +11*v_uint32::nlanes) + vx_load_expand(S1 +11*v_uint32::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_uint32 bh, gh, rh; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; + bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; @@ -2649,19 +2644,19 @@ public: v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); } #else - for ( ; dx <= w - v_uint32::nlanes; dx += v_uint32::nlanes, S0 += v_uint16::nlanes, S1 += v_uint16::nlanes, D += v_uint32::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r0, r1, r2, r3; v_expand(vx_load(S0), r0, r1); v_expand(vx_load(S1), r2, r3); - r0 += r2; r1 += r3; + r0 = v_add(r0, r2); r1 = v_add(r1, r3); v_uint32 v_d; #if CV_SIMD_WIDTH == 16 v_d = r0 + r1; #elif CV_SIMD_WIDTH == 32 v_uint32 t0, t1; v_recombine(r0, r1, t0, t1); - v_d = t0 + t1; + v_d = v_add(t0, t1); #endif v_rshr_pack_store<2>(D, v_d); } @@ -2691,11 +2686,11 @@ public: if (cn == 1) { v_int32 masklow = vx_setall_s32(0x0000ffff); - for (; dx <= w - v_int32::nlanes; dx += v_int32::nlanes, S0 += v_int16::nlanes, S1 += v_int16::nlanes, D += v_int32::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_int32 r0 = v_reinterpret_as_s32(vx_load(S0)); v_int32 r1 = v_reinterpret_as_s32(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 16) + (((r0 & masklow)<<16)>>16) + (r1 >> 16) + (((r1 & masklow)<<16)>>16)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<16>(r0), v_shr<16>(v_shl<16>(v_and(r0, masklow)))), v_shr<16>(r1)), v_shr<16>(v_shl<16>(v_and(r1, masklow))))); } } else if (cn == 3) @@ -2704,38 +2699,38 @@ public: for ( ; dx <= w - 4; dx += 3, S0 += 6, S1 += 6, D += 3) v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 - for ( ; dx <= w - 3*v_int16::nlanes; dx += 3*v_int16::nlanes, S0 += 6*v_int16::nlanes, S1 += 6*v_int16::nlanes, D += 3*v_int16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_int32 t0, t1, t2, t3, t4, t5; v_int32 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_int32::nlanes) + vx_load_expand(S1 + v_int32::nlanes); - s2 = vx_load_expand(S0 + 2*v_int32::nlanes) + vx_load_expand(S1 + 2*v_int32::nlanes); - s3 = vx_load_expand(S0 + 3*v_int32::nlanes) + vx_load_expand(S1 + 3*v_int32::nlanes); - s4 = vx_load_expand(S0 + 4*v_int32::nlanes) + vx_load_expand(S1 + 4*v_int32::nlanes); - s5 = vx_load_expand(S0 + 5*v_int32::nlanes) + vx_load_expand(S1 + 5*v_int32::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_int32 bl, gl, rl; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; + bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; #endif - s0 = vx_load_expand(S0 + 6*v_int32::nlanes) + vx_load_expand(S1 + 6*v_int32::nlanes); - s1 = vx_load_expand(S0 + 7*v_int32::nlanes) + vx_load_expand(S1 + 7*v_int32::nlanes); - s2 = vx_load_expand(S0 + 8*v_int32::nlanes) + vx_load_expand(S1 + 8*v_int32::nlanes); - s3 = vx_load_expand(S0 + 9*v_int32::nlanes) + vx_load_expand(S1 + 9*v_int32::nlanes); - s4 = vx_load_expand(S0 +10*v_int32::nlanes) + vx_load_expand(S1 +10*v_int32::nlanes); - s5 = vx_load_expand(S0 +11*v_int32::nlanes) + vx_load_expand(S1 +11*v_int32::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_int32 bh, gh, rh; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; + bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; @@ -2763,7 +2758,7 @@ public: else { CV_Assert(cn == 4); - for (; dx <= w - v_int16::nlanes; dx += v_int16::nlanes, S0 += 2 * v_int16::nlanes, S1 += 2 * v_int16::nlanes, D += v_int16::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2 * VTraits::vlanes(), S1 += 2 * VTraits::vlanes(), D += VTraits::vlanes()) { #if CV_SIMD_WIDTH >= 64 v_int64 r00, r01, r10, r11; @@ -2778,17 +2773,17 @@ public: v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); #else v_int32 r0, r1, r2, r3; - r0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - r1 = vx_load_expand(S0 + v_int32::nlanes) + vx_load_expand(S1 + v_int32::nlanes); - r2 = vx_load_expand(S0 + 2*v_int32::nlanes) + vx_load_expand(S1 + 2*v_int32::nlanes); - r3 = vx_load_expand(S0 + 3*v_int32::nlanes) + vx_load_expand(S1 + 3*v_int32::nlanes); + r0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + r1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + r2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + r3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); v_int32 dl, dh; #if CV_SIMD_WIDTH == 16 dl = r0 + r1; dh = r2 + r3; #elif CV_SIMD_WIDTH == 32 v_int32 t0, t1, t2, t3; v_recombine(r0, r1, t0, t1); v_recombine(r2, r3, t2, t3); - dl = t0 + t1; dh = t2 + t3; + dl = v_add(t0, t1); dh = v_add(t2, t3); #endif v_store(D, v_rshr_pack<2>(dl, dh)); #endif @@ -2822,12 +2817,12 @@ struct ResizeAreaFastVec_SIMD_32f if (cn == 1) { v_float32 v_025 = vx_setall_f32(0.25f); - for ( ; dx <= w - v_float32::nlanes; dx += v_float32::nlanes, S0 += 2*v_float32::nlanes, S1 += 2*v_float32::nlanes, D += v_float32::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_float32 v_row00, v_row01, v_row10, v_row11; v_load_deinterleave(S0, v_row00, v_row01); v_load_deinterleave(S1, v_row10, v_row11); - v_store(D, ((v_row00 + v_row01) + (v_row10 + v_row11)) * v_025); + v_store(D, v_mul(v_add(v_add(v_row00, v_row01), v_add(v_row10, v_row11)), v_025)); } } else if (cn == 4) @@ -2841,8 +2836,8 @@ struct ResizeAreaFastVec_SIMD_32f for (; dx <= w - v_float32x8::nlanes; dx += v_float32x8::nlanes, S0 += 2*v_float32x8::nlanes, S1 += 2*v_float32x8::nlanes, D += v_float32x8::nlanes) { v_float32x8 dst0, dst1; - v_recombine(v256_load(S0) + v256_load(S1), v256_load(S0 + v_float32x8::nlanes) + v256_load(S1 + v_float32x8::nlanes), dst0, dst1); - v_store(D, (dst0 + dst1) * v_025); + v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + v_float32x8::nlanes), v256_load(S1 + v_float32x8::nlanes)), dst0, dst1); + v_store(D, v_mul(v_add(dst0, dst1), v_025)); } #endif } diff --git a/modules/imgproc/src/smooth.simd.hpp b/modules/imgproc/src/smooth.simd.hpp index 62ff31ac94..33e58d4e80 100644 --- a/modules/imgproc/src/smooth.simd.hpp +++ b/modules/imgproc/src/smooth.simd.hpp @@ -81,11 +81,11 @@ void hlineSmooth1N(const uint8_t* src, int cn, const ufi { int lencn = len*cn; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; - v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + v_uint16 vmul = vx_setall_u16(*((uint16_t*)m)); for (; i <= lencn - VECSZ; i += VECSZ) - v_store((uint16_t*)dst + i, v_mul_wrap(v_mul, vx_load_expand(src + i))); + v_store((uint16_t*)dst + i, v_mul(vmul, vx_load_expand(src + i))); #endif for (; i < lencn; i++) dst[i] = m[0] * src[i]; @@ -101,8 +101,8 @@ void hlineSmooth1N1(const uint8_t* src, int cn, const uf { int lencn = len*cn; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ) v_store((uint16_t*)dst + i, v_shl<8>(vx_load_expand(src + i))); #endif @@ -168,16 +168,14 @@ void hlineSmooth3N(const uint8_t* src, int cn, const ufi src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - cn), v_mul0) + - v_mul_wrap(vx_load_expand(src), v_mul1) + - v_mul_wrap(vx_load_expand(src + cn), v_mul2)); + v_store((uint16_t*)dst, v_add(v_add(v_mul(vx_load_expand(src - cn), v_mul0), v_mul(vx_load_expand(src), v_mul1)), v_mul(vx_load_expand(src + cn), v_mul2))); #endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-cn] + m[1] * src[0] + m[2] * src[cn]; @@ -220,10 +218,10 @@ void hlineSmooth3N121Impl(const ET* src, int cn, const FT*, int, FT* dst, int le src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD - const int VECSZ = VFT::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((typename FT::raw_t*)dst, (vx_load_expand(src - cn) + vx_load_expand(src + cn) + (vx_load_expand(src) << 1)) << (FT::fixedShift-2)); + v_store((typename FT::raw_t*)dst, v_shl<(FT::fixedShift-2)>(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn), v_shl<1>((vx_load_expand(src)))))); #endif for (; i < lencn; i++, src++, dst++) *dst = (FT(src[-cn])>>2) + (FT(src[cn])>>2) + (FT(src[0])>>1); @@ -320,14 +318,13 @@ void hlineSmooth3Naba(const uint8_t* src, int cn, const src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - cn) + vx_load_expand(src + cn), v_mul0) + - v_mul_wrap(vx_load_expand(src), v_mul1)); + v_store((uint16_t*)dst, v_add(v_mul(v_add( vx_load_expand(src - cn), vx_load_expand(src + cn)), v_mul0), v_mul(vx_load_expand(src), v_mul1))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = saturate_cast(((uint16_t*)m)[1] * (uint32_t)(src[0]) + ((uint16_t*)m)[0] * ((uint32_t)(src[-cn]) + (uint32_t)(src[cn]))); @@ -514,20 +511,16 @@ void hlineSmooth5N(const uint8_t* src, int cn, const ufi src += 2 * cn; dst += 2 * cn; int i = 2*cn, lencn = (len - 2)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); v_uint16 v_mul3 = vx_setall_u16(_m[3]); v_uint16 v_mul4 = vx_setall_u16(_m[4]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - 2 * cn), v_mul0) + - v_mul_wrap(vx_load_expand(src - cn), v_mul1) + - v_mul_wrap(vx_load_expand(src), v_mul2) + - v_mul_wrap(vx_load_expand(src + cn), v_mul3) + - v_mul_wrap(vx_load_expand(src + 2 * cn), v_mul4)); + v_store((uint16_t*)dst, v_add(v_add(v_add(v_add(v_mul(vx_load_expand(src - 2 * cn), v_mul0), v_mul(vx_load_expand(src - cn), v_mul1)), v_mul(vx_load_expand(src), v_mul2)), v_mul(vx_load_expand(src + cn), v_mul3)), v_mul(vx_load_expand(src + 2 * cn), v_mul4))); #endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-2*cn] + m[1] * src[-cn] + m[2] * src[0] + m[3] * src[cn] + m[4] * src[2*cn]; @@ -726,11 +719,11 @@ void hlineSmooth5N14641(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_6 = vx_setall_u16(6); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, (v_mul_wrap(vx_load_expand(src), v_6) + ((vx_load_expand(src - cn) + vx_load_expand(src + cn)) << 2) + vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn)) << 4); + v_store((uint16_t*)dst, v_shl<4>(v_add(v_add(v_add(v_mul(vx_load_expand(src), v_6), v_shl<2>(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn)))), vx_load_expand(src - 2 * cn)), vx_load_expand(src + 2 * cn)))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = (uint16_t(src[0]) * 6 + ((uint16_t(src[-cn]) + uint16_t(src[cn])) << 2) + uint16_t(src[-2 * cn]) + uint16_t(src[2 * cn])) << 4; @@ -924,16 +917,14 @@ void hlineSmooth5Nabcba(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn), v_mul0) + - v_mul_wrap(vx_load_expand(src - cn) + vx_load_expand(src + cn), v_mul1) + - v_mul_wrap(vx_load_expand(src), v_mul2)); + v_store((uint16_t*)dst, v_add(v_add(v_mul(v_add(vx_load_expand(src - 2 * cn), vx_load_expand(src + 2 * cn)), v_mul0), v_mul(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn)), v_mul1)), v_mul(vx_load_expand(src), v_mul2))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = saturate_cast(((uint16_t*)m)[0] * ((uint32_t)(src[-2 * cn]) + (uint32_t)(src[2 * cn])) + ((uint16_t*)m)[1] * ((uint32_t)(src[-cn]) + (uint32_t)(src[cn])) + ((uint16_t*)m)[2] * (uint32_t)(src[0])); @@ -1044,13 +1035,13 @@ void hlineSmooth(const uint8_t* src, int cn, const ufixe } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i+=VECSZ, src+=VECSZ, dst+=VECSZ) { - v_uint16 v_res0 = v_mul_wrap(vx_load_expand(src), vx_setall_u16(*((uint16_t*)m))); + v_uint16 v_res0 = v_mul(vx_load_expand(src), vx_setall_u16(*((uint16_t*)m))); for (int j = 1; j < n; j++) - v_res0 += v_mul_wrap(vx_load_expand(src + j * cn), vx_setall_u16(*((uint16_t*)(m + j)))); + v_res0 = v_add(v_res0, v_mul(vx_load_expand(src + j * cn), vx_setall_u16(*((uint16_t *)(m + j))))); v_store((uint16_t*)dst, v_res0); } #endif @@ -1163,13 +1154,13 @@ void hlineSmoothONa_yzy_a(const uint8_t* src, int cn, co } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) { - v_uint16 v_res0 = v_mul_wrap(vx_load_expand(src + pre_shift * cn), vx_setall_u16(*((uint16_t*)(m + pre_shift)))); + v_uint16 v_res0 = v_mul(vx_load_expand(src + pre_shift * cn), vx_setall_u16(*((uint16_t*)(m + pre_shift)))); for (int j = 0; j < pre_shift; j ++) - v_res0 += v_mul_wrap(vx_load_expand(src + j * cn) + vx_load_expand(src + (n - 1 - j)*cn), vx_setall_u16(*((uint16_t*)(m + j)))); + v_res0 = v_add(v_res0, v_mul(v_add(vx_load_expand(src + j * cn), vx_load_expand(src + (n - 1 - j) * cn)), vx_setall_u16(*((uint16_t *)(m + j))))); v_store((uint16_t*)dst, v_res0); } #endif @@ -1228,8 +1219,8 @@ void hlineSmoothONa_yzy_a(const uint16_t* src, int cn, } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ * 2; i += VECSZ * 2, src += VECSZ * 2, dst += VECSZ * 2) { v_uint32 v_res0, v_res1; @@ -1239,11 +1230,11 @@ void hlineSmoothONa_yzy_a(const uint16_t* src, int cn, v_uint16 v_weight = vx_setall_u16((uint16_t) *((uint32_t*)(m + j))); v_uint32 v_add0, v_add1; v_mul_expand(vx_load(src + j * cn), v_weight, v_add0, v_add1); - v_res0 += v_add0; - v_res1 += v_add1; + v_res0 = v_add(v_res0, v_add0); + v_res1 = v_add(v_res1, v_add1); v_mul_expand(vx_load(src + (n - 1 - j)*cn), v_weight, v_add0, v_add1); - v_res0 += v_add0; - v_res1 += v_add1; + v_res0 = v_add(v_res0, v_add0); + v_res1 = v_add(v_res1, v_add1); } v_store((uint32_t*)dst, v_res0); v_store((uint32_t*)dst + VECSZ, v_res1); @@ -1285,8 +1276,8 @@ void vlineSmooth1N(const ufixedpoint16* const * src, con { const ufixedpoint16* src0 = src[0]; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)<<1); for (; i <= len - VECSZ; i += VECSZ) v_rshr_pack_store<1>(dst + i, v_mul_hi(vx_load((uint16_t*)src0 + i), v_mul)); @@ -1306,8 +1297,8 @@ void vlineSmooth1N1(const ufixedpoint16* const * src, co { const ufixedpoint16* src0 = src[0]; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - VECSZ; i += VECSZ) v_rshr_pack_store<8>(dst + i, vx_load((uint16_t*)(src0 + i))); #endif @@ -1324,10 +1315,10 @@ template <> void vlineSmooth3N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2]) * ufixedpoint16((uint8_t)128) }; @@ -1370,26 +1361,26 @@ void vlineSmooth3N(const ufixedpoint16* const * src, con v_src02 = vx_load(src2 + 2*VECSZ); v_src03 = vx_load(src2 + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul2, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul2, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul2, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul2, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1410,8 +1401,8 @@ template <> void vlineSmooth3N121(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; @@ -1421,8 +1412,8 @@ void vlineSmooth3N121(const ufixedpoint16* const * src, v_expand(vx_load((uint16_t*)(src[1]) + i + VECSZ), v_src12, v_src13); v_expand(vx_load((uint16_t*)(src[2]) + i), v_src20, v_src21); v_expand(vx_load((uint16_t*)(src[2]) + i + VECSZ), v_src22, v_src23); - v_store(dst + i, v_pack(v_rshr_pack<10>(v_src00 + v_src20 + (v_src10 + v_src10), v_src01 + v_src21 + (v_src11 + v_src11)), - v_rshr_pack<10>(v_src02 + v_src22 + (v_src12 + v_src12), v_src03 + v_src23 + (v_src13 + v_src13)))); + v_store(dst + i, v_pack(v_rshr_pack<10>(v_add(v_add(v_src00, v_src20), v_add(v_src10, v_src10)), v_add(v_add(v_src01, v_src21), v_add(v_src11, v_src11))), + v_rshr_pack<10>(v_add(v_add(v_src02, v_src22), v_add(v_src12, v_src12)), v_add(v_add(v_src03, v_src23), v_add(v_src13, v_src13))))); } #endif for (; i < len; i++) @@ -1432,8 +1423,8 @@ template <> void vlineSmooth3N121(const ufixedpoint32* const * src, const ufixedpoint32*, int, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint64 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; @@ -1443,8 +1434,8 @@ void vlineSmooth3N121(const ufixedpoint32* const * src, v_expand(vx_load((uint32_t*)(src[1]) + i + VECSZ), v_src12, v_src13); v_expand(vx_load((uint32_t*)(src[2]) + i), v_src20, v_src21); v_expand(vx_load((uint32_t*)(src[2]) + i + VECSZ), v_src22, v_src23); - v_store(dst + i, v_pack(v_rshr_pack<18>(v_src00 + v_src20 + (v_src10 + v_src10), v_src01 + v_src21 + (v_src11 + v_src11)), - v_rshr_pack<18>(v_src02 + v_src22 + (v_src12 + v_src12), v_src03 + v_src23 + (v_src13 + v_src13)))); + v_store(dst + i, v_pack(v_rshr_pack<18>(v_add(v_add(v_src00, v_src20), v_add(v_src10, v_src10)), v_add(v_add(v_src01, v_src21), v_add(v_src11, v_src11))), + v_rshr_pack<18>(v_add(v_add(v_src02, v_src22), v_add(v_src12, v_src12)), v_add(v_add(v_src03, v_src23), v_add(v_src13, v_src13))))); } #endif for (; i < len; i++) @@ -1460,13 +1451,13 @@ template <> void vlineSmooth5N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); if (len >= 4 * VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2] + m[3] + m[4]) * ufixedpoint16((uint8_t)128) }; v_int32 v_128_4 = vx_setall_s32(*((int32_t*)val)); - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int16 v_mul01 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)m))); v_int16 v_mul23 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)(m + 2)))); v_int16 v_mul4 = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + 4)))); @@ -1509,17 +1500,17 @@ void vlineSmooth5N(const ufixedpoint16* const * src, con v_src12 = vx_load(src3 + 2*VECSZ); v_src13 = vx_load(src3 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul23); - v_res1 += v_dotprod(v_tmp1, v_mul23); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul23)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_res2 += v_dotprod(v_tmp0, v_mul23); - v_res3 += v_dotprod(v_tmp1, v_mul23); + v_res2 = v_add(v_res2, v_dotprod(v_tmp0, v_mul23)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_res4 += v_dotprod(v_tmp0, v_mul23); - v_res5 += v_dotprod(v_tmp1, v_mul23); + v_res4 = v_add(v_res4, v_dotprod(v_tmp0, v_mul23)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_res6 += v_dotprod(v_tmp0, v_mul23); - v_res7 += v_dotprod(v_tmp1, v_mul23); + v_res6 = v_add(v_res6, v_dotprod(v_tmp0, v_mul23)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp1, v_mul23)); v_int32 v_resj0, v_resj1; const int16_t* src4 = (const int16_t*)src[4] + i; @@ -1528,26 +1519,26 @@ void vlineSmooth5N(const ufixedpoint16* const * src, con v_src02 = vx_load(src4 + 2*VECSZ); v_src03 = vx_load(src4 + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul4, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul4, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul4, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul4, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1569,9 +1560,9 @@ template <> void vlineSmooth5N14641(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint32 v_6 = vx_setall_u32(6); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src10, v_src20, v_src30, v_src40; @@ -1588,10 +1579,10 @@ void vlineSmooth5N14641(const ufixedpoint16* const * src v_expand(vx_load((uint16_t*)(src[3]) + i + VECSZ), v_src32, v_src33); v_expand(vx_load((uint16_t*)(src[4]) + i), v_src40, v_src41); v_expand(vx_load((uint16_t*)(src[4]) + i + VECSZ), v_src42, v_src43); - v_store(dst + i, v_pack(v_rshr_pack<12>(v_src20*v_6 + ((v_src10 + v_src30) << 2) + v_src00 + v_src40, - v_src21*v_6 + ((v_src11 + v_src31) << 2) + v_src01 + v_src41), - v_rshr_pack<12>(v_src22*v_6 + ((v_src12 + v_src32) << 2) + v_src02 + v_src42, - v_src23*v_6 + ((v_src13 + v_src33) << 2) + v_src03 + v_src43))); + v_store(dst + i, v_pack(v_rshr_pack<12>(v_add(v_add(v_add(v_mul(v_src20, v_6), v_shl<2>(v_add(v_src10, v_src30))), v_src00), v_src40), + v_add(v_add(v_add(v_mul(v_src21, v_6), v_shl<2>(v_add(v_src11, v_src31))), v_src01), v_src41)), + v_rshr_pack<12>(v_add(v_add(v_add(v_mul(v_src22, v_6), v_shl<2>(v_add(v_src12, v_src32))), v_src02), v_src42), + v_add(v_add(v_add(v_mul(v_src23, v_6), v_shl<2>(v_add(v_src13, v_src33))), v_src03), v_src43)))); } #endif for (; i < len; i++) @@ -1603,8 +1594,8 @@ template <> void vlineSmooth5N14641(const ufixedpoint32* const * src, const ufixedpoint32*, int, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint64 v_src00, v_src10, v_src20, v_src30, v_src40; @@ -1621,10 +1612,10 @@ void vlineSmooth5N14641(const ufixedpoint32* const * sr v_expand(vx_load((uint32_t*)(src[3]) + i + VECSZ), v_src32, v_src33); v_expand(vx_load((uint32_t*)(src[4]) + i), v_src40, v_src41); v_expand(vx_load((uint32_t*)(src[4]) + i + VECSZ), v_src42, v_src43); - v_store(dst + i, v_pack(v_rshr_pack<20>((v_src20 << 2) + (v_src20 << 1) + ((v_src10 + v_src30) << 2) + v_src00 + v_src40, - (v_src21 << 2) + (v_src21 << 1) + ((v_src11 + v_src31) << 2) + v_src01 + v_src41), - v_rshr_pack<20>((v_src22 << 2) + (v_src22 << 1) + ((v_src12 + v_src32) << 2) + v_src02 + v_src42, - (v_src23 << 2) + (v_src23 << 1) + ((v_src13 + v_src33) << 2) + v_src03 + v_src43))); + v_store(dst + i, v_pack(v_rshr_pack<20>(v_add(v_add(v_add(v_add(v_shl<2>(v_src20), v_shl<1>(v_src20)), v_shl<2>(v_add(v_src10, v_src30))), v_src00), v_src40), + v_add(v_add(v_add(v_add(v_shl<2>(v_src21), v_shl<1>(v_src21)), v_shl<2>(v_add(v_src11, v_src31))), v_src01), v_src41)), + v_rshr_pack<20>(v_add(v_add(v_add(v_add(v_shl<2>(v_src22), v_shl<1>(v_src22)), v_shl<2>(v_add(v_src12, v_src32))), v_src02), v_src42), + v_add(v_add(v_add(v_add(v_shl<2>(v_src23), v_shl<1>(v_src23)), v_shl<2>(v_add(v_src13, v_src33))), v_src03), v_src43)))); } #endif for (; i < len; i++) @@ -1647,10 +1638,10 @@ template <> void vlineSmooth(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[1]; @@ -1705,17 +1696,17 @@ void vlineSmooth(const ufixedpoint16* const * src, const v_src12 = vx_load(srcj1 + 2*VECSZ); v_src13 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul); - v_res1 += v_dotprod(v_tmp1, v_mul); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_res2 += v_dotprod(v_tmp0, v_mul); - v_res3 += v_dotprod(v_tmp1, v_mul); + v_res2 = v_add(v_res2, v_dotprod(v_tmp0, v_mul)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_res4 += v_dotprod(v_tmp0, v_mul); - v_res5 += v_dotprod(v_tmp1, v_mul); + v_res4 = v_add(v_res4, v_dotprod(v_tmp0, v_mul)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_res6 += v_dotprod(v_tmp0, v_mul); - v_res7 += v_dotprod(v_tmp1, v_mul); + v_res6 = v_add(v_res6, v_dotprod(v_tmp0, v_mul)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp1, v_mul)); } if(j < n) { @@ -1727,26 +1718,26 @@ void vlineSmooth(const ufixedpoint16* const * src, const v_src02 = vx_load(srcj + 2*VECSZ); v_src03 = vx_load(srcj + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); } - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1780,11 +1771,11 @@ template <> void vlineSmoothONa_yzy_a(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int pre_shift = n / 2; - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[pre_shift] + m[n - 1]; @@ -1826,27 +1817,27 @@ void vlineSmoothONa_yzy_a(const ufixedpoint16* const * s v_src21 = vx_load(srcj1 + 2*VECSZ); v_src31 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src01, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul); - v_res1 += v_dotprod(v_tmp1, v_mul); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src10, v_128), v_add_wrap(v_src11, v_128), v_tmp2, v_tmp3); - v_res2 += v_dotprod(v_tmp2, v_mul); - v_res3 += v_dotprod(v_tmp3, v_mul); + v_res2 = v_add(v_res2, v_dotprod(v_tmp2, v_mul)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp3, v_mul)); v_zip(v_add_wrap(v_src20, v_128), v_add_wrap(v_src21, v_128), v_tmp4, v_tmp5); - v_res4 += v_dotprod(v_tmp4, v_mul); - v_res5 += v_dotprod(v_tmp5, v_mul); + v_res4 = v_add(v_res4, v_dotprod(v_tmp4, v_mul)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp5, v_mul)); v_zip(v_add_wrap(v_src30, v_128), v_add_wrap(v_src31, v_128), v_tmp6, v_tmp7); - v_res6 += v_dotprod(v_tmp6, v_mul); - v_res7 += v_dotprod(v_tmp7, v_mul); + v_res6 = v_add(v_res6, v_dotprod(v_tmp6, v_mul)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp7, v_mul)); } - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1868,9 +1859,9 @@ template <> void vlineSmoothONa_yzy_a(const ufixedpoint32* const * src, const ufixedpoint32* m, int n, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int pre_shift = n / 2; - const int VECSZ = v_uint32::nlanes; + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src10, v_src01, v_src11; @@ -1895,15 +1886,15 @@ void vlineSmoothONa_yzy_a(const ufixedpoint32* const * v_src01 = vx_load(srcj1); v_mul_expand(v_src00, v_mul, v_tmp0, v_tmp1); v_mul_expand(v_src01, v_mul, v_tmp2, v_tmp3); - v_res0 += v_tmp0 + v_tmp2; - v_res1 += v_tmp1 + v_tmp3; + v_res0 = v_add(v_res0, v_add(v_tmp0, v_tmp2)); + v_res1 = v_add(v_res1, v_add(v_tmp1, v_tmp3)); v_src10 = vx_load(srcj0 + VECSZ); v_src11 = vx_load(srcj1 + VECSZ); v_mul_expand(v_src10, v_mul, v_tmp4, v_tmp5); v_mul_expand(v_src11, v_mul, v_tmp6, v_tmp7); - v_res2 += v_tmp4 + v_tmp6; - v_res3 += v_tmp5 + v_tmp7; + v_res2 = v_add(v_res2, v_add(v_tmp4, v_tmp6)); + v_res3 = v_add(v_res3, v_add(v_tmp5, v_tmp7)); } v_store(dst + i, v_pack(v_rshr_pack<32>(v_res0, v_res1), diff --git a/modules/imgproc/src/spatialgradient.cpp b/modules/imgproc/src/spatialgradient.cpp index 1aed1fa031..f422609c40 100644 --- a/modules/imgproc/src/spatialgradient.cpp +++ b/modules/imgproc/src/spatialgradient.cpp @@ -57,6 +57,25 @@ namespace cv * 0 0 0 * 1 2 1 */ +#if (CV_SIMD || CV_SIMD_SCALABLE) +template +static inline void spatialGradientKernel_vec( T& vx, T& vy, + const T& v00, const T& v01, const T& v02, + const T& v10, const T& v12, + const T& v20, const T& v21, const T& v22 ) +{ + // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) + // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) + T tmp_add = v_sub(v22, v00), + tmp_sub = v_sub(v02, v20), + tmp_x = v_sub(v12, v10), + tmp_y = v_sub(v21, v01); + + vx = v_add(v_add(v_add(tmp_add, tmp_sub), tmp_x), tmp_x); + vy = v_add(v_add(v_sub(tmp_add, tmp_sub), tmp_y), tmp_y); +} +#endif + template static inline void spatialGradientKernel( T& vx, T& vy, const T& v00, const T& v01, const T& v02, @@ -65,7 +84,6 @@ static inline void spatialGradientKernel( T& vx, T& vy, { // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) - T tmp_add = v22 - v00, tmp_sub = v02 - v20, tmp_x = v12 - v10, @@ -125,7 +143,7 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, int i_start = 0; int j_start = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) // Characters in variable names have the following meanings: // u: unsigned char // s: signed int @@ -148,7 +166,7 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, short *n_dy = dy.ptr(i+1); // Process rest of columns 16-column chunks at a time - for ( j = 1; j < W - v_uint8::nlanes; j += v_uint8::nlanes) + for ( j = 1; j < W - VTraits::vlanes(); j += VTraits::vlanes()) { // Load top row for 3x3 Sobel filter v_uint8 v_um = vx_load(&p_src[j-1]); @@ -195,22 +213,22 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, // dx & dy for rows 1, 2, 3 v_int16 v_sdx1, v_sdy1; - spatialGradientKernel( v_sdx1, v_sdy1, + spatialGradientKernel_vec( v_sdx1, v_sdy1, v_s1m1, v_s1n1, v_s1p1, v_s2m1, v_s2p1, v_s3m1, v_s3n1, v_s3p1 ); v_int16 v_sdx2, v_sdy2; - spatialGradientKernel( v_sdx2, v_sdy2, + spatialGradientKernel_vec( v_sdx2, v_sdy2, v_s1m2, v_s1n2, v_s1p2, v_s2m2, v_s2p2, v_s3m2, v_s3n2, v_s3p2 ); // Store v_store(&c_dx[j], v_sdx1); - v_store(&c_dx[j+v_int16::nlanes], v_sdx2); + v_store(&c_dx[j+VTraits::vlanes()], v_sdx2); v_store(&c_dy[j], v_sdy1); - v_store(&c_dy[j+v_int16::nlanes], v_sdy2); + v_store(&c_dy[j+VTraits::vlanes()], v_sdy2); // Load fourth row for 3x3 Sobel filter v_um = vx_load(&m_src[j-1]); @@ -227,21 +245,21 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, v_int16 v_s4p2 = v_reinterpret_as_s16(v_up2); // dx & dy for rows 2, 3, 4 - spatialGradientKernel( v_sdx1, v_sdy1, + spatialGradientKernel_vec( v_sdx1, v_sdy1, v_s2m1, v_s2n1, v_s2p1, v_s3m1, v_s3p1, v_s4m1, v_s4n1, v_s4p1 ); - spatialGradientKernel( v_sdx2, v_sdy2, + spatialGradientKernel_vec( v_sdx2, v_sdy2, v_s2m2, v_s2n2, v_s2p2, v_s3m2, v_s3p2, v_s4m2, v_s4n2, v_s4p2 ); // Store v_store(&n_dx[j], v_sdx1); - v_store(&n_dx[j+v_int16::nlanes], v_sdx2); + v_store(&n_dx[j+VTraits::vlanes()], v_sdx2); v_store(&n_dy[j], v_sdy1); - v_store(&n_dy[j+v_int16::nlanes], v_sdy2); + v_store(&n_dy[j+VTraits::vlanes()], v_sdy2); } } i_start = i; From d4cb564ce25eb7c251e98178f39af0110f4d128b Mon Sep 17 00:00:00 2001 From: Alexander Lyulkov Date: Fri, 15 Sep 2023 14:22:21 +0700 Subject: [PATCH 10/34] Fixed removePhaseSwitches in tf_graph_simplifier --- modules/dnn/src/tensorflow/tf_graph_simplifier.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 2e8cde2799..5531b28111 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -1120,15 +1120,16 @@ void removePhaseSwitches(tensorflow::GraphDef& net) inpName = inpName.substr(1 + (int)inpName.find('^'), inpName.rfind(':')); nodesMapIt = nodesMap.find(inpName); CV_Assert(nodesMapIt != nodesMap.end()); - int inpNodeId = nodesMapIt->second; + + CV_CheckGT(numConsumers[inpNodeId], 0, + "Input node of the current node should have at least one output node"); if (numConsumers[inpNodeId] == 1) { mergeOpSubgraphNodes.push(inpNodeId); nodesToRemove.push_back(inpNodeId); } - else if (numConsumers[inpNodeId] > 0) - numConsumers[inpNodeId] -= 1; + numConsumers[inpNodeId] -= 1; } } std::sort(nodesToRemove.begin(), nodesToRemove.end()); From 638c575c73d8380fbc784b7cea804d86410d3677 Mon Sep 17 00:00:00 2001 From: Yuriy Chernyshov Date: Fri, 15 Sep 2023 12:37:43 +0300 Subject: [PATCH 11/34] More fixes for iterators-are-pointers case --- modules/calib3d/src/checkchessboard.cpp | 9 +++------ .../src/aruco/apriltag/apriltag_quad_thresh.cpp | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/calib3d/src/checkchessboard.cpp b/modules/calib3d/src/checkchessboard.cpp index 97876d218c..350614e78f 100644 --- a/modules/calib3d/src/checkchessboard.cpp +++ b/modules/calib3d/src/checkchessboard.cpp @@ -55,15 +55,12 @@ static void icvGetQuadrangleHypotheses(const std::vector >::const_iterator iter_t; - iter_t i; - for (i = contours.begin(); i != contours.end(); ++i) + for (size_t i = 0; i < contours.size(); ++i) { - const iter_t::difference_type idx = i - contours.begin(); - if (hierarchy.at(idx)[3] != -1) + if (hierarchy.at(i)[3] != -1) continue; // skip holes - const std::vector< cv::Point > & c = *i; + const std::vector< cv::Point > & c = contours[i]; cv::RotatedRect box = cv::minAreaRect(c); float box_size = MAX(box.size.width, box.size.height); diff --git a/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp b/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp index 38d1b2ffb8..96b7fe517e 100644 --- a/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp +++ b/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp @@ -349,7 +349,7 @@ int quad_segment_maxima(const DetectorParameters &td, int sz, struct line_fit_pt } y[iy] = acc; } - copy(y.begin(), y.end(), errs.begin()); + std::copy(y.begin(), y.end(), errs.begin()); } std::vector maxima(sz); From 687fc11626901cff09d2b3b5f331fd59190ad4c7 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Fri, 15 Sep 2023 13:25:49 +0200 Subject: [PATCH 12/34] Merge pull request #24274 from vrabaud:webp_1.3.2 Merge pull request #24274 from vrabaud:webp_1.3.2 Bump libwebp to 1.3.2 #24274 This is version [c1ffd9a](https://chromium.googlesource.com/webm/libwebp/+/c1ffd9ac7593894c40a1de99d03f0b7af8af2577) It is 1.3.2 with a few patches that were made right after to help compilation. No need for patches on the OpenCV side! ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch --- 3rdparty/libwebp/CMakeLists.txt | 10 +- .../patches/20190910-msa-asm-patch.diff | 22 - 3rdparty/libwebp/sharpyuv/sharpyuv.c | 565 ++++++++++++++ 3rdparty/libwebp/sharpyuv/sharpyuv.h | 174 +++++ 3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c | 14 + 3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h | 22 + 3rdparty/libwebp/sharpyuv/sharpyuv_csp.c | 110 +++ 3rdparty/libwebp/sharpyuv/sharpyuv_csp.h | 60 ++ 3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c | 104 +++ 3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h | 28 + 3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c | 419 +++++++++++ 3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h | 38 + 3rdparty/libwebp/sharpyuv/sharpyuv_neon.c | 181 +++++ 3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c | 201 +++++ 3rdparty/libwebp/src/dec/alpha_dec.c | 37 +- 3rdparty/libwebp/src/dec/buffer_dec.c | 12 +- 3rdparty/libwebp/src/dec/frame_dec.c | 2 +- 3rdparty/libwebp/src/dec/io_dec.c | 98 ++- 3rdparty/libwebp/src/dec/tree_dec.c | 3 +- 3rdparty/libwebp/src/dec/vp8_dec.c | 6 +- 3rdparty/libwebp/src/dec/vp8i_dec.h | 4 +- 3rdparty/libwebp/src/dec/vp8l_dec.c | 244 +++--- 3rdparty/libwebp/src/dec/vp8li_dec.h | 15 +- 3rdparty/libwebp/src/dec/webp_dec.c | 50 +- 3rdparty/libwebp/src/dec/webpi_dec.h | 4 + 3rdparty/libwebp/src/demux/anim_decode.c | 52 +- 3rdparty/libwebp/src/demux/demux.c | 23 +- 3rdparty/libwebp/src/dsp/alpha_processing.c | 66 +- .../libwebp/src/dsp/alpha_processing_neon.c | 27 +- .../libwebp/src/dsp/alpha_processing_sse2.c | 73 +- .../libwebp/src/dsp/alpha_processing_sse41.c | 8 +- 3rdparty/libwebp/src/dsp/cost.c | 5 +- 3rdparty/libwebp/src/dsp/cost_neon.c | 4 +- 3rdparty/libwebp/src/dsp/cpu.c | 21 +- 3rdparty/libwebp/src/dsp/cpu.h | 266 +++++++ 3rdparty/libwebp/src/dsp/dec.c | 7 +- 3rdparty/libwebp/src/dsp/dec_neon.c | 4 +- 3rdparty/libwebp/src/dsp/dec_sse2.c | 93 +-- 3rdparty/libwebp/src/dsp/dec_sse41.c | 2 +- 3rdparty/libwebp/src/dsp/dsp.h | 262 +------ 3rdparty/libwebp/src/dsp/enc.c | 7 +- 3rdparty/libwebp/src/dsp/enc_neon.c | 13 +- 3rdparty/libwebp/src/dsp/enc_sse2.c | 293 ++++++-- 3rdparty/libwebp/src/dsp/filters.c | 14 +- 3rdparty/libwebp/src/dsp/filters_sse2.c | 5 + 3rdparty/libwebp/src/dsp/lossless.c | 81 +- 3rdparty/libwebp/src/dsp/lossless.h | 53 +- 3rdparty/libwebp/src/dsp/lossless_common.h | 8 +- 3rdparty/libwebp/src/dsp/lossless_enc.c | 73 +- .../libwebp/src/dsp/lossless_enc_mips32.c | 22 +- 3rdparty/libwebp/src/dsp/lossless_enc_neon.c | 2 +- 3rdparty/libwebp/src/dsp/lossless_enc_sse2.c | 117 ++- 3rdparty/libwebp/src/dsp/lossless_enc_sse41.c | 173 +++-- .../libwebp/src/dsp/lossless_mips_dsp_r2.c | 37 +- 3rdparty/libwebp/src/dsp/lossless_neon.c | 22 +- 3rdparty/libwebp/src/dsp/lossless_sse2.c | 130 ++-- 3rdparty/libwebp/src/dsp/lossless_sse41.c | 133 ++++ 3rdparty/libwebp/src/dsp/msa_macro.h | 37 +- 3rdparty/libwebp/src/dsp/neon.h | 11 +- 3rdparty/libwebp/src/dsp/quant.h | 16 +- 3rdparty/libwebp/src/dsp/rescaler.c | 12 +- 3rdparty/libwebp/src/dsp/rescaler_sse2.c | 6 +- 3rdparty/libwebp/src/dsp/ssim.c | 3 +- 3rdparty/libwebp/src/dsp/upsampling.c | 11 +- 3rdparty/libwebp/src/dsp/upsampling_neon.c | 2 +- 3rdparty/libwebp/src/dsp/upsampling_sse2.c | 2 +- 3rdparty/libwebp/src/dsp/yuv.c | 85 +-- 3rdparty/libwebp/src/dsp/yuv.h | 2 +- 3rdparty/libwebp/src/dsp/yuv_neon.c | 108 --- 3rdparty/libwebp/src/dsp/yuv_sse2.c | 132 +--- 3rdparty/libwebp/src/dsp/yuv_sse41.c | 6 +- 3rdparty/libwebp/src/enc/alpha_enc.c | 32 +- 3rdparty/libwebp/src/enc/analysis_enc.c | 12 +- .../src/enc/backward_references_cost_enc.c | 77 +- .../libwebp/src/enc/backward_references_enc.c | 83 ++- .../libwebp/src/enc/backward_references_enc.h | 12 +- 3rdparty/libwebp/src/enc/frame_enc.c | 21 +- 3rdparty/libwebp/src/enc/histogram_enc.c | 252 +++---- 3rdparty/libwebp/src/enc/histogram_enc.h | 30 +- 3rdparty/libwebp/src/enc/picture_csp_enc.c | 502 ++----------- 3rdparty/libwebp/src/enc/picture_enc.c | 46 +- .../libwebp/src/enc/picture_rescale_enc.c | 119 ++- 3rdparty/libwebp/src/enc/picture_tools_enc.c | 45 +- 3rdparty/libwebp/src/enc/predictor_enc.c | 52 +- 3rdparty/libwebp/src/enc/quant_enc.c | 120 +-- 3rdparty/libwebp/src/enc/syntax_enc.c | 8 +- 3rdparty/libwebp/src/enc/vp8i_enc.h | 29 +- 3rdparty/libwebp/src/enc/vp8l_enc.c | 696 +++++++++--------- 3rdparty/libwebp/src/enc/vp8li_enc.h | 26 +- 3rdparty/libwebp/src/enc/webp_enc.c | 14 +- 3rdparty/libwebp/src/mux/anim_encode.c | 25 +- 3rdparty/libwebp/src/mux/muxedit.c | 6 +- 3rdparty/libwebp/src/mux/muxi.h | 4 +- 3rdparty/libwebp/src/mux/muxinternal.c | 9 +- 3rdparty/libwebp/src/mux/muxread.c | 11 +- .../libwebp/src/utils/bit_reader_inl_utils.h | 13 +- 3rdparty/libwebp/src/utils/bit_reader_utils.c | 3 +- 3rdparty/libwebp/src/utils/bit_reader_utils.h | 3 +- 3rdparty/libwebp/src/utils/bit_writer_utils.c | 4 +- .../libwebp/src/utils/color_cache_utils.c | 22 +- .../libwebp/src/utils/huffman_encode_utils.c | 5 +- .../libwebp/src/utils/huffman_encode_utils.h | 2 +- 3rdparty/libwebp/src/utils/huffman_utils.c | 99 ++- 3rdparty/libwebp/src/utils/huffman_utils.h | 27 +- 3rdparty/libwebp/src/utils/palette.c | 402 ++++++++++ 3rdparty/libwebp/src/utils/palette.h | 60 ++ .../src/utils/quant_levels_dec_utils.c | 2 +- 3rdparty/libwebp/src/utils/rescaler_utils.c | 114 +-- 3rdparty/libwebp/src/utils/rescaler_utils.h | 13 +- 3rdparty/libwebp/src/utils/utils.c | 77 +- 3rdparty/libwebp/src/utils/utils.h | 43 +- 3rdparty/libwebp/src/webp/decode.h | 44 +- 3rdparty/libwebp/src/webp/encode.h | 6 +- 3rdparty/libwebp/src/webp/format_constants.h | 2 +- 3rdparty/libwebp/src/webp/types.h | 6 +- 115 files changed, 5350 insertions(+), 2780 deletions(-) delete mode 100644 3rdparty/libwebp/patches/20190910-msa-asm-patch.diff create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv.h create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_csp.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_csp.h create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_neon.c create mode 100644 3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c create mode 100644 3rdparty/libwebp/src/dsp/cpu.h create mode 100644 3rdparty/libwebp/src/dsp/lossless_sse41.c create mode 100644 3rdparty/libwebp/src/utils/palette.c create mode 100644 3rdparty/libwebp/src/utils/palette.h diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 723575c8db..f3b6ebd0d6 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -9,8 +9,8 @@ if(ANDROID) ocv_include_directories(${CPUFEATURES_INCLUDE_DIRS}) endif() -file(GLOB lib_srcs src/dec/*.c src/demux/*.c src/dsp/*.c src/enc/*.c src/mux/*.c src/utils/*.c src/webp/*.c) -file(GLOB lib_hdrs src/dec/*.h src/demux/*.h src/dsp/*.h src/enc/*.h src/mux/*.h src/utils/*.h src/webp/*.h) +file(GLOB lib_srcs sharpyuv/*.c src/dec/*.c src/demux/*.c src/dsp/*.c src/enc/*.c src/mux/*.c src/utils/*.c src/webp/*.c) +file(GLOB lib_hdrs sharpyuv/*.h src/dec/*.h src/demux/*.h src/dsp/*.h src/enc/*.h src/mux/*.h src/utils/*.h src/webp/*.h) # FIXIT if(ANDROID AND ARMEABI_V7A AND NOT NEON) @@ -21,12 +21,6 @@ if(ANDROID AND ARMEABI_V7A AND NOT NEON) endforeach() endif() -# FIX for quant.h - requires C99 for() loops -ocv_check_flag_support(C "-std=c99" _varname "${CMAKE_C_FLAGS}") -if(${_varname}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") -endif() - # ---------------------------------------------------------------------------------- # Define the library target: diff --git a/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff b/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff deleted file mode 100644 index 1be2135203..0000000000 --- a/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/3rdparty/libwebp/src/dsp/msa_macro.h b/3rdparty/libwebp/src/dsp/msa_macro.h -index de026a1d9e..a16c0bb300 100644 ---- a/3rdparty/libwebp/src/dsp/msa_macro.h -+++ b/3rdparty/libwebp/src/dsp/msa_macro.h -@@ -73,7 +73,7 @@ - static inline TYPE FUNC_NAME(const void* const psrc) { \ - const uint8_t* const psrc_m = (const uint8_t*)psrc; \ - TYPE val_m; \ -- asm volatile ( \ -+ __asm__ volatile ( \ - "" #INSTR " %[val_m], %[psrc_m] \n\t" \ - : [val_m] "=r" (val_m) \ - : [psrc_m] "m" (*psrc_m)); \ -@@ -86,7 +86,7 @@ - static inline void FUNC_NAME(TYPE val, void* const pdst) { \ - uint8_t* const pdst_m = (uint8_t*)pdst; \ - TYPE val_m = val; \ -- asm volatile ( \ -+ __asm__ volatile ( \ - " " #INSTR " %[val_m], %[pdst_m] \n\t" \ - : [pdst_m] "=m" (*pdst_m) \ - : [val_m] "r" (val_m)); \ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv.c b/3rdparty/libwebp/sharpyuv/sharpyuv.c new file mode 100644 index 0000000000..b94885a6c3 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv.c @@ -0,0 +1,565 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Sharp RGB to YUV conversion. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv.h" + +#include +#include +#include +#include +#include + +#include "src/webp/types.h" +#include "sharpyuv/sharpyuv_cpu.h" +#include "sharpyuv/sharpyuv_dsp.h" +#include "sharpyuv/sharpyuv_gamma.h" + +//------------------------------------------------------------------------------ + +int SharpYuvGetVersion(void) { + return SHARPYUV_VERSION; +} + +//------------------------------------------------------------------------------ +// Sharp RGB->YUV conversion + +static const int kNumIterations = 4; + +#define YUV_FIX 16 // fixed-point precision for RGB->YUV +static const int kYuvHalf = 1 << (YUV_FIX - 1); + +// Max bit depth so that intermediate calculations fit in 16 bits. +static const int kMaxBitDepth = 14; + +// Returns the precision shift to use based on the input rgb_bit_depth. +static int GetPrecisionShift(int rgb_bit_depth) { + // Try to add 2 bits of precision if it fits in kMaxBitDepth. Otherwise remove + // bits if needed. + return ((rgb_bit_depth + 2) <= kMaxBitDepth) ? 2 + : (kMaxBitDepth - rgb_bit_depth); +} + +typedef int16_t fixed_t; // signed type with extra precision for UV +typedef uint16_t fixed_y_t; // unsigned type with extra precision for W + +//------------------------------------------------------------------------------ + +static uint8_t clip_8b(fixed_t v) { + return (!(v & ~0xff)) ? (uint8_t)v : (v < 0) ? 0u : 255u; +} + +static uint16_t clip(fixed_t v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static fixed_y_t clip_bit_depth(int y, int bit_depth) { + const int max = (1 << bit_depth) - 1; + return (!(y & ~max)) ? (fixed_y_t)y : (y < 0) ? 0 : max; +} + +//------------------------------------------------------------------------------ + +static int RGBToGray(int64_t r, int64_t g, int64_t b) { + const int64_t luma = 13933 * r + 46871 * g + 4732 * b + kYuvHalf; + return (int)(luma >> YUV_FIX); +} + +static uint32_t ScaleDown(uint16_t a, uint16_t b, uint16_t c, uint16_t d, + int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + const uint32_t A = SharpYuvGammaToLinear(a, bit_depth, transfer_type); + const uint32_t B = SharpYuvGammaToLinear(b, bit_depth, transfer_type); + const uint32_t C = SharpYuvGammaToLinear(c, bit_depth, transfer_type); + const uint32_t D = SharpYuvGammaToLinear(d, bit_depth, transfer_type); + return SharpYuvLinearToGamma((A + B + C + D + 2) >> 2, bit_depth, + transfer_type); +} + +static WEBP_INLINE void UpdateW(const fixed_y_t* src, fixed_y_t* dst, int w, + int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + int i; + for (i = 0; i < w; ++i) { + const uint32_t R = + SharpYuvGammaToLinear(src[0 * w + i], bit_depth, transfer_type); + const uint32_t G = + SharpYuvGammaToLinear(src[1 * w + i], bit_depth, transfer_type); + const uint32_t B = + SharpYuvGammaToLinear(src[2 * w + i], bit_depth, transfer_type); + const uint32_t Y = RGBToGray(R, G, B); + dst[i] = (fixed_y_t)SharpYuvLinearToGamma(Y, bit_depth, transfer_type); + } +} + +static void UpdateChroma(const fixed_y_t* src1, const fixed_y_t* src2, + fixed_t* dst, int uv_w, int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + int i; + for (i = 0; i < uv_w; ++i) { + const int r = + ScaleDown(src1[0 * uv_w + 0], src1[0 * uv_w + 1], src2[0 * uv_w + 0], + src2[0 * uv_w + 1], rgb_bit_depth, transfer_type); + const int g = + ScaleDown(src1[2 * uv_w + 0], src1[2 * uv_w + 1], src2[2 * uv_w + 0], + src2[2 * uv_w + 1], rgb_bit_depth, transfer_type); + const int b = + ScaleDown(src1[4 * uv_w + 0], src1[4 * uv_w + 1], src2[4 * uv_w + 0], + src2[4 * uv_w + 1], rgb_bit_depth, transfer_type); + const int W = RGBToGray(r, g, b); + dst[0 * uv_w] = (fixed_t)(r - W); + dst[1 * uv_w] = (fixed_t)(g - W); + dst[2 * uv_w] = (fixed_t)(b - W); + dst += 1; + src1 += 2; + src2 += 2; + } +} + +static void StoreGray(const fixed_y_t* rgb, fixed_y_t* y, int w) { + int i; + assert(w > 0); + for (i = 0; i < w; ++i) { + y[i] = RGBToGray(rgb[0 * w + i], rgb[1 * w + i], rgb[2 * w + i]); + } +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE fixed_y_t Filter2(int A, int B, int W0, int bit_depth) { + const int v0 = (A * 3 + B + 2) >> 2; + return clip_bit_depth(v0 + W0, bit_depth); +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE int Shift(int v, int shift) { + return (shift >= 0) ? (v << shift) : (v >> -shift); +} + +static void ImportOneRow(const uint8_t* const r_ptr, + const uint8_t* const g_ptr, + const uint8_t* const b_ptr, + int rgb_step, + int rgb_bit_depth, + int pic_width, + fixed_y_t* const dst) { + // Convert the rgb_step from a number of bytes to a number of uint8_t or + // uint16_t values depending the bit depth. + const int step = (rgb_bit_depth > 8) ? rgb_step / 2 : rgb_step; + int i; + const int w = (pic_width + 1) & ~1; + for (i = 0; i < pic_width; ++i) { + const int off = i * step; + const int shift = GetPrecisionShift(rgb_bit_depth); + if (rgb_bit_depth == 8) { + dst[i + 0 * w] = Shift(r_ptr[off], shift); + dst[i + 1 * w] = Shift(g_ptr[off], shift); + dst[i + 2 * w] = Shift(b_ptr[off], shift); + } else { + dst[i + 0 * w] = Shift(((uint16_t*)r_ptr)[off], shift); + dst[i + 1 * w] = Shift(((uint16_t*)g_ptr)[off], shift); + dst[i + 2 * w] = Shift(((uint16_t*)b_ptr)[off], shift); + } + } + if (pic_width & 1) { // replicate rightmost pixel + dst[pic_width + 0 * w] = dst[pic_width + 0 * w - 1]; + dst[pic_width + 1 * w] = dst[pic_width + 1 * w - 1]; + dst[pic_width + 2 * w] = dst[pic_width + 2 * w - 1]; + } +} + +static void InterpolateTwoRows(const fixed_y_t* const best_y, + const fixed_t* prev_uv, + const fixed_t* cur_uv, + const fixed_t* next_uv, + int w, + fixed_y_t* out1, + fixed_y_t* out2, + int rgb_bit_depth) { + const int uv_w = w >> 1; + const int len = (w - 1) >> 1; // length to filter + int k = 3; + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + while (k-- > 0) { // process each R/G/B segments in turn + // special boundary case for i==0 + out1[0] = Filter2(cur_uv[0], prev_uv[0], best_y[0], bit_depth); + out2[0] = Filter2(cur_uv[0], next_uv[0], best_y[w], bit_depth); + + SharpYuvFilterRow(cur_uv, prev_uv, len, best_y + 0 + 1, out1 + 1, + bit_depth); + SharpYuvFilterRow(cur_uv, next_uv, len, best_y + w + 1, out2 + 1, + bit_depth); + + // special boundary case for i == w - 1 when w is even + if (!(w & 1)) { + out1[w - 1] = Filter2(cur_uv[uv_w - 1], prev_uv[uv_w - 1], + best_y[w - 1 + 0], bit_depth); + out2[w - 1] = Filter2(cur_uv[uv_w - 1], next_uv[uv_w - 1], + best_y[w - 1 + w], bit_depth); + } + out1 += w; + out2 += w; + prev_uv += uv_w; + cur_uv += uv_w; + next_uv += uv_w; + } +} + +static WEBP_INLINE int RGBToYUVComponent(int r, int g, int b, + const int coeffs[4], int sfix) { + const int srounder = 1 << (YUV_FIX + sfix - 1); + const int luma = coeffs[0] * r + coeffs[1] * g + coeffs[2] * b + + coeffs[3] + srounder; + return (luma >> (YUV_FIX + sfix)); +} + +static int ConvertWRGBToYUV(const fixed_y_t* best_y, const fixed_t* best_uv, + uint8_t* y_ptr, int y_stride, uint8_t* u_ptr, + int u_stride, uint8_t* v_ptr, int v_stride, + int rgb_bit_depth, + int yuv_bit_depth, int width, int height, + const SharpYuvConversionMatrix* yuv_matrix) { + int i, j; + const fixed_t* const best_uv_base = best_uv; + const int w = (width + 1) & ~1; + const int h = (height + 1) & ~1; + const int uv_w = w >> 1; + const int uv_h = h >> 1; + const int sfix = GetPrecisionShift(rgb_bit_depth); + const int yuv_max = (1 << yuv_bit_depth) - 1; + + for (best_uv = best_uv_base, j = 0; j < height; ++j) { + for (i = 0; i < width; ++i) { + const int off = (i >> 1); + const int W = best_y[i]; + const int r = best_uv[off + 0 * uv_w] + W; + const int g = best_uv[off + 1 * uv_w] + W; + const int b = best_uv[off + 2 * uv_w] + W; + const int y = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_y, sfix); + if (yuv_bit_depth <= 8) { + y_ptr[i] = clip_8b(y); + } else { + ((uint16_t*)y_ptr)[i] = clip(y, yuv_max); + } + } + best_y += w; + best_uv += (j & 1) * 3 * uv_w; + y_ptr += y_stride; + } + for (best_uv = best_uv_base, j = 0; j < uv_h; ++j) { + for (i = 0; i < uv_w; ++i) { + const int off = i; + // Note r, g and b values here are off by W, but a constant offset on all + // 3 components doesn't change the value of u and v with a YCbCr matrix. + const int r = best_uv[off + 0 * uv_w]; + const int g = best_uv[off + 1 * uv_w]; + const int b = best_uv[off + 2 * uv_w]; + const int u = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_u, sfix); + const int v = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_v, sfix); + if (yuv_bit_depth <= 8) { + u_ptr[i] = clip_8b(u); + v_ptr[i] = clip_8b(v); + } else { + ((uint16_t*)u_ptr)[i] = clip(u, yuv_max); + ((uint16_t*)v_ptr)[i] = clip(v, yuv_max); + } + } + best_uv += 3 * uv_w; + u_ptr += u_stride; + v_ptr += v_stride; + } + return 1; +} + +//------------------------------------------------------------------------------ +// Main function + +static void* SafeMalloc(uint64_t nmemb, size_t size) { + const uint64_t total_size = nmemb * (uint64_t)size; + if (total_size != (size_t)total_size) return NULL; + return malloc((size_t)total_size); +} + +#define SAFE_ALLOC(W, H, T) ((T*)SafeMalloc((W) * (H), sizeof(T))) + +static int DoSharpArgbToYuv(const uint8_t* r_ptr, const uint8_t* g_ptr, + const uint8_t* b_ptr, int rgb_step, int rgb_stride, + int rgb_bit_depth, uint8_t* y_ptr, int y_stride, + uint8_t* u_ptr, int u_stride, uint8_t* v_ptr, + int v_stride, int yuv_bit_depth, int width, + int height, + const SharpYuvConversionMatrix* yuv_matrix, + SharpYuvTransferFunctionType transfer_type) { + // we expand the right/bottom border if needed + const int w = (width + 1) & ~1; + const int h = (height + 1) & ~1; + const int uv_w = w >> 1; + const int uv_h = h >> 1; + uint64_t prev_diff_y_sum = ~0; + int j, iter; + + // TODO(skal): allocate one big memory chunk. But for now, it's easier + // for valgrind debugging to have several chunks. + fixed_y_t* const tmp_buffer = SAFE_ALLOC(w * 3, 2, fixed_y_t); // scratch + fixed_y_t* const best_y_base = SAFE_ALLOC(w, h, fixed_y_t); + fixed_y_t* const target_y_base = SAFE_ALLOC(w, h, fixed_y_t); + fixed_y_t* const best_rgb_y = SAFE_ALLOC(w, 2, fixed_y_t); + fixed_t* const best_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); + fixed_t* const target_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); + fixed_t* const best_rgb_uv = SAFE_ALLOC(uv_w * 3, 1, fixed_t); + fixed_y_t* best_y = best_y_base; + fixed_y_t* target_y = target_y_base; + fixed_t* best_uv = best_uv_base; + fixed_t* target_uv = target_uv_base; + const uint64_t diff_y_threshold = (uint64_t)(3.0 * w * h); + int ok; + assert(w > 0); + assert(h > 0); + + if (best_y_base == NULL || best_uv_base == NULL || + target_y_base == NULL || target_uv_base == NULL || + best_rgb_y == NULL || best_rgb_uv == NULL || + tmp_buffer == NULL) { + ok = 0; + goto End; + } + + // Import RGB samples to W/RGB representation. + for (j = 0; j < height; j += 2) { + const int is_last_row = (j == height - 1); + fixed_y_t* const src1 = tmp_buffer + 0 * w; + fixed_y_t* const src2 = tmp_buffer + 3 * w; + + // prepare two rows of input + ImportOneRow(r_ptr, g_ptr, b_ptr, rgb_step, rgb_bit_depth, width, + src1); + if (!is_last_row) { + ImportOneRow(r_ptr + rgb_stride, g_ptr + rgb_stride, b_ptr + rgb_stride, + rgb_step, rgb_bit_depth, width, src2); + } else { + memcpy(src2, src1, 3 * w * sizeof(*src2)); + } + StoreGray(src1, best_y + 0, w); + StoreGray(src2, best_y + w, w); + + UpdateW(src1, target_y, w, rgb_bit_depth, transfer_type); + UpdateW(src2, target_y + w, w, rgb_bit_depth, transfer_type); + UpdateChroma(src1, src2, target_uv, uv_w, rgb_bit_depth, transfer_type); + memcpy(best_uv, target_uv, 3 * uv_w * sizeof(*best_uv)); + best_y += 2 * w; + best_uv += 3 * uv_w; + target_y += 2 * w; + target_uv += 3 * uv_w; + r_ptr += 2 * rgb_stride; + g_ptr += 2 * rgb_stride; + b_ptr += 2 * rgb_stride; + } + + // Iterate and resolve clipping conflicts. + for (iter = 0; iter < kNumIterations; ++iter) { + const fixed_t* cur_uv = best_uv_base; + const fixed_t* prev_uv = best_uv_base; + uint64_t diff_y_sum = 0; + + best_y = best_y_base; + best_uv = best_uv_base; + target_y = target_y_base; + target_uv = target_uv_base; + for (j = 0; j < h; j += 2) { + fixed_y_t* const src1 = tmp_buffer + 0 * w; + fixed_y_t* const src2 = tmp_buffer + 3 * w; + { + const fixed_t* const next_uv = cur_uv + ((j < h - 2) ? 3 * uv_w : 0); + InterpolateTwoRows(best_y, prev_uv, cur_uv, next_uv, w, + src1, src2, rgb_bit_depth); + prev_uv = cur_uv; + cur_uv = next_uv; + } + + UpdateW(src1, best_rgb_y + 0 * w, w, rgb_bit_depth, transfer_type); + UpdateW(src2, best_rgb_y + 1 * w, w, rgb_bit_depth, transfer_type); + UpdateChroma(src1, src2, best_rgb_uv, uv_w, rgb_bit_depth, transfer_type); + + // update two rows of Y and one row of RGB + diff_y_sum += + SharpYuvUpdateY(target_y, best_rgb_y, best_y, 2 * w, + rgb_bit_depth + GetPrecisionShift(rgb_bit_depth)); + SharpYuvUpdateRGB(target_uv, best_rgb_uv, best_uv, 3 * uv_w); + + best_y += 2 * w; + best_uv += 3 * uv_w; + target_y += 2 * w; + target_uv += 3 * uv_w; + } + // test exit condition + if (iter > 0) { + if (diff_y_sum < diff_y_threshold) break; + if (diff_y_sum > prev_diff_y_sum) break; + } + prev_diff_y_sum = diff_y_sum; + } + + // final reconstruction + ok = ConvertWRGBToYUV(best_y_base, best_uv_base, y_ptr, y_stride, u_ptr, + u_stride, v_ptr, v_stride, rgb_bit_depth, yuv_bit_depth, + width, height, yuv_matrix); + + End: + free(best_y_base); + free(best_uv_base); + free(target_y_base); + free(target_uv_base); + free(best_rgb_y); + free(best_rgb_uv); + free(tmp_buffer); + return ok; +} +#undef SAFE_ALLOC + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include // NOLINT + +#define LOCK_ACCESS \ + static pthread_mutex_t sharpyuv_lock = PTHREAD_MUTEX_INITIALIZER; \ + if (pthread_mutex_lock(&sharpyuv_lock)) return +#define UNLOCK_ACCESS_AND_RETURN \ + do { \ + (void)pthread_mutex_unlock(&sharpyuv_lock); \ + return; \ + } while (0) +#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) +#define LOCK_ACCESS do {} while (0) +#define UNLOCK_ACCESS_AND_RETURN return +#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) + +// Hidden exported init function. +// By default SharpYuvConvert calls it with SharpYuvGetCPUInfo. If needed, +// users can declare it as extern and call it with an alternate VP8CPUInfo +// function. +extern VP8CPUInfo SharpYuvGetCPUInfo; +SHARPYUV_EXTERN void SharpYuvInit(VP8CPUInfo cpu_info_func); +void SharpYuvInit(VP8CPUInfo cpu_info_func) { + static volatile VP8CPUInfo sharpyuv_last_cpuinfo_used = + (VP8CPUInfo)&sharpyuv_last_cpuinfo_used; + LOCK_ACCESS; + // Only update SharpYuvGetCPUInfo when called from external code to avoid a + // race on reading the value in SharpYuvConvert(). + if (cpu_info_func != (VP8CPUInfo)&SharpYuvGetCPUInfo) { + SharpYuvGetCPUInfo = cpu_info_func; + } + if (sharpyuv_last_cpuinfo_used == SharpYuvGetCPUInfo) { + UNLOCK_ACCESS_AND_RETURN; + } + + SharpYuvInitDsp(); + SharpYuvInitGammaTables(); + + sharpyuv_last_cpuinfo_used = SharpYuvGetCPUInfo; + UNLOCK_ACCESS_AND_RETURN; +} + +int SharpYuvConvert(const void* r_ptr, const void* g_ptr, const void* b_ptr, + int rgb_step, int rgb_stride, int rgb_bit_depth, + void* y_ptr, int y_stride, void* u_ptr, int u_stride, + void* v_ptr, int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvConversionMatrix* yuv_matrix) { + SharpYuvOptions options; + options.yuv_matrix = yuv_matrix; + options.transfer_type = kSharpYuvTransferFunctionSrgb; + return SharpYuvConvertWithOptions( + r_ptr, g_ptr, b_ptr, rgb_step, rgb_stride, rgb_bit_depth, y_ptr, y_stride, + u_ptr, u_stride, v_ptr, v_stride, yuv_bit_depth, width, height, &options); +} + +int SharpYuvOptionsInitInternal(const SharpYuvConversionMatrix* yuv_matrix, + SharpYuvOptions* options, int version) { + const int major = (version >> 24); + const int minor = (version >> 16) & 0xff; + if (options == NULL || yuv_matrix == NULL || + (major == SHARPYUV_VERSION_MAJOR && major == 0 && + minor != SHARPYUV_VERSION_MINOR) || + (major != SHARPYUV_VERSION_MAJOR)) { + return 0; + } + options->yuv_matrix = yuv_matrix; + options->transfer_type = kSharpYuvTransferFunctionSrgb; + return 1; +} + +int SharpYuvConvertWithOptions(const void* r_ptr, const void* g_ptr, + const void* b_ptr, int rgb_step, int rgb_stride, + int rgb_bit_depth, void* y_ptr, int y_stride, + void* u_ptr, int u_stride, void* v_ptr, + int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvOptions* options) { + const SharpYuvConversionMatrix* yuv_matrix = options->yuv_matrix; + SharpYuvTransferFunctionType transfer_type = options->transfer_type; + SharpYuvConversionMatrix scaled_matrix; + const int rgb_max = (1 << rgb_bit_depth) - 1; + const int rgb_round = 1 << (rgb_bit_depth - 1); + const int yuv_max = (1 << yuv_bit_depth) - 1; + const int sfix = GetPrecisionShift(rgb_bit_depth); + + if (width < 1 || height < 1 || width == INT_MAX || height == INT_MAX || + r_ptr == NULL || g_ptr == NULL || b_ptr == NULL || y_ptr == NULL || + u_ptr == NULL || v_ptr == NULL) { + return 0; + } + if (rgb_bit_depth != 8 && rgb_bit_depth != 10 && rgb_bit_depth != 12 && + rgb_bit_depth != 16) { + return 0; + } + if (yuv_bit_depth != 8 && yuv_bit_depth != 10 && yuv_bit_depth != 12) { + return 0; + } + if (rgb_bit_depth > 8 && (rgb_step % 2 != 0 || rgb_stride %2 != 0)) { + // Step/stride should be even for uint16_t buffers. + return 0; + } + if (yuv_bit_depth > 8 && + (y_stride % 2 != 0 || u_stride % 2 != 0 || v_stride % 2 != 0)) { + // Stride should be even for uint16_t buffers. + return 0; + } + // The address of the function pointer is used to avoid a read race. + SharpYuvInit((VP8CPUInfo)&SharpYuvGetCPUInfo); + + // Add scaling factor to go from rgb_bit_depth to yuv_bit_depth, to the + // rgb->yuv conversion matrix. + if (rgb_bit_depth == yuv_bit_depth) { + memcpy(&scaled_matrix, yuv_matrix, sizeof(scaled_matrix)); + } else { + int i; + for (i = 0; i < 3; ++i) { + scaled_matrix.rgb_to_y[i] = + (yuv_matrix->rgb_to_y[i] * yuv_max + rgb_round) / rgb_max; + scaled_matrix.rgb_to_u[i] = + (yuv_matrix->rgb_to_u[i] * yuv_max + rgb_round) / rgb_max; + scaled_matrix.rgb_to_v[i] = + (yuv_matrix->rgb_to_v[i] * yuv_max + rgb_round) / rgb_max; + } + } + // Also incorporate precision change scaling. + scaled_matrix.rgb_to_y[3] = Shift(yuv_matrix->rgb_to_y[3], sfix); + scaled_matrix.rgb_to_u[3] = Shift(yuv_matrix->rgb_to_u[3], sfix); + scaled_matrix.rgb_to_v[3] = Shift(yuv_matrix->rgb_to_v[3], sfix); + + return DoSharpArgbToYuv(r_ptr, g_ptr, b_ptr, rgb_step, rgb_stride, + rgb_bit_depth, y_ptr, y_stride, u_ptr, u_stride, + v_ptr, v_stride, yuv_bit_depth, width, height, + &scaled_matrix, transfer_type); +} + +//------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv.h b/3rdparty/libwebp/sharpyuv/sharpyuv.h new file mode 100644 index 0000000000..23a69ce39c --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv.h @@ -0,0 +1,174 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Sharp RGB to YUV conversion. + +#ifndef WEBP_SHARPYUV_SHARPYUV_H_ +#define WEBP_SHARPYUV_SHARPYUV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SHARPYUV_EXTERN +#ifdef WEBP_EXTERN +#define SHARPYUV_EXTERN WEBP_EXTERN +#else +// This explicitly marks library functions and allows for changing the +// signature for e.g., Windows DLL builds. +#if defined(__GNUC__) && __GNUC__ >= 4 +#define SHARPYUV_EXTERN extern __attribute__((visibility("default"))) +#else +#if defined(_MSC_VER) && defined(WEBP_DLL) +#define SHARPYUV_EXTERN __declspec(dllexport) +#else +#define SHARPYUV_EXTERN extern +#endif /* _MSC_VER && WEBP_DLL */ +#endif /* __GNUC__ >= 4 */ +#endif /* WEBP_EXTERN */ +#endif /* SHARPYUV_EXTERN */ + +#ifndef SHARPYUV_INLINE +#ifdef WEBP_INLINE +#define SHARPYUV_INLINE WEBP_INLINE +#else +#ifndef _MSC_VER +#if defined(__cplusplus) || !defined(__STRICT_ANSI__) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +#define SHARPYUV_INLINE inline +#else +#define SHARPYUV_INLINE +#endif +#else +#define SHARPYUV_INLINE __forceinline +#endif /* _MSC_VER */ +#endif /* WEBP_INLINE */ +#endif /* SHARPYUV_INLINE */ + +// SharpYUV API version following the convention from semver.org +#define SHARPYUV_VERSION_MAJOR 0 +#define SHARPYUV_VERSION_MINOR 4 +#define SHARPYUV_VERSION_PATCH 0 +// Version as a uint32_t. The major number is the high 8 bits. +// The minor number is the middle 8 bits. The patch number is the low 16 bits. +#define SHARPYUV_MAKE_VERSION(MAJOR, MINOR, PATCH) \ + (((MAJOR) << 24) | ((MINOR) << 16) | (PATCH)) +#define SHARPYUV_VERSION \ + SHARPYUV_MAKE_VERSION(SHARPYUV_VERSION_MAJOR, SHARPYUV_VERSION_MINOR, \ + SHARPYUV_VERSION_PATCH) + +// Returns the library's version number, packed in hexadecimal. See +// SHARPYUV_VERSION. +SHARPYUV_EXTERN int SharpYuvGetVersion(void); + +// RGB to YUV conversion matrix, in 16 bit fixed point. +// y = rgb_to_y[0] * r + rgb_to_y[1] * g + rgb_to_y[2] * b + rgb_to_y[3] +// u = rgb_to_u[0] * r + rgb_to_u[1] * g + rgb_to_u[2] * b + rgb_to_u[3] +// v = rgb_to_v[0] * r + rgb_to_v[1] * g + rgb_to_v[2] * b + rgb_to_v[3] +// Then y, u and v values are divided by 1<<16 and rounded. +typedef struct { + int rgb_to_y[4]; + int rgb_to_u[4]; + int rgb_to_v[4]; +} SharpYuvConversionMatrix; + +typedef struct SharpYuvOptions SharpYuvOptions; + +// Enums for transfer functions, as defined in H.273, +// https://www.itu.int/rec/T-REC-H.273-202107-I/en +typedef enum SharpYuvTransferFunctionType { + // 0 is reserved + kSharpYuvTransferFunctionBt709 = 1, + // 2 is unspecified + // 3 is reserved + kSharpYuvTransferFunctionBt470M = 4, + kSharpYuvTransferFunctionBt470Bg = 5, + kSharpYuvTransferFunctionBt601 = 6, + kSharpYuvTransferFunctionSmpte240 = 7, + kSharpYuvTransferFunctionLinear = 8, + kSharpYuvTransferFunctionLog100 = 9, + kSharpYuvTransferFunctionLog100_Sqrt10 = 10, + kSharpYuvTransferFunctionIec61966 = 11, + kSharpYuvTransferFunctionBt1361 = 12, + kSharpYuvTransferFunctionSrgb = 13, + kSharpYuvTransferFunctionBt2020_10Bit = 14, + kSharpYuvTransferFunctionBt2020_12Bit = 15, + kSharpYuvTransferFunctionSmpte2084 = 16, // PQ + kSharpYuvTransferFunctionSmpte428 = 17, + kSharpYuvTransferFunctionHlg = 18, + kSharpYuvTransferFunctionNum +} SharpYuvTransferFunctionType; + +// Converts RGB to YUV420 using a downsampling algorithm that minimizes +// artefacts caused by chroma subsampling. +// This is slower than standard downsampling (averaging of 4 UV values). +// Assumes that the image will be upsampled using a bilinear filter. If nearest +// neighbor is used instead, the upsampled image might look worse than with +// standard downsampling. +// r_ptr, g_ptr, b_ptr: pointers to the source r, g and b channels. Should point +// to uint8_t buffers if rgb_bit_depth is 8, or uint16_t buffers otherwise. +// rgb_step: distance in bytes between two horizontally adjacent pixels on the +// r, g and b channels. If rgb_bit_depth is > 8, it should be a +// multiple of 2. +// rgb_stride: distance in bytes between two vertically adjacent pixels on the +// r, g, and b channels. If rgb_bit_depth is > 8, it should be a +// multiple of 2. +// rgb_bit_depth: number of bits for each r/g/b value. One of: 8, 10, 12, 16. +// Note: 16 bit input is truncated to 14 bits before conversion to yuv. +// yuv_bit_depth: number of bits for each y/u/v value. One of: 8, 10, 12. +// y_ptr, u_ptr, v_ptr: pointers to the destination y, u and v channels. Should +// point to uint8_t buffers if yuv_bit_depth is 8, or uint16_t buffers +// otherwise. +// y_stride, u_stride, v_stride: distance in bytes between two vertically +// adjacent pixels on the y, u and v channels. If yuv_bit_depth > 8, they +// should be multiples of 2. +// width, height: width and height of the image in pixels +// This function calls SharpYuvConvertWithOptions with a default transfer +// function of kSharpYuvTransferFunctionSrgb. +SHARPYUV_EXTERN int SharpYuvConvert(const void* r_ptr, const void* g_ptr, + const void* b_ptr, int rgb_step, + int rgb_stride, int rgb_bit_depth, + void* y_ptr, int y_stride, void* u_ptr, + int u_stride, void* v_ptr, int v_stride, + int yuv_bit_depth, int width, int height, + const SharpYuvConversionMatrix* yuv_matrix); + +struct SharpYuvOptions { + // This matrix cannot be NULL and can be initialized by + // SharpYuvComputeConversionMatrix. + const SharpYuvConversionMatrix* yuv_matrix; + SharpYuvTransferFunctionType transfer_type; +}; + +// Internal, version-checked, entry point +SHARPYUV_EXTERN int SharpYuvOptionsInitInternal(const SharpYuvConversionMatrix*, + SharpYuvOptions*, int); + +// Should always be called, to initialize a fresh SharpYuvOptions +// structure before modification. SharpYuvOptionsInit() must have succeeded +// before using the 'options' object. +static SHARPYUV_INLINE int SharpYuvOptionsInit( + const SharpYuvConversionMatrix* yuv_matrix, SharpYuvOptions* options) { + return SharpYuvOptionsInitInternal(yuv_matrix, options, SHARPYUV_VERSION); +} + +SHARPYUV_EXTERN int SharpYuvConvertWithOptions( + const void* r_ptr, const void* g_ptr, const void* b_ptr, int rgb_step, + int rgb_stride, int rgb_bit_depth, void* y_ptr, int y_stride, void* u_ptr, + int u_stride, void* v_ptr, int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvOptions* options); + +// TODO(b/194336375): Add YUV444 to YUV420 conversion. Maybe also add 422 +// support (it's rarely used in practice, especially for images). + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c new file mode 100644 index 0000000000..29425a0c49 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c @@ -0,0 +1,14 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +#include "sharpyuv/sharpyuv_cpu.h" + +// Include src/dsp/cpu.c to create SharpYuvGetCPUInfo from VP8GetCPUInfo. The +// function pointer is renamed in sharpyuv_cpu.h. +#include "src/dsp/cpu.c" diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h new file mode 100644 index 0000000000..176ca3eb16 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h @@ -0,0 +1,22 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +#ifndef WEBP_SHARPYUV_SHARPYUV_CPU_H_ +#define WEBP_SHARPYUV_SHARPYUV_CPU_H_ + +#include "sharpyuv/sharpyuv.h" + +// Avoid exporting SharpYuvGetCPUInfo in shared object / DLL builds. +// SharpYuvInit() replaces the use of the function pointer. +#undef WEBP_EXTERN +#define WEBP_EXTERN extern +#define VP8GetCPUInfo SharpYuvGetCPUInfo +#include "src/dsp/cpu.h" + +#endif // WEBP_SHARPYUV_SHARPYUV_CPU_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c new file mode 100644 index 0000000000..0ad22be945 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c @@ -0,0 +1,110 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Colorspace utilities. + +#include "sharpyuv/sharpyuv_csp.h" + +#include +#include +#include + +static int ToFixed16(float f) { return (int)floor(f * (1 << 16) + 0.5f); } + +void SharpYuvComputeConversionMatrix(const SharpYuvColorSpace* yuv_color_space, + SharpYuvConversionMatrix* matrix) { + const float kr = yuv_color_space->kr; + const float kb = yuv_color_space->kb; + const float kg = 1.0f - kr - kb; + const float cr = 0.5f / (1.0f - kb); + const float cb = 0.5f / (1.0f - kr); + + const int shift = yuv_color_space->bit_depth - 8; + + const float denom = (float)((1 << yuv_color_space->bit_depth) - 1); + float scale_y = 1.0f; + float add_y = 0.0f; + float scale_u = cr; + float scale_v = cb; + float add_uv = (float)(128 << shift); + assert(yuv_color_space->bit_depth >= 8); + + if (yuv_color_space->range == kSharpYuvRangeLimited) { + scale_y *= (219 << shift) / denom; + scale_u *= (224 << shift) / denom; + scale_v *= (224 << shift) / denom; + add_y = (float)(16 << shift); + } + + matrix->rgb_to_y[0] = ToFixed16(kr * scale_y); + matrix->rgb_to_y[1] = ToFixed16(kg * scale_y); + matrix->rgb_to_y[2] = ToFixed16(kb * scale_y); + matrix->rgb_to_y[3] = ToFixed16(add_y); + + matrix->rgb_to_u[0] = ToFixed16(-kr * scale_u); + matrix->rgb_to_u[1] = ToFixed16(-kg * scale_u); + matrix->rgb_to_u[2] = ToFixed16((1 - kb) * scale_u); + matrix->rgb_to_u[3] = ToFixed16(add_uv); + + matrix->rgb_to_v[0] = ToFixed16((1 - kr) * scale_v); + matrix->rgb_to_v[1] = ToFixed16(-kg * scale_v); + matrix->rgb_to_v[2] = ToFixed16(-kb * scale_v); + matrix->rgb_to_v[3] = ToFixed16(add_uv); +} + +// Matrices are in YUV_FIX fixed point precision. +// WebP's matrix, similar but not identical to kRec601LimitedMatrix. +static const SharpYuvConversionMatrix kWebpMatrix = { + {16839, 33059, 6420, 16 << 16}, + {-9719, -19081, 28800, 128 << 16}, + {28800, -24116, -4684, 128 << 16}, +}; +// Kr=0.2990f Kb=0.1140f bits=8 range=kSharpYuvRangeLimited +static const SharpYuvConversionMatrix kRec601LimitedMatrix = { + {16829, 33039, 6416, 16 << 16}, + {-9714, -19071, 28784, 128 << 16}, + {28784, -24103, -4681, 128 << 16}, +}; +// Kr=0.2990f Kb=0.1140f bits=8 range=kSharpYuvRangeFull +static const SharpYuvConversionMatrix kRec601FullMatrix = { + {19595, 38470, 7471, 0}, + {-11058, -21710, 32768, 128 << 16}, + {32768, -27439, -5329, 128 << 16}, +}; +// Kr=0.2126f Kb=0.0722f bits=8 range=kSharpYuvRangeLimited +static const SharpYuvConversionMatrix kRec709LimitedMatrix = { + {11966, 40254, 4064, 16 << 16}, + {-6596, -22189, 28784, 128 << 16}, + {28784, -26145, -2639, 128 << 16}, +}; +// Kr=0.2126f Kb=0.0722f bits=8 range=kSharpYuvRangeFull +static const SharpYuvConversionMatrix kRec709FullMatrix = { + {13933, 46871, 4732, 0}, + {-7509, -25259, 32768, 128 << 16}, + {32768, -29763, -3005, 128 << 16}, +}; + +const SharpYuvConversionMatrix* SharpYuvGetConversionMatrix( + SharpYuvMatrixType matrix_type) { + switch (matrix_type) { + case kSharpYuvMatrixWebp: + return &kWebpMatrix; + case kSharpYuvMatrixRec601Limited: + return &kRec601LimitedMatrix; + case kSharpYuvMatrixRec601Full: + return &kRec601FullMatrix; + case kSharpYuvMatrixRec709Limited: + return &kRec709LimitedMatrix; + case kSharpYuvMatrixRec709Full: + return &kRec709FullMatrix; + case kSharpYuvMatrixNum: + return NULL; + } + return NULL; +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h new file mode 100644 index 0000000000..3214e3ac60 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h @@ -0,0 +1,60 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Colorspace utilities. + +#ifndef WEBP_SHARPYUV_SHARPYUV_CSP_H_ +#define WEBP_SHARPYUV_SHARPYUV_CSP_H_ + +#include "sharpyuv/sharpyuv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Range of YUV values. +typedef enum { + kSharpYuvRangeFull, // YUV values between [0;255] (for 8 bit) + kSharpYuvRangeLimited // Y in [16;235], YUV in [16;240] (for 8 bit) +} SharpYuvRange; + +// Constants that define a YUV color space. +typedef struct { + // Kr and Kb are defined such that: + // Y = Kr * r + Kg * g + Kb * b where Kg = 1 - Kr - Kb. + float kr; + float kb; + int bit_depth; // 8, 10 or 12 + SharpYuvRange range; +} SharpYuvColorSpace; + +// Fills in 'matrix' for the given YUVColorSpace. +SHARPYUV_EXTERN void SharpYuvComputeConversionMatrix( + const SharpYuvColorSpace* yuv_color_space, + SharpYuvConversionMatrix* matrix); + +// Enums for precomputed conversion matrices. +typedef enum { + kSharpYuvMatrixWebp = 0, + kSharpYuvMatrixRec601Limited, + kSharpYuvMatrixRec601Full, + kSharpYuvMatrixRec709Limited, + kSharpYuvMatrixRec709Full, + kSharpYuvMatrixNum +} SharpYuvMatrixType; + +// Returns a pointer to a matrix for one of the predefined colorspaces. +SHARPYUV_EXTERN const SharpYuvConversionMatrix* SharpYuvGetConversionMatrix( + SharpYuvMatrixType matrix_type); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_CSP_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c new file mode 100644 index 0000000000..0da3efc0b8 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c @@ -0,0 +1,104 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#include +#include + +#include "sharpyuv/sharpyuv_cpu.h" + +//----------------------------------------------------------------------------- + +#if !WEBP_NEON_OMIT_C_CODE +static uint16_t clip(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_C(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + uint64_t diff = 0; + int i; + const int max_y = (1 << bit_depth) - 1; + for (i = 0; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip(new_y, max_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYuvUpdateRGB_C(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i; + for (i = 0; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow_C(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + int i; + const int max_y = (1 << bit_depth) - 1; + for (i = 0; i < len; ++i, ++A, ++B) { + const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4; + const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4; + out[2 * i + 0] = clip(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip(best_y[2 * i + 1] + v1, max_y); + } +} +#endif // !WEBP_NEON_OMIT_C_CODE + +//----------------------------------------------------------------------------- + +uint64_t (*SharpYuvUpdateY)(const uint16_t* src, const uint16_t* ref, + uint16_t* dst, int len, int bit_depth); +void (*SharpYuvUpdateRGB)(const int16_t* src, const int16_t* ref, int16_t* dst, + int len); +void (*SharpYuvFilterRow)(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth); + +extern VP8CPUInfo SharpYuvGetCPUInfo; +extern void InitSharpYuvSSE2(void); +extern void InitSharpYuvNEON(void); + +void SharpYuvInitDsp(void) { +#if !WEBP_NEON_OMIT_C_CODE + SharpYuvUpdateY = SharpYuvUpdateY_C; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_C; + SharpYuvFilterRow = SharpYuvFilterRow_C; +#endif + + if (SharpYuvGetCPUInfo != NULL) { +#if defined(WEBP_HAVE_SSE2) + if (SharpYuvGetCPUInfo(kSSE2)) { + InitSharpYuvSSE2(); + } +#endif // WEBP_HAVE_SSE2 + } + +#if defined(WEBP_HAVE_NEON) + if (WEBP_NEON_OMIT_C_CODE || + (SharpYuvGetCPUInfo != NULL && SharpYuvGetCPUInfo(kNEON))) { + InitSharpYuvNEON(); + } +#endif // WEBP_HAVE_NEON + + assert(SharpYuvUpdateY != NULL); + assert(SharpYuvUpdateRGB != NULL); + assert(SharpYuvFilterRow != NULL); +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h new file mode 100644 index 0000000000..805fbadbf6 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h @@ -0,0 +1,28 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. + +#ifndef WEBP_SHARPYUV_SHARPYUV_DSP_H_ +#define WEBP_SHARPYUV_SHARPYUV_DSP_H_ + +#include "sharpyuv/sharpyuv_cpu.h" +#include "src/webp/types.h" + +extern uint64_t (*SharpYuvUpdateY)(const uint16_t* src, const uint16_t* ref, + uint16_t* dst, int len, int bit_depth); +extern void (*SharpYuvUpdateRGB)(const int16_t* src, const int16_t* ref, + int16_t* dst, int len); +extern void (*SharpYuvFilterRow)(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth); + +void SharpYuvInitDsp(void); + +#endif // WEBP_SHARPYUV_SHARPYUV_DSP_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c new file mode 100644 index 0000000000..fecadc6480 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c @@ -0,0 +1,419 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Gamma correction utilities. + +#include "sharpyuv/sharpyuv_gamma.h" + +#include +#include +#include + +#include "src/webp/types.h" + +// Gamma correction compensates loss of resolution during chroma subsampling. +// Size of pre-computed table for converting from gamma to linear. +#define GAMMA_TO_LINEAR_TAB_BITS 10 +#define GAMMA_TO_LINEAR_TAB_SIZE (1 << GAMMA_TO_LINEAR_TAB_BITS) +static uint32_t kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE + 2]; +#define LINEAR_TO_GAMMA_TAB_BITS 9 +#define LINEAR_TO_GAMMA_TAB_SIZE (1 << LINEAR_TO_GAMMA_TAB_BITS) +static uint32_t kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE + 2]; + +static const double kGammaF = 1. / 0.45; +#define GAMMA_TO_LINEAR_BITS 16 + +static volatile int kGammaTablesSOk = 0; +void SharpYuvInitGammaTables(void) { + assert(GAMMA_TO_LINEAR_BITS <= 16); + if (!kGammaTablesSOk) { + int v; + const double a = 0.09929682680944; + const double thresh = 0.018053968510807; + const double final_scale = 1 << GAMMA_TO_LINEAR_BITS; + // Precompute gamma to linear table. + { + const double norm = 1. / GAMMA_TO_LINEAR_TAB_SIZE; + const double a_rec = 1. / (1. + a); + for (v = 0; v <= GAMMA_TO_LINEAR_TAB_SIZE; ++v) { + const double g = norm * v; + double value; + if (g <= thresh * 4.5) { + value = g / 4.5; + } else { + value = pow(a_rec * (g + a), kGammaF); + } + kGammaToLinearTabS[v] = (uint32_t)(value * final_scale + .5); + } + // to prevent small rounding errors to cause read-overflow: + kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE + 1] = + kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE]; + } + // Precompute linear to gamma table. + { + const double scale = 1. / LINEAR_TO_GAMMA_TAB_SIZE; + for (v = 0; v <= LINEAR_TO_GAMMA_TAB_SIZE; ++v) { + const double g = scale * v; + double value; + if (g <= thresh) { + value = 4.5 * g; + } else { + value = (1. + a) * pow(g, 1. / kGammaF) - a; + } + kLinearToGammaTabS[v] = + (uint32_t)(final_scale * value + 0.5); + } + // to prevent small rounding errors to cause read-overflow: + kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE + 1] = + kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE]; + } + kGammaTablesSOk = 1; + } +} + +static WEBP_INLINE int Shift(int v, int shift) { + return (shift >= 0) ? (v << shift) : (v >> -shift); +} + +static WEBP_INLINE uint32_t FixedPointInterpolation(int v, uint32_t* tab, + int tab_pos_shift_right, + int tab_value_shift) { + const uint32_t tab_pos = Shift(v, -tab_pos_shift_right); + // fractional part, in 'tab_pos_shift' fixed-point precision + const uint32_t x = v - (tab_pos << tab_pos_shift_right); // fractional part + // v0 / v1 are in kGammaToLinearBits fixed-point precision (range [0..1]) + const uint32_t v0 = Shift(tab[tab_pos + 0], tab_value_shift); + const uint32_t v1 = Shift(tab[tab_pos + 1], tab_value_shift); + // Final interpolation. + const uint32_t v2 = (v1 - v0) * x; // note: v1 >= v0. + const int half = + (tab_pos_shift_right > 0) ? 1 << (tab_pos_shift_right - 1) : 0; + const uint32_t result = v0 + ((v2 + half) >> tab_pos_shift_right); + return result; +} + +static uint32_t ToLinearSrgb(uint16_t v, int bit_depth) { + const int shift = GAMMA_TO_LINEAR_TAB_BITS - bit_depth; + if (shift > 0) { + return kGammaToLinearTabS[v << shift]; + } + return FixedPointInterpolation(v, kGammaToLinearTabS, -shift, 0); +} + +static uint16_t FromLinearSrgb(uint32_t value, int bit_depth) { + return FixedPointInterpolation( + value, kLinearToGammaTabS, + (GAMMA_TO_LINEAR_BITS - LINEAR_TO_GAMMA_TAB_BITS), + bit_depth - GAMMA_TO_LINEAR_BITS); +} + +//////////////////////////////////////////////////////////////////////////////// + +#define CLAMP(x, low, high) \ + (((x) < (low)) ? (low) : (((high) < (x)) ? (high) : (x))) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +static WEBP_INLINE float Roundf(float x) { + if (x < 0) + return (float)ceil((double)(x - 0.5f)); + else + return (float)floor((double)(x + 0.5f)); +} + +static WEBP_INLINE float Powf(float base, float exp) { + return (float)pow((double)base, (double)exp); +} + +static WEBP_INLINE float Log10f(float x) { return (float)log10((double)x); } + +static float ToLinear709(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinear709(float linear) { + if (linear < 0.f) { + return 0.f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } else if (linear < 1.f) { + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; + } + return 1.f; +} + +static float ToLinear470M(float gamma) { + return Powf(CLAMP(gamma, 0.f, 1.f), 1.f / 2.2f); +} + +static float FromLinear470M(float linear) { + return Powf(CLAMP(linear, 0.f, 1.f), 2.2f); +} + +static float ToLinear470Bg(float gamma) { + return Powf(CLAMP(gamma, 0.f, 1.f), 1.f / 2.8f); +} + +static float FromLinear470Bg(float linear) { + return Powf(CLAMP(linear, 0.f, 1.f), 2.8f); +} + +static float ToLinearSmpte240(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma < 4.f * 0.022821585529445f) { + return gamma / 4.f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.111572195921731f) / 1.111572195921731f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinearSmpte240(float linear) { + if (linear < 0.f) { + return 0.f; + } else if (linear < 0.022821585529445f) { + return linear * 4.f; + } else if (linear < 1.f) { + return 1.111572195921731f * Powf(linear, 0.45f) - 0.111572195921731f; + } + return 1.f; +} + +static float ToLinearLog100(float gamma) { + return (gamma < 0.01f) ? 0.0f : 1.0f + Log10f(MIN(gamma, 1.f)) / 2.0f; +} + +static float FromLinearLog100(float linear) { + // The function is non-bijective so choose the middle of [0, 0.01]. + const float mid_interval = 0.01f / 2.f; + return (linear <= 0.0f) ? mid_interval + : Powf(10.0f, 2.f * (MIN(linear, 1.f) - 1.0f)); +} + +static float ToLinearLog100Sqrt10(float gamma) { + return (gamma < 0.00316227766f) ? 0.0f + : 1.0f + Log10f(MIN(gamma, 1.f)) / 2.5f; +} + +static float FromLinearLog100Sqrt10(float linear) { + // The function is non-bijective so choose the middle of [0, 0.00316227766f[. + const float mid_interval = 0.00316227766f / 2.f; + return (linear < 0.0f) ? mid_interval + : Powf(10.0f, 2.5f * (MIN(linear, 1.f) - 1.0f)); +} + +static float ToLinearIec61966(float gamma) { + if (gamma <= -4.5f * 0.018053968510807f) { + return Powf((-gamma + 0.09929682680944f) / -1.09929682680944f, 1.f / 0.45f); + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); +} + +static float FromLinearIec61966(float linear) { + if (linear <= -0.018053968510807f) { + return -1.09929682680944f * Powf(-linear, 0.45f) + 0.09929682680944f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; +} + +static float ToLinearBt1361(float gamma) { + if (gamma < -0.25f) { + return -0.25f; + } else if (gamma < 0.f) { + return Powf((gamma - 0.02482420670236f) / -0.27482420670236f, 1.f / 0.45f) / + -4.f; + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinearBt1361(float linear) { + if (linear < -0.25f) { + return -0.25f; + } else if (linear < 0.f) { + return -0.27482420670236f * Powf(-4.f * linear, 0.45f) + 0.02482420670236f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } else if (linear < 1.f) { + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; + } + return 1.f; +} + +static float ToLinearPq(float gamma) { + if (gamma > 0.f) { + const float pow_gamma = Powf(gamma, 32.f / 2523.f); + const float num = MAX(pow_gamma - 107.f / 128.f, 0.0f); + const float den = MAX(2413.f / 128.f - 2392.f / 128.f * pow_gamma, FLT_MIN); + return Powf(num / den, 4096.f / 653.f); + } + return 0.f; +} + +static float FromLinearPq(float linear) { + if (linear > 0.f) { + const float pow_linear = Powf(linear, 653.f / 4096.f); + const float num = 107.f / 128.f + 2413.f / 128.f * pow_linear; + const float den = 1.0f + 2392.f / 128.f * pow_linear; + return Powf(num / den, 2523.f / 32.f); + } + return 0.f; +} + +static float ToLinearSmpte428(float gamma) { + return Powf(0.91655527974030934f * MAX(gamma, 0.f), 1.f / 2.6f); +} + +static float FromLinearSmpte428(float linear) { + return Powf(MAX(linear, 0.f), 2.6f) / 0.91655527974030934f; +} + +// Conversion in BT.2100 requires RGB info. Simplify to gamma correction here. +static float ToLinearHlg(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma <= 0.5f) { + return Powf((gamma * gamma) * (1.f / 3.f), 1.2f); + } + return Powf((expf((gamma - 0.55991073f) / 0.17883277f) + 0.28466892f) / 12.0f, + 1.2f); +} + +static float FromLinearHlg(float linear) { + linear = Powf(linear, 1.f / 1.2f); + if (linear < 0.f) { + return 0.f; + } else if (linear <= (1.f / 12.f)) { + return sqrtf(3.f * linear); + } + return 0.17883277f * logf(12.f * linear - 0.28466892f) + 0.55991073f; +} + +uint32_t SharpYuvGammaToLinear(uint16_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type) { + float v_float, linear; + if (transfer_type == kSharpYuvTransferFunctionSrgb) { + return ToLinearSrgb(v, bit_depth); + } + v_float = (float)v / ((1 << bit_depth) - 1); + switch (transfer_type) { + case kSharpYuvTransferFunctionBt709: + case kSharpYuvTransferFunctionBt601: + case kSharpYuvTransferFunctionBt2020_10Bit: + case kSharpYuvTransferFunctionBt2020_12Bit: + linear = ToLinear709(v_float); + break; + case kSharpYuvTransferFunctionBt470M: + linear = ToLinear470M(v_float); + break; + case kSharpYuvTransferFunctionBt470Bg: + linear = ToLinear470Bg(v_float); + break; + case kSharpYuvTransferFunctionSmpte240: + linear = ToLinearSmpte240(v_float); + break; + case kSharpYuvTransferFunctionLinear: + return v; + case kSharpYuvTransferFunctionLog100: + linear = ToLinearLog100(v_float); + break; + case kSharpYuvTransferFunctionLog100_Sqrt10: + linear = ToLinearLog100Sqrt10(v_float); + break; + case kSharpYuvTransferFunctionIec61966: + linear = ToLinearIec61966(v_float); + break; + case kSharpYuvTransferFunctionBt1361: + linear = ToLinearBt1361(v_float); + break; + case kSharpYuvTransferFunctionSmpte2084: + linear = ToLinearPq(v_float); + break; + case kSharpYuvTransferFunctionSmpte428: + linear = ToLinearSmpte428(v_float); + break; + case kSharpYuvTransferFunctionHlg: + linear = ToLinearHlg(v_float); + break; + default: + assert(0); + linear = 0; + break; + } + return (uint32_t)Roundf(linear * ((1 << 16) - 1)); +} + +uint16_t SharpYuvLinearToGamma(uint32_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type) { + float v_float, linear; + if (transfer_type == kSharpYuvTransferFunctionSrgb) { + return FromLinearSrgb(v, bit_depth); + } + v_float = (float)v / ((1 << 16) - 1); + switch (transfer_type) { + case kSharpYuvTransferFunctionBt709: + case kSharpYuvTransferFunctionBt601: + case kSharpYuvTransferFunctionBt2020_10Bit: + case kSharpYuvTransferFunctionBt2020_12Bit: + linear = FromLinear709(v_float); + break; + case kSharpYuvTransferFunctionBt470M: + linear = FromLinear470M(v_float); + break; + case kSharpYuvTransferFunctionBt470Bg: + linear = FromLinear470Bg(v_float); + break; + case kSharpYuvTransferFunctionSmpte240: + linear = FromLinearSmpte240(v_float); + break; + case kSharpYuvTransferFunctionLinear: + return v; + case kSharpYuvTransferFunctionLog100: + linear = FromLinearLog100(v_float); + break; + case kSharpYuvTransferFunctionLog100_Sqrt10: + linear = FromLinearLog100Sqrt10(v_float); + break; + case kSharpYuvTransferFunctionIec61966: + linear = FromLinearIec61966(v_float); + break; + case kSharpYuvTransferFunctionBt1361: + linear = FromLinearBt1361(v_float); + break; + case kSharpYuvTransferFunctionSmpte2084: + linear = FromLinearPq(v_float); + break; + case kSharpYuvTransferFunctionSmpte428: + linear = FromLinearSmpte428(v_float); + break; + case kSharpYuvTransferFunctionHlg: + linear = FromLinearHlg(v_float); + break; + default: + assert(0); + linear = 0; + break; + } + return (uint16_t)Roundf(linear * ((1 << bit_depth) - 1)); +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h new file mode 100644 index 0000000000..b8ba7e9870 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h @@ -0,0 +1,38 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Gamma correction utilities. + +#ifndef WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ +#define WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ + +#include "sharpyuv/sharpyuv.h" +#include "src/webp/types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Initializes precomputed tables. Must be called once before calling +// SharpYuvGammaToLinear or SharpYuvLinearToGamma. +void SharpYuvInitGammaTables(void); + +// Converts a 'bit_depth'-bit gamma color value to a 16-bit linear value. +uint32_t SharpYuvGammaToLinear(uint16_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type); + +// Converts a 16-bit linear color value to a 'bit_depth'-bit gamma value. +uint16_t SharpYuvLinearToGamma(uint32_t value, int bit_depth, + SharpYuvTransferFunctionType transfer_type); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c b/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c new file mode 100644 index 0000000000..5840914865 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c @@ -0,0 +1,181 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#if defined(WEBP_USE_NEON) +#include +#include +#include + +static uint16_t clip_NEON(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_NEON(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const int16x8_t zero = vdupq_n_s16(0); + const int16x8_t max = vdupq_n_s16(max_y); + uint64x2_t sum = vdupq_n_u64(0); + uint64_t diff; + + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t A = vreinterpretq_s16_u16(vld1q_u16(ref + i)); + const int16x8_t B = vreinterpretq_s16_u16(vld1q_u16(src + i)); + const int16x8_t C = vreinterpretq_s16_u16(vld1q_u16(dst + i)); + const int16x8_t D = vsubq_s16(A, B); // diff_y + const int16x8_t F = vaddq_s16(C, D); // new_y + const uint16x8_t H = + vreinterpretq_u16_s16(vmaxq_s16(vminq_s16(F, max), zero)); + const int16x8_t I = vabsq_s16(D); // abs(diff_y) + vst1q_u16(dst + i, H); + sum = vpadalq_u32(sum, vpaddlq_u16(vreinterpretq_u16_s16(I))); + } + diff = vgetq_lane_u64(sum, 0) + vgetq_lane_u64(sum, 1); + for (; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)(dst[i]) + diff_y; + dst[i] = clip_NEON(new_y, max_y); + diff += (uint64_t)(abs(diff_y)); + } + return diff; +} + +static void SharpYuvUpdateRGB_NEON(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i; + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t A = vld1q_s16(ref + i); + const int16x8_t B = vld1q_s16(src + i); + const int16x8_t C = vld1q_s16(dst + i); + const int16x8_t D = vsubq_s16(A, B); // diff_uv + const int16x8_t E = vaddq_s16(C, D); // new_uv + vst1q_s16(dst + i, E); + } + for (; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow16_NEON(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const int16x8_t max = vdupq_n_s16(max_y); + const int16x8_t zero = vdupq_n_s16(0); + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t a0 = vld1q_s16(A + i + 0); + const int16x8_t a1 = vld1q_s16(A + i + 1); + const int16x8_t b0 = vld1q_s16(B + i + 0); + const int16x8_t b1 = vld1q_s16(B + i + 1); + const int16x8_t a0b1 = vaddq_s16(a0, b1); + const int16x8_t a1b0 = vaddq_s16(a1, b0); + const int16x8_t a0a1b0b1 = vaddq_s16(a0b1, a1b0); // A0+A1+B0+B1 + const int16x8_t a0b1_2 = vaddq_s16(a0b1, a0b1); // 2*(A0+B1) + const int16x8_t a1b0_2 = vaddq_s16(a1b0, a1b0); // 2*(A1+B0) + const int16x8_t c0 = vshrq_n_s16(vaddq_s16(a0b1_2, a0a1b0b1), 3); + const int16x8_t c1 = vshrq_n_s16(vaddq_s16(a1b0_2, a0a1b0b1), 3); + const int16x8_t e0 = vrhaddq_s16(c1, a0); + const int16x8_t e1 = vrhaddq_s16(c0, a1); + const int16x8x2_t f = vzipq_s16(e0, e1); + const int16x8_t g0 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 0)); + const int16x8_t g1 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 8)); + const int16x8_t h0 = vaddq_s16(g0, f.val[0]); + const int16x8_t h1 = vaddq_s16(g1, f.val[1]); + const int16x8_t i0 = vmaxq_s16(vminq_s16(h0, max), zero); + const int16x8_t i1 = vmaxq_s16(vminq_s16(h1, max), zero); + vst1q_u16(out + 2 * i + 0, vreinterpretq_u16_s16(i0)); + vst1q_u16(out + 2 * i + 8, vreinterpretq_u16_s16(i1)); + } + for (; i < len; ++i) { + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_NEON(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_NEON(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow32_NEON(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const uint16x8_t max = vdupq_n_u16(max_y); + for (i = 0; i + 4 <= len; i += 4) { + const int16x4_t a0 = vld1_s16(A + i + 0); + const int16x4_t a1 = vld1_s16(A + i + 1); + const int16x4_t b0 = vld1_s16(B + i + 0); + const int16x4_t b1 = vld1_s16(B + i + 1); + const int32x4_t a0b1 = vaddl_s16(a0, b1); + const int32x4_t a1b0 = vaddl_s16(a1, b0); + const int32x4_t a0a1b0b1 = vaddq_s32(a0b1, a1b0); // A0+A1+B0+B1 + const int32x4_t a0b1_2 = vaddq_s32(a0b1, a0b1); // 2*(A0+B1) + const int32x4_t a1b0_2 = vaddq_s32(a1b0, a1b0); // 2*(A1+B0) + const int32x4_t c0 = vshrq_n_s32(vaddq_s32(a0b1_2, a0a1b0b1), 3); + const int32x4_t c1 = vshrq_n_s32(vaddq_s32(a1b0_2, a0a1b0b1), 3); + const int32x4_t e0 = vrhaddq_s32(c1, vmovl_s16(a0)); + const int32x4_t e1 = vrhaddq_s32(c0, vmovl_s16(a1)); + const int32x4x2_t f = vzipq_s32(e0, e1); + + const int16x8_t g = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i)); + const int32x4_t h0 = vaddw_s16(f.val[0], vget_low_s16(g)); + const int32x4_t h1 = vaddw_s16(f.val[1], vget_high_s16(g)); + const uint16x8_t i_16 = vcombine_u16(vqmovun_s32(h0), vqmovun_s32(h1)); + const uint16x8_t i_clamped = vminq_u16(i_16, max); + vst1q_u16(out + 2 * i + 0, i_clamped); + } + for (; i < len; ++i) { + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_NEON(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_NEON(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow_NEON(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + if (bit_depth <= 10) { + SharpYuvFilterRow16_NEON(A, B, len, best_y, out, bit_depth); + } else { + SharpYuvFilterRow32_NEON(A, B, len, best_y, out, bit_depth); + } +} + +//------------------------------------------------------------------------------ + +extern void InitSharpYuvNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvNEON(void) { + SharpYuvUpdateY = SharpYuvUpdateY_NEON; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_NEON; + SharpYuvFilterRow = SharpYuvFilterRow_NEON; +} + +#else // !WEBP_USE_NEON + +extern void InitSharpYuvNEON(void); + +void InitSharpYuvNEON(void) {} + +#endif // WEBP_USE_NEON diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c b/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c new file mode 100644 index 0000000000..9744d1bb6c --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c @@ -0,0 +1,201 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#if defined(WEBP_USE_SSE2) +#include +#include + +static uint16_t clip_SSE2(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + uint64_t diff = 0; + uint32_t tmp[4]; + int i; + const __m128i zero = _mm_setzero_si128(); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i one = _mm_set1_epi16(1); + __m128i sum = zero; + + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_y + const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) + const __m128i F = _mm_add_epi16(C, D); // new_y + const __m128i G = _mm_or_si128(E, one); // -1 or 1 + const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); + const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) + _mm_storeu_si128((__m128i*)(dst + i), H); + sum = _mm_add_epi32(sum, I); + } + _mm_storeu_si128((__m128i*)tmp, sum); + diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; + for (; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip_SSE2(new_y, max_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYuvUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i = 0; + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_uv + const __m128i E = _mm_add_epi16(C, D); // new_uv + _mm_storeu_si128((__m128i*)(dst + i), E); + } + for (; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow16_SSE2(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const __m128i kCst8 = _mm_set1_epi16(8); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 8 <= len; i += 8) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); + const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); + const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); + const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); + const __m128i a0b1 = _mm_add_epi16(a0, b1); + const __m128i a1b0 = _mm_add_epi16(a1, b0); + const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 + const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); + const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) + const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) + const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); + const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); + const __m128i d0 = _mm_add_epi16(c1, a0); + const __m128i d1 = _mm_add_epi16(c0, a1); + const __m128i e0 = _mm_srai_epi16(d0, 1); + const __m128i e1 = _mm_srai_epi16(d1, 1); + const __m128i f0 = _mm_unpacklo_epi16(e0, e1); + const __m128i f1 = _mm_unpackhi_epi16(e0, e1); + const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); + const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); + const __m128i h0 = _mm_add_epi16(g0, f0); + const __m128i h1 = _mm_add_epi16(g1, f1); + const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); + const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); + _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); + _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); + } + for (; i < len; ++i) { + // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = + // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 + // We reuse the common sub-expressions. + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); + } +} + +static WEBP_INLINE __m128i s16_to_s32(__m128i in) { + return _mm_srai_epi32(_mm_unpacklo_epi16(in, in), 16); +} + +static void SharpYuvFilterRow32_SSE2(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const __m128i kCst8 = _mm_set1_epi32(8); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 4 <= len; i += 4) { + const __m128i a0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 0))); + const __m128i a1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 1))); + const __m128i b0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 0))); + const __m128i b1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 1))); + const __m128i a0b1 = _mm_add_epi32(a0, b1); + const __m128i a1b0 = _mm_add_epi32(a1, b0); + const __m128i a0a1b0b1 = _mm_add_epi32(a0b1, a1b0); // A0+A1+B0+B1 + const __m128i a0a1b0b1_8 = _mm_add_epi32(a0a1b0b1, kCst8); + const __m128i a0b1_2 = _mm_add_epi32(a0b1, a0b1); // 2*(A0+B1) + const __m128i a1b0_2 = _mm_add_epi32(a1b0, a1b0); // 2*(A1+B0) + const __m128i c0 = _mm_srai_epi32(_mm_add_epi32(a0b1_2, a0a1b0b1_8), 3); + const __m128i c1 = _mm_srai_epi32(_mm_add_epi32(a1b0_2, a0a1b0b1_8), 3); + const __m128i d0 = _mm_add_epi32(c1, a0); + const __m128i d1 = _mm_add_epi32(c0, a1); + const __m128i e0 = _mm_srai_epi32(d0, 1); + const __m128i e1 = _mm_srai_epi32(d1, 1); + const __m128i f0 = _mm_unpacklo_epi32(e0, e1); + const __m128i f1 = _mm_unpackhi_epi32(e0, e1); + const __m128i g = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); + const __m128i h_16 = _mm_add_epi16(g, _mm_packs_epi32(f0, f1)); + const __m128i final = _mm_max_epi16(_mm_min_epi16(h_16, max), zero); + _mm_storeu_si128((__m128i*)(out + 2 * i + 0), final); + } + for (; i < len; ++i) { + // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = + // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 + // We reuse the common sub-expressions. + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + if (bit_depth <= 10) { + SharpYuvFilterRow16_SSE2(A, B, len, best_y, out, bit_depth); + } else { + SharpYuvFilterRow32_SSE2(A, B, len, best_y, out, bit_depth); + } +} + +//------------------------------------------------------------------------------ + +extern void InitSharpYuvSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvSSE2(void) { + SharpYuvUpdateY = SharpYuvUpdateY_SSE2; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_SSE2; + SharpYuvFilterRow = SharpYuvFilterRow_SSE2; +} +#else // !WEBP_USE_SSE2 + +extern void InitSharpYuvSSE2(void); + +void InitSharpYuvSSE2(void) {} + +#endif // WEBP_USE_SSE2 diff --git a/3rdparty/libwebp/src/dec/alpha_dec.c b/3rdparty/libwebp/src/dec/alpha_dec.c index bce735bfc2..663255c42f 100644 --- a/3rdparty/libwebp/src/dec/alpha_dec.c +++ b/3rdparty/libwebp/src/dec/alpha_dec.c @@ -117,21 +117,12 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) { const uint8_t* deltas = dec->alpha_data_ + ALPHA_HEADER_LEN + row * width; uint8_t* dst = dec->alpha_plane_ + row * width; assert(deltas <= &dec->alpha_data_[dec->alpha_data_size_]); - if (alph_dec->filter_ != WEBP_FILTER_NONE) { - assert(WebPUnfilters[alph_dec->filter_] != NULL); - for (y = 0; y < num_rows; ++y) { - WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width); - prev_line = dst; - dst += width; - deltas += width; - } - } else { - for (y = 0; y < num_rows; ++y) { - memcpy(dst, deltas, width * sizeof(*dst)); - prev_line = dst; - dst += width; - deltas += width; - } + assert(WebPUnfilters[alph_dec->filter_] != NULL); + for (y = 0; y < num_rows; ++y) { + WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width); + prev_line = dst; + dst += width; + deltas += width; } dec->alpha_prev_line_ = prev_line; } else { // alph_dec->method_ == ALPHA_LOSSLESS_COMPRESSION @@ -155,7 +146,8 @@ static int AllocateAlphaPlane(VP8Decoder* const dec, const VP8Io* const io) { dec->alpha_plane_mem_ = (uint8_t*)WebPSafeMalloc(alpha_size, sizeof(*dec->alpha_plane_)); if (dec->alpha_plane_mem_ == NULL) { - return 0; + return VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY, + "Alpha decoder initialization failed."); } dec->alpha_plane_ = dec->alpha_plane_mem_; dec->alpha_prev_line_ = NULL; @@ -183,16 +175,25 @@ const uint8_t* VP8DecompressAlphaRows(VP8Decoder* const dec, assert(dec != NULL && io != NULL); if (row < 0 || num_rows <= 0 || row + num_rows > height) { - return NULL; // sanity check. + return NULL; } if (!dec->is_alpha_decoded_) { if (dec->alph_dec_ == NULL) { // Initialize decoder. dec->alph_dec_ = ALPHNew(); - if (dec->alph_dec_ == NULL) return NULL; + if (dec->alph_dec_ == NULL) { + VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY, + "Alpha decoder initialization failed."); + return NULL; + } if (!AllocateAlphaPlane(dec, io)) goto Error; if (!ALPHInit(dec->alph_dec_, dec->alpha_data_, dec->alpha_data_size_, io, dec->alpha_plane_)) { + VP8LDecoder* const vp8l_dec = dec->alph_dec_->vp8l_dec_; + VP8SetError(dec, + (vp8l_dec == NULL) ? VP8_STATUS_OUT_OF_MEMORY + : vp8l_dec->status_, + "Alpha decoder initialization failed."); goto Error; } // if we allowed use of alpha dithering, check whether it's needed at all diff --git a/3rdparty/libwebp/src/dec/buffer_dec.c b/3rdparty/libwebp/src/dec/buffer_dec.c index 3cd94eb4d9..11ce76f19e 100644 --- a/3rdparty/libwebp/src/dec/buffer_dec.c +++ b/3rdparty/libwebp/src/dec/buffer_dec.c @@ -75,7 +75,7 @@ static VP8StatusCode CheckDecBuffer(const WebPDecBuffer* const buffer) { const WebPRGBABuffer* const buf = &buffer->u.RGBA; const int stride = abs(buf->stride); const uint64_t size = - MIN_BUFFER_SIZE(width * kModeBpp[mode], height, stride); + MIN_BUFFER_SIZE((uint64_t)width * kModeBpp[mode], height, stride); ok &= (size <= buf->size); ok &= (stride >= width * kModeBpp[mode]); ok &= (buf->rgba != NULL); @@ -102,7 +102,7 @@ static VP8StatusCode AllocateBuffer(WebPDecBuffer* const buffer) { int stride; uint64_t size; - if ((uint64_t)w * kModeBpp[mode] >= (1ull << 32)) { + if ((uint64_t)w * kModeBpp[mode] >= (1ull << 31)) { return VP8_STATUS_INVALID_PARAM; } stride = w * kModeBpp[mode]; @@ -117,7 +117,6 @@ static VP8StatusCode AllocateBuffer(WebPDecBuffer* const buffer) { } total_size = size + 2 * uv_size + a_size; - // Security/sanity checks output = (uint8_t*)WebPSafeMalloc(total_size, sizeof(*output)); if (output == NULL) { return VP8_STATUS_OUT_OF_MEMORY; @@ -156,11 +155,11 @@ VP8StatusCode WebPFlipBuffer(WebPDecBuffer* const buffer) { } if (WebPIsRGBMode(buffer->colorspace)) { WebPRGBABuffer* const buf = &buffer->u.RGBA; - buf->rgba += (buffer->height - 1) * buf->stride; + buf->rgba += (int64_t)(buffer->height - 1) * buf->stride; buf->stride = -buf->stride; } else { WebPYUVABuffer* const buf = &buffer->u.YUVA; - const int H = buffer->height; + const int64_t H = buffer->height; buf->y += (H - 1) * buf->y_stride; buf->y_stride = -buf->y_stride; buf->u += ((H - 1) >> 1) * buf->u_stride; @@ -188,8 +187,7 @@ VP8StatusCode WebPAllocateDecBuffer(int width, int height, const int ch = options->crop_height; const int x = options->crop_left & ~1; const int y = options->crop_top & ~1; - if (x < 0 || y < 0 || cw <= 0 || ch <= 0 || - x + cw > width || y + ch > height) { + if (!WebPCheckCropDimensions(width, height, x, y, cw, ch)) { return VP8_STATUS_INVALID_PARAM; // out of frame boundary. } width = cw; diff --git a/3rdparty/libwebp/src/dec/frame_dec.c b/3rdparty/libwebp/src/dec/frame_dec.c index 04609a8e56..91ca1f8609 100644 --- a/3rdparty/libwebp/src/dec/frame_dec.c +++ b/3rdparty/libwebp/src/dec/frame_dec.c @@ -705,7 +705,7 @@ static int AllocateMemory(VP8Decoder* const dec) { + cache_size + alpha_size + WEBP_ALIGN_CST; uint8_t* mem; - if (needed != (size_t)needed) return 0; // check for overflow + if (!CheckSizeOverflow(needed)) return 0; // check for overflow if (needed > dec->mem_size_) { WebPSafeFree(dec->mem_); dec->mem_size_ = 0; diff --git a/3rdparty/libwebp/src/dec/io_dec.c b/3rdparty/libwebp/src/dec/io_dec.c index 29dc6345df..5ef6298886 100644 --- a/3rdparty/libwebp/src/dec/io_dec.c +++ b/3rdparty/libwebp/src/dec/io_dec.c @@ -298,46 +298,57 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) { const int uv_out_height = (out_height + 1) >> 1; const int uv_in_width = (io->mb_w + 1) >> 1; const int uv_in_height = (io->mb_h + 1) >> 1; - const size_t work_size = 2 * out_width; // scratch memory for luma rescaler + // scratch memory for luma rescaler + const size_t work_size = 2 * (size_t)out_width; const size_t uv_work_size = 2 * uv_out_width; // and for each u/v ones - size_t tmp_size, rescaler_size; + uint64_t total_size; + size_t rescaler_size; rescaler_t* work; WebPRescaler* scalers; const int num_rescalers = has_alpha ? 4 : 3; - tmp_size = (work_size + 2 * uv_work_size) * sizeof(*work); + total_size = ((uint64_t)work_size + 2 * uv_work_size) * sizeof(*work); if (has_alpha) { - tmp_size += work_size * sizeof(*work); + total_size += (uint64_t)work_size * sizeof(*work); } rescaler_size = num_rescalers * sizeof(*p->scaler_y) + WEBP_ALIGN_CST; + total_size += rescaler_size; + if (!CheckSizeOverflow(total_size)) { + return 0; + } - p->memory = WebPSafeMalloc(1ULL, tmp_size + rescaler_size); + p->memory = WebPSafeMalloc(1ULL, (size_t)total_size); if (p->memory == NULL) { return 0; // memory error } work = (rescaler_t*)p->memory; - scalers = (WebPRescaler*)WEBP_ALIGN((const uint8_t*)work + tmp_size); + scalers = (WebPRescaler*)WEBP_ALIGN( + (const uint8_t*)work + total_size - rescaler_size); p->scaler_y = &scalers[0]; p->scaler_u = &scalers[1]; p->scaler_v = &scalers[2]; p->scaler_a = has_alpha ? &scalers[3] : NULL; - WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, - buf->y, out_width, out_height, buf->y_stride, 1, - work); - WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, - buf->u, uv_out_width, uv_out_height, buf->u_stride, 1, - work + work_size); - WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, - buf->v, uv_out_width, uv_out_height, buf->v_stride, 1, - work + work_size + uv_work_size); + if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, + buf->y, out_width, out_height, buf->y_stride, 1, + work) || + !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, + buf->u, uv_out_width, uv_out_height, buf->u_stride, 1, + work + work_size) || + !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, + buf->v, uv_out_width, uv_out_height, buf->v_stride, 1, + work + work_size + uv_work_size)) { + return 0; + } p->emit = EmitRescaledYUV; if (has_alpha) { - WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, - buf->a, out_width, out_height, buf->a_stride, 1, - work + work_size + 2 * uv_work_size); + if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, + buf->a, out_width, out_height, buf->a_stride, 1, + work + work_size + 2 * uv_work_size)) { + return 0; + } p->emit_alpha = EmitRescaledAlphaYUV; WebPInitAlphaProcessing(); } @@ -480,51 +491,58 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) { const int out_height = io->scaled_height; const int uv_in_width = (io->mb_w + 1) >> 1; const int uv_in_height = (io->mb_h + 1) >> 1; - const size_t work_size = 2 * out_width; // scratch memory for one rescaler + // scratch memory for one rescaler + const size_t work_size = 2 * (size_t)out_width; rescaler_t* work; // rescalers work area uint8_t* tmp; // tmp storage for scaled YUV444 samples before RGB conversion - size_t tmp_size1, tmp_size2, total_size, rescaler_size; + uint64_t tmp_size1, tmp_size2, total_size; + size_t rescaler_size; WebPRescaler* scalers; const int num_rescalers = has_alpha ? 4 : 3; - tmp_size1 = 3 * work_size; - tmp_size2 = 3 * out_width; - if (has_alpha) { - tmp_size1 += work_size; - tmp_size2 += out_width; - } + tmp_size1 = (uint64_t)num_rescalers * work_size; + tmp_size2 = (uint64_t)num_rescalers * out_width; total_size = tmp_size1 * sizeof(*work) + tmp_size2 * sizeof(*tmp); rescaler_size = num_rescalers * sizeof(*p->scaler_y) + WEBP_ALIGN_CST; + total_size += rescaler_size; + if (!CheckSizeOverflow(total_size)) { + return 0; + } - p->memory = WebPSafeMalloc(1ULL, total_size + rescaler_size); + p->memory = WebPSafeMalloc(1ULL, (size_t)total_size); if (p->memory == NULL) { return 0; // memory error } work = (rescaler_t*)p->memory; tmp = (uint8_t*)(work + tmp_size1); - scalers = (WebPRescaler*)WEBP_ALIGN((const uint8_t*)work + total_size); + scalers = (WebPRescaler*)WEBP_ALIGN( + (const uint8_t*)work + total_size - rescaler_size); p->scaler_y = &scalers[0]; p->scaler_u = &scalers[1]; p->scaler_v = &scalers[2]; p->scaler_a = has_alpha ? &scalers[3] : NULL; - WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, - tmp + 0 * out_width, out_width, out_height, 0, 1, - work + 0 * work_size); - WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, - tmp + 1 * out_width, out_width, out_height, 0, 1, - work + 1 * work_size); - WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, - tmp + 2 * out_width, out_width, out_height, 0, 1, - work + 2 * work_size); + if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, + tmp + 0 * out_width, out_width, out_height, 0, 1, + work + 0 * work_size) || + !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, + tmp + 1 * out_width, out_width, out_height, 0, 1, + work + 1 * work_size) || + !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, + tmp + 2 * out_width, out_width, out_height, 0, 1, + work + 2 * work_size)) { + return 0; + } p->emit = EmitRescaledRGB; WebPInitYUV444Converters(); if (has_alpha) { - WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, - tmp + 3 * out_width, out_width, out_height, 0, 1, - work + 3 * work_size); + if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, + tmp + 3 * out_width, out_width, out_height, 0, 1, + work + 3 * work_size)) { + return 0; + } p->emit_alpha = EmitRescaledAlphaRGB; if (p->output->colorspace == MODE_RGBA_4444 || p->output->colorspace == MODE_rgbA_4444) { diff --git a/3rdparty/libwebp/src/dec/tree_dec.c b/3rdparty/libwebp/src/dec/tree_dec.c index 1c6fdea27c..2434605953 100644 --- a/3rdparty/libwebp/src/dec/tree_dec.c +++ b/3rdparty/libwebp/src/dec/tree_dec.c @@ -12,10 +12,11 @@ // Author: Skal (pascal.massimino@gmail.com) #include "src/dec/vp8i_dec.h" +#include "src/dsp/cpu.h" #include "src/utils/bit_reader_inl_utils.h" #if !defined(USE_GENERIC_TREE) -#if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) +#if !defined(__arm__) && !defined(_M_ARM) && !WEBP_AARCH64 // using a table is ~1-2% slower on ARM. Prefer the coded-tree approach then. #define USE_GENERIC_TREE 1 // ALTERNATE_CODE #else diff --git a/3rdparty/libwebp/src/dec/vp8_dec.c b/3rdparty/libwebp/src/dec/vp8_dec.c index 8f73697478..20b92e84c4 100644 --- a/3rdparty/libwebp/src/dec/vp8_dec.c +++ b/3rdparty/libwebp/src/dec/vp8_dec.c @@ -335,7 +335,7 @@ int VP8GetHeaders(VP8Decoder* const dec, VP8Io* const io) { io->scaled_width = io->width; io->scaled_height = io->height; - io->mb_w = io->width; // sanity check + io->mb_w = io->width; // for soundness io->mb_h = io->height; // ditto VP8ResetProba(&dec->proba_); @@ -403,7 +403,7 @@ static const uint8_t kZigzag[16] = { 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 }; -// See section 13-2: http://tools.ietf.org/html/rfc6386#section-13.2 +// See section 13-2: https://datatracker.ietf.org/doc/html/rfc6386#section-13.2 static int GetLargeValue(VP8BitReader* const br, const uint8_t* const p) { int v; if (!VP8GetBit(br, p[3], "coeffs")) { @@ -494,6 +494,8 @@ static int GetCoeffsAlt(VP8BitReader* const br, return 16; } +extern VP8CPUInfo VP8GetCPUInfo; + WEBP_DSP_INIT_FUNC(InitGetCoeffs) { if (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kSlowSSSE3)) { GetCoeffs = GetCoeffsAlt; diff --git a/3rdparty/libwebp/src/dec/vp8i_dec.h b/3rdparty/libwebp/src/dec/vp8i_dec.h index a0c0af1579..1ae4ff62f2 100644 --- a/3rdparty/libwebp/src/dec/vp8i_dec.h +++ b/3rdparty/libwebp/src/dec/vp8i_dec.h @@ -31,8 +31,8 @@ extern "C" { // version numbers #define DEC_MAJ_VERSION 1 -#define DEC_MIN_VERSION 2 -#define DEC_REV_VERSION 0 +#define DEC_MIN_VERSION 3 +#define DEC_REV_VERSION 1 // YUV-cache parameters. Cache is 32-bytes wide (= one cacheline). // Constraints are: We need to store one 16x16 block of luma samples (y), diff --git a/3rdparty/libwebp/src/dec/vp8l_dec.c b/3rdparty/libwebp/src/dec/vp8l_dec.c index 2d603b4379..11c00ea964 100644 --- a/3rdparty/libwebp/src/dec/vp8l_dec.c +++ b/3rdparty/libwebp/src/dec/vp8l_dec.c @@ -12,6 +12,7 @@ // Authors: Vikas Arora (vikaas.arora@gmail.com) // Jyrki Alakuijala (jyrki@google.com) +#include #include #include "src/dec/alphai_dec.h" @@ -84,7 +85,7 @@ static const uint8_t kCodeToPlane[CODE_TO_PLANE_CODES] = { // to 256 (green component values) + 24 (length prefix values) // + color_cache_size (between 0 and 2048). // All values computed for 8-bit first level lookup with Mark Adler's tool: -// http://www.hdfgroup.org/ftp/lib-external/zlib/zlib-1.2.5/examples/enough.c +// https://github.com/madler/zlib/blob/v1.2.5/examples/enough.c #define FIXED_TABLE_SIZE (630 * 3 + 410) static const uint16_t kTableSize[12] = { FIXED_TABLE_SIZE + 654, @@ -101,6 +102,14 @@ static const uint16_t kTableSize[12] = { FIXED_TABLE_SIZE + 2704 }; +static int VP8LSetError(VP8LDecoder* const dec, VP8StatusCode error) { + // The oldest error reported takes precedence over the new one. + if (dec->status_ == VP8_STATUS_OK || dec->status_ == VP8_STATUS_SUSPENDED) { + dec->status_ = error; + } + return 0; +} + static int DecodeImageStream(int xsize, int ysize, int is_level0, VP8LDecoder* const dec, @@ -178,7 +187,7 @@ static WEBP_INLINE int PlaneCodeToDistance(int xsize, int plane_code) { //------------------------------------------------------------------------------ // Decodes the next Huffman code from bit-stream. -// FillBitWindow(br) needs to be called at minimum every second call +// VP8LFillBitWindow(br) needs to be called at minimum every second call // to ReadSymbol, in order to pre-fetch enough bits. static WEBP_INLINE int ReadSymbol(const HuffmanCode* table, VP8LBitReader* const br) { @@ -253,11 +262,11 @@ static int ReadHuffmanCodeLengths( int symbol; int max_symbol; int prev_code_len = DEFAULT_CODE_LENGTH; - HuffmanCode table[1 << LENGTHS_TABLE_BITS]; + HuffmanTables tables; - if (!VP8LBuildHuffmanTable(table, LENGTHS_TABLE_BITS, - code_length_code_lengths, - NUM_CODE_LENGTH_CODES)) { + if (!VP8LHuffmanTablesAllocate(1 << LENGTHS_TABLE_BITS, &tables) || + !VP8LBuildHuffmanTable(&tables, LENGTHS_TABLE_BITS, + code_length_code_lengths, NUM_CODE_LENGTH_CODES)) { goto End; } @@ -277,7 +286,7 @@ static int ReadHuffmanCodeLengths( int code_len; if (max_symbol-- == 0) break; VP8LFillBitWindow(br); - p = &table[VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK]; + p = &tables.curr_segment->start[VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK]; VP8LSetBitPos(br, br->bit_pos_ + p->bits); code_len = p->value; if (code_len < kCodeLengthLiterals) { @@ -300,14 +309,16 @@ static int ReadHuffmanCodeLengths( ok = 1; End: - if (!ok) dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LHuffmanTablesDeallocate(&tables); + if (!ok) return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); return ok; } // 'code_lengths' is pre-allocated temporary buffer, used for creating Huffman // tree. static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, - int* const code_lengths, HuffmanCode* const table) { + int* const code_lengths, + HuffmanTables* const table) { int ok = 0; int size = 0; VP8LBitReader* const br = &dec->br_; @@ -321,7 +332,7 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, // The first code is either 1 bit or 8 bit code. int symbol = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8); code_lengths[symbol] = 1; - // The second code (if present), is always 8 bit long. + // The second code (if present), is always 8 bits long. if (num_symbols == 2) { symbol = VP8LReadBits(br, 8); code_lengths[symbol] = 1; @@ -331,10 +342,7 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, int i; int code_length_code_lengths[NUM_CODE_LENGTH_CODES] = { 0 }; const int num_codes = VP8LReadBits(br, 4) + 4; - if (num_codes > NUM_CODE_LENGTH_CODES) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; - } + assert(num_codes <= NUM_CODE_LENGTH_CODES); for (i = 0; i < num_codes; ++i) { code_length_code_lengths[kCodeLengthCodeOrder[i]] = VP8LReadBits(br, 3); @@ -349,36 +357,35 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, code_lengths, alphabet_size); } if (!ok || size == 0) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; + return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); } return size; } static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, int color_cache_bits, int allow_recursion) { - int i, j; + int i; VP8LBitReader* const br = &dec->br_; VP8LMetadata* const hdr = &dec->hdr_; uint32_t* huffman_image = NULL; HTreeGroup* htree_groups = NULL; - HuffmanCode* huffman_tables = NULL; - HuffmanCode* huffman_table = NULL; + HuffmanTables* huffman_tables = &hdr->huffman_tables_; int num_htree_groups = 1; int num_htree_groups_max = 1; - int max_alphabet_size = 0; - int* code_lengths = NULL; - const int table_size = kTableSize[color_cache_bits]; int* mapping = NULL; int ok = 0; + // Check the table has been 0 initialized (through InitMetadata). + assert(huffman_tables->root.start == NULL); + assert(huffman_tables->curr_segment == NULL); + if (allow_recursion && VP8LReadBits(br, 1)) { // use meta Huffman codes. const int huffman_precision = VP8LReadBits(br, 3) + 2; const int huffman_xsize = VP8LSubSampleSize(xsize, huffman_precision); const int huffman_ysize = VP8LSubSampleSize(ysize, huffman_precision); const int huffman_pixs = huffman_xsize * huffman_ysize; - if (!DecodeImageStream(huffman_xsize, huffman_ysize, 0, dec, + if (!DecodeImageStream(huffman_xsize, huffman_ysize, /*is_level0=*/0, dec, &huffman_image)) { goto Error; } @@ -402,7 +409,7 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, // values [0, num_htree_groups) mapping = (int*)WebPSafeMalloc(num_htree_groups_max, sizeof(*mapping)); if (mapping == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Error; } // -1 means a value is unmapped, and therefore unused in the Huffman @@ -421,29 +428,55 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, if (br->eos_) goto Error; - // Find maximum alphabet size for the htree group. - for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { - int alphabet_size = kAlphabetSize[j]; - if (j == 0 && color_cache_bits > 0) { - alphabet_size += 1 << color_cache_bits; - } - if (max_alphabet_size < alphabet_size) { - max_alphabet_size = alphabet_size; - } + if (!ReadHuffmanCodesHelper(color_cache_bits, num_htree_groups, + num_htree_groups_max, mapping, dec, + huffman_tables, &htree_groups)) { + goto Error; } + ok = 1; - code_lengths = (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, - sizeof(*code_lengths)); - huffman_tables = (HuffmanCode*)WebPSafeMalloc(num_htree_groups * table_size, - sizeof(*huffman_tables)); - htree_groups = VP8LHtreeGroupsNew(num_htree_groups); + // All OK. Finalize pointers. + hdr->huffman_image_ = huffman_image; + hdr->num_htree_groups_ = num_htree_groups; + hdr->htree_groups_ = htree_groups; - if (htree_groups == NULL || code_lengths == NULL || huffman_tables == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + Error: + WebPSafeFree(mapping); + if (!ok) { + WebPSafeFree(huffman_image); + VP8LHuffmanTablesDeallocate(huffman_tables); + VP8LHtreeGroupsFree(htree_groups); + } + return ok; +} + +int ReadHuffmanCodesHelper(int color_cache_bits, int num_htree_groups, + int num_htree_groups_max, const int* const mapping, + VP8LDecoder* const dec, + HuffmanTables* const huffman_tables, + HTreeGroup** const htree_groups) { + int i, j, ok = 0; + const int max_alphabet_size = + kAlphabetSize[0] + ((color_cache_bits > 0) ? 1 << color_cache_bits : 0); + const int table_size = kTableSize[color_cache_bits]; + int* code_lengths = NULL; + + if ((mapping == NULL && num_htree_groups != num_htree_groups_max) || + num_htree_groups > num_htree_groups_max) { + goto Error; + } + + code_lengths = + (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, sizeof(*code_lengths)); + *htree_groups = VP8LHtreeGroupsNew(num_htree_groups); + + if (*htree_groups == NULL || code_lengths == NULL || + !VP8LHuffmanTablesAllocate(num_htree_groups * table_size, + huffman_tables)) { + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Error; } - huffman_table = huffman_tables; for (i = 0; i < num_htree_groups_max; ++i) { // If the index "i" is unused in the Huffman image, just make sure the // coefficients are valid but do not store them. @@ -460,7 +493,7 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, } } else { HTreeGroup* const htree_group = - &htree_groups[(mapping == NULL) ? i : mapping[i]]; + &(*htree_groups)[(mapping == NULL) ? i : mapping[i]]; HuffmanCode** const htrees = htree_group->htrees; int size; int total_size = 0; @@ -468,19 +501,20 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, int max_bits = 0; for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { int alphabet_size = kAlphabetSize[j]; - htrees[j] = huffman_table; if (j == 0 && color_cache_bits > 0) { alphabet_size += (1 << color_cache_bits); } - size = ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_table); + size = + ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_tables); + htrees[j] = huffman_tables->curr_segment->curr_table; if (size == 0) { goto Error; } if (is_trivial_literal && kLiteralMap[j] == 1) { - is_trivial_literal = (huffman_table->bits == 0); + is_trivial_literal = (htrees[j]->bits == 0); } - total_size += huffman_table->bits; - huffman_table += size; + total_size += htrees[j]->bits; + huffman_tables->curr_segment->curr_table += size; if (j <= ALPHA) { int local_max_bits = code_lengths[0]; int k; @@ -511,19 +545,12 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, } ok = 1; - // All OK. Finalize pointers. - hdr->huffman_image_ = huffman_image; - hdr->num_htree_groups_ = num_htree_groups; - hdr->htree_groups_ = htree_groups; - hdr->huffman_tables_ = huffman_tables; - Error: WebPSafeFree(code_lengths); - WebPSafeFree(mapping); if (!ok) { - WebPSafeFree(huffman_image); - WebPSafeFree(huffman_tables); - VP8LHtreeGroupsFree(htree_groups); + VP8LHuffmanTablesDeallocate(huffman_tables); + VP8LHtreeGroupsFree(*htree_groups); + *htree_groups = NULL; } return ok; } @@ -547,8 +574,7 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { scaled_data_size * sizeof(*scaled_data); uint8_t* memory = (uint8_t*)WebPSafeMalloc(memory_size, sizeof(*memory)); if (memory == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } assert(dec->rescaler_memory == NULL); dec->rescaler_memory = memory; @@ -559,8 +585,11 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { memory += work_size * sizeof(*work); scaled_data = (uint32_t*)memory; - WebPRescalerInit(dec->rescaler, in_width, in_height, (uint8_t*)scaled_data, - out_width, out_height, 0, num_channels, work); + if (!WebPRescalerInit(dec->rescaler, in_width, in_height, + (uint8_t*)scaled_data, out_width, out_height, + 0, num_channels, work)) { + return 0; + } return 1; } #endif // WEBP_REDUCE_SIZE @@ -574,13 +603,14 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { static int Export(WebPRescaler* const rescaler, WEBP_CSP_MODE colorspace, int rgba_stride, uint8_t* const rgba) { uint32_t* const src = (uint32_t*)rescaler->dst; + uint8_t* dst = rgba; const int dst_width = rescaler->dst_width; int num_lines_out = 0; while (WebPRescalerHasPendingOutput(rescaler)) { - uint8_t* const dst = rgba + num_lines_out * rgba_stride; WebPRescalerExportRow(rescaler); WebPMultARGBRow(src, dst_width, 1); VP8LConvertFromBGRA(src, dst_width, colorspace, dst); + dst += rgba_stride; ++num_lines_out; } return num_lines_out; @@ -594,8 +624,8 @@ static int EmitRescaledRowsRGBA(const VP8LDecoder* const dec, int num_lines_in = 0; int num_lines_out = 0; while (num_lines_in < mb_h) { - uint8_t* const row_in = in + num_lines_in * in_stride; - uint8_t* const row_out = out + num_lines_out * out_stride; + uint8_t* const row_in = in + (uint64_t)num_lines_in * in_stride; + uint8_t* const row_out = out + (uint64_t)num_lines_out * out_stride; const int lines_left = mb_h - num_lines_in; const int needed_lines = WebPRescaleNeededLines(dec->rescaler, lines_left); int lines_imported; @@ -796,7 +826,8 @@ static void ProcessRows(VP8LDecoder* const dec, int row) { const WebPDecBuffer* const output = dec->output_; if (WebPIsRGBMode(output->colorspace)) { // convert to RGBA const WebPRGBABuffer* const buf = &output->u.RGBA; - uint8_t* const rgba = buf->rgba + dec->last_out_row_ * buf->stride; + uint8_t* const rgba = + buf->rgba + (int64_t)dec->last_out_row_ * buf->stride; const int num_rows_out = #if !defined(WEBP_REDUCE_SIZE) io->use_scaling ? @@ -1077,12 +1108,10 @@ static int DecodeAlphaData(VP8LDecoder* const dec, uint8_t* const data, End: br->eos_ = VP8LIsEndOfStream(br); if (!ok || (br->eos_ && pos < end)) { - ok = 0; - dec->status_ = br->eos_ ? VP8_STATUS_SUSPENDED - : VP8_STATUS_BITSTREAM_ERROR; - } else { - dec->last_pixel_ = pos; + return VP8LSetError( + dec, br->eos_ ? VP8_STATUS_SUSPENDED : VP8_STATUS_BITSTREAM_ERROR); } + dec->last_pixel_ = pos; return ok; } @@ -1232,9 +1261,20 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data, } br->eos_ = VP8LIsEndOfStream(br); - if (dec->incremental_ && br->eos_ && src < src_end) { + // In incremental decoding: + // br->eos_ && src < src_last: if 'br' reached the end of the buffer and + // 'src_last' has not been reached yet, there is not enough data. 'dec' has to + // be reset until there is more data. + // !br->eos_ && src < src_last: this cannot happen as either the buffer is + // fully read, either enough has been read to reach 'src_last'. + // src >= src_last: 'src_last' is reached, all is fine. 'src' can actually go + // beyond 'src_last' in case the image is cropped and an LZ77 goes further. + // The buffer might have been enough or there is some left. 'br->eos_' does + // not matter. + assert(!dec->incremental_ || (br->eos_ && src < src_last) || src >= src_last); + if (dec->incremental_ && br->eos_ && src < src_last) { RestoreState(dec); - } else if (!br->eos_) { + } else if ((dec->incremental_ && src >= src_last) || !br->eos_) { // Process the remaining rows corresponding to last row-block. if (process_func != NULL) { process_func(dec, row > last_row ? last_row : row); @@ -1249,8 +1289,7 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data, return 1; Error: - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; + return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); } // ----------------------------------------------------------------------------- @@ -1276,7 +1315,7 @@ static int ExpandColorMap(int num_colors, VP8LTransform* const transform) { uint8_t* const new_data = (uint8_t*)new_color_map; new_color_map[0] = transform->data_[0]; for (i = 4; i < 4 * num_colors; ++i) { - // Equivalent to AddPixelEq(), on a byte-basis. + // Equivalent to VP8LAddPixels(), on a byte-basis. new_data[i] = (data[i] + new_data[i - 4]) & 0xff; } for (; i < 4 * final_num_colors; ++i) { @@ -1317,7 +1356,7 @@ static int ReadTransform(int* const xsize, int const* ysize, transform->bits_), VP8LSubSampleSize(transform->ysize_, transform->bits_), - 0, dec, &transform->data_); + /*is_level0=*/0, dec, &transform->data_); break; case COLOR_INDEXING_TRANSFORM: { const int num_colors = VP8LReadBits(br, 8) + 1; @@ -1327,11 +1366,14 @@ static int ReadTransform(int* const xsize, int const* ysize, : 3; *xsize = VP8LSubSampleSize(transform->xsize_, bits); transform->bits_ = bits; - ok = DecodeImageStream(num_colors, 1, 0, dec, &transform->data_); - ok = ok && ExpandColorMap(num_colors, transform); + ok = DecodeImageStream(num_colors, /*ysize=*/1, /*is_level0=*/0, dec, + &transform->data_); + if (ok && !ExpandColorMap(num_colors, transform)) { + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); + } break; } - case SUBTRACT_GREEN: + case SUBTRACT_GREEN_TRANSFORM: break; default: assert(0); // can't happen @@ -1353,7 +1395,7 @@ static void ClearMetadata(VP8LMetadata* const hdr) { assert(hdr != NULL); WebPSafeFree(hdr->huffman_image_); - WebPSafeFree(hdr->huffman_tables_); + VP8LHuffmanTablesDeallocate(&hdr->huffman_tables_); VP8LHtreeGroupsFree(hdr->htree_groups_); VP8LColorCacheClear(&hdr->color_cache_); VP8LColorCacheClear(&hdr->saved_color_cache_); @@ -1434,7 +1476,7 @@ static int DecodeImageStream(int xsize, int ysize, color_cache_bits = VP8LReadBits(br, 4); ok = (color_cache_bits >= 1 && color_cache_bits <= MAX_CACHE_BITS); if (!ok) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto End; } } @@ -1443,7 +1485,7 @@ static int DecodeImageStream(int xsize, int ysize, ok = ok && ReadHuffmanCodes(dec, transform_xsize, transform_ysize, color_cache_bits, is_level0); if (!ok) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto End; } @@ -1451,8 +1493,7 @@ static int DecodeImageStream(int xsize, int ysize, if (color_cache_bits > 0) { hdr->color_cache_size_ = 1 << color_cache_bits; if (!VP8LColorCacheInit(&hdr->color_cache_, color_cache_bits)) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - ok = 0; + ok = VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto End; } } else { @@ -1469,8 +1510,7 @@ static int DecodeImageStream(int xsize, int ysize, const uint64_t total_size = (uint64_t)transform_xsize * transform_ysize; data = (uint32_t*)WebPSafeMalloc(total_size, sizeof(*data)); if (data == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - ok = 0; + ok = VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto End; } } @@ -1514,9 +1554,8 @@ static int AllocateInternalBuffers32b(VP8LDecoder* const dec, int final_width) { assert(dec->width_ <= final_width); dec->pixels_ = (uint32_t*)WebPSafeMalloc(total_num_pixels, sizeof(uint32_t)); if (dec->pixels_ == NULL) { - dec->argb_cache_ = NULL; // for sanity check - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + dec->argb_cache_ = NULL; // for soundness + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } dec->argb_cache_ = dec->pixels_ + num_pixels + cache_top_pixels; return 1; @@ -1524,11 +1563,10 @@ static int AllocateInternalBuffers32b(VP8LDecoder* const dec, int final_width) { static int AllocateInternalBuffers8b(VP8LDecoder* const dec) { const uint64_t total_num_pixels = (uint64_t)dec->width_ * dec->height_; - dec->argb_cache_ = NULL; // for sanity check + dec->argb_cache_ = NULL; // for soundness dec->pixels_ = (uint32_t*)WebPSafeMalloc(total_num_pixels, sizeof(uint8_t)); if (dec->pixels_ == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } return 1; } @@ -1583,7 +1621,8 @@ int VP8LDecodeAlphaHeader(ALPHDecoder* const alph_dec, dec->status_ = VP8_STATUS_OK; VP8LInitBitReader(&dec->br_, data, data_size); - if (!DecodeImageStream(alph_dec->width_, alph_dec->height_, 1, dec, NULL)) { + if (!DecodeImageStream(alph_dec->width_, alph_dec->height_, /*is_level0=*/1, + dec, /*decoded_data=*/NULL)) { goto Err; } @@ -1638,22 +1677,24 @@ int VP8LDecodeHeader(VP8LDecoder* const dec, VP8Io* const io) { if (dec == NULL) return 0; if (io == NULL) { - dec->status_ = VP8_STATUS_INVALID_PARAM; - return 0; + return VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); } dec->io_ = io; dec->status_ = VP8_STATUS_OK; VP8LInitBitReader(&dec->br_, io->data, io->data_size); if (!ReadImageInfo(&dec->br_, &width, &height, &has_alpha)) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto Error; } dec->state_ = READ_DIM; io->width = width; io->height = height; - if (!DecodeImageStream(width, height, 1, dec, NULL)) goto Error; + if (!DecodeImageStream(width, height, /*is_level0=*/1, dec, + /*decoded_data=*/NULL)) { + goto Error; + } return 1; Error: @@ -1666,10 +1707,9 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { VP8Io* io = NULL; WebPDecParams* params = NULL; - // Sanity checks. if (dec == NULL) return 0; - assert(dec->hdr_.huffman_tables_ != NULL); + assert(dec->hdr_.huffman_tables_.root.start != NULL); assert(dec->hdr_.htree_groups_ != NULL); assert(dec->hdr_.num_htree_groups_ > 0); @@ -1684,7 +1724,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { assert(dec->output_ != NULL); if (!WebPIoInitFromOptions(params->options, io, MODE_BGRA)) { - dec->status_ = VP8_STATUS_INVALID_PARAM; + VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); goto Err; } @@ -1694,7 +1734,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { if (io->use_scaling && !AllocateAndInitRescaler(dec, io)) goto Err; #else if (io->use_scaling) { - dec->status_ = VP8_STATUS_INVALID_PARAM; + VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); goto Err; } #endif @@ -1712,7 +1752,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { dec->hdr_.saved_color_cache_.colors_ == NULL) { if (!VP8LColorCacheInit(&dec->hdr_.saved_color_cache_, dec->hdr_.color_cache_.hash_bits_)) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Err; } } diff --git a/3rdparty/libwebp/src/dec/vp8li_dec.h b/3rdparty/libwebp/src/dec/vp8li_dec.h index 72b2e86120..b057573f6c 100644 --- a/3rdparty/libwebp/src/dec/vp8li_dec.h +++ b/3rdparty/libwebp/src/dec/vp8li_dec.h @@ -51,7 +51,7 @@ typedef struct { uint32_t* huffman_image_; int num_htree_groups_; HTreeGroup* htree_groups_; - HuffmanCode* huffman_tables_; + HuffmanTables huffman_tables_; } VP8LMetadata; typedef struct VP8LDecoder VP8LDecoder; @@ -126,6 +126,19 @@ void VP8LClear(VP8LDecoder* const dec); // Clears and deallocate a lossless decoder instance. void VP8LDelete(VP8LDecoder* const dec); +// Helper function for reading the different Huffman codes and storing them in +// 'huffman_tables' and 'htree_groups'. +// If mapping is NULL 'num_htree_groups_max' must equal 'num_htree_groups'. +// If it is not NULL, it maps 'num_htree_groups_max' indices to the +// 'num_htree_groups' groups. If 'num_htree_groups_max' > 'num_htree_groups', +// some of those indices map to -1. This is used for non-balanced codes to +// limit memory usage. +int ReadHuffmanCodesHelper(int color_cache_bits, int num_htree_groups, + int num_htree_groups_max, const int* const mapping, + VP8LDecoder* const dec, + HuffmanTables* const huffman_tables, + HTreeGroup** const htree_groups); + //------------------------------------------------------------------------------ #ifdef __cplusplus diff --git a/3rdparty/libwebp/src/dec/webp_dec.c b/3rdparty/libwebp/src/dec/webp_dec.c index 42d098874d..f557868b99 100644 --- a/3rdparty/libwebp/src/dec/webp_dec.c +++ b/3rdparty/libwebp/src/dec/webp_dec.c @@ -179,7 +179,7 @@ static VP8StatusCode ParseOptionalChunks(const uint8_t** const data, return VP8_STATUS_BITSTREAM_ERROR; // Not a valid chunk size. } // For odd-sized chunk-payload, there's one byte padding at the end. - disk_chunk_size = (CHUNK_HEADER_SIZE + chunk_size + 1) & ~1; + disk_chunk_size = (CHUNK_HEADER_SIZE + chunk_size + 1) & ~1u; total_size += disk_chunk_size; // Check that total bytes skipped so far does not exceed riff_size. @@ -658,19 +658,26 @@ uint8_t* WebPDecodeBGRA(const uint8_t* data, size_t data_size, uint8_t* WebPDecodeYUV(const uint8_t* data, size_t data_size, int* width, int* height, uint8_t** u, uint8_t** v, int* stride, int* uv_stride) { - WebPDecBuffer output; // only to preserve the side-infos - uint8_t* const out = Decode(MODE_YUV, data, data_size, - width, height, &output); - - if (out != NULL) { - const WebPYUVABuffer* const buf = &output.u.YUVA; - *u = buf->u; - *v = buf->v; - *stride = buf->y_stride; - *uv_stride = buf->u_stride; - assert(buf->u_stride == buf->v_stride); + // data, width and height are checked by Decode(). + if (u == NULL || v == NULL || stride == NULL || uv_stride == NULL) { + return NULL; + } + + { + WebPDecBuffer output; // only to preserve the side-infos + uint8_t* const out = Decode(MODE_YUV, data, data_size, + width, height, &output); + + if (out != NULL) { + const WebPYUVABuffer* const buf = &output.u.YUVA; + *u = buf->u; + *v = buf->v; + *stride = buf->y_stride; + *uv_stride = buf->u_stride; + assert(buf->u_stride == buf->v_stride); + } + return out; } - return out; } static void DefaultFeatures(WebPBitstreamFeatures* const features) { @@ -785,6 +792,13 @@ VP8StatusCode WebPDecode(const uint8_t* data, size_t data_size, //------------------------------------------------------------------------------ // Cropping and rescaling. +int WebPCheckCropDimensions(int image_width, int image_height, + int x, int y, int w, int h) { + return !(x < 0 || y < 0 || w <= 0 || h <= 0 || + x >= image_width || w > image_width || w > image_width - x || + y >= image_height || h > image_height || h > image_height - y); +} + int WebPIoInitFromOptions(const WebPDecoderOptions* const options, VP8Io* const io, WEBP_CSP_MODE src_colorspace) { const int W = io->width; @@ -792,7 +806,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, int x = 0, y = 0, w = W, h = H; // Cropping - io->use_cropping = (options != NULL) && (options->use_cropping > 0); + io->use_cropping = (options != NULL) && options->use_cropping; if (io->use_cropping) { w = options->crop_width; h = options->crop_height; @@ -802,7 +816,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, x &= ~1; y &= ~1; } - if (x < 0 || y < 0 || w <= 0 || h <= 0 || x + w > W || y + h > H) { + if (!WebPCheckCropDimensions(W, H, x, y, w, h)) { return 0; // out of frame boundary error } } @@ -814,7 +828,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, io->mb_h = h; // Scaling - io->use_scaling = (options != NULL) && (options->use_scaling > 0); + io->use_scaling = (options != NULL) && options->use_scaling; if (io->use_scaling) { int scaled_width = options->scaled_width; int scaled_height = options->scaled_height; @@ -835,8 +849,8 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, if (io->use_scaling) { // disable filter (only for large downscaling ratio). - io->bypass_filtering = (io->scaled_width < W * 3 / 4) && - (io->scaled_height < H * 3 / 4); + io->bypass_filtering |= (io->scaled_width < W * 3 / 4) && + (io->scaled_height < H * 3 / 4); io->fancy_upsampling = 0; } return 1; diff --git a/3rdparty/libwebp/src/dec/webpi_dec.h b/3rdparty/libwebp/src/dec/webpi_dec.h index 24baff5d27..3b97388c71 100644 --- a/3rdparty/libwebp/src/dec/webpi_dec.h +++ b/3rdparty/libwebp/src/dec/webpi_dec.h @@ -77,6 +77,10 @@ VP8StatusCode WebPParseHeaders(WebPHeaderStructure* const headers); //------------------------------------------------------------------------------ // Misc utils +// Returns true if crop dimensions are within image bounds. +int WebPCheckCropDimensions(int image_width, int image_height, + int x, int y, int w, int h); + // Initializes VP8Io with custom setup, io and teardown functions. The default // hooks will use the supplied 'params' as io->opaque handle. void WebPInitCustomIo(WebPDecParams* const params, VP8Io* const io); diff --git a/3rdparty/libwebp/src/demux/anim_decode.c b/3rdparty/libwebp/src/demux/anim_decode.c index 3dcacc35d6..e077ffb536 100644 --- a/3rdparty/libwebp/src/demux/anim_decode.c +++ b/3rdparty/libwebp/src/demux/anim_decode.c @@ -23,6 +23,14 @@ #define NUM_CHANNELS 4 +// Channel extraction from a uint32_t representation of a uint8_t RGBA/BGRA +// buffer. +#ifdef WORDS_BIGENDIAN +#define CHANNEL_SHIFT(i) (24 - (i) * 8) +#else +#define CHANNEL_SHIFT(i) ((i) * 8) +#endif + typedef void (*BlendRowFunc)(uint32_t* const, const uint32_t* const, int); static void BlendPixelRowNonPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels); @@ -87,11 +95,19 @@ WebPAnimDecoder* WebPAnimDecoderNewInternal( int abi_version) { WebPAnimDecoderOptions options; WebPAnimDecoder* dec = NULL; + WebPBitstreamFeatures features; if (webp_data == NULL || WEBP_ABI_IS_INCOMPATIBLE(abi_version, WEBP_DEMUX_ABI_VERSION)) { return NULL; } + // Validate the bitstream before doing expensive allocations. The demuxer may + // be more tolerant than the decoder. + if (WebPGetFeatures(webp_data->bytes, webp_data->size, &features) != + VP8_STATUS_OK) { + return NULL; + } + // Note: calloc() so that the pointer members are initialized to NULL. dec = (WebPAnimDecoder*)WebPSafeCalloc(1ULL, sizeof(*dec)); if (dec == NULL) goto Error; @@ -145,7 +161,7 @@ static int ZeroFillCanvas(uint8_t* buf, uint32_t canvas_width, uint32_t canvas_height) { const uint64_t size = (uint64_t)canvas_width * canvas_height * NUM_CHANNELS * sizeof(*buf); - if (size != (size_t)size) return 0; + if (!CheckSizeOverflow(size)) return 0; memset(buf, 0, (size_t)size); return 1; } @@ -166,7 +182,7 @@ static void ZeroFillFrameRect(uint8_t* buf, int buf_stride, int x_offset, static int CopyCanvas(const uint8_t* src, uint8_t* dst, uint32_t width, uint32_t height) { const uint64_t size = (uint64_t)width * height * NUM_CHANNELS; - if (size != (size_t)size) return 0; + if (!CheckSizeOverflow(size)) return 0; assert(src != NULL && dst != NULL); memcpy(dst, src, (size_t)size); return 1; @@ -201,35 +217,35 @@ static uint8_t BlendChannelNonPremult(uint32_t src, uint8_t src_a, const uint8_t dst_channel = (dst >> shift) & 0xff; const uint32_t blend_unscaled = src_channel * src_a + dst_channel * dst_a; assert(blend_unscaled < (1ULL << 32) / scale); - return (blend_unscaled * scale) >> 24; + return (blend_unscaled * scale) >> CHANNEL_SHIFT(3); } // Blend 'src' over 'dst' assuming they are NOT pre-multiplied by alpha. static uint32_t BlendPixelNonPremult(uint32_t src, uint32_t dst) { - const uint8_t src_a = (src >> 24) & 0xff; + const uint8_t src_a = (src >> CHANNEL_SHIFT(3)) & 0xff; if (src_a == 0) { return dst; } else { - const uint8_t dst_a = (dst >> 24) & 0xff; + const uint8_t dst_a = (dst >> CHANNEL_SHIFT(3)) & 0xff; // This is the approximate integer arithmetic for the actual formula: // dst_factor_a = (dst_a * (255 - src_a)) / 255. const uint8_t dst_factor_a = (dst_a * (256 - src_a)) >> 8; const uint8_t blend_a = src_a + dst_factor_a; const uint32_t scale = (1UL << 24) / blend_a; - const uint8_t blend_r = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 0); - const uint8_t blend_g = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 8); - const uint8_t blend_b = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 16); + const uint8_t blend_r = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(0)); + const uint8_t blend_g = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(1)); + const uint8_t blend_b = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(2)); assert(src_a + dst_factor_a < 256); - return (blend_r << 0) | - (blend_g << 8) | - (blend_b << 16) | - ((uint32_t)blend_a << 24); + return ((uint32_t)blend_r << CHANNEL_SHIFT(0)) | + ((uint32_t)blend_g << CHANNEL_SHIFT(1)) | + ((uint32_t)blend_b << CHANNEL_SHIFT(2)) | + ((uint32_t)blend_a << CHANNEL_SHIFT(3)); } } @@ -239,7 +255,7 @@ static void BlendPixelRowNonPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const uint8_t src_alpha = (src[i] >> 24) & 0xff; + const uint8_t src_alpha = (src[i] >> CHANNEL_SHIFT(3)) & 0xff; if (src_alpha != 0xff) { src[i] = BlendPixelNonPremult(src[i], dst[i]); } @@ -256,7 +272,7 @@ static WEBP_INLINE uint32_t ChannelwiseMultiply(uint32_t pix, uint32_t scale) { // Blend 'src' over 'dst' assuming they are pre-multiplied by alpha. static uint32_t BlendPixelPremult(uint32_t src, uint32_t dst) { - const uint8_t src_a = (src >> 24) & 0xff; + const uint8_t src_a = (src >> CHANNEL_SHIFT(3)) & 0xff; return src + ChannelwiseMultiply(dst, 256 - src_a); } @@ -266,7 +282,7 @@ static void BlendPixelRowPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const uint8_t src_alpha = (src[i] >> 24) & 0xff; + const uint8_t src_alpha = (src[i] >> CHANNEL_SHIFT(3)) & 0xff; if (src_alpha != 0xff) { src[i] = BlendPixelPremult(src[i], dst[i]); } diff --git a/3rdparty/libwebp/src/demux/demux.c b/3rdparty/libwebp/src/demux/demux.c index 860e2ce761..fd45a2500e 100644 --- a/3rdparty/libwebp/src/demux/demux.c +++ b/3rdparty/libwebp/src/demux/demux.c @@ -24,8 +24,8 @@ #include "src/webp/format_constants.h" #define DMUX_MAJ_VERSION 1 -#define DMUX_MIN_VERSION 2 -#define DMUX_REV_VERSION 0 +#define DMUX_MIN_VERSION 3 +#define DMUX_REV_VERSION 1 typedef struct { size_t start_; // start location of the data @@ -221,12 +221,16 @@ static ParseStatus StoreFrame(int frame_num, uint32_t min_size, const size_t chunk_start_offset = mem->start_; const uint32_t fourcc = ReadLE32(mem); const uint32_t payload_size = ReadLE32(mem); - const uint32_t payload_size_padded = payload_size + (payload_size & 1); - const size_t payload_available = (payload_size_padded > MemDataSize(mem)) - ? MemDataSize(mem) : payload_size_padded; - const size_t chunk_size = CHUNK_HEADER_SIZE + payload_available; + uint32_t payload_size_padded; + size_t payload_available; + size_t chunk_size; if (payload_size > MAX_CHUNK_PAYLOAD) return PARSE_ERROR; + + payload_size_padded = payload_size + (payload_size & 1); + payload_available = (payload_size_padded > MemDataSize(mem)) + ? MemDataSize(mem) : payload_size_padded; + chunk_size = CHUNK_HEADER_SIZE + payload_available; if (SizeIsInvalid(mem, payload_size_padded)) return PARSE_ERROR; if (payload_size_padded > MemDataSize(mem)) status = PARSE_NEED_MORE_DATA; @@ -451,9 +455,11 @@ static ParseStatus ParseVP8XChunks(WebPDemuxer* const dmux) { const size_t chunk_start_offset = mem->start_; const uint32_t fourcc = ReadLE32(mem); const uint32_t chunk_size = ReadLE32(mem); - const uint32_t chunk_size_padded = chunk_size + (chunk_size & 1); + uint32_t chunk_size_padded; if (chunk_size > MAX_CHUNK_PAYLOAD) return PARSE_ERROR; + + chunk_size_padded = chunk_size + (chunk_size & 1); if (SizeIsInvalid(mem, chunk_size_padded)) return PARSE_ERROR; switch (fourcc) { @@ -608,7 +614,6 @@ static int IsValidExtendedFormat(const WebPDemuxer* const dmux) { while (f != NULL) { const int cur_frame_set = f->frame_num_; - int frame_count = 0; // Check frame properties. for (; f != NULL && f->frame_num_ == cur_frame_set; f = f->next_) { @@ -643,8 +648,6 @@ static int IsValidExtendedFormat(const WebPDemuxer* const dmux) { dmux->canvas_width_, dmux->canvas_height_)) { return 0; } - - ++frame_count; } } return 1; diff --git a/3rdparty/libwebp/src/dsp/alpha_processing.c b/3rdparty/libwebp/src/dsp/alpha_processing.c index 3a27990ddc..1d152f24da 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing.c @@ -157,7 +157,8 @@ void WebPMultARGBRow_C(uint32_t* const ptr, int width, int inverse) { } } -void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, +void WebPMultRow_C(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse) { int x; for (x = 0; x < width; ++x) { @@ -178,7 +179,8 @@ void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, #undef MFIX void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse); -void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, +void (*WebPMultRow)(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); //------------------------------------------------------------------------------ @@ -193,8 +195,8 @@ void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, } } -void WebPMultRows(uint8_t* ptr, int stride, - const uint8_t* alpha, int alpha_stride, +void WebPMultRows(uint8_t* WEBP_RESTRICT ptr, int stride, + const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int num_rows, int inverse) { int n; for (n = 0; n < num_rows; ++n) { @@ -290,9 +292,9 @@ static void ApplyAlphaMultiply_16b_C(uint8_t* rgba4444, } #if !WEBP_NEON_OMIT_C_CODE -static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride, +static int DispatchAlpha_C(const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int height, - uint8_t* dst, int dst_stride) { + uint8_t* WEBP_RESTRICT dst, int dst_stride) { uint32_t alpha_mask = 0xff; int i, j; @@ -309,9 +311,10 @@ static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride, return (alpha_mask != 0xff); } -static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_C(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; for (j = 0; j < height; ++j) { for (i = 0; i < width; ++i) { @@ -322,9 +325,9 @@ static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_C(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_C(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { uint8_t alpha_mask = 0xff; int i, j; @@ -340,7 +343,8 @@ static int ExtractAlpha_C(const uint8_t* argb, int argb_stride, return (alpha_mask == 0xff); } -static void ExtractGreen_C(const uint32_t* argb, uint8_t* alpha, int size) { +static void ExtractGreen_C(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { int i; for (i = 0; i < size; ++i) alpha[i] = argb[i] >> 8; } @@ -372,8 +376,11 @@ static WEBP_INLINE uint32_t MakeARGB32(int a, int r, int g, int b) { } #ifdef WORDS_BIGENDIAN -static void PackARGB_C(const uint8_t* a, const uint8_t* r, const uint8_t* g, - const uint8_t* b, int len, uint32_t* out) { +static void PackARGB_C(const uint8_t* WEBP_RESTRICT a, + const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, uint32_t* WEBP_RESTRICT out) { int i; for (i = 0; i < len; ++i) { out[i] = MakeARGB32(a[4 * i], r[4 * i], g[4 * i], b[4 * i]); @@ -381,8 +388,10 @@ static void PackARGB_C(const uint8_t* a, const uint8_t* r, const uint8_t* g, } #endif -static void PackRGB_C(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out) { +static void PackRGB_C(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out) { int i, offset = 0; for (i = 0; i < len; ++i) { out[i] = MakeARGB32(0xff, r[offset], g[offset], b[offset]); @@ -392,16 +401,22 @@ static void PackRGB_C(const uint8_t* r, const uint8_t* g, const uint8_t* b, void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int); void (*WebPApplyAlphaMultiply4444)(uint8_t*, int, int, int); -int (*WebPDispatchAlpha)(const uint8_t*, int, int, int, uint8_t*, int); -void (*WebPDispatchAlphaToGreen)(const uint8_t*, int, int, int, uint32_t*, int); -int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int); -void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); +int (*WebPDispatchAlpha)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint8_t* WEBP_RESTRICT, int); +void (*WebPDispatchAlphaToGreen)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint32_t* WEBP_RESTRICT, int); +int (*WebPExtractAlpha)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint8_t* WEBP_RESTRICT, int); +void (*WebPExtractGreen)(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size); #ifdef WORDS_BIGENDIAN void (*WebPPackARGB)(const uint8_t* a, const uint8_t* r, const uint8_t* g, const uint8_t* b, int, uint32_t*); #endif -void (*WebPPackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out); +void (*WebPPackRGB)(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out); int (*WebPHasAlpha8b)(const uint8_t* src, int length); int (*WebPHasAlpha32b)(const uint8_t* src, int length); @@ -410,6 +425,7 @@ void (*WebPAlphaReplace)(uint32_t* src, int length, uint32_t color); //------------------------------------------------------------------------------ // Init function +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitAlphaProcessingMIPSdspR2(void); extern void WebPInitAlphaProcessingSSE2(void); extern void WebPInitAlphaProcessingSSE41(void); @@ -438,10 +454,10 @@ WEBP_DSP_INIT_FUNC(WebPInitAlphaProcessing) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitAlphaProcessingSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitAlphaProcessingSSE41(); } @@ -455,7 +471,7 @@ WEBP_DSP_INIT_FUNC(WebPInitAlphaProcessing) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitAlphaProcessingNEON(); diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_neon.c b/3rdparty/libwebp/src/dsp/alpha_processing_neon.c index 9d55421704..6716fb77f0 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_neon.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_neon.c @@ -80,10 +80,10 @@ static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first, //------------------------------------------------------------------------------ -static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride) { - uint32_t alpha_mask = 0xffffffffu; +static int DispatchAlpha_NEON(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride) { + uint32_t alpha_mask = 0xffu; uint8x8_t mask8 = vdup_n_u8(0xff); uint32_t tmp[2]; int i, j; @@ -107,14 +107,16 @@ static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride, dst += dst_stride; } vst1_u8((uint8_t*)tmp, mask8); + alpha_mask *= 0x01010101; alpha_mask &= tmp[0]; alpha_mask &= tmp[1]; return (alpha_mask != 0xffffffffu); } -static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_NEON(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; uint8x8x4_t greens; // leave A/R/B channels zero'd. greens.val[0] = vdup_n_u8(0); @@ -131,10 +133,10 @@ static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_NEON(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { - uint32_t alpha_mask = 0xffffffffu; + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { + uint32_t alpha_mask = 0xffu; uint8x8_t mask8 = vdup_n_u8(0xff); uint32_t tmp[2]; int i, j; @@ -156,13 +158,14 @@ static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride, alpha += alpha_stride; } vst1_u8((uint8_t*)tmp, mask8); + alpha_mask *= 0x01010101; alpha_mask &= tmp[0]; alpha_mask &= tmp[1]; return (alpha_mask == 0xffffffffu); } -static void ExtractGreen_NEON(const uint32_t* argb, - uint8_t* alpha, int size) { +static void ExtractGreen_NEON(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { int i; for (i = 0; i + 16 <= size; i += 16) { const uint8x16x4_t rgbX = vld4q_u8((const uint8_t*)(argb + i)); diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c b/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c index f6c6e0fb1a..aa0cc2848a 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c @@ -18,16 +18,16 @@ //------------------------------------------------------------------------------ -static int DispatchAlpha_SSE2(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride) { +static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; const __m128i zero = _mm_setzero_si128(); - const __m128i rgb_mask = _mm_set1_epi32(0xffffff00u); // to preserve RGB - const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + const __m128i rgb_mask = _mm_set1_epi32((int)0xffffff00); // to preserve RGB + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte @@ -72,9 +72,10 @@ static int DispatchAlpha_SSE2(const uint8_t* alpha, int alpha_stride, return (alpha_and != 0xff); } -static void DispatchAlphaToGreen_SSE2(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_SSE2(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; const __m128i zero = _mm_setzero_si128(); const int limit = width & ~15; @@ -98,15 +99,15 @@ static void DispatchAlphaToGreen_SSE2(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_SSE2(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; - const __m128i a_mask = _mm_set1_epi32(0xffu); // to preserve alpha - const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + const __m128i a_mask = _mm_set1_epi32(0xff); // to preserve alpha + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte @@ -143,6 +144,46 @@ static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, return (alpha_and == 0xff); } +static void ExtractGreen_SSE2(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { + int i; + const __m128i mask = _mm_set1_epi32(0xff); + const __m128i* src = (const __m128i*)argb; + + for (i = 0; i + 16 <= size; i += 16, src += 4) { + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i a2 = _mm_loadu_si128(src + 2); + const __m128i a3 = _mm_loadu_si128(src + 3); + const __m128i b0 = _mm_srli_epi32(a0, 8); + const __m128i b1 = _mm_srli_epi32(a1, 8); + const __m128i b2 = _mm_srli_epi32(a2, 8); + const __m128i b3 = _mm_srli_epi32(a3, 8); + const __m128i c0 = _mm_and_si128(b0, mask); + const __m128i c1 = _mm_and_si128(b1, mask); + const __m128i c2 = _mm_and_si128(b2, mask); + const __m128i c3 = _mm_and_si128(b3, mask); + const __m128i d0 = _mm_packs_epi32(c0, c1); + const __m128i d1 = _mm_packs_epi32(c2, c3); + const __m128i e = _mm_packus_epi16(d0, d1); + // store + _mm_storeu_si128((__m128i*)&alpha[i], e); + } + if (i + 8 <= size) { + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i b0 = _mm_srli_epi32(a0, 8); + const __m128i b1 = _mm_srli_epi32(a1, 8); + const __m128i c0 = _mm_and_si128(b0, mask); + const __m128i c1 = _mm_and_si128(b1, mask); + const __m128i d = _mm_packs_epi32(c0, c1); + const __m128i e = _mm_packus_epi16(d, d); + _mm_storel_epi64((__m128i*)&alpha[i], e); + i += 8; + } + for (; i < size; ++i) alpha[i] = argb[i] >> 8; +} + //------------------------------------------------------------------------------ // Non-dither premultiplied modes @@ -177,7 +218,7 @@ static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first, int w, int h, int stride) { const __m128i zero = _mm_setzero_si128(); - const __m128i kMult = _mm_set1_epi16(0x8081u); + const __m128i kMult = _mm_set1_epi16((short)0x8081); const __m128i kMask = _mm_set_epi16(0, 0xff, 0xff, 0, 0, 0xff, 0xff, 0); const int kSpan = 4; while (h-- > 0) { @@ -266,7 +307,7 @@ static int HasAlpha32b_SSE2(const uint8_t* src, int length) { } static void AlphaReplace_SSE2(uint32_t* src, int length, uint32_t color) { - const __m128i m_color = _mm_set1_epi32(color); + const __m128i m_color = _mm_set1_epi32((int)color); const __m128i zero = _mm_setzero_si128(); int i = 0; for (; i + 8 <= length; i += 8) { @@ -317,7 +358,8 @@ static void MultARGBRow_SSE2(uint32_t* const ptr, int width, int inverse) { if (width > 0) WebPMultARGBRow_C(ptr + x, width, inverse); } -static void MultRow_SSE2(uint8_t* const ptr, const uint8_t* const alpha, +static void MultRow_SSE2(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse) { int x = 0; if (!inverse) { @@ -352,6 +394,7 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) { WebPDispatchAlpha = DispatchAlpha_SSE2; WebPDispatchAlphaToGreen = DispatchAlphaToGreen_SSE2; WebPExtractAlpha = ExtractAlpha_SSE2; + WebPExtractGreen = ExtractGreen_SSE2; WebPHasAlpha8b = HasAlpha8b_SSE2; WebPHasAlpha32b = HasAlpha32b_SSE2; diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c b/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c index 56040f9c88..1156ac3417 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c @@ -19,14 +19,14 @@ //------------------------------------------------------------------------------ -static int ExtractAlpha_SSE41(const uint8_t* argb, int argb_stride, - int width, int height, - uint8_t* alpha, int alpha_stride) { +static int ExtractAlpha_SSE41(const uint8_t* WEBP_RESTRICT argb, + int argb_stride, int width, int height, + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; - const __m128i all_0xff = _mm_set1_epi32(~0u); + const __m128i all_0xff = _mm_set1_epi32(~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte diff --git a/3rdparty/libwebp/src/dsp/cost.c b/3rdparty/libwebp/src/dsp/cost.c index cc681cdd4b..73d2140177 100644 --- a/3rdparty/libwebp/src/dsp/cost.c +++ b/3rdparty/libwebp/src/dsp/cost.c @@ -374,6 +374,7 @@ static void SetResidualCoeffs_C(const int16_t* const coeffs, VP8GetResidualCostFunc VP8GetResidualCost; VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8EncDspCostInitMIPS32(void); extern void VP8EncDspCostInitMIPSdspR2(void); extern void VP8EncDspCostInitSSE2(void); @@ -395,12 +396,12 @@ WEBP_DSP_INIT_FUNC(VP8EncDspCostInit) { VP8EncDspCostInitMIPSdspR2(); } #endif -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8EncDspCostInitSSE2(); } #endif -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (VP8GetCPUInfo(kNEON)) { VP8EncDspCostInitNEON(); } diff --git a/3rdparty/libwebp/src/dsp/cost_neon.c b/3rdparty/libwebp/src/dsp/cost_neon.c index 8cc8ce58aa..6582669cb3 100644 --- a/3rdparty/libwebp/src/dsp/cost_neon.c +++ b/3rdparty/libwebp/src/dsp/cost_neon.c @@ -29,7 +29,7 @@ static void SetResidualCoeffs_NEON(const int16_t* const coeffs, const uint8x16_t eob = vcombine_u8(vqmovn_u16(eob_0), vqmovn_u16(eob_1)); const uint8x16_t masked = vandq_u8(eob, vld1q_u8(position)); -#ifdef __aarch64__ +#if WEBP_AARCH64 res->last = vmaxvq_u8(masked) - 1; #else const uint8x8_t eob_8x8 = vmax_u8(vget_low_u8(masked), vget_high_u8(masked)); @@ -43,7 +43,7 @@ static void SetResidualCoeffs_NEON(const int16_t* const coeffs, vst1_lane_s32(&res->last, vreinterpret_s32_u32(eob_32x2), 0); --res->last; -#endif // __aarch64__ +#endif // WEBP_AARCH64 res->coeffs = coeffs; } diff --git a/3rdparty/libwebp/src/dsp/cpu.c b/3rdparty/libwebp/src/dsp/cpu.c index 4ca90d88bf..2234c77b35 100644 --- a/3rdparty/libwebp/src/dsp/cpu.c +++ b/3rdparty/libwebp/src/dsp/cpu.c @@ -11,7 +11,7 @@ // // Author: Christian Duvivier (cduvivier@google.com) -#include "src/dsp/dsp.h" +#include "src/dsp/cpu.h" #if defined(WEBP_HAVE_NEON_RTCD) #include @@ -173,6 +173,7 @@ static int x86CPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = x86CPUInfo; #elif defined(WEBP_ANDROID_NEON) // NB: needs to be before generic NEON test. static int AndroidCPUInfo(CPUFeature feature) { @@ -184,22 +185,23 @@ static int AndroidCPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = AndroidCPUInfo; #elif defined(EMSCRIPTEN) // also needs to be before generic NEON test // Use compile flags as an indicator of SIMD support instead of a runtime check. static int wasmCPUInfo(CPUFeature feature) { switch (feature) { -#ifdef WEBP_USE_SSE2 +#ifdef WEBP_HAVE_SSE2 case kSSE2: return 1; #endif -#ifdef WEBP_USE_SSE41 +#ifdef WEBP_HAVE_SSE41 case kSSE3: case kSlowSSSE3: case kSSE4_1: return 1; #endif -#ifdef WEBP_USE_NEON +#ifdef WEBP_HAVE_NEON case kNEON: return 1; #endif @@ -208,10 +210,12 @@ static int wasmCPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = wasmCPUInfo; -#elif defined(WEBP_USE_NEON) -// define a dummy function to enable turning off NEON at runtime by setting -// VP8DecGetCPUInfo = NULL +#elif defined(WEBP_HAVE_NEON) +// In most cases this function doesn't check for NEON support (it's assumed by +// the configuration), but enables turning off NEON at runtime, for testing +// purposes, by setting VP8GetCPUInfo = NULL. static int armCPUInfo(CPUFeature feature) { if (feature != kNEON) return 0; #if defined(__linux__) && defined(WEBP_HAVE_NEON_RTCD) @@ -235,6 +239,7 @@ static int armCPUInfo(CPUFeature feature) { return 1; #endif } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = armCPUInfo; #elif defined(WEBP_USE_MIPS32) || defined(WEBP_USE_MIPS_DSP_R2) || \ defined(WEBP_USE_MSA) @@ -246,7 +251,9 @@ static int mipsCPUInfo(CPUFeature feature) { } } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = mipsCPUInfo; #else +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = NULL; #endif diff --git a/3rdparty/libwebp/src/dsp/cpu.h b/3rdparty/libwebp/src/dsp/cpu.h new file mode 100644 index 0000000000..c86540f280 --- /dev/null +++ b/3rdparty/libwebp/src/dsp/cpu.h @@ -0,0 +1,266 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// CPU detection functions and macros. +// +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef WEBP_DSP_CPU_H_ +#define WEBP_DSP_CPU_H_ + +#include + +#ifdef HAVE_CONFIG_H +#include "src/webp/config.h" +#endif + +#include "src/webp/types.h" + +#if defined(__GNUC__) +#define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) +#define LOCAL_GCC_PREREQ(maj, min) (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#else +#define LOCAL_GCC_VERSION 0 +#define LOCAL_GCC_PREREQ(maj, min) 0 +#endif + +#if defined(__clang__) +#define LOCAL_CLANG_VERSION ((__clang_major__ << 8) | __clang_minor__) +#define LOCAL_CLANG_PREREQ(maj, min) \ + (LOCAL_CLANG_VERSION >= (((maj) << 8) | (min))) +#else +#define LOCAL_CLANG_VERSION 0 +#define LOCAL_CLANG_PREREQ(maj, min) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +//------------------------------------------------------------------------------ +// x86 defines. + +#if !defined(HAVE_CONFIG_H) +#if defined(_MSC_VER) && _MSC_VER > 1310 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE2 // Visual C++ SSE2 targets +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1500 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets +#endif +#endif + +// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp +// files without intrinsics, allowing the corresponding Init() to be called. +// Files containing intrinsics will need to be built targeting the instruction +// set so should succeed on one of the earlier tests. +#if (defined(__SSE2__) || defined(WEBP_MSC_SSE2)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_SSE2)) +#define WEBP_USE_SSE2 +#endif + +#if defined(WEBP_USE_SSE2) && !defined(WEBP_HAVE_SSE2) +#define WEBP_HAVE_SSE2 +#endif + +#if (defined(__SSE4_1__) || defined(WEBP_MSC_SSE41)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_SSE41)) +#define WEBP_USE_SSE41 +#endif + +#if defined(WEBP_USE_SSE41) && !defined(WEBP_HAVE_SSE41) +#define WEBP_HAVE_SSE41 +#endif + +#undef WEBP_MSC_SSE41 +#undef WEBP_MSC_SSE2 + +//------------------------------------------------------------------------------ +// Arm defines. + +// The intrinsics currently cause compiler errors with arm-nacl-gcc and the +// inline assembly would need to be modified for use with Native Client. +#if ((defined(__ARM_NEON__) || defined(__aarch64__)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_NEON))) && \ + !defined(__native_client__) +#define WEBP_USE_NEON +#endif + +#if !defined(WEBP_USE_NEON) && defined(__ANDROID__) && \ + defined(__ARM_ARCH_7A__) && defined(HAVE_CPU_FEATURES_H) +#define WEBP_ANDROID_NEON // Android targets that may have NEON +#define WEBP_USE_NEON +#endif + +// Note: ARM64 is supported in Visual Studio 2017, but requires the direct +// inclusion of arm64_neon.h; Visual Studio 2019 includes this file in +// arm_neon.h. Compile errors were seen with Visual Studio 2019 16.4 with +// vtbl4_u8(); a fix was made in 16.6. +#if defined(_MSC_VER) && \ + ((_MSC_VER >= 1700 && defined(_M_ARM)) || \ + (_MSC_VER >= 1926 && (defined(_M_ARM64) || defined(_M_ARM64EC)))) +#define WEBP_USE_NEON +#define WEBP_USE_INTRINSICS +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#define WEBP_AARCH64 1 +#else +#define WEBP_AARCH64 0 +#endif + +#if defined(WEBP_USE_NEON) && !defined(WEBP_HAVE_NEON) +#define WEBP_HAVE_NEON +#endif + +//------------------------------------------------------------------------------ +// MIPS defines. + +#if defined(__mips__) && !defined(__mips64) && defined(__mips_isa_rev) && \ + (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) +#define WEBP_USE_MIPS32 +#if (__mips_isa_rev >= 2) +#define WEBP_USE_MIPS32_R2 +#if defined(__mips_dspr2) || (defined(__mips_dsp_rev) && __mips_dsp_rev >= 2) +#define WEBP_USE_MIPS_DSP_R2 +#endif +#endif +#endif + +#if defined(__mips_msa) && defined(__mips_isa_rev) && (__mips_isa_rev >= 5) +#define WEBP_USE_MSA +#endif + +//------------------------------------------------------------------------------ + +#ifndef WEBP_DSP_OMIT_C_CODE +#define WEBP_DSP_OMIT_C_CODE 1 +#endif + +#if defined(WEBP_USE_NEON) && WEBP_DSP_OMIT_C_CODE +#define WEBP_NEON_OMIT_C_CODE 1 +#else +#define WEBP_NEON_OMIT_C_CODE 0 +#endif + +#if !(LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 8) || WEBP_AARCH64) +#define WEBP_NEON_WORK_AROUND_GCC 1 +#else +#define WEBP_NEON_WORK_AROUND_GCC 0 +#endif + +//------------------------------------------------------------------------------ + +// This macro prevents thread_sanitizer from reporting known concurrent writes. +#define WEBP_TSAN_IGNORE_FUNCTION +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#undef WEBP_TSAN_IGNORE_FUNCTION +#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) +#endif +#endif + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define WEBP_MSAN +#endif +#endif + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include // NOLINT + +#define WEBP_DSP_INIT(func) \ + do { \ + static volatile VP8CPUInfo func##_last_cpuinfo_used = \ + (VP8CPUInfo)&func##_last_cpuinfo_used; \ + static pthread_mutex_t func##_lock = PTHREAD_MUTEX_INITIALIZER; \ + if (pthread_mutex_lock(&func##_lock)) break; \ + if (func##_last_cpuinfo_used != VP8GetCPUInfo) func(); \ + func##_last_cpuinfo_used = VP8GetCPUInfo; \ + (void)pthread_mutex_unlock(&func##_lock); \ + } while (0) +#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) +#define WEBP_DSP_INIT(func) \ + do { \ + static volatile VP8CPUInfo func##_last_cpuinfo_used = \ + (VP8CPUInfo)&func##_last_cpuinfo_used; \ + if (func##_last_cpuinfo_used == VP8GetCPUInfo) break; \ + func(); \ + func##_last_cpuinfo_used = VP8GetCPUInfo; \ + } while (0) +#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) + +// Defines an Init + helper function that control multiple initialization of +// function pointers / tables. +/* Usage: + WEBP_DSP_INIT_FUNC(InitFunc) { + ...function body + } +*/ +#define WEBP_DSP_INIT_FUNC(name) \ + static WEBP_TSAN_IGNORE_FUNCTION void name##_body(void); \ + WEBP_TSAN_IGNORE_FUNCTION void name(void) { WEBP_DSP_INIT(name##_body); } \ + static WEBP_TSAN_IGNORE_FUNCTION void name##_body(void) + +#define WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#if defined(__clang__) && defined(__has_attribute) +#if __has_attribute(no_sanitize) +// This macro prevents the undefined behavior sanitizer from reporting +// failures. This is only meant to silence unaligned loads on platforms that +// are known to support them. +#undef WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNDEF __attribute__((no_sanitize("undefined"))) + +// This macro prevents the undefined behavior sanitizer from reporting +// failures related to unsigned integer overflows. This is only meant to +// silence cases where this well defined behavior is expected. +#undef WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW \ + __attribute__((no_sanitize("unsigned-integer-overflow"))) +#endif +#endif + +// If 'ptr' is NULL, returns NULL. Otherwise returns 'ptr + off'. +// Prevents undefined behavior sanitizer nullptr-with-nonzero-offset warning. +#if !defined(WEBP_OFFSET_PTR) +#define WEBP_OFFSET_PTR(ptr, off) (((ptr) == NULL) ? NULL : ((ptr) + (off))) +#endif + +// Regularize the definition of WEBP_SWAP_16BIT_CSP (backward compatibility) +#if !defined(WEBP_SWAP_16BIT_CSP) +#define WEBP_SWAP_16BIT_CSP 0 +#endif + +// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) +#if !defined(WORDS_BIGENDIAN) && \ + (defined(__BIG_ENDIAN__) || defined(_M_PPC) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) +#define WORDS_BIGENDIAN +#endif + +typedef enum { + kSSE2, + kSSE3, + kSlowSSSE3, // special feature for slow SSSE3 architectures + kSSE4_1, + kAVX, + kAVX2, + kNEON, + kMIPS32, + kMIPSdspR2, + kMSA +} CPUFeature; + +// returns true if the CPU supports the feature. +typedef int (*VP8CPUInfo)(CPUFeature feature); + +#endif // WEBP_DSP_CPU_H_ diff --git a/3rdparty/libwebp/src/dsp/dec.c b/3rdparty/libwebp/src/dsp/dec.c index 1119842dd3..33d8df8a62 100644 --- a/3rdparty/libwebp/src/dsp/dec.c +++ b/3rdparty/libwebp/src/dsp/dec.c @@ -734,6 +734,7 @@ VP8SimpleFilterFunc VP8SimpleHFilter16i; void (*VP8DitherCombine8x8)(const uint8_t* dither, uint8_t* dst, int dst_stride); +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8DspInitSSE2(void); extern void VP8DspInitSSE41(void); extern void VP8DspInitNEON(void); @@ -807,10 +808,10 @@ WEBP_DSP_INIT_FUNC(VP8DspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8DspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8DspInitSSE41(); } @@ -834,7 +835,7 @@ WEBP_DSP_INIT_FUNC(VP8DspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8DspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/dec_neon.c b/3rdparty/libwebp/src/dsp/dec_neon.c index fa851707e2..22784cf15a 100644 --- a/3rdparty/libwebp/src/dsp/dec_neon.c +++ b/3rdparty/libwebp/src/dsp/dec_neon.c @@ -1428,7 +1428,7 @@ static WEBP_INLINE void DC8_NEON(uint8_t* dst, int do_top, int do_left) { if (do_top) { const uint8x8_t A = vld1_u8(dst - BPS); // top row -#if defined(__aarch64__) +#if WEBP_AARCH64 const uint16_t p2 = vaddlv_u8(A); sum_top = vdupq_n_u16(p2); #else @@ -1511,7 +1511,7 @@ static WEBP_INLINE void DC16_NEON(uint8_t* dst, int do_top, int do_left) { if (do_top) { const uint8x16_t A = vld1q_u8(dst - BPS); // top row -#if defined(__aarch64__) +#if WEBP_AARCH64 const uint16_t p3 = vaddlvq_u8(A); sum_top = vdupq_n_u16(p3); #else diff --git a/3rdparty/libwebp/src/dsp/dec_sse2.c b/3rdparty/libwebp/src/dsp/dec_sse2.c index 873aa59e8a..01e6bcb636 100644 --- a/3rdparty/libwebp/src/dsp/dec_sse2.c +++ b/3rdparty/libwebp/src/dsp/dec_sse2.c @@ -158,10 +158,10 @@ static void Transform_SSE2(const int16_t* in, uint8_t* dst, int do_two) { dst3 = _mm_loadl_epi64((__m128i*)(dst + 3 * BPS)); } else { // Load four bytes/pixels per line. - dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); - dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); - dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); - dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + dst0 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 0 * BPS)); + dst1 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 1 * BPS)); + dst2 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 2 * BPS)); + dst3 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 3 * BPS)); } // Convert to 16b. dst0 = _mm_unpacklo_epi8(dst0, zero); @@ -187,10 +187,10 @@ static void Transform_SSE2(const int16_t* in, uint8_t* dst, int do_two) { _mm_storel_epi64((__m128i*)(dst + 3 * BPS), dst3); } else { // Store four bytes/pixels per line. - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); } } } @@ -213,10 +213,10 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) { const __m128i m3 = _mm_subs_epi16(B, d4); const __m128i zero = _mm_setzero_si128(); // Load the source pixels. - __m128i dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); - __m128i dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); - __m128i dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); - __m128i dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + __m128i dst0 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 0 * BPS)); + __m128i dst1 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 1 * BPS)); + __m128i dst2 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 2 * BPS)); + __m128i dst3 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 3 * BPS)); // Convert to 16b. dst0 = _mm_unpacklo_epi8(dst0, zero); dst1 = _mm_unpacklo_epi8(dst1, zero); @@ -233,10 +233,10 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) { dst2 = _mm_packus_epi16(dst2, dst2); dst3 = _mm_packus_epi16(dst3, dst3); // Store the results. - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); } #undef MUL #endif // USE_TRANSFORM_AC3 @@ -477,11 +477,11 @@ static WEBP_INLINE void Load8x4_SSE2(const uint8_t* const b, int stride, // A0 = 63 62 61 60 23 22 21 20 43 42 41 40 03 02 01 00 // A1 = 73 72 71 70 33 32 31 30 53 52 51 50 13 12 11 10 const __m128i A0 = _mm_set_epi32( - WebPMemToUint32(&b[6 * stride]), WebPMemToUint32(&b[2 * stride]), - WebPMemToUint32(&b[4 * stride]), WebPMemToUint32(&b[0 * stride])); + WebPMemToInt32(&b[6 * stride]), WebPMemToInt32(&b[2 * stride]), + WebPMemToInt32(&b[4 * stride]), WebPMemToInt32(&b[0 * stride])); const __m128i A1 = _mm_set_epi32( - WebPMemToUint32(&b[7 * stride]), WebPMemToUint32(&b[3 * stride]), - WebPMemToUint32(&b[5 * stride]), WebPMemToUint32(&b[1 * stride])); + WebPMemToInt32(&b[7 * stride]), WebPMemToInt32(&b[3 * stride]), + WebPMemToInt32(&b[5 * stride]), WebPMemToInt32(&b[1 * stride])); // B0 = 53 43 52 42 51 41 50 40 13 03 12 02 11 01 10 00 // B1 = 73 63 72 62 71 61 70 60 33 23 32 22 31 21 30 20 @@ -540,7 +540,7 @@ static WEBP_INLINE void Store4x4_SSE2(__m128i* const x, uint8_t* dst, int stride) { int i; for (i = 0; i < 4; ++i, dst += stride) { - WebPUint32ToMem(dst, _mm_cvtsi128_si32(*x)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(*x)); *x = _mm_srli_si128(*x, 4); } } @@ -908,10 +908,10 @@ static void VE4_SSE2(uint8_t* dst) { // vertical const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); const __m128i b = _mm_subs_epu8(a, lsb); const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); - const uint32_t vals = _mm_cvtsi128_si32(avg); + const int vals = _mm_cvtsi128_si32(avg); int i; for (i = 0; i < 4; ++i) { - WebPUint32ToMem(dst + i * BPS, vals); + WebPInt32ToMem(dst + i * BPS, vals); } } @@ -925,10 +925,10 @@ static void LD4_SSE2(uint8_t* dst) { // Down-Left const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static void VR4_SSE2(uint8_t* dst) { // Vertical-Right @@ -946,10 +946,10 @@ static void VR4_SSE2(uint8_t* dst) { // Vertical-Right const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i efgh = _mm_avg_epu8(avg2, XABCD); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); // these two are hard to implement in SSE2, so we keep the C-version: DST(0, 2) = AVG3(J, I, X); @@ -970,11 +970,12 @@ static void VL4_SSE2(uint8_t* dst) { // Vertical-Left const __m128i abbc = _mm_or_si128(ab, bc); const __m128i lsb2 = _mm_and_si128(abbc, lsb1); const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); - const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + const uint32_t extra_out = + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); // these two are hard to get and irregular DST(3, 2) = (extra_out >> 0) & 0xff; @@ -990,7 +991,7 @@ static void RD4_SSE2(uint8_t* dst) { // Down-right const uint32_t K = dst[-1 + 2 * BPS]; const uint32_t L = dst[-1 + 3 * BPS]; const __m128i LKJI_____ = - _mm_cvtsi32_si128(L | (K << 8) | (J << 16) | (I << 24)); + _mm_cvtsi32_si128((int)(L | (K << 8) | (J << 16) | (I << 24))); const __m128i LKJIXABCD = _mm_or_si128(LKJI_____, ____XABCD); const __m128i KJIXABCD_ = _mm_srli_si128(LKJIXABCD, 1); const __m128i JIXABCD__ = _mm_srli_si128(LKJIXABCD, 2); @@ -998,10 +999,10 @@ static void RD4_SSE2(uint8_t* dst) { // Down-right const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } #undef DST @@ -1015,13 +1016,13 @@ static WEBP_INLINE void TrueMotion_SSE2(uint8_t* dst, int size) { const __m128i zero = _mm_setzero_si128(); int y; if (size == 4) { - const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToInt32(top)); const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); for (y = 0; y < 4; ++y, dst += BPS) { const int val = dst[-1] - top[-1]; const __m128i base = _mm_set1_epi16(val); const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); - WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(out)); } } else if (size == 8) { const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); @@ -1062,7 +1063,7 @@ static void VE16_SSE2(uint8_t* dst) { static void HE16_SSE2(uint8_t* dst) { // horizontal int j; for (j = 16; j > 0; --j) { - const __m128i values = _mm_set1_epi8(dst[-1]); + const __m128i values = _mm_set1_epi8((char)dst[-1]); _mm_storeu_si128((__m128i*)dst, values); dst += BPS; } @@ -1070,7 +1071,7 @@ static void HE16_SSE2(uint8_t* dst) { // horizontal static WEBP_INLINE void Put16_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 16; ++j) { _mm_storeu_si128((__m128i*)(dst + j * BPS), values); } @@ -1130,7 +1131,7 @@ static void VE8uv_SSE2(uint8_t* dst) { // vertical // helper for chroma-DC predictions static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 8; ++j) { _mm_storel_epi64((__m128i*)(dst + j * BPS), values); } diff --git a/3rdparty/libwebp/src/dsp/dec_sse41.c b/3rdparty/libwebp/src/dsp/dec_sse41.c index 8f18506d54..08a3630272 100644 --- a/3rdparty/libwebp/src/dsp/dec_sse41.c +++ b/3rdparty/libwebp/src/dsp/dec_sse41.c @@ -23,7 +23,7 @@ static void HE16_SSE41(uint8_t* dst) { // horizontal int j; const __m128i kShuffle3 = _mm_set1_epi8(3); for (j = 16; j > 0; --j) { - const __m128i in = _mm_cvtsi32_si128(WebPMemToUint32(dst - 4)); + const __m128i in = _mm_cvtsi32_si128(WebPMemToInt32(dst - 4)); const __m128i values = _mm_shuffle_epi8(in, kShuffle3); _mm_storeu_si128((__m128i*)dst, values); dst += BPS; diff --git a/3rdparty/libwebp/src/dsp/dsp.h b/3rdparty/libwebp/src/dsp/dsp.h index 298c721ae2..d2000b8efc 100644 --- a/3rdparty/libwebp/src/dsp/dsp.h +++ b/3rdparty/libwebp/src/dsp/dsp.h @@ -18,6 +18,7 @@ #include "src/webp/config.h" #endif +#include "src/dsp/cpu.h" #include "src/webp/types.h" #ifdef __cplusplus @@ -27,205 +28,22 @@ extern "C" { #define BPS 32 // this is the common stride for enc/dec //------------------------------------------------------------------------------ -// CPU detection +// WEBP_RESTRICT +// Declares a pointer with the restrict type qualifier if available. +// This allows code to hint to the compiler that only this pointer references a +// particular object or memory region within the scope of the block in which it +// is declared. This may allow for improved optimizations due to the lack of +// pointer aliasing. See also: +// https://en.cppreference.com/w/c/language/restrict #if defined(__GNUC__) -# define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) -# define LOCAL_GCC_PREREQ(maj, min) \ - (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#define WEBP_RESTRICT __restrict__ +#elif defined(_MSC_VER) +#define WEBP_RESTRICT __restrict #else -# define LOCAL_GCC_VERSION 0 -# define LOCAL_GCC_PREREQ(maj, min) 0 +#define WEBP_RESTRICT #endif -#if defined(__clang__) -# define LOCAL_CLANG_VERSION ((__clang_major__ << 8) | __clang_minor__) -# define LOCAL_CLANG_PREREQ(maj, min) \ - (LOCAL_CLANG_VERSION >= (((maj) << 8) | (min))) -#else -# define LOCAL_CLANG_VERSION 0 -# define LOCAL_CLANG_PREREQ(maj, min) 0 -#endif - -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -#if !defined(HAVE_CONFIG_H) -#if defined(_MSC_VER) && _MSC_VER > 1310 && \ - (defined(_M_X64) || defined(_M_IX86)) -#define WEBP_MSC_SSE2 // Visual C++ SSE2 targets -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1500 && \ - (defined(_M_X64) || defined(_M_IX86)) -#define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets -#endif -#endif - -// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp -// files without intrinsics, allowing the corresponding Init() to be called. -// Files containing intrinsics will need to be built targeting the instruction -// set so should succeed on one of the earlier tests. -#if defined(__SSE2__) || defined(WEBP_MSC_SSE2) || defined(WEBP_HAVE_SSE2) -#define WEBP_USE_SSE2 -#endif - -#if defined(__SSE4_1__) || defined(WEBP_MSC_SSE41) || defined(WEBP_HAVE_SSE41) -#define WEBP_USE_SSE41 -#endif - -#undef WEBP_MSC_SSE41 -#undef WEBP_MSC_SSE2 - -// The intrinsics currently cause compiler errors with arm-nacl-gcc and the -// inline assembly would need to be modified for use with Native Client. -#if (defined(__ARM_NEON__) || \ - defined(__aarch64__) || defined(WEBP_HAVE_NEON)) && \ - !defined(__native_client__) -#define WEBP_USE_NEON -#endif - -#if !defined(WEBP_USE_NEON) && defined(__ANDROID__) && \ - defined(__ARM_ARCH_7A__) && defined(HAVE_CPU_FEATURES_H) -#define WEBP_ANDROID_NEON // Android targets that may have NEON -#define WEBP_USE_NEON -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) -#define WEBP_USE_NEON -#define WEBP_USE_INTRINSICS -#endif - -#if defined(__mips__) && !defined(__mips64) && \ - defined(__mips_isa_rev) && (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) -#define WEBP_USE_MIPS32 -#if (__mips_isa_rev >= 2) -#define WEBP_USE_MIPS32_R2 -#if defined(__mips_dspr2) || (defined(__mips_dsp_rev) && __mips_dsp_rev >= 2) -#define WEBP_USE_MIPS_DSP_R2 -#endif -#endif -#endif - -#if defined(__mips_msa) && defined(__mips_isa_rev) && (__mips_isa_rev >= 5) -#define WEBP_USE_MSA -#endif - -#ifndef WEBP_DSP_OMIT_C_CODE -#define WEBP_DSP_OMIT_C_CODE 1 -#endif - -#if (defined(__aarch64__) || defined(__ARM_NEON__)) && WEBP_DSP_OMIT_C_CODE -#define WEBP_NEON_OMIT_C_CODE 1 -#else -#define WEBP_NEON_OMIT_C_CODE 0 -#endif - -#if !(LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,8) || defined(__aarch64__)) -#define WEBP_NEON_WORK_AROUND_GCC 1 -#else -#define WEBP_NEON_WORK_AROUND_GCC 0 -#endif - -// This macro prevents thread_sanitizer from reporting known concurrent writes. -#define WEBP_TSAN_IGNORE_FUNCTION -#if defined(__has_feature) -#if __has_feature(thread_sanitizer) -#undef WEBP_TSAN_IGNORE_FUNCTION -#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) -#endif -#endif - -#if defined(WEBP_USE_THREAD) && !defined(_WIN32) -#include // NOLINT - -#define WEBP_DSP_INIT(func) do { \ - static volatile VP8CPUInfo func ## _last_cpuinfo_used = \ - (VP8CPUInfo)&func ## _last_cpuinfo_used; \ - static pthread_mutex_t func ## _lock = PTHREAD_MUTEX_INITIALIZER; \ - if (pthread_mutex_lock(&func ## _lock)) break; \ - if (func ## _last_cpuinfo_used != VP8GetCPUInfo) func(); \ - func ## _last_cpuinfo_used = VP8GetCPUInfo; \ - (void)pthread_mutex_unlock(&func ## _lock); \ -} while (0) -#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) -#define WEBP_DSP_INIT(func) do { \ - static volatile VP8CPUInfo func ## _last_cpuinfo_used = \ - (VP8CPUInfo)&func ## _last_cpuinfo_used; \ - if (func ## _last_cpuinfo_used == VP8GetCPUInfo) break; \ - func(); \ - func ## _last_cpuinfo_used = VP8GetCPUInfo; \ -} while (0) -#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) - -// Defines an Init + helper function that control multiple initialization of -// function pointers / tables. -/* Usage: - WEBP_DSP_INIT_FUNC(InitFunc) { - ...function body - } -*/ -#define WEBP_DSP_INIT_FUNC(name) \ - static WEBP_TSAN_IGNORE_FUNCTION void name ## _body(void); \ - WEBP_TSAN_IGNORE_FUNCTION void name(void) { \ - WEBP_DSP_INIT(name ## _body); \ - } \ - static WEBP_TSAN_IGNORE_FUNCTION void name ## _body(void) - -#define WEBP_UBSAN_IGNORE_UNDEF -#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW -#if defined(__clang__) && defined(__has_attribute) -#if __has_attribute(no_sanitize) -// This macro prevents the undefined behavior sanitizer from reporting -// failures. This is only meant to silence unaligned loads on platforms that -// are known to support them. -#undef WEBP_UBSAN_IGNORE_UNDEF -#define WEBP_UBSAN_IGNORE_UNDEF \ - __attribute__((no_sanitize("undefined"))) - -// This macro prevents the undefined behavior sanitizer from reporting -// failures related to unsigned integer overflows. This is only meant to -// silence cases where this well defined behavior is expected. -#undef WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW -#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW \ - __attribute__((no_sanitize("unsigned-integer-overflow"))) -#endif -#endif - -// If 'ptr' is NULL, returns NULL. Otherwise returns 'ptr + off'. -// Prevents undefined behavior sanitizer nullptr-with-nonzero-offset warning. -#if !defined(WEBP_OFFSET_PTR) -#define WEBP_OFFSET_PTR(ptr, off) (((ptr) == NULL) ? NULL : ((ptr) + (off))) -#endif - -// Regularize the definition of WEBP_SWAP_16BIT_CSP (backward compatibility) -#if !defined(WEBP_SWAP_16BIT_CSP) -#define WEBP_SWAP_16BIT_CSP 0 -#endif - -// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) -#if !defined(WORDS_BIGENDIAN) && \ - (defined(__BIG_ENDIAN__) || defined(_M_PPC) || \ - (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) -#define WORDS_BIGENDIAN -#endif - -typedef enum { - kSSE2, - kSSE3, - kSlowSSSE3, // special feature for slow SSSE3 architectures - kSSE4_1, - kAVX, - kAVX2, - kNEON, - kMIPS32, - kMIPSdspR2, - kMSA -} CPUFeature; -// returns true if the CPU supports the feature. -typedef int (*VP8CPUInfo)(CPUFeature feature); -WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; //------------------------------------------------------------------------------ // Init stub generator @@ -514,15 +332,6 @@ extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, uint8_t* u, uint8_t* v, int width); -// utilities for accurate RGB->YUV conversion -extern uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* src, const uint16_t* ref, - uint16_t* dst, int len); -extern void (*WebPSharpYUVUpdateRGB)(const int16_t* src, const int16_t* ref, - int16_t* dst, int len); -extern void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, - int len, - const uint16_t* best_y, uint16_t* out); - // Must be called before using the above. void WebPInitConvertARGBToYUV(void); @@ -578,26 +387,29 @@ extern void (*WebPApplyAlphaMultiply4444)( // Dispatch the values from alpha[] plane to the ARGB destination 'dst'. // Returns true if alpha[] plane has non-trivial values different from 0xff. -extern int (*WebPDispatchAlpha)(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride); +extern int (*WebPDispatchAlpha)(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride); // Transfer packed 8b alpha[] values to green channel in dst[], zero'ing the // A/R/B values. 'dst_stride' is the stride for dst[] in uint32_t units. -extern void (*WebPDispatchAlphaToGreen)(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride); +extern void (*WebPDispatchAlphaToGreen)(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride); // Extract the alpha values from 32b values in argb[] and pack them into alpha[] // (this is the opposite of WebPDispatchAlpha). // Returns true if there's only trivial 0xff alpha values. -extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride, - int width, int height, - uint8_t* alpha, int alpha_stride); +extern int (*WebPExtractAlpha)(const uint8_t* WEBP_RESTRICT argb, + int argb_stride, int width, int height, + uint8_t* WEBP_RESTRICT alpha, + int alpha_stride); // Extract the green values from 32b values in argb[] and pack them into alpha[] // (this is the opposite of WebPDispatchAlphaToGreen). -extern void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); +extern void (*WebPExtractGreen)(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size); // Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B). // Un-Multiply operation transforms x into x * 255 / A. @@ -610,29 +422,35 @@ void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, int inverse); // Same for a row of single values, with side alpha values. -extern void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, +extern void (*WebPMultRow)(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); // Same a WebPMultRow(), but for several 'num_rows' rows. -void WebPMultRows(uint8_t* ptr, int stride, - const uint8_t* alpha, int alpha_stride, +void WebPMultRows(uint8_t* WEBP_RESTRICT ptr, int stride, + const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int num_rows, int inverse); // Plain-C versions, used as fallback by some implementations. -void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, +void WebPMultRow_C(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); void WebPMultARGBRow_C(uint32_t* const ptr, int width, int inverse); #ifdef WORDS_BIGENDIAN // ARGB packing function: a/r/g/b input is rgba or bgra order. -extern void (*WebPPackARGB)(const uint8_t* a, const uint8_t* r, - const uint8_t* g, const uint8_t* b, int len, - uint32_t* out); +extern void (*WebPPackARGB)(const uint8_t* WEBP_RESTRICT a, + const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, uint32_t* WEBP_RESTRICT out); #endif // RGB packing function. 'step' can be 3 or 4. r/g/b input is rgb or bgr order. -extern void (*WebPPackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out); +extern void (*WebPPackRGB)(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out); // This function returns true if src[i] contains a value different from 0xff. extern int (*WebPHasAlpha8b)(const uint8_t* src, int length); diff --git a/3rdparty/libwebp/src/dsp/enc.c b/3rdparty/libwebp/src/dsp/enc.c index 2fddbc4c52..2ba97ba8d6 100644 --- a/3rdparty/libwebp/src/dsp/enc.c +++ b/3rdparty/libwebp/src/dsp/enc.c @@ -732,6 +732,7 @@ VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; VP8BlockCopy VP8Copy4x4; VP8BlockCopy VP8Copy16x8; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8EncDspInitSSE2(void); extern void VP8EncDspInitSSE41(void); extern void VP8EncDspInitNEON(void); @@ -773,10 +774,10 @@ WEBP_DSP_INIT_FUNC(VP8EncDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8EncDspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8EncDspInitSSE41(); } @@ -800,7 +801,7 @@ WEBP_DSP_INIT_FUNC(VP8EncDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8EncDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/enc_neon.c b/3rdparty/libwebp/src/dsp/enc_neon.c index 43bf1245c5..714800367b 100644 --- a/3rdparty/libwebp/src/dsp/enc_neon.c +++ b/3rdparty/libwebp/src/dsp/enc_neon.c @@ -9,7 +9,7 @@ // // ARM NEON version of speed-critical encoding functions. // -// adapted from libvpx (http://www.webmproject.org/code/) +// adapted from libvpx (https://www.webmproject.org/code/) #include "src/dsp/dsp.h" @@ -764,9 +764,14 @@ static WEBP_INLINE void AccumulateSSE16_NEON(const uint8_t* const a, // Horizontal sum of all four uint32_t values in 'sum'. static int SumToInt_NEON(uint32x4_t sum) { +#if WEBP_AARCH64 + return (int)vaddvq_u32(sum); +#else const uint64x2_t sum2 = vpaddlq_u32(sum); - const uint64_t sum3 = vgetq_lane_u64(sum2, 0) + vgetq_lane_u64(sum2, 1); - return (int)sum3; + const uint32x2_t sum3 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(sum2)), + vreinterpret_u32_u64(vget_high_u64(sum2))); + return (int)vget_lane_u32(sum3, 0); +#endif } static int SSE16x16_NEON(const uint8_t* a, const uint8_t* b) { @@ -860,7 +865,7 @@ static int QuantizeBlock_NEON(int16_t in[16], int16_t out[16], uint8x8x4_t shuffles; // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) uint8x16x2_t all_out; INIT_VECTOR2(all_out, vreinterpretq_u8_s16(out0), vreinterpretq_u8_s16(out1)); diff --git a/3rdparty/libwebp/src/dsp/enc_sse2.c b/3rdparty/libwebp/src/dsp/enc_sse2.c index b2e78ed941..010624a2f7 100644 --- a/3rdparty/libwebp/src/dsp/enc_sse2.c +++ b/3rdparty/libwebp/src/dsp/enc_sse2.c @@ -25,9 +25,160 @@ //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) -// Does one or two inverse transforms. -static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, - int do_two) { +// Does one inverse transform. +static void ITransform_One_SSE2(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + // This implementation makes use of 16-bit fixed point versions of two + // multiply constants: + // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 + // K2 = sqrt(2) * sin (pi/8) ~= 35468 / 2^16 + // + // To be able to use signed 16-bit integers, we use the following trick to + // have constants within range: + // - Associated constants are obtained by subtracting the 16-bit fixed point + // version of one: + // k = K - (1 << 16) => K = k + (1 << 16) + // K1 = 85267 => k1 = 20091 + // K2 = 35468 => k2 = -30068 + // - The multiplication of a variable by a constant become the sum of the + // variable and the multiplication of that variable by the associated + // constant: + // (x * K) >> 16 = (x * (k + (1 << 16))) >> 16 = ((x * k ) >> 16) + x + const __m128i k1k2 = _mm_set_epi16(-30068, -30068, -30068, -30068, + 20091, 20091, 20091, 20091); + const __m128i k2k1 = _mm_set_epi16(20091, 20091, 20091, 20091, + -30068, -30068, -30068, -30068); + const __m128i zero = _mm_setzero_si128(); + const __m128i zero_four = _mm_set_epi16(0, 0, 0, 0, 4, 4, 4, 4); + __m128i T01, T23; + + // Load and concatenate the transform coefficients. + const __m128i in01 = _mm_loadu_si128((const __m128i*)&in[0]); + const __m128i in23 = _mm_loadu_si128((const __m128i*)&in[8]); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + + // Vertical pass and subsequent transpose. + { + const __m128i in1 = _mm_unpackhi_epi64(in01, in01); + const __m128i in3 = _mm_unpackhi_epi64(in23, in23); + + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + // c = MUL(in1, K2) - MUL(in3, K1) = MUL(in1, k2) - MUL(in3, k1) + in1 - in3 + // d = MUL(in1, K1) + MUL(in3, K2) = MUL(in1, k1) + MUL(in3, k2) + in1 + in3 + const __m128i a_d3 = _mm_add_epi16(in01, in23); + const __m128i b_c3 = _mm_sub_epi16(in01, in23); + const __m128i c1d1 = _mm_mulhi_epi16(in1, k2k1); + const __m128i c2d2 = _mm_mulhi_epi16(in3, k1k2); + const __m128i c3 = _mm_unpackhi_epi64(b_c3, b_c3); + const __m128i c4 = _mm_sub_epi16(c1d1, c2d2); + const __m128i c = _mm_add_epi16(c3, c4); + const __m128i d4u = _mm_add_epi16(c1d1, c2d2); + const __m128i du = _mm_add_epi16(a_d3, d4u); + const __m128i d = _mm_unpackhi_epi64(du, du); + + // Second pass. + const __m128i comb_ab = _mm_unpacklo_epi64(a_d3, b_c3); + const __m128i comb_dc = _mm_unpacklo_epi64(d, c); + + const __m128i tmp01 = _mm_add_epi16(comb_ab, comb_dc); + const __m128i tmp32 = _mm_sub_epi16(comb_ab, comb_dc); + const __m128i tmp23 = _mm_shuffle_epi32(tmp32, _MM_SHUFFLE(1, 0, 3, 2)); + + const __m128i transpose_0 = _mm_unpacklo_epi16(tmp01, tmp23); + const __m128i transpose_1 = _mm_unpackhi_epi16(tmp01, tmp23); + // a00 a20 a01 a21 a02 a22 a03 a23 + // a10 a30 a11 a31 a12 a32 a13 a33 + + T01 = _mm_unpacklo_epi16(transpose_0, transpose_1); + T23 = _mm_unpackhi_epi16(transpose_0, transpose_1); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + } + + // Horizontal pass and subsequent transpose. + { + const __m128i T1 = _mm_unpackhi_epi64(T01, T01); + const __m128i T3 = _mm_unpackhi_epi64(T23, T23); + + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i dc = _mm_add_epi16(T01, zero_four); + + // c = MUL(T1, K2) - MUL(T3, K1) = MUL(T1, k2) - MUL(T3, k1) + T1 - T3 + // d = MUL(T1, K1) + MUL(T3, K2) = MUL(T1, k1) + MUL(T3, k2) + T1 + T3 + const __m128i a_d3 = _mm_add_epi16(dc, T23); + const __m128i b_c3 = _mm_sub_epi16(dc, T23); + const __m128i c1d1 = _mm_mulhi_epi16(T1, k2k1); + const __m128i c2d2 = _mm_mulhi_epi16(T3, k1k2); + const __m128i c3 = _mm_unpackhi_epi64(b_c3, b_c3); + const __m128i c4 = _mm_sub_epi16(c1d1, c2d2); + const __m128i c = _mm_add_epi16(c3, c4); + const __m128i d4u = _mm_add_epi16(c1d1, c2d2); + const __m128i du = _mm_add_epi16(a_d3, d4u); + const __m128i d = _mm_unpackhi_epi64(du, du); + + // Second pass. + const __m128i comb_ab = _mm_unpacklo_epi64(a_d3, b_c3); + const __m128i comb_dc = _mm_unpacklo_epi64(d, c); + + const __m128i tmp01 = _mm_add_epi16(comb_ab, comb_dc); + const __m128i tmp32 = _mm_sub_epi16(comb_ab, comb_dc); + const __m128i tmp23 = _mm_shuffle_epi32(tmp32, _MM_SHUFFLE(1, 0, 3, 2)); + + const __m128i shifted01 = _mm_srai_epi16(tmp01, 3); + const __m128i shifted23 = _mm_srai_epi16(tmp23, 3); + // a00 a01 a02 a03 a10 a11 a12 a13 + // a20 a21 a22 a23 a30 a31 a32 a33 + + const __m128i transpose_0 = _mm_unpacklo_epi16(shifted01, shifted23); + const __m128i transpose_1 = _mm_unpackhi_epi16(shifted01, shifted23); + // a00 a20 a01 a21 a02 a22 a03 a23 + // a10 a30 a11 a31 a12 a32 a13 a33 + + T01 = _mm_unpacklo_epi16(transpose_0, transpose_1); + T23 = _mm_unpackhi_epi16(transpose_0, transpose_1); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + } + + // Add inverse transform to 'ref' and store. + { + // Load the reference(s). + __m128i ref01, ref23, ref0123; + int32_t buf[4]; + + // Load four bytes/pixels per line. + const __m128i ref0 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[0 * BPS])); + const __m128i ref1 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[1 * BPS])); + const __m128i ref2 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[2 * BPS])); + const __m128i ref3 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[3 * BPS])); + ref01 = _mm_unpacklo_epi32(ref0, ref1); + ref23 = _mm_unpacklo_epi32(ref2, ref3); + + // Convert to 16b. + ref01 = _mm_unpacklo_epi8(ref01, zero); + ref23 = _mm_unpacklo_epi8(ref23, zero); + // Add the inverse transform(s). + ref01 = _mm_add_epi16(ref01, T01); + ref23 = _mm_add_epi16(ref23, T23); + // Unsigned saturate to 8b. + ref0123 = _mm_packus_epi16(ref01, ref23); + + _mm_storeu_si128((__m128i *)buf, ref0123); + + // Store four bytes/pixels per line. + WebPInt32ToMem(&dst[0 * BPS], buf[0]); + WebPInt32ToMem(&dst[1 * BPS], buf[1]); + WebPInt32ToMem(&dst[2 * BPS], buf[2]); + WebPInt32ToMem(&dst[3 * BPS], buf[3]); + } +} + +// Does two inverse transforms. +static void ITransform_Two_SSE2(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { // This implementation makes use of 16-bit fixed point versions of two // multiply constants: // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 @@ -49,33 +200,21 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, __m128i T0, T1, T2, T3; // Load and concatenate the transform coefficients (we'll do two inverse - // transforms in parallel). In the case of only one inverse transform, the - // second half of the vectors will just contain random value we'll never - // use nor store. + // transforms in parallel). __m128i in0, in1, in2, in3; { - in0 = _mm_loadl_epi64((const __m128i*)&in[0]); - in1 = _mm_loadl_epi64((const __m128i*)&in[4]); - in2 = _mm_loadl_epi64((const __m128i*)&in[8]); - in3 = _mm_loadl_epi64((const __m128i*)&in[12]); - // a00 a10 a20 a30 x x x x - // a01 a11 a21 a31 x x x x - // a02 a12 a22 a32 x x x x - // a03 a13 a23 a33 x x x x - if (do_two) { - const __m128i inB0 = _mm_loadl_epi64((const __m128i*)&in[16]); - const __m128i inB1 = _mm_loadl_epi64((const __m128i*)&in[20]); - const __m128i inB2 = _mm_loadl_epi64((const __m128i*)&in[24]); - const __m128i inB3 = _mm_loadl_epi64((const __m128i*)&in[28]); - in0 = _mm_unpacklo_epi64(in0, inB0); - in1 = _mm_unpacklo_epi64(in1, inB1); - in2 = _mm_unpacklo_epi64(in2, inB2); - in3 = _mm_unpacklo_epi64(in3, inB3); - // a00 a10 a20 a30 b00 b10 b20 b30 - // a01 a11 a21 a31 b01 b11 b21 b31 - // a02 a12 a22 a32 b02 b12 b22 b32 - // a03 a13 a23 a33 b03 b13 b23 b33 - } + const __m128i tmp0 = _mm_loadu_si128((const __m128i*)&in[0]); + const __m128i tmp1 = _mm_loadu_si128((const __m128i*)&in[8]); + const __m128i tmp2 = _mm_loadu_si128((const __m128i*)&in[16]); + const __m128i tmp3 = _mm_loadu_si128((const __m128i*)&in[24]); + in0 = _mm_unpacklo_epi64(tmp0, tmp2); + in1 = _mm_unpackhi_epi64(tmp0, tmp2); + in2 = _mm_unpacklo_epi64(tmp1, tmp3); + in3 = _mm_unpackhi_epi64(tmp1, tmp3); + // a00 a10 a20 a30 b00 b10 b20 b30 + // a01 a11 a21 a31 b01 b11 b21 b31 + // a02 a12 a22 a32 b02 b12 b22 b32 + // a03 a13 a23 a33 b03 b13 b23 b33 } // Vertical pass and subsequent transpose. @@ -148,19 +287,11 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, const __m128i zero = _mm_setzero_si128(); // Load the reference(s). __m128i ref0, ref1, ref2, ref3; - if (do_two) { - // Load eight bytes/pixels per line. - ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); - ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); - ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); - ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); - } else { - // Load four bytes/pixels per line. - ref0 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[0 * BPS])); - ref1 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[1 * BPS])); - ref2 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[2 * BPS])); - ref3 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[3 * BPS])); - } + // Load eight bytes/pixels per line. + ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); + ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); + ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); + ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); // Convert to 16b. ref0 = _mm_unpacklo_epi8(ref0, zero); ref1 = _mm_unpacklo_epi8(ref1, zero); @@ -176,20 +307,21 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, ref1 = _mm_packus_epi16(ref1, ref1); ref2 = _mm_packus_epi16(ref2, ref2); ref3 = _mm_packus_epi16(ref3, ref3); - // Store the results. - if (do_two) { - // Store eight bytes/pixels per line. - _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); - _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); - _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); - _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); - } else { - // Store four bytes/pixels per line. - WebPUint32ToMem(&dst[0 * BPS], _mm_cvtsi128_si32(ref0)); - WebPUint32ToMem(&dst[1 * BPS], _mm_cvtsi128_si32(ref1)); - WebPUint32ToMem(&dst[2 * BPS], _mm_cvtsi128_si32(ref2)); - WebPUint32ToMem(&dst[3 * BPS], _mm_cvtsi128_si32(ref3)); - } + // Store eight bytes/pixels per line. + _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); + _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); + _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); + _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); + } +} + +// Does one or two inverse transforms. +static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + if (do_two) { + ITransform_Two_SSE2(ref, in, dst); + } else { + ITransform_One_SSE2(ref, in, dst); } } @@ -481,7 +613,7 @@ static void CollectHistogram_SSE2(const uint8_t* ref, const uint8_t* pred, // helper for chroma-DC predictions static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 8; ++j) { _mm_storel_epi64((__m128i*)(dst + j * BPS), values); } @@ -489,7 +621,7 @@ static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { static WEBP_INLINE void Put16_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 16; ++j) { _mm_store_si128((__m128i*)(dst + j * BPS), values); } @@ -540,7 +672,7 @@ static WEBP_INLINE void VerticalPred_SSE2(uint8_t* dst, static WEBP_INLINE void HE8uv_SSE2(uint8_t* dst, const uint8_t* left) { int j; for (j = 0; j < 8; ++j) { - const __m128i values = _mm_set1_epi8(left[j]); + const __m128i values = _mm_set1_epi8((char)left[j]); _mm_storel_epi64((__m128i*)dst, values); dst += BPS; } @@ -549,7 +681,7 @@ static WEBP_INLINE void HE8uv_SSE2(uint8_t* dst, const uint8_t* left) { static WEBP_INLINE void HE16_SSE2(uint8_t* dst, const uint8_t* left) { int j; for (j = 0; j < 16; ++j) { - const __m128i values = _mm_set1_epi8(left[j]); + const __m128i values = _mm_set1_epi8((char)left[j]); _mm_store_si128((__m128i*)dst, values); dst += BPS; } @@ -722,10 +854,10 @@ static WEBP_INLINE void VE4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); const __m128i b = _mm_subs_epu8(a, lsb); const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); - const uint32_t vals = _mm_cvtsi128_si32(avg); + const int vals = _mm_cvtsi128_si32(avg); int i; for (i = 0; i < 4; ++i) { - WebPUint32ToMem(dst + i * BPS, vals); + WebPInt32ToMem(dst + i * BPS, vals); } } @@ -760,10 +892,10 @@ static WEBP_INLINE void LD4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static WEBP_INLINE void VR4_SSE2(uint8_t* dst, @@ -782,10 +914,10 @@ static WEBP_INLINE void VR4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i efgh = _mm_avg_epu8(avg2, XABCD); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); // these two are hard to implement in SSE2, so we keep the C-version: DST(0, 2) = AVG3(J, I, X); @@ -807,11 +939,12 @@ static WEBP_INLINE void VL4_SSE2(uint8_t* dst, const __m128i abbc = _mm_or_si128(ab, bc); const __m128i lsb2 = _mm_and_si128(abbc, lsb1); const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); - const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + const uint32_t extra_out = + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); // these two are hard to get and irregular DST(3, 2) = (extra_out >> 0) & 0xff; @@ -829,10 +962,10 @@ static WEBP_INLINE void RD4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static WEBP_INLINE void HU4_SSE2(uint8_t* dst, const uint8_t* top) { @@ -875,14 +1008,14 @@ static WEBP_INLINE void HD4_SSE2(uint8_t* dst, const uint8_t* top) { static WEBP_INLINE void TM4_SSE2(uint8_t* dst, const uint8_t* top) { const __m128i zero = _mm_setzero_si128(); - const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToInt32(top)); const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); int y; for (y = 0; y < 4; ++y, dst += BPS) { const int val = top[-2 - y] - top[-1]; const __m128i base = _mm_set1_epi16(val); const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); - WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(out)); } } diff --git a/3rdparty/libwebp/src/dsp/filters.c b/3rdparty/libwebp/src/dsp/filters.c index 9e910d99c9..c1350d5c9d 100644 --- a/3rdparty/libwebp/src/dsp/filters.c +++ b/3rdparty/libwebp/src/dsp/filters.c @@ -189,6 +189,12 @@ static void GradientFilter_C(const uint8_t* data, int width, int height, //------------------------------------------------------------------------------ +static void NoneUnfilter_C(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + (void)prev; + if (out != in) memcpy(out, in, width * sizeof(*out)); +} + static void HorizontalUnfilter_C(const uint8_t* prev, const uint8_t* in, uint8_t* out, int width) { uint8_t pred = (prev == NULL) ? 0 : prev[0]; @@ -233,13 +239,14 @@ static void GradientUnfilter_C(const uint8_t* prev, const uint8_t* in, WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8FiltersInitMIPSdspR2(void); extern void VP8FiltersInitMSA(void); extern void VP8FiltersInitNEON(void); extern void VP8FiltersInitSSE2(void); WEBP_DSP_INIT_FUNC(VP8FiltersInit) { - WebPUnfilters[WEBP_FILTER_NONE] = NULL; + WebPUnfilters[WEBP_FILTER_NONE] = NoneUnfilter_C; #if !WEBP_NEON_OMIT_C_CODE WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_C; WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_C; @@ -254,7 +261,7 @@ WEBP_DSP_INIT_FUNC(VP8FiltersInit) { #endif if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8FiltersInitSSE2(); } @@ -271,13 +278,14 @@ WEBP_DSP_INIT_FUNC(VP8FiltersInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8FiltersInitNEON(); } #endif + assert(WebPUnfilters[WEBP_FILTER_NONE] != NULL); assert(WebPUnfilters[WEBP_FILTER_HORIZONTAL] != NULL); assert(WebPUnfilters[WEBP_FILTER_VERTICAL] != NULL); assert(WebPUnfilters[WEBP_FILTER_GRADIENT] != NULL); diff --git a/3rdparty/libwebp/src/dsp/filters_sse2.c b/3rdparty/libwebp/src/dsp/filters_sse2.c index 4b3f2d020f..5c33ec15e2 100644 --- a/3rdparty/libwebp/src/dsp/filters_sse2.c +++ b/3rdparty/libwebp/src/dsp/filters_sse2.c @@ -320,7 +320,12 @@ extern void VP8FiltersInitSSE2(void); WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2; +#if defined(CHROMIUM) + // TODO(crbug.com/654974) + (void)VerticalUnfilter_SSE2; +#else WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2; +#endif WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2; WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2; diff --git a/3rdparty/libwebp/src/dsp/lossless.c b/3rdparty/libwebp/src/dsp/lossless.c index 46b220e2ed..9f81209453 100644 --- a/3rdparty/libwebp/src/dsp/lossless.c +++ b/3rdparty/libwebp/src/dsp/lossless.c @@ -49,7 +49,7 @@ static WEBP_INLINE uint32_t Clip255(uint32_t a) { } static WEBP_INLINE int AddSubtractComponentFull(int a, int b, int c) { - return Clip255(a + b - c); + return Clip255((uint32_t)(a + b - c)); } static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, @@ -66,7 +66,7 @@ static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, } static WEBP_INLINE int AddSubtractComponentHalf(int a, int b) { - return Clip255(a + (a - b) / 2); + return Clip255((uint32_t)(a + (a - b) / 2)); } static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, @@ -107,63 +107,77 @@ static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { //------------------------------------------------------------------------------ // Predictors -uint32_t VP8LPredictor0_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor0_C(const uint32_t* const left, + const uint32_t* const top) { (void)top; (void)left; return ARGB_BLACK; } -uint32_t VP8LPredictor1_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor1_C(const uint32_t* const left, + const uint32_t* const top) { (void)top; - return left; + return *left; } -uint32_t VP8LPredictor2_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor2_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[0]; } -uint32_t VP8LPredictor3_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor3_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[1]; } -uint32_t VP8LPredictor4_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor4_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[-1]; } -uint32_t VP8LPredictor5_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average3(left, top[0], top[1]); +uint32_t VP8LPredictor5_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average3(*left, top[0], top[1]); return pred; } -uint32_t VP8LPredictor6_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2(left, top[-1]); +uint32_t VP8LPredictor6_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2(*left, top[-1]); return pred; } -uint32_t VP8LPredictor7_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2(left, top[0]); +uint32_t VP8LPredictor7_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2(*left, top[0]); return pred; } -uint32_t VP8LPredictor8_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor8_C(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2(top[-1], top[0]); (void)left; return pred; } -uint32_t VP8LPredictor9_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor9_C(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2(top[0], top[1]); (void)left; return pred; } -uint32_t VP8LPredictor10_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average4(left, top[-1], top[0], top[1]); +uint32_t VP8LPredictor10_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average4(*left, top[-1], top[0], top[1]); return pred; } -uint32_t VP8LPredictor11_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Select(top[0], left, top[-1]); +uint32_t VP8LPredictor11_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Select(top[0], *left, top[-1]); return pred; } -uint32_t VP8LPredictor12_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); +uint32_t VP8LPredictor12_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull(*left, top[0], top[-1]); return pred; } -uint32_t VP8LPredictor13_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); +uint32_t VP8LPredictor13_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf(*left, top[0], top[-1]); return pred; } @@ -279,10 +293,10 @@ void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, const uint32_t red = argb >> 16; int new_red = red & 0xff; int new_blue = argb & 0xff; - new_red += ColorTransformDelta(m->green_to_red_, green); + new_red += ColorTransformDelta((int8_t)m->green_to_red_, green); new_red &= 0xff; - new_blue += ColorTransformDelta(m->green_to_blue_, green); - new_blue += ColorTransformDelta(m->red_to_blue_, (int8_t)new_red); + new_blue += ColorTransformDelta((int8_t)m->green_to_blue_, green); + new_blue += ColorTransformDelta((int8_t)m->red_to_blue_, (int8_t)new_red); new_blue &= 0xff; dst[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } @@ -381,7 +395,7 @@ void VP8LInverseTransform(const VP8LTransform* const transform, assert(row_start < row_end); assert(row_end <= transform->ysize_); switch (transform->type_) { - case SUBTRACT_GREEN: + case SUBTRACT_GREEN_TRANSFORM: VP8LAddGreenToBlueAndRed(in, (row_end - row_start) * width, out); break; case PREDICTOR_TRANSFORM: @@ -574,7 +588,9 @@ VP8LConvertFunc VP8LConvertBGRAToBGR; VP8LMapARGBFunc VP8LMapColor32b; VP8LMapAlphaFunc VP8LMapColor8b; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8LDspInitSSE2(void); +extern void VP8LDspInitSSE41(void); extern void VP8LDspInitNEON(void); extern void VP8LDspInitMIPSdspR2(void); extern void VP8LDspInitMSA(void); @@ -621,9 +637,14 @@ WEBP_DSP_INIT_FUNC(VP8LDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8LDspInitSSE2(); +#if defined(WEBP_HAVE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + VP8LDspInitSSE41(); + } +#endif } #endif #if defined(WEBP_USE_MIPS_DSP_R2) @@ -638,7 +659,7 @@ WEBP_DSP_INIT_FUNC(VP8LDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8LDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/lossless.h b/3rdparty/libwebp/src/dsp/lossless.h index ebd316d1ed..0bf10a1a3d 100644 --- a/3rdparty/libwebp/src/dsp/lossless.h +++ b/3rdparty/libwebp/src/dsp/lossless.h @@ -28,23 +28,38 @@ extern "C" { //------------------------------------------------------------------------------ // Decoding -typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top); +typedef uint32_t (*VP8LPredictorFunc)(const uint32_t* const left, + const uint32_t* const top); extern VP8LPredictorFunc VP8LPredictors[16]; -uint32_t VP8LPredictor0_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor1_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor2_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor3_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor4_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor5_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor6_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor7_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor8_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor9_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor10_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor11_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor12_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor13_C(uint32_t left, const uint32_t* const top); +uint32_t VP8LPredictor0_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor1_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor2_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor3_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor4_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor5_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor6_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor7_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor8_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor9_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor10_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor11_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor12_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor13_C(const uint32_t* const left, + const uint32_t* const top); // These Add/Sub function expects upper[-1] and out[-1] to be readable. typedef void (*VP8LPredictorAddSubFunc)(const uint32_t* in, @@ -167,9 +182,9 @@ extern VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; // ----------------------------------------------------------------------------- // Huffman-cost related functions. -typedef double (*VP8LCostFunc)(const uint32_t* population, int length); -typedef double (*VP8LCostCombinedFunc)(const uint32_t* X, const uint32_t* Y, - int length); +typedef uint32_t (*VP8LCostFunc)(const uint32_t* population, int length); +typedef uint32_t (*VP8LCostCombinedFunc)(const uint32_t* X, const uint32_t* Y, + int length); typedef float (*VP8LCombinedShannonEntropyFunc)(const int X[256], const int Y[256]); @@ -183,7 +198,7 @@ typedef struct { // small struct to hold counters } VP8LStreaks; typedef struct { // small struct to hold bit entropy results - double entropy; // entropy + float entropy; // entropy uint32_t sum; // sum of the population int nonzeros; // number of non-zero elements in the population uint32_t max_val; // maximum value in the population diff --git a/3rdparty/libwebp/src/dsp/lossless_common.h b/3rdparty/libwebp/src/dsp/lossless_common.h index 96a106f9ee..d6139b2b57 100644 --- a/3rdparty/libwebp/src/dsp/lossless_common.h +++ b/3rdparty/libwebp/src/dsp/lossless_common.h @@ -16,9 +16,9 @@ #ifndef WEBP_DSP_LOSSLESS_COMMON_H_ #define WEBP_DSP_LOSSLESS_COMMON_H_ -#include "src/webp/types.h" - +#include "src/dsp/cpu.h" #include "src/utils/utils.h" +#include "src/webp/types.h" #ifdef __cplusplus extern "C" { @@ -166,7 +166,7 @@ uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { } //------------------------------------------------------------------------------ -// Transform-related functions use din both encoding and decoding. +// Transform-related functions used in both encoding and decoding. // Macros used to create a batch predictor that iteratively uses a // one-pixel predictor. @@ -179,7 +179,7 @@ static void PREDICTOR_ADD(const uint32_t* in, const uint32_t* upper, \ int x; \ assert(upper != NULL); \ for (x = 0; x < num_pixels; ++x) { \ - const uint32_t pred = (PREDICTOR)(out[x - 1], upper + x); \ + const uint32_t pred = (PREDICTOR)(&out[x - 1], upper + x); \ out[x] = VP8LAddPixels(in[x], pred); \ } \ } diff --git a/3rdparty/libwebp/src/dsp/lossless_enc.c b/3rdparty/libwebp/src/dsp/lossless_enc.c index a0c7ab9117..997d56c2ad 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc.c @@ -329,6 +329,15 @@ const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX] = { static float FastSLog2Slow_C(uint32_t v) { assert(v >= LOG_LOOKUP_IDX_MAX); if (v < APPROX_LOG_WITH_CORRECTION_MAX) { +#if !defined(WEBP_HAVE_SLOW_CLZ_CTZ) + // use clz if available + const int log_cnt = BitsLog2Floor(v) - 7; + const uint32_t y = 1 << log_cnt; + int correction = 0; + const float v_f = (float)v; + const uint32_t orig_v = v; + v >>= log_cnt; +#else int log_cnt = 0; uint32_t y = 1; int correction = 0; @@ -339,6 +348,7 @@ static float FastSLog2Slow_C(uint32_t v) { v = v >> 1; y = y << 1; } while (v >= LOG_LOOKUP_IDX_MAX); +#endif // vf = (2^log_cnt) * Xf; where y = 2^log_cnt and Xf < 256 // Xf = floor(Xf) * (1 + (v % y) / v) // log2(Xf) = log2(floor(Xf)) + log2(1 + (v % y) / v) @@ -355,6 +365,14 @@ static float FastSLog2Slow_C(uint32_t v) { static float FastLog2Slow_C(uint32_t v) { assert(v >= LOG_LOOKUP_IDX_MAX); if (v < APPROX_LOG_WITH_CORRECTION_MAX) { +#if !defined(WEBP_HAVE_SLOW_CLZ_CTZ) + // use clz if available + const int log_cnt = BitsLog2Floor(v) - 7; + const uint32_t y = 1 << log_cnt; + const uint32_t orig_v = v; + double log_2; + v >>= log_cnt; +#else int log_cnt = 0; uint32_t y = 1; const uint32_t orig_v = v; @@ -364,6 +382,7 @@ static float FastLog2Slow_C(uint32_t v) { v = v >> 1; y = y << 1; } while (v >= LOG_LOOKUP_IDX_MAX); +#endif log_2 = kLog2Table[v] + log_cnt; if (orig_v >= APPROX_LOG_MAX) { // Since the division is still expensive, add this correction factor only @@ -383,7 +402,7 @@ static float FastLog2Slow_C(uint32_t v) { // Compute the combined Shanon's entropy for distribution {X} and {X+Y} static float CombinedShannonEntropy_C(const int X[256], const int Y[256]) { int i; - double retval = 0.; + float retval = 0.f; int sumX = 0, sumXY = 0; for (i = 0; i < 256; ++i) { const int x = X[i]; @@ -399,7 +418,7 @@ static float CombinedShannonEntropy_C(const int X[256], const int Y[256]) { } } retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); - return (float)retval; + return retval; } void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) { @@ -503,11 +522,11 @@ static void GetCombinedEntropyUnrefined_C(const uint32_t X[], void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const int argb = argb_data[i]; + const int argb = (int)argb_data[i]; const int green = (argb >> 8) & 0xff; const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff; const uint32_t new_b = (((argb >> 0) & 0xff) - green) & 0xff; - argb_data[i] = (argb & 0xff00ff00u) | (new_r << 16) | new_b; + argb_data[i] = ((uint32_t)argb & 0xff00ff00u) | (new_r << 16) | new_b; } } @@ -528,10 +547,10 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, const int8_t red = U32ToS8(argb >> 16); int new_red = red & 0xff; int new_blue = argb & 0xff; - new_red -= ColorTransformDelta(m->green_to_red_, green); + new_red -= ColorTransformDelta((int8_t)m->green_to_red_, green); new_red &= 0xff; - new_blue -= ColorTransformDelta(m->green_to_blue_, green); - new_blue -= ColorTransformDelta(m->red_to_blue_, red); + new_blue -= ColorTransformDelta((int8_t)m->green_to_blue_, green); + new_blue -= ColorTransformDelta((int8_t)m->red_to_blue_, red); new_blue &= 0xff; data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } @@ -541,7 +560,7 @@ static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, uint32_t argb) { const int8_t green = U32ToS8(argb >> 8); int new_red = argb >> 16; - new_red -= ColorTransformDelta(green_to_red, green); + new_red -= ColorTransformDelta((int8_t)green_to_red, green); return (new_red & 0xff); } @@ -550,9 +569,9 @@ static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, uint32_t argb) { const int8_t green = U32ToS8(argb >> 8); const int8_t red = U32ToS8(argb >> 16); - uint8_t new_blue = argb & 0xff; - new_blue -= ColorTransformDelta(green_to_blue, green); - new_blue -= ColorTransformDelta(red_to_blue, red); + int new_blue = argb & 0xff; + new_blue -= ColorTransformDelta((int8_t)green_to_blue, green); + new_blue -= ColorTransformDelta((int8_t)red_to_blue, red); return (new_blue & 0xff); } @@ -617,20 +636,25 @@ void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits, //------------------------------------------------------------------------------ -static double ExtraCost_C(const uint32_t* population, int length) { +static uint32_t ExtraCost_C(const uint32_t* population, int length) { int i; - double cost = 0.; - for (i = 2; i < length - 2; ++i) cost += (i >> 1) * population[i + 2]; + uint32_t cost = population[4] + population[5]; + assert(length % 2 == 0); + for (i = 2; i < length / 2 - 1; ++i) { + cost += i * (population[2 * i + 2] + population[2 * i + 3]); + } return cost; } -static double ExtraCostCombined_C(const uint32_t* X, const uint32_t* Y, - int length) { +static uint32_t ExtraCostCombined_C(const uint32_t* X, const uint32_t* Y, + int length) { int i; - double cost = 0.; - for (i = 2; i < length - 2; ++i) { - const int xy = X[i + 2] + Y[i + 2]; - cost += (i >> 1) * xy; + uint32_t cost = X[4] + Y[4] + X[5] + Y[5]; + assert(length % 2 == 0); + for (i = 2; i < length / 2 - 1; ++i) { + const int xy0 = X[2 * i + 2] + Y[2 * i + 2]; + const int xy1 = X[2 * i + 3] + Y[2 * i + 3]; + cost += i * (xy0 + xy1); } return cost; } @@ -726,7 +750,7 @@ static void PredictorSub##PREDICTOR_I##_C(const uint32_t* in, \ assert(upper != NULL); \ for (x = 0; x < num_pixels; ++x) { \ const uint32_t pred = \ - VP8LPredictor##PREDICTOR_I##_C(in[x - 1], upper + x); \ + VP8LPredictor##PREDICTOR_I##_C(&in[x - 1], upper + x); \ out[x] = VP8LSubPixels(in[x], pred); \ } \ } @@ -772,6 +796,7 @@ VP8LBundleColorMapFunc VP8LBundleColorMap; VP8LPredictorAddSubFunc VP8LPredictorsSub[16]; VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8LEncDspInitSSE2(void); extern void VP8LEncDspInitSSE41(void); extern void VP8LEncDspInitNEON(void); @@ -843,10 +868,10 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8LEncDspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8LEncDspInitSSE41(); } @@ -870,7 +895,7 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8LEncDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c b/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c index 0412a093cf..e10f12da9d 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c @@ -103,8 +103,8 @@ static float FastLog2Slow_MIPS32(uint32_t v) { // cost += i * *(pop + 1); // pop += 2; // } -// return (double)cost; -static double ExtraCost_MIPS32(const uint32_t* const population, int length) { +// return cost; +static uint32_t ExtraCost_MIPS32(const uint32_t* const population, int length) { int i, temp0, temp1; const uint32_t* pop = &population[4]; const uint32_t* const LoopEnd = &population[length]; @@ -130,7 +130,7 @@ static double ExtraCost_MIPS32(const uint32_t* const population, int length) { : "memory", "hi", "lo" ); - return (double)((int64_t)temp0 << 32 | temp1); + return ((int64_t)temp0 << 32 | temp1); } // C version of this function: @@ -148,9 +148,9 @@ static double ExtraCost_MIPS32(const uint32_t* const population, int length) { // pX += 2; // pY += 2; // } -// return (double)cost; -static double ExtraCostCombined_MIPS32(const uint32_t* const X, - const uint32_t* const Y, int length) { +// return cost; +static uint32_t ExtraCostCombined_MIPS32(const uint32_t* const X, + const uint32_t* const Y, int length) { int i, temp0, temp1, temp2, temp3; const uint32_t* pX = &X[4]; const uint32_t* pY = &Y[4]; @@ -183,7 +183,7 @@ static double ExtraCostCombined_MIPS32(const uint32_t* const X, : "memory", "hi", "lo" ); - return (double)((int64_t)temp0 << 32 | temp1); + return ((int64_t)temp0 << 32 | temp1); } #define HUFFMAN_COST_PASS \ @@ -347,24 +347,24 @@ static void GetCombinedEntropyUnrefined_MIPS32(const uint32_t X[], static void AddVector_MIPS32(const uint32_t* pa, const uint32_t* pb, uint32_t* pout, int size) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - const uint32_t end = ((size) / 4) * 4; + const int end = ((size) / 4) * 4; const uint32_t* const LoopEnd = pa + end; int i; ASM_START ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout) ASM_END_0 - for (i = end; i < size; ++i) pout[i] = pa[i] + pb[i]; + for (i = 0; i < size - end; ++i) pout[i] = pa[i] + pb[i]; } static void AddVectorEq_MIPS32(const uint32_t* pa, uint32_t* pout, int size) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - const uint32_t end = ((size) / 4) * 4; + const int end = ((size) / 4) * 4; const uint32_t* const LoopEnd = pa + end; int i; ASM_START ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout) ASM_END_1 - for (i = end; i < size; ++i) pout[i] += pa[i]; + for (i = 0; i < size - end; ++i) pout[i] += pa[i]; } #undef ASM_END_1 diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_neon.c b/3rdparty/libwebp/src/dsp/lossless_enc_neon.c index 7c7b73f8b6..e32c7961a2 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_neon.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_neon.c @@ -25,7 +25,7 @@ // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) #define USE_VTBLQ #endif diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c b/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c index 90c263735f..66cbaab772 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c @@ -54,8 +54,8 @@ static void TransformColor_SSE2(const VP8LMultipliers* const m, const __m128i mults_rb = MK_CST_16(CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_)); const __m128i mults_b2 = MK_CST_16(CST_5b(m->red_to_blue_), 0); - const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks - const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); // alpha-green masks + const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks int i; for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb @@ -232,79 +232,55 @@ static void AddVectorEq_SSE2(const uint32_t* a, uint32_t* out, int size) { //------------------------------------------------------------------------------ // Entropy -// Checks whether the X or Y contribution is worth computing and adding. -// Used in loop unrolling. -#define ANALYZE_X_OR_Y(x_or_y, j) \ - do { \ - if ((x_or_y)[i + (j)] != 0) retval -= VP8LFastSLog2((x_or_y)[i + (j)]); \ - } while (0) +// TODO(https://crbug.com/webp/499): this function produces different results +// from the C code due to use of double/float resulting in output differences +// when compared to -noasm. +#if !(defined(WEBP_HAVE_SLOW_CLZ_CTZ) || defined(__i386__) || defined(_M_IX86)) -// Checks whether the X + Y contribution is worth computing and adding. -// Used in loop unrolling. -#define ANALYZE_XY(j) \ - do { \ - if (tmp[j] != 0) { \ - retval -= VP8LFastSLog2(tmp[j]); \ - ANALYZE_X_OR_Y(X, j); \ - } \ - } while (0) - -#if !(defined(__i386__) || defined(_M_IX86)) static float CombinedShannonEntropy_SSE2(const int X[256], const int Y[256]) { int i; - double retval = 0.; - int sumX, sumXY; - int32_t tmp[4]; - __m128i zero = _mm_setzero_si128(); - // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY). - __m128i sumXY_128 = zero; - __m128i sumX_128 = zero; + float retval = 0.f; + int sumX = 0, sumXY = 0; + const __m128i zero = _mm_setzero_si128(); - for (i = 0; i < 256; i += 4) { - const __m128i x = _mm_loadu_si128((const __m128i*)(X + i)); - const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i)); - - // Check if any X is non-zero: this actually provides a speedup as X is - // usually sparse. - if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) { - const __m128i xy_128 = _mm_add_epi32(x, y); - sumXY_128 = _mm_add_epi32(sumXY_128, xy_128); - - sumX_128 = _mm_add_epi32(sumX_128, x); - - // Analyze the different X + Y. - _mm_storeu_si128((__m128i*)tmp, xy_128); - - ANALYZE_XY(0); - ANALYZE_XY(1); - ANALYZE_XY(2); - ANALYZE_XY(3); - } else { - // X is fully 0, so only deal with Y. - sumXY_128 = _mm_add_epi32(sumXY_128, y); - - ANALYZE_X_OR_Y(Y, 0); - ANALYZE_X_OR_Y(Y, 1); - ANALYZE_X_OR_Y(Y, 2); - ANALYZE_X_OR_Y(Y, 3); + for (i = 0; i < 256; i += 16) { + const __m128i x0 = _mm_loadu_si128((const __m128i*)(X + i + 0)); + const __m128i y0 = _mm_loadu_si128((const __m128i*)(Y + i + 0)); + const __m128i x1 = _mm_loadu_si128((const __m128i*)(X + i + 4)); + const __m128i y1 = _mm_loadu_si128((const __m128i*)(Y + i + 4)); + const __m128i x2 = _mm_loadu_si128((const __m128i*)(X + i + 8)); + const __m128i y2 = _mm_loadu_si128((const __m128i*)(Y + i + 8)); + const __m128i x3 = _mm_loadu_si128((const __m128i*)(X + i + 12)); + const __m128i y3 = _mm_loadu_si128((const __m128i*)(Y + i + 12)); + const __m128i x4 = _mm_packs_epi16(_mm_packs_epi32(x0, x1), + _mm_packs_epi32(x2, x3)); + const __m128i y4 = _mm_packs_epi16(_mm_packs_epi32(y0, y1), + _mm_packs_epi32(y2, y3)); + const int32_t mx = _mm_movemask_epi8(_mm_cmpgt_epi8(x4, zero)); + int32_t my = _mm_movemask_epi8(_mm_cmpgt_epi8(y4, zero)) | mx; + while (my) { + const int32_t j = BitsCtz(my); + int xy; + if ((mx >> j) & 1) { + const int x = X[i + j]; + sumXY += x; + retval -= VP8LFastSLog2(x); + } + xy = X[i + j] + Y[i + j]; + sumX += xy; + retval -= VP8LFastSLog2(xy); + my &= my - 1; } } - - // Sum up sumX_128 to get sumX. - _mm_storeu_si128((__m128i*)tmp, sumX_128); - sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - - // Sum up sumXY_128 to get sumXY. - _mm_storeu_si128((__m128i*)tmp, sumXY_128); - sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); - return (float)retval; + return retval; } -#endif // !(defined(__i386__) || defined(_M_IX86)) -#undef ANALYZE_X_OR_Y -#undef ANALYZE_XY +#else + +#define DONT_USE_COMBINED_SHANNON_ENTROPY_SSE2_FUNC // won't be faster + +#endif //------------------------------------------------------------------------------ @@ -400,7 +376,7 @@ static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, break; } case 2: { - const __m128i mask_or = _mm_set1_epi32(0xff000000); + const __m128i mask_or = _mm_set1_epi32((int)0xff000000); const __m128i mul_cst = _mm_set1_epi16(0x0104); const __m128i mask_mul = _mm_set1_epi16(0x0f00); for (x = 0; x + 16 <= width; x += 16, dst += 4) { @@ -451,7 +427,7 @@ static WEBP_INLINE void Average2_m128i(const __m128i* const a0, static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - const __m128i black = _mm_set1_epi32(ARGB_BLACK); + const __m128i black = _mm_set1_epi32((int)ARGB_BLACK); for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); const __m128i res = _mm_sub_epi8(src, black); @@ -662,10 +638,7 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE2; VP8LAddVector = AddVector_SSE2; VP8LAddVectorEq = AddVectorEq_SSE2; - // TODO(https://crbug.com/webp/499): this function produces different results - // from the C code due to use of double/float resulting in output differences - // when compared to -noasm. -#if !(defined(__i386__) || defined(_M_IX86)) +#if !defined(DONT_USE_COMBINED_SHANNON_ENTROPY_SSE2_FUNC) VP8LCombinedShannonEntropy = CombinedShannonEntropy_SSE2; #endif VP8LVectorMismatch = VectorMismatch_SSE2; diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c b/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c index 719d8ed25e..7ab83c2604 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c @@ -18,8 +18,53 @@ #include #include "src/dsp/lossless.h" -// For sign-extended multiplying constants, pre-shifted by 5: -#define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5) +//------------------------------------------------------------------------------ +// Cost operations. + +static WEBP_INLINE uint32_t HorizontalSum_SSE41(__m128i cost) { + cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 8)); + cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 4)); + return _mm_cvtsi128_si32(cost); +} + +static uint32_t ExtraCost_SSE41(const uint32_t* const a, int length) { + int i; + __m128i cost = _mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]); + assert(length % 8 == 0); + + for (i = 8; i + 8 <= length; i += 8) { + const int j = (i - 2) >> 1; + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); + const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j); + const __m128i a2 = _mm_hadd_epi32(a0, a1); + const __m128i mul = _mm_mullo_epi32(a2, w); + cost = _mm_add_epi32(mul, cost); + } + return HorizontalSum_SSE41(cost); +} + +static uint32_t ExtraCostCombined_SSE41(const uint32_t* const a, + const uint32_t* const b, int length) { + int i; + __m128i cost = _mm_add_epi32(_mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]), + _mm_set_epi32(2 * b[7], 2 * b[6], b[5], b[4])); + assert(length % 8 == 0); + + for (i = 8; i + 8 <= length; i += 8) { + const int j = (i - 2) >> 1; + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); + const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]); + const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j); + const __m128i a2 = _mm_hadd_epi32(a0, a1); + const __m128i b2 = _mm_hadd_epi32(b0, b1); + const __m128i mul = _mm_mullo_epi32(_mm_add_epi32(a2, b2), w); + cost = _mm_add_epi32(mul, cost); + } + return HorizontalSum_SSE41(cost); +} //------------------------------------------------------------------------------ // Subtract-Green Transform @@ -44,46 +89,50 @@ static void SubtractGreenFromBlueAndRed_SSE41(uint32_t* argb_data, //------------------------------------------------------------------------------ // Color Transform -#define SPAN 8 +// For sign-extended multiplying constants, pre-shifted by 5: +#define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5) + +#define MK_CST_16(HI, LO) \ + _mm_set1_epi32((int)(((uint32_t)(HI) << 16) | ((LO) & 0xffff))) + static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_blue, int red_to_blue, int histo[]) { - const __m128i mults_r = _mm_set1_epi16(CST_5b(red_to_blue)); - const __m128i mults_g = _mm_set1_epi16(CST_5b(green_to_blue)); - const __m128i mask_g = _mm_set1_epi16((short)0xff00); // green mask - const __m128i mask_gb = _mm_set1_epi32(0xffff); // green/blue mask - const __m128i mask_b = _mm_set1_epi16(0x00ff); // blue mask - const __m128i shuffler_lo = _mm_setr_epi8(-1, 2, -1, 6, -1, 10, -1, 14, -1, - -1, -1, -1, -1, -1, -1, -1); - const __m128i shuffler_hi = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, - 2, -1, 6, -1, 10, -1, 14); - int y; - for (y = 0; y < tile_height; ++y) { - const uint32_t* const src = argb + y * stride; - int i, x; - for (x = 0; x + SPAN <= tile_width; x += SPAN) { - uint16_t values[SPAN]; - const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); - const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); - const __m128i r0 = _mm_shuffle_epi8(in0, shuffler_lo); - const __m128i r1 = _mm_shuffle_epi8(in1, shuffler_hi); - const __m128i r = _mm_or_si128(r0, r1); // r 0 - const __m128i gb0 = _mm_and_si128(in0, mask_gb); - const __m128i gb1 = _mm_and_si128(in1, mask_gb); - const __m128i gb = _mm_packus_epi32(gb0, gb1); // g b - const __m128i g = _mm_and_si128(gb, mask_g); // g 0 - const __m128i A = _mm_mulhi_epi16(r, mults_r); // x dbr - const __m128i B = _mm_mulhi_epi16(g, mults_g); // x dbg - const __m128i C = _mm_sub_epi8(gb, B); // x b' - const __m128i D = _mm_sub_epi8(C, A); // x b'' - const __m128i E = _mm_and_si128(D, mask_b); // 0 b'' - _mm_storeu_si128((__m128i*)values, E); - for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + const __m128i mult = + MK_CST_16(CST_5b(red_to_blue) + 256,CST_5b(green_to_blue)); + const __m128i perm = + _mm_setr_epi8(-1, 1, -1, 2, -1, 5, -1, 6, -1, 9, -1, 10, -1, 13, -1, 14); + if (tile_width >= 4) { + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + const __m128i A1 = _mm_loadu_si128((const __m128i*)src); + const __m128i B1 = _mm_shuffle_epi8(A1, perm); + const __m128i C1 = _mm_mulhi_epi16(B1, mult); + const __m128i D1 = _mm_sub_epi16(A1, C1); + __m128i E = _mm_add_epi16(_mm_srli_epi32(D1, 16), D1); + int x; + for (x = 4; x + 4 <= tile_width; x += 4) { + const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x)); + __m128i B2, C2, D2; + ++histo[_mm_extract_epi8(E, 0)]; + B2 = _mm_shuffle_epi8(A2, perm); + ++histo[_mm_extract_epi8(E, 4)]; + C2 = _mm_mulhi_epi16(B2, mult); + ++histo[_mm_extract_epi8(E, 8)]; + D2 = _mm_sub_epi16(A2, C2); + ++histo[_mm_extract_epi8(E, 12)]; + E = _mm_add_epi16(_mm_srli_epi32(D2, 16), D2); + } + ++histo[_mm_extract_epi8(E, 0)]; + ++histo[_mm_extract_epi8(E, 4)]; + ++histo[_mm_extract_epi8(E, 8)]; + ++histo[_mm_extract_epi8(E, 12)]; } } { - const int left_over = tile_width & (SPAN - 1); + const int left_over = tile_width & 3; if (left_over > 0) { VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride, left_over, tile_height, @@ -95,33 +144,37 @@ static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride, static void CollectColorRedTransforms_SSE41(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_red, int histo[]) { - const __m128i mults_g = _mm_set1_epi16(CST_5b(green_to_red)); - const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask - const __m128i mask = _mm_set1_epi16(0xff); - int y; - for (y = 0; y < tile_height; ++y) { - const uint32_t* const src = argb + y * stride; - int i, x; - for (x = 0; x + SPAN <= tile_width; x += SPAN) { - uint16_t values[SPAN]; - const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); - const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); - const __m128i g0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 - const __m128i g1 = _mm_and_si128(in1, mask_g); - const __m128i g = _mm_packus_epi32(g0, g1); // g 0 - const __m128i A0 = _mm_srli_epi32(in0, 16); // 0 0 | x r - const __m128i A1 = _mm_srli_epi32(in1, 16); - const __m128i A = _mm_packus_epi32(A0, A1); // x r - const __m128i B = _mm_mulhi_epi16(g, mults_g); // x dr - const __m128i C = _mm_sub_epi8(A, B); // x r' - const __m128i D = _mm_and_si128(C, mask); // 0 r' - _mm_storeu_si128((__m128i*)values, D); - for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + const __m128i mult = MK_CST_16(0, CST_5b(green_to_red)); + const __m128i mask_g = _mm_set1_epi32(0x0000ff00); + if (tile_width >= 4) { + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + const __m128i A1 = _mm_loadu_si128((const __m128i*)src); + const __m128i B1 = _mm_and_si128(A1, mask_g); + const __m128i C1 = _mm_madd_epi16(B1, mult); + __m128i D = _mm_sub_epi16(A1, C1); + int x; + for (x = 4; x + 4 <= tile_width; x += 4) { + const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x)); + __m128i B2, C2; + ++histo[_mm_extract_epi8(D, 2)]; + B2 = _mm_and_si128(A2, mask_g); + ++histo[_mm_extract_epi8(D, 6)]; + C2 = _mm_madd_epi16(B2, mult); + ++histo[_mm_extract_epi8(D, 10)]; + ++histo[_mm_extract_epi8(D, 14)]; + D = _mm_sub_epi16(A2, C2); + } + ++histo[_mm_extract_epi8(D, 2)]; + ++histo[_mm_extract_epi8(D, 6)]; + ++histo[_mm_extract_epi8(D, 10)]; + ++histo[_mm_extract_epi8(D, 14)]; } } { - const int left_over = tile_width & (SPAN - 1); + const int left_over = tile_width & 3; if (left_over > 0) { VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride, left_over, tile_height, green_to_red, @@ -130,12 +183,16 @@ static void CollectColorRedTransforms_SSE41(const uint32_t* argb, int stride, } } +#undef MK_CST_16 + //------------------------------------------------------------------------------ // Entry point extern void VP8LEncDspInitSSE41(void); WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE41(void) { + VP8LExtraCost = ExtraCost_SSE41; + VP8LExtraCostCombined = ExtraCostCombined_SSE41; VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed_SSE41; VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE41; VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE41; diff --git a/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c b/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c index 9888854d57..bfe5ea6b38 100644 --- a/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c +++ b/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c @@ -188,46 +188,51 @@ static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, return Average2(Average2(a0, a1), Average2(a2, a3)); } -static uint32_t Predictor5_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average3(left, top[0], top[1]); +static uint32_t Predictor5_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average3(*left, top[0], top[1]); } -static uint32_t Predictor6_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average2(left, top[-1]); +static uint32_t Predictor6_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average2(*left, top[-1]); } -static uint32_t Predictor7_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average2(left, top[0]); +static uint32_t Predictor7_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average2(*left, top[0]); } -static uint32_t Predictor8_MIPSdspR2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor8_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { (void)left; return Average2(top[-1], top[0]); } -static uint32_t Predictor9_MIPSdspR2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor9_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { (void)left; return Average2(top[0], top[1]); } -static uint32_t Predictor10_MIPSdspR2(uint32_t left, +static uint32_t Predictor10_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return Average4(left, top[-1], top[0], top[1]); + return Average4(*left, top[-1], top[0], top[1]); } -static uint32_t Predictor11_MIPSdspR2(uint32_t left, +static uint32_t Predictor11_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return Select(top[0], left, top[-1]); + return Select(top[0], *left, top[-1]); } -static uint32_t Predictor12_MIPSdspR2(uint32_t left, +static uint32_t Predictor12_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return ClampedAddSubtractFull(left, top[0], top[-1]); + return ClampedAddSubtractFull(*left, top[0], top[-1]); } -static uint32_t Predictor13_MIPSdspR2(uint32_t left, +static uint32_t Predictor13_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return ClampedAddSubtractHalf(left, top[0], top[-1]); + return ClampedAddSubtractHalf(*left, top[0], top[-1]); } // Add green to blue and red channels (i.e. perform the inverse transform of diff --git a/3rdparty/libwebp/src/dsp/lossless_neon.c b/3rdparty/libwebp/src/dsp/lossless_neon.c index 76a1b6f873..ddc9b61711 100644 --- a/3rdparty/libwebp/src/dsp/lossless_neon.c +++ b/3rdparty/libwebp/src/dsp/lossless_neon.c @@ -188,17 +188,21 @@ static WEBP_INLINE uint32_t Average3_NEON(uint32_t a0, uint32_t a1, return avg; } -static uint32_t Predictor5_NEON(uint32_t left, const uint32_t* const top) { - return Average3_NEON(left, top[0], top[1]); +static uint32_t Predictor5_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average3_NEON(*left, top[0], top[1]); } -static uint32_t Predictor6_NEON(uint32_t left, const uint32_t* const top) { - return Average2_NEON(left, top[-1]); +static uint32_t Predictor6_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average2_NEON(*left, top[-1]); } -static uint32_t Predictor7_NEON(uint32_t left, const uint32_t* const top) { - return Average2_NEON(left, top[0]); +static uint32_t Predictor7_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average2_NEON(*left, top[0]); } -static uint32_t Predictor13_NEON(uint32_t left, const uint32_t* const top) { - return ClampedAddSubtractHalf_NEON(left, top[0], top[-1]); +static uint32_t Predictor13_NEON(const uint32_t* const left, + const uint32_t* const top) { + return ClampedAddSubtractHalf_NEON(*left, top[0], top[-1]); } // Batch versions of those functions. @@ -494,7 +498,7 @@ static void PredictorAdd13_NEON(const uint32_t* in, const uint32_t* upper, // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) #define USE_VTBLQ #endif diff --git a/3rdparty/libwebp/src/dsp/lossless_sse2.c b/3rdparty/libwebp/src/dsp/lossless_sse2.c index aef0cee1b3..4b6a532c23 100644 --- a/3rdparty/libwebp/src/dsp/lossless_sse2.c +++ b/3rdparty/libwebp/src/dsp/lossless_sse2.c @@ -18,7 +18,6 @@ #include "src/dsp/common_sse2.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" -#include #include //------------------------------------------------------------------------------ @@ -28,23 +27,22 @@ static WEBP_INLINE uint32_t ClampedAddSubtractFull_SSE2(uint32_t c0, uint32_t c1, uint32_t c2) { const __m128i zero = _mm_setzero_si128(); - const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); - const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); - const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); + const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); const __m128i V1 = _mm_add_epi16(C0, C1); const __m128i V2 = _mm_sub_epi16(V1, C2); const __m128i b = _mm_packus_epi16(V2, V2); - const uint32_t output = _mm_cvtsi128_si32(b); - return output; + return (uint32_t)_mm_cvtsi128_si32(b); } static WEBP_INLINE uint32_t ClampedAddSubtractHalf_SSE2(uint32_t c0, uint32_t c1, uint32_t c2) { const __m128i zero = _mm_setzero_si128(); - const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); - const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); - const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); + const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); const __m128i avg = _mm_add_epi16(C1, C0); const __m128i A0 = _mm_srli_epi16(avg, 1); const __m128i A1 = _mm_sub_epi16(A0, B0); @@ -53,16 +51,15 @@ static WEBP_INLINE uint32_t ClampedAddSubtractHalf_SSE2(uint32_t c0, const __m128i A3 = _mm_srai_epi16(A2, 1); const __m128i A4 = _mm_add_epi16(A0, A3); const __m128i A5 = _mm_packus_epi16(A4, A4); - const uint32_t output = _mm_cvtsi128_si32(A5); - return output; + return (uint32_t)_mm_cvtsi128_si32(A5); } static WEBP_INLINE uint32_t Select_SSE2(uint32_t a, uint32_t b, uint32_t c) { int pa_minus_pb; const __m128i zero = _mm_setzero_si128(); - const __m128i A0 = _mm_cvtsi32_si128(a); - const __m128i B0 = _mm_cvtsi32_si128(b); - const __m128i C0 = _mm_cvtsi32_si128(c); + const __m128i A0 = _mm_cvtsi32_si128((int)a); + const __m128i B0 = _mm_cvtsi32_si128((int)b); + const __m128i C0 = _mm_cvtsi32_si128((int)c); const __m128i AC0 = _mm_subs_epu8(A0, C0); const __m128i CA0 = _mm_subs_epu8(C0, A0); const __m128i BC0 = _mm_subs_epu8(B0, C0); @@ -95,8 +92,8 @@ static WEBP_INLINE void Average2_uint32_SSE2(const uint32_t a0, __m128i* const avg) { // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) const __m128i ones = _mm_set1_epi8(1); - const __m128i A0 = _mm_cvtsi32_si128(a0); - const __m128i A1 = _mm_cvtsi32_si128(a1); + const __m128i A0 = _mm_cvtsi32_si128((int)a0); + const __m128i A1 = _mm_cvtsi32_si128((int)a1); const __m128i avg1 = _mm_avg_epu8(A0, A1); const __m128i one = _mm_and_si128(_mm_xor_si128(A0, A1), ones); *avg = _mm_sub_epi8(avg1, one); @@ -104,8 +101,8 @@ static WEBP_INLINE void Average2_uint32_SSE2(const uint32_t a0, static WEBP_INLINE __m128i Average2_uint32_16_SSE2(uint32_t a0, uint32_t a1) { const __m128i zero = _mm_setzero_si128(); - const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a0), zero); - const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a0), zero); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); const __m128i sum = _mm_add_epi16(A1, A0); return _mm_srli_epi16(sum, 1); } @@ -113,19 +110,18 @@ static WEBP_INLINE __m128i Average2_uint32_16_SSE2(uint32_t a0, uint32_t a1) { static WEBP_INLINE uint32_t Average2_SSE2(uint32_t a0, uint32_t a1) { __m128i output; Average2_uint32_SSE2(a0, a1, &output); - return _mm_cvtsi128_si32(output); + return (uint32_t)_mm_cvtsi128_si32(output); } static WEBP_INLINE uint32_t Average3_SSE2(uint32_t a0, uint32_t a1, uint32_t a2) { const __m128i zero = _mm_setzero_si128(); const __m128i avg1 = Average2_uint32_16_SSE2(a0, a2); - const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); const __m128i sum = _mm_add_epi16(avg1, A1); const __m128i avg2 = _mm_srli_epi16(sum, 1); const __m128i A2 = _mm_packus_epi16(avg2, avg2); - const uint32_t output = _mm_cvtsi128_si32(A2); - return output; + return (uint32_t)_mm_cvtsi128_si32(A2); } static WEBP_INLINE uint32_t Average4_SSE2(uint32_t a0, uint32_t a1, @@ -135,46 +131,54 @@ static WEBP_INLINE uint32_t Average4_SSE2(uint32_t a0, uint32_t a1, const __m128i sum = _mm_add_epi16(avg2, avg1); const __m128i avg3 = _mm_srli_epi16(sum, 1); const __m128i A0 = _mm_packus_epi16(avg3, avg3); - const uint32_t output = _mm_cvtsi128_si32(A0); - return output; + return (uint32_t)_mm_cvtsi128_si32(A0); } -static uint32_t Predictor5_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average3_SSE2(left, top[0], top[1]); +static uint32_t Predictor5_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average3_SSE2(*left, top[0], top[1]); return pred; } -static uint32_t Predictor6_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2_SSE2(left, top[-1]); +static uint32_t Predictor6_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2_SSE2(*left, top[-1]); return pred; } -static uint32_t Predictor7_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2_SSE2(left, top[0]); +static uint32_t Predictor7_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2_SSE2(*left, top[0]); return pred; } -static uint32_t Predictor8_SSE2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor8_SSE2(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2_SSE2(top[-1], top[0]); (void)left; return pred; } -static uint32_t Predictor9_SSE2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor9_SSE2(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2_SSE2(top[0], top[1]); (void)left; return pred; } -static uint32_t Predictor10_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average4_SSE2(left, top[-1], top[0], top[1]); +static uint32_t Predictor10_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average4_SSE2(*left, top[-1], top[0], top[1]); return pred; } -static uint32_t Predictor11_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Select_SSE2(top[0], left, top[-1]); +static uint32_t Predictor11_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Select_SSE2(top[0], *left, top[-1]); return pred; } -static uint32_t Predictor12_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractFull_SSE2(left, top[0], top[-1]); +static uint32_t Predictor12_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull_SSE2(*left, top[0], top[-1]); return pred; } -static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractHalf_SSE2(left, top[0], top[-1]); +static uint32_t Predictor13_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf_SSE2(*left, top[0], top[-1]); return pred; } @@ -184,7 +188,7 @@ static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) { static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - const __m128i black = _mm_set1_epi32(ARGB_BLACK); + const __m128i black = _mm_set1_epi32((int)ARGB_BLACK); for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); const __m128i res = _mm_add_epi8(src, black); @@ -200,7 +204,7 @@ static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, static void PredictorAdd1_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - __m128i prev = _mm_set1_epi32(out[-1]); + __m128i prev = _mm_set1_epi32((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { // a | b | c | d const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); @@ -277,12 +281,12 @@ GENERATE_PREDICTOR_2(9, upper[i + 1]) #undef GENERATE_PREDICTOR_2 // Predictor10: average of (average of (L,TL), average of (T, TR)). -#define DO_PRED10(OUT) do { \ - __m128i avgLTL, avg; \ - Average2_m128i(&L, &TL, &avgLTL); \ - Average2_m128i(&avgTTR, &avgLTL, &avg); \ - L = _mm_add_epi8(avg, src); \ - out[i + (OUT)] = _mm_cvtsi128_si32(L); \ +#define DO_PRED10(OUT) do { \ + __m128i avgLTL, avg; \ + Average2_m128i(&L, &TL, &avgLTL); \ + Average2_m128i(&avgTTR, &avgLTL, &avg); \ + L = _mm_add_epi8(avg, src); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ } while (0) #define DO_PRED10_SHIFT do { \ @@ -295,7 +299,7 @@ GENERATE_PREDICTOR_2(9, upper[i + 1]) static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - __m128i L = _mm_cvtsi32_si128(out[-1]); + __m128i L = _mm_cvtsi32_si128((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); @@ -328,7 +332,7 @@ static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, const __m128i B = _mm_andnot_si128(mask, T); \ const __m128i pred = _mm_or_si128(A, B); /* pred = (pa > b)? L : T*/ \ L = _mm_add_epi8(src, pred); \ - out[i + (OUT)] = _mm_cvtsi128_si32(L); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ } while (0) #define DO_PRED11_SHIFT do { \ @@ -343,7 +347,7 @@ static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; __m128i pa; - __m128i L = _mm_cvtsi32_si128(out[-1]); + __m128i L = _mm_cvtsi32_si128((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); @@ -376,12 +380,12 @@ static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, #undef DO_PRED11_SHIFT // Predictor12: ClampedAddSubtractFull. -#define DO_PRED12(DIFF, LANE, OUT) do { \ - const __m128i all = _mm_add_epi16(L, (DIFF)); \ - const __m128i alls = _mm_packus_epi16(all, all); \ - const __m128i res = _mm_add_epi8(src, alls); \ - out[i + (OUT)] = _mm_cvtsi128_si32(res); \ - L = _mm_unpacklo_epi8(res, zero); \ +#define DO_PRED12(DIFF, LANE, OUT) do { \ + const __m128i all = _mm_add_epi16(L, (DIFF)); \ + const __m128i alls = _mm_packus_epi16(all, all); \ + const __m128i res = _mm_add_epi8(src, alls); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(res); \ + L = _mm_unpacklo_epi8(res, zero); \ } while (0) #define DO_PRED12_SHIFT(DIFF, LANE) do { \ @@ -394,7 +398,7 @@ static void PredictorAdd12_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; const __m128i zero = _mm_setzero_si128(); - const __m128i L8 = _mm_cvtsi32_si128(out[-1]); + const __m128i L8 = _mm_cvtsi32_si128((int)out[-1]); __m128i L = _mm_unpacklo_epi8(L8, zero); for (i = 0; i + 4 <= num_pixels; i += 4) { // Load 4 pixels at a time. @@ -460,7 +464,7 @@ static void TransformColorInverse_SSE2(const VP8LMultipliers* const m, const __m128i mults_b2 = MK_CST_16(CST(red_to_blue_), 0); #undef MK_CST_16 #undef CST - const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); // alpha-green masks int i; for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb @@ -524,7 +528,7 @@ static void ConvertBGRAToRGB_SSE2(const uint32_t* src, int num_pixels, static void ConvertBGRAToRGBA_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { - const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ffu); + const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ff); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; while (num_pixels >= 8) { @@ -553,7 +557,7 @@ static void ConvertBGRAToRGBA_SSE2(const uint32_t* src, static void ConvertBGRAToRGBA4444_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { const __m128i mask_0x0f = _mm_set1_epi8(0x0f); - const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i mask_0xf0 = _mm_set1_epi8((char)0xf0); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; while (num_pixels >= 8) { @@ -588,8 +592,8 @@ static void ConvertBGRAToRGBA4444_SSE2(const uint32_t* src, static void ConvertBGRAToRGB565_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { - const __m128i mask_0xe0 = _mm_set1_epi8(0xe0); - const __m128i mask_0xf8 = _mm_set1_epi8(0xf8); + const __m128i mask_0xe0 = _mm_set1_epi8((char)0xe0); + const __m128i mask_0xf8 = _mm_set1_epi8((char)0xf8); const __m128i mask_0x07 = _mm_set1_epi8(0x07); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; diff --git a/3rdparty/libwebp/src/dsp/lossless_sse41.c b/3rdparty/libwebp/src/dsp/lossless_sse41.c new file mode 100644 index 0000000000..bb7ce7611f --- /dev/null +++ b/3rdparty/libwebp/src/dsp/lossless_sse41.c @@ -0,0 +1,133 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE41 variant of methods for lossless decoder + +#include "src/dsp/dsp.h" + +#if defined(WEBP_USE_SSE41) + +#include "src/dsp/common_sse41.h" +#include "src/dsp/lossless.h" +#include "src/dsp/lossless_common.h" + +//------------------------------------------------------------------------------ +// Color-space conversion functions + +static void TransformColorInverse_SSE41(const VP8LMultipliers* const m, + const uint32_t* const src, + int num_pixels, uint32_t* dst) { +// sign-extended multiplying constants, pre-shifted by 5. +#define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend + const __m128i mults_rb = + _mm_set1_epi32((int)((uint32_t)CST(green_to_red_) << 16 | + (CST(green_to_blue_) & 0xffff))); + const __m128i mults_b2 = _mm_set1_epi32(CST(red_to_blue_)); +#undef CST + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); + const __m128i perm1 = _mm_setr_epi8(-1, 1, -1, 1, -1, 5, -1, 5, + -1, 9, -1, 9, -1, 13, -1, 13); + const __m128i perm2 = _mm_setr_epi8(-1, 2, -1, -1, -1, 6, -1, -1, + -1, 10, -1, -1, -1, 14, -1, -1); + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i A = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i B = _mm_shuffle_epi8(A, perm1); // argb -> g0g0 + const __m128i C = _mm_mulhi_epi16(B, mults_rb); + const __m128i D = _mm_add_epi8(A, C); + const __m128i E = _mm_shuffle_epi8(D, perm2); + const __m128i F = _mm_mulhi_epi16(E, mults_b2); + const __m128i G = _mm_add_epi8(D, F); + const __m128i out = _mm_blendv_epi8(G, A, mask_ag); + _mm_storeu_si128((__m128i*)&dst[i], out); + } + // Fall-back to C-version for left-overs. + if (i != num_pixels) { + VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i); + } +} + +//------------------------------------------------------------------------------ + +#define ARGB_TO_RGB_SSE41 do { \ + while (num_pixels >= 16) { \ + const __m128i in0 = _mm_loadu_si128(in + 0); \ + const __m128i in1 = _mm_loadu_si128(in + 1); \ + const __m128i in2 = _mm_loadu_si128(in + 2); \ + const __m128i in3 = _mm_loadu_si128(in + 3); \ + const __m128i a0 = _mm_shuffle_epi8(in0, perm0); \ + const __m128i a1 = _mm_shuffle_epi8(in1, perm1); \ + const __m128i a2 = _mm_shuffle_epi8(in2, perm2); \ + const __m128i a3 = _mm_shuffle_epi8(in3, perm3); \ + const __m128i b0 = _mm_blend_epi16(a0, a1, 0xc0); \ + const __m128i b1 = _mm_blend_epi16(a1, a2, 0xf0); \ + const __m128i b2 = _mm_blend_epi16(a2, a3, 0xfc); \ + _mm_storeu_si128(out + 0, b0); \ + _mm_storeu_si128(out + 1, b1); \ + _mm_storeu_si128(out + 2, b2); \ + in += 4; \ + out += 3; \ + num_pixels -= 16; \ + } \ +} while (0) + +static void ConvertBGRAToRGB_SSE41(const uint32_t* src, int num_pixels, + uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + const __m128i perm0 = _mm_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, + 8, 14, 13, 12, -1, -1, -1, -1); + const __m128i perm1 = _mm_shuffle_epi32(perm0, 0x39); + const __m128i perm2 = _mm_shuffle_epi32(perm0, 0x4e); + const __m128i perm3 = _mm_shuffle_epi32(perm0, 0x93); + + ARGB_TO_RGB_SSE41; + + // left-overs + if (num_pixels > 0) { + VP8LConvertBGRAToRGB_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } +} + +static void ConvertBGRAToBGR_SSE41(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + const __m128i perm0 = _mm_setr_epi8(0, 1, 2, 4, 5, 6, 8, 9, 10, + 12, 13, 14, -1, -1, -1, -1); + const __m128i perm1 = _mm_shuffle_epi32(perm0, 0x39); + const __m128i perm2 = _mm_shuffle_epi32(perm0, 0x4e); + const __m128i perm3 = _mm_shuffle_epi32(perm0, 0x93); + + ARGB_TO_RGB_SSE41; + + // left-overs + if (num_pixels > 0) { + VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } +} + +#undef ARGB_TO_RGB_SSE41 + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitSSE41(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE41(void) { + VP8LTransformColorInverse = TransformColorInverse_SSE41; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB_SSE41; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR_SSE41; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(VP8LDspInitSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/3rdparty/libwebp/src/dsp/msa_macro.h b/3rdparty/libwebp/src/dsp/msa_macro.h index a16c0bb300..90adbbc319 100644 --- a/3rdparty/libwebp/src/dsp/msa_macro.h +++ b/3rdparty/libwebp/src/dsp/msa_macro.h @@ -14,6 +14,10 @@ #ifndef WEBP_DSP_MSA_MACRO_H_ #define WEBP_DSP_MSA_MACRO_H_ +#include "src/dsp/dsp.h" + +#if defined(WEBP_USE_MSA) + #include #include @@ -69,27 +73,25 @@ #define ST_UW(...) ST_W(v4u32, __VA_ARGS__) #define ST_SW(...) ST_W(v4i32, __VA_ARGS__) -#define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ - static inline TYPE FUNC_NAME(const void* const psrc) { \ - const uint8_t* const psrc_m = (const uint8_t*)psrc; \ - TYPE val_m; \ - __asm__ volatile ( \ - "" #INSTR " %[val_m], %[psrc_m] \n\t" \ - : [val_m] "=r" (val_m) \ - : [psrc_m] "m" (*psrc_m)); \ - return val_m; \ +#define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline TYPE FUNC_NAME(const void* const psrc) { \ + const uint8_t* const psrc_m = (const uint8_t*)psrc; \ + TYPE val_m; \ + __asm__ volatile("" #INSTR " %[val_m], %[psrc_m] \n\t" \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + return val_m; \ } #define MSA_LOAD(psrc, FUNC_NAME) FUNC_NAME(psrc) -#define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ - static inline void FUNC_NAME(TYPE val, void* const pdst) { \ - uint8_t* const pdst_m = (uint8_t*)pdst; \ - TYPE val_m = val; \ - __asm__ volatile ( \ - " " #INSTR " %[val_m], %[pdst_m] \n\t" \ - : [pdst_m] "=m" (*pdst_m) \ - : [val_m] "r" (val_m)); \ +#define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline void FUNC_NAME(TYPE val, void* const pdst) { \ + uint8_t* const pdst_m = (uint8_t*)pdst; \ + TYPE val_m = val; \ + __asm__ volatile(" " #INSTR " %[val_m], %[pdst_m] \n\t" \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ } #define MSA_STORE(val, pdst, FUNC_NAME) FUNC_NAME(val, pdst) @@ -1389,4 +1391,5 @@ static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) { } while (0) #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) +#endif // WEBP_USE_MSA #endif // WEBP_DSP_MSA_MACRO_H_ diff --git a/3rdparty/libwebp/src/dsp/neon.h b/3rdparty/libwebp/src/dsp/neon.h index aa1dea1301..14acb4044b 100644 --- a/3rdparty/libwebp/src/dsp/neon.h +++ b/3rdparty/libwebp/src/dsp/neon.h @@ -12,14 +12,16 @@ #ifndef WEBP_DSP_NEON_H_ #define WEBP_DSP_NEON_H_ -#include - #include "src/dsp/dsp.h" +#if defined(WEBP_USE_NEON) + +#include + // Right now, some intrinsics functions seem slower, so we disable them // everywhere except newer clang/gcc or aarch64 where the inline assembly is // incompatible. -#if LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,9) || defined(__aarch64__) +#if LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 9) || WEBP_AARCH64 #define WEBP_USE_INTRINSICS // use intrinsics when possible #endif @@ -44,7 +46,7 @@ // if using intrinsics, this flag avoids some functions that make gcc-4.6.3 // crash ("internal compiler error: in immed_double_const, at emit-rtl."). // (probably similar to gcc.gnu.org/bugzilla/show_bug.cgi?id=48183) -#if !(LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,8) || defined(__aarch64__)) +#if !(LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 8) || WEBP_AARCH64) #define WORK_AROUND_GCC #endif @@ -98,4 +100,5 @@ static WEBP_INLINE int32x4x4_t Transpose4x4_NEON(const int32x4x4_t rows) { } while (0) #endif +#endif // WEBP_USE_NEON #endif // WEBP_DSP_NEON_H_ diff --git a/3rdparty/libwebp/src/dsp/quant.h b/3rdparty/libwebp/src/dsp/quant.h index 5e8dba8d19..dcbc11c77c 100644 --- a/3rdparty/libwebp/src/dsp/quant.h +++ b/3rdparty/libwebp/src/dsp/quant.h @@ -21,18 +21,24 @@ #define IsFlat IsFlat_NEON -static uint32x2_t horizontal_add_uint32x4(const uint32x4_t a) { +static uint32_t horizontal_add_uint32x4(const uint32x4_t a) { +#if WEBP_AARCH64 + return vaddvq_u32(a); +#else const uint64x2_t b = vpaddlq_u32(a); - return vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), - vreinterpret_u32_u64(vget_high_u64(b))); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +#endif } static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks, int thresh) { const int16x8_t tst_ones = vdupq_n_s16(-1); uint32x4_t sum = vdupq_n_u32(0); + int i; - for (int i = 0; i < num_blocks; ++i) { + for (i = 0; i < num_blocks; ++i) { // Set DC to zero. const int16x8_t a_0 = vsetq_lane_s16(0, vld1q_s16(levels), 0); const int16x8_t a_1 = vld1q_s16(levels + 8); @@ -45,7 +51,7 @@ static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks, levels += 16; } - return thresh >= (int32_t)vget_lane_u32(horizontal_add_uint32x4(sum), 0); + return thresh >= (int)horizontal_add_uint32x4(sum); } #else diff --git a/3rdparty/libwebp/src/dsp/rescaler.c b/3rdparty/libwebp/src/dsp/rescaler.c index c5a01e82df..325d8be180 100644 --- a/3rdparty/libwebp/src/dsp/rescaler.c +++ b/3rdparty/libwebp/src/dsp/rescaler.c @@ -38,8 +38,9 @@ void WebPRescalerImportRowExpand_C(WebPRescaler* const wrk, int x_out = channel; // simple bilinear interpolation int accum = wrk->x_add; - int left = src[x_in]; - int right = (wrk->src_width > 1) ? src[x_in + x_stride] : left; + rescaler_t left = (rescaler_t)src[x_in]; + rescaler_t right = + (wrk->src_width > 1) ? (rescaler_t)src[x_in + x_stride] : left; x_in += x_stride; while (1) { wrk->frow[x_out] = right * wrk->x_add + (left - right) * accum; @@ -50,7 +51,7 @@ void WebPRescalerImportRowExpand_C(WebPRescaler* const wrk, left = right; x_in += x_stride; assert(x_in < wrk->src_width * x_stride); - right = src[x_in]; + right = (rescaler_t)src[x_in]; accum += wrk->x_add; } } @@ -196,6 +197,7 @@ WebPRescalerImportRowFunc WebPRescalerImportRowShrink; WebPRescalerExportRowFunc WebPRescalerExportRowExpand; WebPRescalerExportRowFunc WebPRescalerExportRowShrink; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPRescalerDspInitSSE2(void); extern void WebPRescalerDspInitMIPS32(void); extern void WebPRescalerDspInitMIPSdspR2(void); @@ -213,7 +215,7 @@ WEBP_DSP_INIT_FUNC(WebPRescalerDspInit) { WebPRescalerImportRowShrink = WebPRescalerImportRowShrink_C; if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPRescalerDspInitSSE2(); } @@ -235,7 +237,7 @@ WEBP_DSP_INIT_FUNC(WebPRescalerDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPRescalerDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/rescaler_sse2.c b/3rdparty/libwebp/src/dsp/rescaler_sse2.c index d7effea16e..3f18e94e93 100644 --- a/3rdparty/libwebp/src/dsp/rescaler_sse2.c +++ b/3rdparty/libwebp/src/dsp/rescaler_sse2.c @@ -85,7 +85,7 @@ static void RescalerImportRowExpand_SSE2(WebPRescaler* const wrk, const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum); const __m128i out = _mm_madd_epi16(cur_pixels, mult); assert(sizeof(*frow) == sizeof(uint32_t)); - WebPUint32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); + WebPInt32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); frow += 1; if (frow >= frow_end) break; accum -= wrk->x_sub; @@ -132,7 +132,7 @@ static void RescalerImportRowShrink_SSE2(WebPRescaler* const wrk, __m128i base = zero; accum += wrk->x_add; while (accum > 0) { - const __m128i A = _mm_cvtsi32_si128(WebPMemToUint32(src)); + const __m128i A = _mm_cvtsi32_si128(WebPMemToInt32(src)); src += 4; base = _mm_unpacklo_epi8(A, zero); // To avoid overflow, we need: base * x_add / x_sub < 32768 @@ -198,7 +198,7 @@ static WEBP_INLINE void ProcessRow_SSE2(const __m128i* const A0, const __m128i* const mult, uint8_t* const dst) { const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); - const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0); + const __m128i mask = _mm_set_epi32(~0, 0, ~0, 0); const __m128i B0 = _mm_mul_epu32(*A0, *mult); const __m128i B1 = _mm_mul_epu32(*A1, *mult); const __m128i B2 = _mm_mul_epu32(*A2, *mult); diff --git a/3rdparty/libwebp/src/dsp/ssim.c b/3rdparty/libwebp/src/dsp/ssim.c index 989ce8254c..9a1341ed95 100644 --- a/3rdparty/libwebp/src/dsp/ssim.c +++ b/3rdparty/libwebp/src/dsp/ssim.c @@ -137,6 +137,7 @@ VP8SSIMGetClippedFunc VP8SSIMGetClipped; VP8AccumulateSSEFunc VP8AccumulateSSE; #endif +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8SSIMDspInitSSE2(void); WEBP_DSP_INIT_FUNC(VP8SSIMDspInit) { @@ -150,7 +151,7 @@ WEBP_DSP_INIT_FUNC(VP8SSIMDspInit) { #endif if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8SSIMDspInitSSE2(); } diff --git a/3rdparty/libwebp/src/dsp/upsampling.c b/3rdparty/libwebp/src/dsp/upsampling.c index 9b60da5bbb..983b9c42d3 100644 --- a/3rdparty/libwebp/src/dsp/upsampling.c +++ b/3rdparty/libwebp/src/dsp/upsampling.c @@ -215,6 +215,7 @@ static void EmptyYuv444Func(const uint8_t* y, WebPYUV444Converter WebPYUV444Converters[MODE_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitYUV444ConvertersMIPSdspR2(void); extern void WebPInitYUV444ConvertersSSE2(void); extern void WebPInitYUV444ConvertersSSE41(void); @@ -233,12 +234,12 @@ WEBP_DSP_INIT_FUNC(WebPInitYUV444Converters) { WebPYUV444Converters[MODE_rgbA_4444] = WebPYuv444ToRgba4444_C; if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitYUV444ConvertersSSE2(); } #endif -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitYUV444ConvertersSSE41(); } @@ -278,12 +279,12 @@ WEBP_DSP_INIT_FUNC(WebPInitUpsamplers) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitUpsamplersSSE2(); } #endif -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitUpsamplersSSE41(); } @@ -300,7 +301,7 @@ WEBP_DSP_INIT_FUNC(WebPInitUpsamplers) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitUpsamplersNEON(); diff --git a/3rdparty/libwebp/src/dsp/upsampling_neon.c b/3rdparty/libwebp/src/dsp/upsampling_neon.c index 6ba71a7de5..bbc000ca2d 100644 --- a/3rdparty/libwebp/src/dsp/upsampling_neon.c +++ b/3rdparty/libwebp/src/dsp/upsampling_neon.c @@ -111,7 +111,7 @@ static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; vst4_u8(out, v255_r_g_b); \ } while (0) -#if !defined(WEBP_SWAP_16BIT_CSP) +#if (WEBP_SWAP_16BIT_CSP == 0) #define ZIP_U8(lo, hi) vzip_u8((lo), (hi)) #else #define ZIP_U8(lo, hi) vzip_u8((hi), (lo)) diff --git a/3rdparty/libwebp/src/dsp/upsampling_sse2.c b/3rdparty/libwebp/src/dsp/upsampling_sse2.c index 340f1e2ac2..08b6d0b1cf 100644 --- a/3rdparty/libwebp/src/dsp/upsampling_sse2.c +++ b/3rdparty/libwebp/src/dsp/upsampling_sse2.c @@ -121,7 +121,7 @@ static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ int uv_pos, pos; \ /* 16byte-aligned array to cache reconstructed u and v */ \ uint8_t uv_buf[14 * 32 + 15] = { 0 }; \ - uint8_t* const r_u = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + uint8_t* const r_u = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~(uintptr_t)15); \ uint8_t* const r_v = r_u + 32; \ \ assert(top_y != NULL); \ diff --git a/3rdparty/libwebp/src/dsp/yuv.c b/3rdparty/libwebp/src/dsp/yuv.c index 14e67fc28e..8a04b85d82 100644 --- a/3rdparty/libwebp/src/dsp/yuv.c +++ b/3rdparty/libwebp/src/dsp/yuv.c @@ -70,6 +70,7 @@ void WebPSamplerProcessPlane(const uint8_t* y, int y_stride, WebPSamplerRowFunc WebPSamplers[MODE_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitSamplersSSE2(void); extern void WebPInitSamplersSSE41(void); extern void WebPInitSamplersMIPS32(void); @@ -90,16 +91,16 @@ WEBP_DSP_INIT_FUNC(WebPInitSamplers) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitSamplersSSE2(); } -#endif // WEBP_USE_SSE2 -#if defined(WEBP_USE_SSE41) +#endif // WEBP_HAVE_SSE2 +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitSamplersSSE41(); } -#endif // WEBP_USE_SSE41 +#endif // WEBP_HAVE_SSE41 #if defined(WEBP_USE_MIPS32) if (VP8GetCPUInfo(kMIPS32)) { WebPInitSamplersMIPS32(); @@ -194,50 +195,6 @@ void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, //----------------------------------------------------------------------------- -#if !WEBP_NEON_OMIT_C_CODE -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_C(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - uint64_t diff = 0; - int i; - for (i = 0; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)dst[i] + diff_y; - dst[i] = clip_y(new_y); - diff += (uint64_t)abs(diff_y); - } - return diff; -} - -static void SharpYUVUpdateRGB_C(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i; - for (i = 0; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_C(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - for (i = 0; i < len; ++i, ++A, ++B) { - const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4; - const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4; - out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); - } -} -#endif // !WEBP_NEON_OMIT_C_CODE - -#undef MAX_Y - -//----------------------------------------------------------------------------- - void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, @@ -247,18 +204,9 @@ void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, int src_width, int do_store); -uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len); -void (*WebPSharpYUVUpdateRGB)(const int16_t* ref, const int16_t* src, - int16_t* dst, int len); -void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out); - extern void WebPInitConvertARGBToYUVSSE2(void); extern void WebPInitConvertARGBToYUVSSE41(void); extern void WebPInitConvertARGBToYUVNEON(void); -extern void WebPInitSharpYUVSSE2(void); -extern void WebPInitSharpYUVNEON(void); WEBP_DSP_INIT_FUNC(WebPInitConvertARGBToYUV) { WebPConvertARGBToY = ConvertARGBToY_C; @@ -269,40 +217,29 @@ WEBP_DSP_INIT_FUNC(WebPInitConvertARGBToYUV) { WebPConvertRGBA32ToUV = WebPConvertRGBA32ToUV_C; -#if !WEBP_NEON_OMIT_C_CODE - WebPSharpYUVUpdateY = SharpYUVUpdateY_C; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_C; - WebPSharpYUVFilterRow = SharpYUVFilterRow_C; -#endif - if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitConvertARGBToYUVSSE2(); - WebPInitSharpYUVSSE2(); } -#endif // WEBP_USE_SSE2 -#if defined(WEBP_USE_SSE41) +#endif // WEBP_HAVE_SSE2 +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitConvertARGBToYUVSSE41(); } -#endif // WEBP_USE_SSE41 +#endif // WEBP_HAVE_SSE41 } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitConvertARGBToYUVNEON(); - WebPInitSharpYUVNEON(); } -#endif // WEBP_USE_NEON +#endif // WEBP_HAVE_NEON assert(WebPConvertARGBToY != NULL); assert(WebPConvertARGBToUV != NULL); assert(WebPConvertRGB24ToY != NULL); assert(WebPConvertBGR24ToY != NULL); assert(WebPConvertRGBA32ToUV != NULL); - assert(WebPSharpYUVUpdateY != NULL); - assert(WebPSharpYUVUpdateRGB != NULL); - assert(WebPSharpYUVFilterRow != NULL); } diff --git a/3rdparty/libwebp/src/dsp/yuv.h b/3rdparty/libwebp/src/dsp/yuv.h index c12be1d094..66a397d117 100644 --- a/3rdparty/libwebp/src/dsp/yuv.h +++ b/3rdparty/libwebp/src/dsp/yuv.h @@ -10,7 +10,7 @@ // inline YUV<->RGB conversion function // // The exact naming is Y'CbCr, following the ITU-R BT.601 standard. -// More information at: http://en.wikipedia.org/wiki/YCbCr +// More information at: https://en.wikipedia.org/wiki/YCbCr // Y = 0.2569 * R + 0.5044 * G + 0.0979 * B + 16 // U = -0.1483 * R - 0.2911 * G + 0.4394 * B + 128 // V = 0.4394 * R - 0.3679 * G - 0.0715 * B + 128 diff --git a/3rdparty/libwebp/src/dsp/yuv_neon.c b/3rdparty/libwebp/src/dsp/yuv_neon.c index a34d60248f..ff77b00980 100644 --- a/3rdparty/libwebp/src/dsp/yuv_neon.c +++ b/3rdparty/libwebp/src/dsp/yuv_neon.c @@ -173,116 +173,8 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVNEON(void) { WebPConvertRGBA32ToUV = ConvertRGBA32ToUV_NEON; } -//------------------------------------------------------------------------------ - -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y_NEON(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_NEON(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - int i; - const int16x8_t zero = vdupq_n_s16(0); - const int16x8_t max = vdupq_n_s16(MAX_Y); - uint64x2_t sum = vdupq_n_u64(0); - uint64_t diff; - - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t A = vreinterpretq_s16_u16(vld1q_u16(ref + i)); - const int16x8_t B = vreinterpretq_s16_u16(vld1q_u16(src + i)); - const int16x8_t C = vreinterpretq_s16_u16(vld1q_u16(dst + i)); - const int16x8_t D = vsubq_s16(A, B); // diff_y - const int16x8_t F = vaddq_s16(C, D); // new_y - const uint16x8_t H = - vreinterpretq_u16_s16(vmaxq_s16(vminq_s16(F, max), zero)); - const int16x8_t I = vabsq_s16(D); // abs(diff_y) - vst1q_u16(dst + i, H); - sum = vpadalq_u32(sum, vpaddlq_u16(vreinterpretq_u16_s16(I))); - } - diff = vgetq_lane_u64(sum, 0) + vgetq_lane_u64(sum, 1); - for (; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)(dst[i]) + diff_y; - dst[i] = clip_y_NEON(new_y); - diff += (uint64_t)(abs(diff_y)); - } - return diff; -} - -static void SharpYUVUpdateRGB_NEON(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i; - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t A = vld1q_s16(ref + i); - const int16x8_t B = vld1q_s16(src + i); - const int16x8_t C = vld1q_s16(dst + i); - const int16x8_t D = vsubq_s16(A, B); // diff_uv - const int16x8_t E = vaddq_s16(C, D); // new_uv - vst1q_s16(dst + i, E); - } - for (; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_NEON(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - const int16x8_t max = vdupq_n_s16(MAX_Y); - const int16x8_t zero = vdupq_n_s16(0); - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t a0 = vld1q_s16(A + i + 0); - const int16x8_t a1 = vld1q_s16(A + i + 1); - const int16x8_t b0 = vld1q_s16(B + i + 0); - const int16x8_t b1 = vld1q_s16(B + i + 1); - const int16x8_t a0b1 = vaddq_s16(a0, b1); - const int16x8_t a1b0 = vaddq_s16(a1, b0); - const int16x8_t a0a1b0b1 = vaddq_s16(a0b1, a1b0); // A0+A1+B0+B1 - const int16x8_t a0b1_2 = vaddq_s16(a0b1, a0b1); // 2*(A0+B1) - const int16x8_t a1b0_2 = vaddq_s16(a1b0, a1b0); // 2*(A1+B0) - const int16x8_t c0 = vshrq_n_s16(vaddq_s16(a0b1_2, a0a1b0b1), 3); - const int16x8_t c1 = vshrq_n_s16(vaddq_s16(a1b0_2, a0a1b0b1), 3); - const int16x8_t d0 = vaddq_s16(c1, a0); - const int16x8_t d1 = vaddq_s16(c0, a1); - const int16x8_t e0 = vrshrq_n_s16(d0, 1); - const int16x8_t e1 = vrshrq_n_s16(d1, 1); - const int16x8x2_t f = vzipq_s16(e0, e1); - const int16x8_t g0 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 0)); - const int16x8_t g1 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 8)); - const int16x8_t h0 = vaddq_s16(g0, f.val[0]); - const int16x8_t h1 = vaddq_s16(g1, f.val[1]); - const int16x8_t i0 = vmaxq_s16(vminq_s16(h0, max), zero); - const int16x8_t i1 = vmaxq_s16(vminq_s16(h1, max), zero); - vst1q_u16(out + 2 * i + 0, vreinterpretq_u16_s16(i0)); - vst1q_u16(out + 2 * i + 8, vreinterpretq_u16_s16(i1)); - } - for (; i < len; ++i) { - const int a0b1 = A[i + 0] + B[i + 1]; - const int a1b0 = A[i + 1] + B[i + 0]; - const int a0a1b0b1 = a0b1 + a1b0 + 8; - const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; - const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; - out[2 * i + 0] = clip_y_NEON(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y_NEON(best_y[2 * i + 1] + v1); - } -} -#undef MAX_Y - -//------------------------------------------------------------------------------ - -extern void WebPInitSharpYUVNEON(void); - -WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVNEON(void) { - WebPSharpYUVUpdateY = SharpYUVUpdateY_NEON; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_NEON; - WebPSharpYUVFilterRow = SharpYUVFilterRow_NEON; -} - #else // !WEBP_USE_NEON WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVNEON) -WEBP_DSP_INIT_STUB(WebPInitSharpYUVNEON) #endif // WEBP_USE_NEON diff --git a/3rdparty/libwebp/src/dsp/yuv_sse2.c b/3rdparty/libwebp/src/dsp/yuv_sse2.c index baa48d5371..01a48f9af2 100644 --- a/3rdparty/libwebp/src/dsp/yuv_sse2.c +++ b/3rdparty/libwebp/src/dsp/yuv_sse2.c @@ -15,10 +15,12 @@ #if defined(WEBP_USE_SSE2) -#include "src/dsp/common_sse2.h" #include #include +#include "src/dsp/common_sse2.h" +#include "src/utils/utils.h" + //----------------------------------------------------------------------------- // Convert spans of 32 pixels to various RGB formats for the fancy upsampler. @@ -74,7 +76,7 @@ static WEBP_INLINE __m128i Load_HI_16_SSE2(const uint8_t* src) { // Load and replicate the U/V samples static WEBP_INLINE __m128i Load_UV_HI_8_SSE2(const uint8_t* src) { const __m128i zero = _mm_setzero_si128(); - const __m128i tmp0 = _mm_cvtsi32_si128(*(const uint32_t*)src); + const __m128i tmp0 = _mm_cvtsi32_si128(WebPMemToInt32(src)); const __m128i tmp1 = _mm_unpacklo_epi8(zero, tmp0); return _mm_unpacklo_epi16(tmp1, tmp1); // replicate samples } @@ -130,7 +132,7 @@ static WEBP_INLINE void PackAndStore4444_SSE2(const __m128i* const R, const __m128i rg0 = _mm_packus_epi16(*B, *A); const __m128i ba0 = _mm_packus_epi16(*R, *G); #endif - const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i mask_0xf0 = _mm_set1_epi8((char)0xf0); const __m128i rb1 = _mm_unpacklo_epi8(rg0, ba0); // rbrbrbrbrb... const __m128i ga1 = _mm_unpackhi_epi8(rg0, ba0); // gagagagaga... const __m128i rb2 = _mm_and_si128(rb1, mask_0xf0); @@ -147,9 +149,10 @@ static WEBP_INLINE void PackAndStore565_SSE2(const __m128i* const R, const __m128i r0 = _mm_packus_epi16(*R, *R); const __m128i g0 = _mm_packus_epi16(*G, *G); const __m128i b0 = _mm_packus_epi16(*B, *B); - const __m128i r1 = _mm_and_si128(r0, _mm_set1_epi8(0xf8)); + const __m128i r1 = _mm_and_si128(r0, _mm_set1_epi8((char)0xf8)); const __m128i b1 = _mm_and_si128(_mm_srli_epi16(b0, 3), _mm_set1_epi8(0x1f)); - const __m128i g1 = _mm_srli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0xe0)), 5); + const __m128i g1 = + _mm_srli_epi16(_mm_and_si128(g0, _mm_set1_epi8((char)0xe0)), 5); const __m128i g2 = _mm_slli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0x1c)), 3); const __m128i rg = _mm_or_si128(r1, g1); const __m128i gb = _mm_or_si128(g2, b1); @@ -747,128 +750,9 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVSSE2(void) { WebPConvertRGBA32ToUV = ConvertRGBA32ToUV_SSE2; } -//------------------------------------------------------------------------------ - -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - uint64_t diff = 0; - uint32_t tmp[4]; - int i; - const __m128i zero = _mm_setzero_si128(); - const __m128i max = _mm_set1_epi16(MAX_Y); - const __m128i one = _mm_set1_epi16(1); - __m128i sum = zero; - - for (i = 0; i + 8 <= len; i += 8) { - const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); - const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); - const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); - const __m128i D = _mm_sub_epi16(A, B); // diff_y - const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) - const __m128i F = _mm_add_epi16(C, D); // new_y - const __m128i G = _mm_or_si128(E, one); // -1 or 1 - const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); - const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) - _mm_storeu_si128((__m128i*)(dst + i), H); - sum = _mm_add_epi32(sum, I); - } - _mm_storeu_si128((__m128i*)tmp, sum); - diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - for (; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)dst[i] + diff_y; - dst[i] = clip_y(new_y); - diff += (uint64_t)abs(diff_y); - } - return diff; -} - -static void SharpYUVUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i = 0; - for (i = 0; i + 8 <= len; i += 8) { - const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); - const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); - const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); - const __m128i D = _mm_sub_epi16(A, B); // diff_uv - const __m128i E = _mm_add_epi16(C, D); // new_uv - _mm_storeu_si128((__m128i*)(dst + i), E); - } - for (; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - const __m128i kCst8 = _mm_set1_epi16(8); - const __m128i max = _mm_set1_epi16(MAX_Y); - const __m128i zero = _mm_setzero_si128(); - for (i = 0; i + 8 <= len; i += 8) { - const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); - const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); - const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); - const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); - const __m128i a0b1 = _mm_add_epi16(a0, b1); - const __m128i a1b0 = _mm_add_epi16(a1, b0); - const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 - const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); - const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) - const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) - const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); - const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); - const __m128i d0 = _mm_add_epi16(c1, a0); - const __m128i d1 = _mm_add_epi16(c0, a1); - const __m128i e0 = _mm_srai_epi16(d0, 1); - const __m128i e1 = _mm_srai_epi16(d1, 1); - const __m128i f0 = _mm_unpacklo_epi16(e0, e1); - const __m128i f1 = _mm_unpackhi_epi16(e0, e1); - const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); - const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); - const __m128i h0 = _mm_add_epi16(g0, f0); - const __m128i h1 = _mm_add_epi16(g1, f1); - const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); - const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); - _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); - _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); - } - for (; i < len; ++i) { - // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = - // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 - // We reuse the common sub-expressions. - const int a0b1 = A[i + 0] + B[i + 1]; - const int a1b0 = A[i + 1] + B[i + 0]; - const int a0a1b0b1 = a0b1 + a1b0 + 8; - const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; - const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; - out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); - } -} - -#undef MAX_Y - -//------------------------------------------------------------------------------ - -extern void WebPInitSharpYUVSSE2(void); - -WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVSSE2(void) { - WebPSharpYUVUpdateY = SharpYUVUpdateY_SSE2; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_SSE2; - WebPSharpYUVFilterRow = SharpYUVFilterRow_SSE2; -} - #else // !WEBP_USE_SSE2 WEBP_DSP_INIT_STUB(WebPInitSamplersSSE2) WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVSSE2) -WEBP_DSP_INIT_STUB(WebPInitSharpYUVSSE2) #endif // WEBP_USE_SSE2 diff --git a/3rdparty/libwebp/src/dsp/yuv_sse41.c b/3rdparty/libwebp/src/dsp/yuv_sse41.c index 579d1f7402..f79b802e47 100644 --- a/3rdparty/libwebp/src/dsp/yuv_sse41.c +++ b/3rdparty/libwebp/src/dsp/yuv_sse41.c @@ -15,10 +15,12 @@ #if defined(WEBP_USE_SSE41) -#include "src/dsp/common_sse41.h" #include #include +#include "src/dsp/common_sse41.h" +#include "src/utils/utils.h" + //----------------------------------------------------------------------------- // Convert spans of 32 pixels to various RGB formats for the fancy upsampler. @@ -74,7 +76,7 @@ static WEBP_INLINE __m128i Load_HI_16_SSE41(const uint8_t* src) { // Load and replicate the U/V samples static WEBP_INLINE __m128i Load_UV_HI_8_SSE41(const uint8_t* src) { const __m128i zero = _mm_setzero_si128(); - const __m128i tmp0 = _mm_cvtsi32_si128(*(const uint32_t*)src); + const __m128i tmp0 = _mm_cvtsi32_si128(WebPMemToInt32(src)); const __m128i tmp1 = _mm_unpacklo_epi8(zero, tmp0); return _mm_unpacklo_epi16(tmp1, tmp1); // replicate samples } diff --git a/3rdparty/libwebp/src/enc/alpha_enc.c b/3rdparty/libwebp/src/enc/alpha_enc.c index dce9ca957d..4a599f88a9 100644 --- a/3rdparty/libwebp/src/enc/alpha_enc.c +++ b/3rdparty/libwebp/src/enc/alpha_enc.c @@ -13,6 +13,7 @@ #include #include +#include #include "src/enc/vp8i_enc.h" #include "src/dsp/dsp.h" @@ -54,7 +55,7 @@ static int EncodeLossless(const uint8_t* const data, int width, int height, WebPConfig config; WebPPicture picture; - WebPPictureInit(&picture); + if (!WebPPictureInit(&picture)) return 0; picture.width = width; picture.height = height; picture.use_argb = 1; @@ -86,7 +87,7 @@ static int EncodeLossless(const uint8_t* const data, int width, int height, // a decoder bug related to alpha with color cache. // See: https://code.google.com/p/webp/issues/detail?id=239 // Need to re-enable this later. - ok = (VP8LEncodeStream(&config, &picture, bw, 0 /*use_cache*/) == VP8_ENC_OK); + ok = VP8LEncodeStream(&config, &picture, bw, /*use_cache=*/0); WebPPictureFree(&picture); ok = ok && !bw->error_; if (!ok) { @@ -140,6 +141,11 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, !reduce_levels, &tmp_bw, &result->stats); if (ok) { output = VP8LBitWriterFinish(&tmp_bw); + if (tmp_bw.error_) { + VP8LBitWriterWipeOut(&tmp_bw); + memset(&result->bw, 0, sizeof(result->bw)); + return 0; + } output_size = VP8LBitWriterNumBytes(&tmp_bw); if (output_size > data_size) { // compressed size is larger than source! Revert to uncompressed mode. @@ -148,6 +154,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, } } else { VP8LBitWriterWipeOut(&tmp_bw); + memset(&result->bw, 0, sizeof(result->bw)); return 0; } } @@ -162,7 +169,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, header = method | (filter << 2); if (reduce_levels) header |= ALPHA_PREPROCESSED_LEVELS << 4; - VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size); + if (!VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size)) ok = 0; ok = ok && VP8BitWriterAppend(&result->bw, &header, ALPHA_HEADER_LEN); ok = ok && VP8BitWriterAppend(&result->bw, output, output_size); @@ -303,7 +310,7 @@ static int EncodeAlpha(VP8Encoder* const enc, int ok = 1; const int reduce_levels = (quality < 100); - // quick sanity checks + // quick correctness checks assert((uint64_t)data_size == (uint64_t)width * height); // as per spec assert(enc != NULL && pic != NULL && pic->a != NULL); assert(output != NULL && output_size != NULL); @@ -312,11 +319,11 @@ static int EncodeAlpha(VP8Encoder* const enc, assert(filter >= WEBP_FILTER_NONE && filter <= WEBP_FILTER_FAST); if (quality < 0 || quality > 100) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } if (method < ALPHA_NO_COMPRESSION || method > ALPHA_LOSSLESS_COMPRESSION) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } if (method == ALPHA_NO_COMPRESSION) { @@ -326,7 +333,7 @@ static int EncodeAlpha(VP8Encoder* const enc, quant_alpha = (uint8_t*)WebPSafeMalloc(1ULL, data_size); if (quant_alpha == NULL) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); } // Extract alpha data (width x height) from raw_data (stride x height). @@ -346,6 +353,9 @@ static int EncodeAlpha(VP8Encoder* const enc, ok = ApplyFiltersAndEncode(quant_alpha, width, height, data_size, method, filter, reduce_levels, effort_level, output, output_size, pic->stats); + if (!ok) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); // imprecise + } #if !defined(WEBP_DISABLE_STATS) if (pic->stats != NULL) { // need stats? pic->stats->coded_size += (int)(*output_size); @@ -361,7 +371,7 @@ static int EncodeAlpha(VP8Encoder* const enc, //------------------------------------------------------------------------------ // Main calls -static int CompressAlphaJob(void* arg1, void* dummy) { +static int CompressAlphaJob(void* arg1, void* unused) { VP8Encoder* const enc = (VP8Encoder*)arg1; const WebPConfig* config = enc->config_; uint8_t* alpha_data = NULL; @@ -375,13 +385,13 @@ static int CompressAlphaJob(void* arg1, void* dummy) { filter, effort_level, &alpha_data, &alpha_size)) { return 0; } - if (alpha_size != (uint32_t)alpha_size) { // Sanity check. + if (alpha_size != (uint32_t)alpha_size) { // Soundness check. WebPSafeFree(alpha_data); return 0; } enc->alpha_data_size_ = (uint32_t)alpha_size; enc->alpha_data_ = alpha_data; - (void)dummy; + (void)unused; return 1; } @@ -405,7 +415,7 @@ int VP8EncStartAlpha(VP8Encoder* const enc) { WebPWorker* const worker = &enc->alpha_worker_; // Makes sure worker is good to go. if (!WebPGetWorkerInterface()->Reset(worker)) { - return 0; + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } WebPGetWorkerInterface()->Launch(worker); return 1; diff --git a/3rdparty/libwebp/src/enc/analysis_enc.c b/3rdparty/libwebp/src/enc/analysis_enc.c index ebb784261c..962eaa998f 100644 --- a/3rdparty/libwebp/src/enc/analysis_enc.c +++ b/3rdparty/libwebp/src/enc/analysis_enc.c @@ -391,12 +391,14 @@ static int DoSegmentsJob(void* arg1, void* arg2) { return ok; } +#ifdef WEBP_USE_THREAD static void MergeJobs(const SegmentJob* const src, SegmentJob* const dst) { int i; for (i = 0; i <= MAX_ALPHA; ++i) dst->alphas[i] += src->alphas[i]; dst->alpha += src->alpha; dst->uv_alpha += src->uv_alpha; } +#endif // initialize the job struct with some tasks to perform static void InitSegmentJob(VP8Encoder* const enc, SegmentJob* const job, @@ -425,10 +427,10 @@ int VP8EncAnalyze(VP8Encoder* const enc) { (enc->method_ <= 1); // for method 0 - 1, we need preds_[] to be filled. if (do_segments) { const int last_row = enc->mb_h_; - // We give a little more than a half work to the main thread. - const int split_row = (9 * last_row + 15) >> 4; const int total_mb = last_row * enc->mb_w_; #ifdef WEBP_USE_THREAD + // We give a little more than a half work to the main thread. + const int split_row = (9 * last_row + 15) >> 4; const int kMinSplitRow = 2; // minimal rows needed for mt to be worth it const int do_mt = (enc->thread_level_ > 0) && (split_row >= kMinSplitRow); #else @@ -438,6 +440,7 @@ int VP8EncAnalyze(VP8Encoder* const enc) { WebPGetWorkerInterface(); SegmentJob main_job; if (do_mt) { +#ifdef WEBP_USE_THREAD SegmentJob side_job; // Note the use of '&' instead of '&&' because we must call the functions // no matter what. @@ -455,6 +458,7 @@ int VP8EncAnalyze(VP8Encoder* const enc) { } worker_interface->End(&side_job.worker); if (ok) MergeJobs(&side_job, &main_job); // merge results together +#endif // WEBP_USE_THREAD } else { // Even for single-thread case, we use the generic Worker tools. InitSegmentJob(enc, &main_job, 0, last_row); @@ -470,6 +474,10 @@ int VP8EncAnalyze(VP8Encoder* const enc) { } else { // Use only one default segment. ResetAllMBInfo(enc); } + if (!ok) { + return WebPEncodingSetError(enc->pic_, + VP8_ENC_ERROR_OUT_OF_MEMORY); // imprecise + } return ok; } diff --git a/3rdparty/libwebp/src/enc/backward_references_cost_enc.c b/3rdparty/libwebp/src/enc/backward_references_cost_enc.c index 516abd73eb..6968ef3c9f 100644 --- a/3rdparty/libwebp/src/enc/backward_references_cost_enc.c +++ b/3rdparty/libwebp/src/enc/backward_references_cost_enc.c @@ -15,10 +15,11 @@ // #include +#include +#include "src/dsp/lossless_common.h" #include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" -#include "src/dsp/lossless_common.h" #include "src/utils/color_cache_utils.h" #include "src/utils/utils.h" @@ -30,15 +31,15 @@ extern void VP8LBackwardRefsCursorAdd(VP8LBackwardRefs* const refs, const PixOrCopy v); typedef struct { - double alpha_[VALUES_IN_BYTE]; - double red_[VALUES_IN_BYTE]; - double blue_[VALUES_IN_BYTE]; - double distance_[NUM_DISTANCE_CODES]; - double* literal_; + float alpha_[VALUES_IN_BYTE]; + float red_[VALUES_IN_BYTE]; + float blue_[VALUES_IN_BYTE]; + float distance_[NUM_DISTANCE_CODES]; + float* literal_; } CostModel; static void ConvertPopulationCountTableToBitEstimates( - int num_symbols, const uint32_t population_counts[], double output[]) { + int num_symbols, const uint32_t population_counts[], float output[]) { uint32_t sum = 0; int nonzeros = 0; int i; @@ -51,7 +52,7 @@ static void ConvertPopulationCountTableToBitEstimates( if (nonzeros <= 1) { memset(output, 0, num_symbols * sizeof(*output)); } else { - const double logsum = VP8LFastLog2(sum); + const float logsum = VP8LFastLog2(sum); for (i = 0; i < num_symbols; ++i) { output[i] = logsum - VP8LFastLog2(population_counts[i]); } @@ -75,8 +76,8 @@ static int CostModelBuild(CostModel* const m, int xsize, int cache_bits, } ConvertPopulationCountTableToBitEstimates( - VP8LHistogramNumCodes(histo->palette_code_bits_), - histo->literal_, m->literal_); + VP8LHistogramNumCodes(histo->palette_code_bits_), histo->literal_, + m->literal_); ConvertPopulationCountTableToBitEstimates( VALUES_IN_BYTE, histo->red_, m->red_); ConvertPopulationCountTableToBitEstimates( @@ -92,27 +93,27 @@ static int CostModelBuild(CostModel* const m, int xsize, int cache_bits, return ok; } -static WEBP_INLINE double GetLiteralCost(const CostModel* const m, uint32_t v) { +static WEBP_INLINE float GetLiteralCost(const CostModel* const m, uint32_t v) { return m->alpha_[v >> 24] + m->red_[(v >> 16) & 0xff] + m->literal_[(v >> 8) & 0xff] + m->blue_[v & 0xff]; } -static WEBP_INLINE double GetCacheCost(const CostModel* const m, uint32_t idx) { +static WEBP_INLINE float GetCacheCost(const CostModel* const m, uint32_t idx) { const int literal_idx = VALUES_IN_BYTE + NUM_LENGTH_CODES + idx; return m->literal_[literal_idx]; } -static WEBP_INLINE double GetLengthCost(const CostModel* const m, - uint32_t length) { +static WEBP_INLINE float GetLengthCost(const CostModel* const m, + uint32_t length) { int code, extra_bits; VP8LPrefixEncodeBits(length, &code, &extra_bits); return m->literal_[VALUES_IN_BYTE + code] + extra_bits; } -static WEBP_INLINE double GetDistanceCost(const CostModel* const m, - uint32_t distance) { +static WEBP_INLINE float GetDistanceCost(const CostModel* const m, + uint32_t distance) { int code, extra_bits; VP8LPrefixEncodeBits(distance, &code, &extra_bits); return m->distance_[code] + extra_bits; @@ -122,20 +123,20 @@ static WEBP_INLINE void AddSingleLiteralWithCostModel( const uint32_t* const argb, VP8LColorCache* const hashers, const CostModel* const cost_model, int idx, int use_color_cache, float prev_cost, float* const cost, uint16_t* const dist_array) { - double cost_val = prev_cost; + float cost_val = prev_cost; const uint32_t color = argb[idx]; const int ix = use_color_cache ? VP8LColorCacheContains(hashers, color) : -1; if (ix >= 0) { // use_color_cache is true and hashers contains color - const double mul0 = 0.68; + const float mul0 = 0.68f; cost_val += GetCacheCost(cost_model, ix) * mul0; } else { - const double mul1 = 0.82; + const float mul1 = 0.82f; if (use_color_cache) VP8LColorCacheInsert(hashers, color); cost_val += GetLiteralCost(cost_model, color) * mul1; } if (cost[idx] > cost_val) { - cost[idx] = (float)cost_val; + cost[idx] = cost_val; dist_array[idx] = 1; // only one is inserted. } } @@ -172,7 +173,7 @@ struct CostInterval { // The GetLengthCost(cost_model, k) are cached in a CostCacheInterval. typedef struct { - double cost_; + float cost_; int start_; int end_; // Exclusive. } CostCacheInterval; @@ -187,7 +188,7 @@ typedef struct { int count_; // The number of stored intervals. CostCacheInterval* cache_intervals_; size_t cache_intervals_size_; - double cost_cache_[MAX_LENGTH]; // Contains the GetLengthCost(cost_model, k). + float cost_cache_[MAX_LENGTH]; // Contains the GetLengthCost(cost_model, k). float* costs_; uint16_t* dist_array_; // Most of the time, we only need few intervals -> use a free-list, to avoid @@ -262,10 +263,13 @@ static int CostManagerInit(CostManager* const manager, CostManagerInitFreeList(manager); // Fill in the cost_cache_. - manager->cache_intervals_size_ = 1; - manager->cost_cache_[0] = GetLengthCost(cost_model, 0); - for (i = 1; i < cost_cache_size; ++i) { + // Has to be done in two passes due to a GCC bug on i686 + // related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323 + for (i = 0; i < cost_cache_size; ++i) { manager->cost_cache_[i] = GetLengthCost(cost_model, i); + } + manager->cache_intervals_size_ = 1; + for (i = 1; i < cost_cache_size; ++i) { // Get the number of bound intervals. if (manager->cost_cache_[i] != manager->cost_cache_[i - 1]) { ++manager->cache_intervals_size_; @@ -294,7 +298,7 @@ static int CostManagerInit(CostManager* const manager, cur->end_ = 1; cur->cost_ = manager->cost_cache_[0]; for (i = 1; i < cost_cache_size; ++i) { - const double cost_val = manager->cost_cache_[i]; + const float cost_val = manager->cost_cache_[i]; if (cost_val != cur->cost_) { ++cur; // Initialize an interval. @@ -303,6 +307,8 @@ static int CostManagerInit(CostManager* const manager, } cur->end_ = i + 1; } + assert((size_t)(cur - manager->cache_intervals_) + 1 == + manager->cache_intervals_size_); } manager->costs_ = (float*)WebPSafeMalloc(pix_count, sizeof(*manager->costs_)); @@ -311,7 +317,7 @@ static int CostManagerInit(CostManager* const manager, return 0; } // Set the initial costs_ high for every pixel as we will keep the minimum. - for (i = 0; i < pix_count; ++i) manager->costs_[i] = 1e38f; + for (i = 0; i < pix_count; ++i) manager->costs_[i] = FLT_MAX; return 1; } @@ -457,7 +463,7 @@ static WEBP_INLINE void InsertInterval(CostManager* const manager, // If handling the interval or one of its subintervals becomes to heavy, its // contribution is added to the costs right away. static WEBP_INLINE void PushInterval(CostManager* const manager, - double distance_cost, int position, + float distance_cost, int position, int len) { size_t i; CostInterval* interval = manager->head_; @@ -474,7 +480,7 @@ static WEBP_INLINE void PushInterval(CostManager* const manager, const int k = j - position; float cost_tmp; assert(k >= 0 && k < MAX_LENGTH); - cost_tmp = (float)(distance_cost + manager->cost_cache_[k]); + cost_tmp = distance_cost + manager->cost_cache_[k]; if (manager->costs_[j] > cost_tmp) { manager->costs_[j] = cost_tmp; @@ -492,7 +498,7 @@ static WEBP_INLINE void PushInterval(CostManager* const manager, const int end = position + (cost_cache_intervals[i].end_ > len ? len : cost_cache_intervals[i].end_); - const float cost = (float)(distance_cost + cost_cache_intervals[i].cost_); + const float cost = distance_cost + cost_cache_intervals[i].cost_; for (; interval != NULL && interval->start_ < end; interval = interval_next) { @@ -570,22 +576,21 @@ static int BackwardReferencesHashChainDistanceOnly( const int pix_count = xsize * ysize; const int use_color_cache = (cache_bits > 0); const size_t literal_array_size = - sizeof(double) * (NUM_LITERAL_CODES + NUM_LENGTH_CODES + - ((cache_bits > 0) ? (1 << cache_bits) : 0)); + sizeof(float) * (VP8LHistogramNumCodes(cache_bits)); const size_t cost_model_size = sizeof(CostModel) + literal_array_size; CostModel* const cost_model = (CostModel*)WebPSafeCalloc(1ULL, cost_model_size); VP8LColorCache hashers; CostManager* cost_manager = - (CostManager*)WebPSafeMalloc(1ULL, sizeof(*cost_manager)); + (CostManager*)WebPSafeCalloc(1ULL, sizeof(*cost_manager)); int offset_prev = -1, len_prev = -1; - double offset_cost = -1; + float offset_cost = -1.f; int first_offset_is_constant = -1; // initialized with 'impossible' value int reach = 0; if (cost_model == NULL || cost_manager == NULL) goto Error; - cost_model->literal_ = (double*)(cost_model + 1); + cost_model->literal_ = (float*)(cost_model + 1); if (use_color_cache) { cc_init = VP8LColorCacheInit(&hashers, cache_bits); if (!cc_init) goto Error; @@ -675,7 +680,7 @@ static int BackwardReferencesHashChainDistanceOnly( } ok = !refs->error_; -Error: + Error: if (cc_init) VP8LColorCacheClear(&hashers); CostManagerClear(cost_manager); WebPSafeFree(cost_model); diff --git a/3rdparty/libwebp/src/enc/backward_references_enc.c b/3rdparty/libwebp/src/enc/backward_references_enc.c index 519b36a091..dc98bf1719 100644 --- a/3rdparty/libwebp/src/enc/backward_references_enc.c +++ b/3rdparty/libwebp/src/enc/backward_references_enc.c @@ -10,6 +10,8 @@ // Author: Jyrki Alakuijala (jyrki@google.com) // +#include "src/enc/backward_references_enc.h" + #include #include #include @@ -17,10 +19,11 @@ #include "src/dsp/dsp.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" -#include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" +#include "src/enc/vp8i_enc.h" #include "src/utils/color_cache_utils.h" #include "src/utils/utils.h" +#include "src/webp/encode.h" #define MIN_BLOCK_SIZE 256 // minimum block size for backward references @@ -255,10 +258,13 @@ static WEBP_INLINE int MaxFindCopyLength(int len) { int VP8LHashChainFill(VP8LHashChain* const p, int quality, const uint32_t* const argb, int xsize, int ysize, - int low_effort) { + int low_effort, const WebPPicture* const pic, + int percent_range, int* const percent) { const int size = xsize * ysize; const int iter_max = GetMaxItersForQuality(quality); const uint32_t window_size = GetWindowSizeForHashChain(quality, xsize); + int remaining_percent = percent_range; + int percent_start = *percent; int pos; int argb_comp; uint32_t base_position; @@ -276,7 +282,12 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, hash_to_first_index = (int32_t*)WebPSafeMalloc(HASH_SIZE, sizeof(*hash_to_first_index)); - if (hash_to_first_index == NULL) return 0; + if (hash_to_first_index == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + + percent_range = remaining_percent / 2; + remaining_percent -= percent_range; // Set the int32_t array to -1. memset(hash_to_first_index, 0xff, HASH_SIZE * sizeof(*hash_to_first_index)); @@ -323,12 +334,22 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, hash_to_first_index[hash_code] = pos++; argb_comp = argb_comp_next; } + + if (!WebPReportProgress( + pic, percent_start + percent_range * pos / (size - 2), percent)) { + WebPSafeFree(hash_to_first_index); + return 0; + } } // Process the penultimate pixel. chain[pos] = hash_to_first_index[GetPixPairHash64(argb + pos)]; WebPSafeFree(hash_to_first_index); + percent_start += percent_range; + if (!WebPReportProgress(pic, percent_start, percent)) return 0; + percent_range = remaining_percent; + // Find the best match interval at each pixel, defined by an offset to the // pixel and a length. The right-most pixel cannot match anything to the right // (hence a best length of 0) and the left-most pixel nothing to the left @@ -417,8 +438,17 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, max_base_position = base_position; } } + + if (!WebPReportProgress(pic, + percent_start + percent_range * + (size - 2 - base_position) / + (size - 2), + percent)) { + return 0; + } } - return 1; + + return WebPReportProgress(pic, percent_start + percent_range, percent); } static WEBP_INLINE void AddSingleLiteral(uint32_t pixel, int use_color_cache, @@ -728,7 +758,7 @@ static int CalculateBestCacheSize(const uint32_t* argb, int quality, int* const best_cache_bits) { int i; const int cache_bits_max = (quality <= 25) ? 0 : *best_cache_bits; - double entropy_min = MAX_ENTROPY; + float entropy_min = MAX_ENTROPY; int cc_init[MAX_COLOR_CACHE_BITS + 1] = { 0 }; VP8LColorCache hashers[MAX_COLOR_CACHE_BITS + 1]; VP8LRefsCursor c = VP8LRefsCursorInit(refs); @@ -813,14 +843,14 @@ static int CalculateBestCacheSize(const uint32_t* argb, int quality, } for (i = 0; i <= cache_bits_max; ++i) { - const double entropy = VP8LHistogramEstimateBits(histos[i]); + const float entropy = VP8LHistogramEstimateBits(histos[i]); if (i == 0 || entropy < entropy_min) { entropy_min = entropy; *best_cache_bits = i; } } ok = 1; -Error: + Error: for (i = 0; i <= cache_bits_max; ++i) { if (cc_init[i]) VP8LColorCacheClear(&hashers[i]); VP8LFreeHistogram(histos[i]); @@ -890,7 +920,7 @@ static int GetBackwardReferences(int width, int height, int i, lz77_type; // Index 0 is for a color cache, index 1 for no cache (if needed). int lz77_types_best[2] = {0, 0}; - double bit_costs_best[2] = {DBL_MAX, DBL_MAX}; + float bit_costs_best[2] = {FLT_MAX, FLT_MAX}; VP8LHashChain hash_chain_box; VP8LBackwardRefs* const refs_tmp = &refs[do_no_cache ? 2 : 1]; int status = 0; @@ -902,7 +932,7 @@ static int GetBackwardReferences(int width, int height, for (lz77_type = 1; lz77_types_to_try; lz77_types_to_try &= ~lz77_type, lz77_type <<= 1) { int res = 0; - double bit_cost = 0.; + float bit_cost = 0.f; if ((lz77_types_to_try & lz77_type) == 0) continue; switch (lz77_type) { case kLZ77RLE: @@ -976,15 +1006,16 @@ static int GetBackwardReferences(int width, int height, const VP8LHashChain* const hash_chain_tmp = (lz77_types_best[i] == kLZ77Standard) ? hash_chain : &hash_chain_box; const int cache_bits = (i == 1) ? 0 : *cache_bits_best; - if (VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, - hash_chain_tmp, &refs[i], - refs_tmp)) { - double bit_cost_trace; - VP8LHistogramCreate(histo, refs_tmp, cache_bits); - bit_cost_trace = VP8LHistogramEstimateBits(histo); - if (bit_cost_trace < bit_costs_best[i]) { - BackwardRefsSwap(refs_tmp, &refs[i]); - } + float bit_cost_trace; + if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, + hash_chain_tmp, &refs[i], + refs_tmp)) { + goto Error; + } + VP8LHistogramCreate(histo, refs_tmp, cache_bits); + bit_cost_trace = VP8LHistogramEstimateBits(histo); + if (bit_cost_trace < bit_costs_best[i]) { + BackwardRefsSwap(refs_tmp, &refs[i]); } } @@ -1000,31 +1031,35 @@ static int GetBackwardReferences(int width, int height, } status = 1; -Error: + Error: VP8LHashChainClear(&hash_chain_box); VP8LFreeHistogram(histo); return status; } -WebPEncodingError VP8LGetBackwardReferences( +int VP8LGetBackwardReferences( int width, int height, const uint32_t* const argb, int quality, int low_effort, int lz77_types_to_try, int cache_bits_max, int do_no_cache, const VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs, - int* const cache_bits_best) { + int* const cache_bits_best, const WebPPicture* const pic, int percent_range, + int* const percent) { if (low_effort) { VP8LBackwardRefs* refs_best; *cache_bits_best = cache_bits_max; refs_best = GetBackwardReferencesLowEffort( width, height, argb, cache_bits_best, hash_chain, refs); - if (refs_best == NULL) return VP8_ENC_ERROR_OUT_OF_MEMORY; + if (refs_best == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } // Set it in first position. BackwardRefsSwap(refs_best, &refs[0]); } else { if (!GetBackwardReferences(width, height, argb, quality, lz77_types_to_try, cache_bits_max, do_no_cache, hash_chain, refs, cache_bits_best)) { - return VP8_ENC_ERROR_OUT_OF_MEMORY; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); } } - return VP8_ENC_OK; + + return WebPReportProgress(pic, *percent + percent_range, percent); } diff --git a/3rdparty/libwebp/src/enc/backward_references_enc.h b/3rdparty/libwebp/src/enc/backward_references_enc.h index 4c0267b41e..4dff1c27b5 100644 --- a/3rdparty/libwebp/src/enc/backward_references_enc.h +++ b/3rdparty/libwebp/src/enc/backward_references_enc.h @@ -134,10 +134,11 @@ struct VP8LHashChain { // Must be called first, to set size. int VP8LHashChainInit(VP8LHashChain* const p, int size); -// Pre-compute the best matches for argb. +// Pre-compute the best matches for argb. pic and percent are for progress. int VP8LHashChainFill(VP8LHashChain* const p, int quality, const uint32_t* const argb, int xsize, int ysize, - int low_effort); + int low_effort, const WebPPicture* const pic, + int percent_range, int* const percent); void VP8LHashChainClear(VP8LHashChain* const p); // release memory static WEBP_INLINE int VP8LHashChainFindOffset(const VP8LHashChain* const p, @@ -227,11 +228,14 @@ enum VP8LLZ77Type { // VP8LBackwardRefs is put in the first element, the best value with no-cache in // the second element. // In both cases, the last element is used as temporary internally. -WebPEncodingError VP8LGetBackwardReferences( +// pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). +int VP8LGetBackwardReferences( int width, int height, const uint32_t* const argb, int quality, int low_effort, int lz77_types_to_try, int cache_bits_max, int do_no_cache, const VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs, - int* const cache_bits_best); + int* const cache_bits_best, const WebPPicture* const pic, int percent_range, + int* const percent); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/enc/frame_enc.c b/3rdparty/libwebp/src/enc/frame_enc.c index af538d83ba..01860ca757 100644 --- a/3rdparty/libwebp/src/enc/frame_enc.c +++ b/3rdparty/libwebp/src/enc/frame_enc.c @@ -578,7 +578,7 @@ static uint64_t OneStatPass(VP8Encoder* const enc, VP8RDLevel rd_opt, uint64_t size = 0; uint64_t size_p0 = 0; uint64_t distortion = 0; - const uint64_t pixel_count = nb_mbs * 384; + const uint64_t pixel_count = (uint64_t)nb_mbs * 384; VP8IteratorInit(enc, &it); SetLoopParams(enc, s->q); @@ -689,7 +689,7 @@ static int PreLoopInitialize(VP8Encoder* const enc) { } if (!ok) { VP8EncFreeBitWriters(enc); // malloc error occurred - WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } return ok; } @@ -719,6 +719,7 @@ static int PostLoopFinalize(VP8EncIterator* const it, int ok) { } else { // Something bad happened -> need to do some memory cleanup. VP8EncFreeBitWriters(enc); + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } return ok; } @@ -754,6 +755,11 @@ int VP8EncLoop(VP8Encoder* const enc) { // *then* decide how to code the skip decision if there's one. if (!VP8Decimate(&it, &info, rd_opt) || dont_use_skip) { CodeResiduals(it.bw_, &it, &info); + if (it.bw_->error_) { + // enc->pic_->error_code is set in PostLoopFinalize(). + ok = 0; + break; + } } else { // reset predictors after a skip ResetAfterSkip(&it); } @@ -778,11 +784,12 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { // Roughly refresh the proba eight times per pass int max_count = (enc->mb_w_ * enc->mb_h_) >> 3; int num_pass_left = enc->config_->pass; + int remaining_progress = 40; // percents const int do_search = enc->do_search_; VP8EncIterator it; VP8EncProba* const proba = &enc->proba_; const VP8RDLevel rd_opt = enc->rd_opt_level_; - const uint64_t pixel_count = enc->mb_w_ * enc->mb_h_ * 384; + const uint64_t pixel_count = (uint64_t)enc->mb_w_ * enc->mb_h_ * 384; PassStats stats; int ok; @@ -805,6 +812,9 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { uint64_t size_p0 = 0; uint64_t distortion = 0; int cnt = max_count; + // The final number of passes is not trivial to know in advance. + const int pass_progress = remaining_progress / (2 + num_pass_left); + remaining_progress -= pass_progress; VP8IteratorInit(enc, &it); SetLoopParams(enc, stats.q); if (is_last_pass) { @@ -832,7 +842,7 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { StoreSideInfo(&it); VP8StoreFilterStats(&it); VP8IteratorExport(&it); - ok = VP8IteratorProgress(&it, 20); + ok = VP8IteratorProgress(&it, pass_progress); } VP8IteratorSaveBoundary(&it); } while (ok && VP8IteratorNext(&it)); @@ -878,7 +888,8 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { ok = VP8EmitTokens(&enc->tokens_, enc->parts_ + 0, (const uint8_t*)proba->coeffs_, 1); } - ok = ok && WebPReportProgress(enc->pic_, enc->percent_ + 20, &enc->percent_); + ok = ok && WebPReportProgress(enc->pic_, enc->percent_ + remaining_progress, + &enc->percent_); return PostLoopFinalize(&it, ok); } diff --git a/3rdparty/libwebp/src/enc/histogram_enc.c b/3rdparty/libwebp/src/enc/histogram_enc.c index edc6e4faa4..3ca67b3ad0 100644 --- a/3rdparty/libwebp/src/enc/histogram_enc.c +++ b/3rdparty/libwebp/src/enc/histogram_enc.c @@ -13,15 +13,17 @@ #include "src/webp/config.h" #endif +#include #include -#include "src/enc/backward_references_enc.h" -#include "src/enc/histogram_enc.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" +#include "src/enc/backward_references_enc.h" +#include "src/enc/histogram_enc.h" +#include "src/enc/vp8i_enc.h" #include "src/utils/utils.h" -#define MAX_COST 1.e38 +#define MAX_BIT_COST FLT_MAX // Number of partitions for the three dominant (literal, red and blue) symbol // costs. @@ -228,8 +230,8 @@ void VP8LHistogramAddSinglePixOrCopy(VP8LHistogram* const histo, // ----------------------------------------------------------------------------- // Entropy-related functions. -static WEBP_INLINE double BitsEntropyRefine(const VP8LBitEntropy* entropy) { - double mix; +static WEBP_INLINE float BitsEntropyRefine(const VP8LBitEntropy* entropy) { + float mix; if (entropy->nonzeros < 5) { if (entropy->nonzeros <= 1) { return 0; @@ -238,67 +240,67 @@ static WEBP_INLINE double BitsEntropyRefine(const VP8LBitEntropy* entropy) { // Let's mix in a bit of entropy to favor good clustering when // distributions of these are combined. if (entropy->nonzeros == 2) { - return 0.99 * entropy->sum + 0.01 * entropy->entropy; + return 0.99f * entropy->sum + 0.01f * entropy->entropy; } // No matter what the entropy says, we cannot be better than min_limit // with Huffman coding. I am mixing a bit of entropy into the // min_limit since it produces much better (~0.5 %) compression results // perhaps because of better entropy clustering. if (entropy->nonzeros == 3) { - mix = 0.95; + mix = 0.95f; } else { - mix = 0.7; // nonzeros == 4. + mix = 0.7f; // nonzeros == 4. } } else { - mix = 0.627; + mix = 0.627f; } { - double min_limit = 2 * entropy->sum - entropy->max_val; - min_limit = mix * min_limit + (1.0 - mix) * entropy->entropy; + float min_limit = 2.f * entropy->sum - entropy->max_val; + min_limit = mix * min_limit + (1.f - mix) * entropy->entropy; return (entropy->entropy < min_limit) ? min_limit : entropy->entropy; } } -double VP8LBitsEntropy(const uint32_t* const array, int n) { +float VP8LBitsEntropy(const uint32_t* const array, int n) { VP8LBitEntropy entropy; VP8LBitsEntropyUnrefined(array, n, &entropy); return BitsEntropyRefine(&entropy); } -static double InitialHuffmanCost(void) { +static float InitialHuffmanCost(void) { // Small bias because Huffman code length is typically not stored in // full length. static const int kHuffmanCodeOfHuffmanCodeSize = CODE_LENGTH_CODES * 3; - static const double kSmallBias = 9.1; + static const float kSmallBias = 9.1f; return kHuffmanCodeOfHuffmanCodeSize - kSmallBias; } // Finalize the Huffman cost based on streak numbers and length type (<3 or >=3) -static double FinalHuffmanCost(const VP8LStreaks* const stats) { +static float FinalHuffmanCost(const VP8LStreaks* const stats) { // The constants in this function are experimental and got rounded from // their original values in 1/8 when switched to 1/1024. - double retval = InitialHuffmanCost(); + float retval = InitialHuffmanCost(); // Second coefficient: Many zeros in the histogram are covered efficiently // by a run-length encode. Originally 2/8. - retval += stats->counts[0] * 1.5625 + 0.234375 * stats->streaks[0][1]; + retval += stats->counts[0] * 1.5625f + 0.234375f * stats->streaks[0][1]; // Second coefficient: Constant values are encoded less efficiently, but still // RLE'ed. Originally 6/8. - retval += stats->counts[1] * 2.578125 + 0.703125 * stats->streaks[1][1]; + retval += stats->counts[1] * 2.578125f + 0.703125f * stats->streaks[1][1]; // 0s are usually encoded more efficiently than non-0s. // Originally 15/8. - retval += 1.796875 * stats->streaks[0][0]; + retval += 1.796875f * stats->streaks[0][0]; // Originally 26/8. - retval += 3.28125 * stats->streaks[1][0]; + retval += 3.28125f * stats->streaks[1][0]; return retval; } // Get the symbol entropy for the distribution 'population'. // Set 'trivial_sym', if there's only one symbol present in the distribution. -static double PopulationCost(const uint32_t* const population, int length, - uint32_t* const trivial_sym, - uint8_t* const is_used) { +static float PopulationCost(const uint32_t* const population, int length, + uint32_t* const trivial_sym, + uint8_t* const is_used) { VP8LBitEntropy bit_entropy; VP8LStreaks stats; VP8LGetEntropyUnrefined(population, length, &bit_entropy, &stats); @@ -314,11 +316,10 @@ static double PopulationCost(const uint32_t* const population, int length, // trivial_at_end is 1 if the two histograms only have one element that is // non-zero: both the zero-th one, or both the last one. -static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X, - const uint32_t* const Y, - int length, int is_X_used, - int is_Y_used, - int trivial_at_end) { +static WEBP_INLINE float GetCombinedEntropy(const uint32_t* const X, + const uint32_t* const Y, int length, + int is_X_used, int is_Y_used, + int trivial_at_end) { VP8LStreaks stats; if (trivial_at_end) { // This configuration is due to palettization that transforms an indexed @@ -356,16 +357,18 @@ static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X, } // Estimates the Entropy + Huffman + other block overhead size cost. -double VP8LHistogramEstimateBits(VP8LHistogram* const p) { - return - PopulationCost(p->literal_, VP8LHistogramNumCodes(p->palette_code_bits_), - NULL, &p->is_used_[0]) - + PopulationCost(p->red_, NUM_LITERAL_CODES, NULL, &p->is_used_[1]) - + PopulationCost(p->blue_, NUM_LITERAL_CODES, NULL, &p->is_used_[2]) - + PopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL, &p->is_used_[3]) - + PopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL, &p->is_used_[4]) - + VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES) - + VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); +float VP8LHistogramEstimateBits(VP8LHistogram* const p) { + return PopulationCost(p->literal_, + VP8LHistogramNumCodes(p->palette_code_bits_), NULL, + &p->is_used_[0]) + + PopulationCost(p->red_, NUM_LITERAL_CODES, NULL, &p->is_used_[1]) + + PopulationCost(p->blue_, NUM_LITERAL_CODES, NULL, &p->is_used_[2]) + + PopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL, &p->is_used_[3]) + + PopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL, + &p->is_used_[4]) + + (float)VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, + NUM_LENGTH_CODES) + + (float)VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); } // ----------------------------------------------------------------------------- @@ -373,17 +376,16 @@ double VP8LHistogramEstimateBits(VP8LHistogram* const p) { static int GetCombinedHistogramEntropy(const VP8LHistogram* const a, const VP8LHistogram* const b, - double cost_threshold, - double* cost) { + float cost_threshold, float* cost) { const int palette_code_bits = a->palette_code_bits_; int trivial_at_end = 0; assert(a->palette_code_bits_ == b->palette_code_bits_); *cost += GetCombinedEntropy(a->literal_, b->literal_, VP8LHistogramNumCodes(palette_code_bits), a->is_used_[0], b->is_used_[0], 0); - *cost += VP8LExtraCostCombined(a->literal_ + NUM_LITERAL_CODES, - b->literal_ + NUM_LITERAL_CODES, - NUM_LENGTH_CODES); + *cost += (float)VP8LExtraCostCombined(a->literal_ + NUM_LITERAL_CODES, + b->literal_ + NUM_LITERAL_CODES, + NUM_LENGTH_CODES); if (*cost > cost_threshold) return 0; if (a->trivial_symbol_ != VP8L_NON_TRIVIAL_SYM && @@ -417,8 +419,8 @@ static int GetCombinedHistogramEntropy(const VP8LHistogram* const a, *cost += GetCombinedEntropy(a->distance_, b->distance_, NUM_DISTANCE_CODES, a->is_used_[4], b->is_used_[4], 0); - *cost += - VP8LExtraCostCombined(a->distance_, b->distance_, NUM_DISTANCE_CODES); + *cost += (float)VP8LExtraCostCombined(a->distance_, b->distance_, + NUM_DISTANCE_CODES); if (*cost > cost_threshold) return 0; return 1; @@ -439,12 +441,11 @@ static WEBP_INLINE void HistogramAdd(const VP8LHistogram* const a, // Since the previous score passed is 'cost_threshold', we only need to compare // the partial cost against 'cost_threshold + C(a) + C(b)' to possibly bail-out // early. -static double HistogramAddEval(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out, - double cost_threshold) { - double cost = 0; - const double sum_cost = a->bit_cost_ + b->bit_cost_; +static float HistogramAddEval(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out, float cost_threshold) { + float cost = 0; + const float sum_cost = a->bit_cost_ + b->bit_cost_; cost_threshold += sum_cost; if (GetCombinedHistogramEntropy(a, b, cost_threshold, &cost)) { @@ -459,10 +460,10 @@ static double HistogramAddEval(const VP8LHistogram* const a, // Same as HistogramAddEval(), except that the resulting histogram // is not stored. Only the cost C(a+b) - C(a) is evaluated. We omit // the term C(b) which is constant over all the evaluations. -static double HistogramAddThresh(const VP8LHistogram* const a, - const VP8LHistogram* const b, - double cost_threshold) { - double cost; +static float HistogramAddThresh(const VP8LHistogram* const a, + const VP8LHistogram* const b, + float cost_threshold) { + float cost; assert(a != NULL && b != NULL); cost = -a->bit_cost_; GetCombinedHistogramEntropy(a, b, cost_threshold, &cost); @@ -473,24 +474,22 @@ static double HistogramAddThresh(const VP8LHistogram* const a, // The structure to keep track of cost range for the three dominant entropy // symbols. -// TODO(skal): Evaluate if float can be used here instead of double for -// representing the entropy costs. typedef struct { - double literal_max_; - double literal_min_; - double red_max_; - double red_min_; - double blue_max_; - double blue_min_; + float literal_max_; + float literal_min_; + float red_max_; + float red_min_; + float blue_max_; + float blue_min_; } DominantCostRange; static void DominantCostRangeInit(DominantCostRange* const c) { c->literal_max_ = 0.; - c->literal_min_ = MAX_COST; + c->literal_min_ = MAX_BIT_COST; c->red_max_ = 0.; - c->red_min_ = MAX_COST; + c->red_min_ = MAX_BIT_COST; c->blue_max_ = 0.; - c->blue_min_ = MAX_COST; + c->blue_min_ = MAX_BIT_COST; } static void UpdateDominantCostRange( @@ -505,16 +504,15 @@ static void UpdateDominantCostRange( static void UpdateHistogramCost(VP8LHistogram* const h) { uint32_t alpha_sym, red_sym, blue_sym; - const double alpha_cost = - PopulationCost(h->alpha_, NUM_LITERAL_CODES, &alpha_sym, - &h->is_used_[3]); - const double distance_cost = + const float alpha_cost = + PopulationCost(h->alpha_, NUM_LITERAL_CODES, &alpha_sym, &h->is_used_[3]); + const float distance_cost = PopulationCost(h->distance_, NUM_DISTANCE_CODES, NULL, &h->is_used_[4]) + - VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES); + (float)VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES); const int num_codes = VP8LHistogramNumCodes(h->palette_code_bits_); h->literal_cost_ = PopulationCost(h->literal_, num_codes, NULL, &h->is_used_[0]) + - VP8LExtraCost(h->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); + (float)VP8LExtraCost(h->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); h->red_cost_ = PopulationCost(h->red_, NUM_LITERAL_CODES, &red_sym, &h->is_used_[1]); h->blue_cost_ = @@ -529,10 +527,10 @@ static void UpdateHistogramCost(VP8LHistogram* const h) { } } -static int GetBinIdForEntropy(double min, double max, double val) { - const double range = max - min; +static int GetBinIdForEntropy(float min, float max, float val) { + const float range = max - min; if (range > 0.) { - const double delta = val - min; + const float delta = val - min; return (int)((NUM_PARTITIONS - 1e-6) * delta / range); } else { return 0; @@ -641,15 +639,11 @@ static void HistogramAnalyzeEntropyBin(VP8LHistogramSet* const image_histo, // Merges some histograms with same bin_id together if it's advantageous. // Sets the remaining histograms to NULL. -static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo, - int* num_used, - const uint16_t* const clusters, - uint16_t* const cluster_mappings, - VP8LHistogram* cur_combo, - const uint16_t* const bin_map, - int num_bins, - double combine_cost_factor, - int low_effort) { +static void HistogramCombineEntropyBin( + VP8LHistogramSet* const image_histo, int* num_used, + const uint16_t* const clusters, uint16_t* const cluster_mappings, + VP8LHistogram* cur_combo, const uint16_t* const bin_map, int num_bins, + float combine_cost_factor, int low_effort) { VP8LHistogram** const histograms = image_histo->histograms; int idx; struct { @@ -679,11 +673,10 @@ static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo, cluster_mappings[clusters[idx]] = clusters[first]; } else { // try to merge #idx into #first (both share the same bin_id) - const double bit_cost = histograms[idx]->bit_cost_; - const double bit_cost_thresh = -bit_cost * combine_cost_factor; - const double curr_cost_diff = - HistogramAddEval(histograms[first], histograms[idx], - cur_combo, bit_cost_thresh); + const float bit_cost = histograms[idx]->bit_cost_; + const float bit_cost_thresh = -bit_cost * combine_cost_factor; + const float curr_cost_diff = HistogramAddEval( + histograms[first], histograms[idx], cur_combo, bit_cost_thresh); if (curr_cost_diff < bit_cost_thresh) { // Try to merge two histograms only if the combo is a trivial one or // the two candidate histograms are already non-trivial. @@ -731,8 +724,8 @@ static uint32_t MyRand(uint32_t* const seed) { typedef struct { int idx1; int idx2; - double cost_diff; - double cost_combo; + float cost_diff; + float cost_combo; } HistogramPair; typedef struct { @@ -787,10 +780,9 @@ static void HistoQueueUpdateHead(HistoQueue* const histo_queue, // Update the cost diff and combo of a pair of histograms. This needs to be // called when the the histograms have been merged with a third one. static void HistoQueueUpdatePair(const VP8LHistogram* const h1, - const VP8LHistogram* const h2, - double threshold, + const VP8LHistogram* const h2, float threshold, HistogramPair* const pair) { - const double sum_cost = h1->bit_cost_ + h2->bit_cost_; + const float sum_cost = h1->bit_cost_ + h2->bit_cost_; pair->cost_combo = 0.; GetCombinedHistogramEntropy(h1, h2, sum_cost + threshold, &pair->cost_combo); pair->cost_diff = pair->cost_combo - sum_cost; @@ -799,9 +791,9 @@ static void HistoQueueUpdatePair(const VP8LHistogram* const h1, // Create a pair from indices "idx1" and "idx2" provided its cost // is inferior to "threshold", a negative entropy. // It returns the cost of the pair, or 0. if it superior to threshold. -static double HistoQueuePush(HistoQueue* const histo_queue, - VP8LHistogram** const histograms, int idx1, - int idx2, double threshold) { +static float HistoQueuePush(HistoQueue* const histo_queue, + VP8LHistogram** const histograms, int idx1, + int idx2, float threshold) { const VP8LHistogram* h1; const VP8LHistogram* h2; HistogramPair pair; @@ -945,8 +937,8 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, ++tries_with_no_success < num_tries_no_success; ++iter) { int* mapping_index; - double best_cost = - (histo_queue.size == 0) ? 0. : histo_queue.queue[0].cost_diff; + float best_cost = + (histo_queue.size == 0) ? 0.f : histo_queue.queue[0].cost_diff; int best_idx1 = -1, best_idx2 = 1; const uint32_t rand_range = (*num_used - 1) * (*num_used); // (*num_used) / 2 was chosen empirically. Less means faster but worse @@ -955,7 +947,7 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, // Pick random samples. for (j = 0; *num_used >= 2 && j < num_tries; ++j) { - double curr_cost; + float curr_cost; // Choose two different histograms at random and try to combine them. const uint32_t tmp = MyRand(&seed) % rand_range; uint32_t idx1 = tmp / (*num_used - 1); @@ -1034,7 +1026,7 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, *do_greedy = (*num_used <= min_cluster_size); ok = 1; -End: + End: HistoQueueClear(&histo_queue); WebPSafeFree(mappings); return ok; @@ -1057,7 +1049,7 @@ static void HistogramRemap(const VP8LHistogramSet* const in, if (out_size > 1) { for (i = 0; i < in_size; ++i) { int best_out = 0; - double best_bits = MAX_COST; + float best_bits = MAX_BIT_COST; int k; if (in_histo[i] == NULL) { // Arbitrarily set to the previous value if unused to help future LZ77. @@ -1065,7 +1057,7 @@ static void HistogramRemap(const VP8LHistogramSet* const in, continue; } for (k = 0; k < out_size; ++k) { - double cur_bits; + float cur_bits; cur_bits = HistogramAddThresh(out_histo[k], in_histo[i], best_bits); if (k == 0 || cur_bits < best_bits) { best_bits = cur_bits; @@ -1093,13 +1085,13 @@ static void HistogramRemap(const VP8LHistogramSet* const in, } } -static double GetCombineCostFactor(int histo_size, int quality) { - double combine_cost_factor = 0.16; +static float GetCombineCostFactor(int histo_size, int quality) { + float combine_cost_factor = 0.16f; if (quality < 90) { - if (histo_size > 256) combine_cost_factor /= 2.; - if (histo_size > 512) combine_cost_factor /= 2.; - if (histo_size > 1024) combine_cost_factor /= 2.; - if (quality <= 50) combine_cost_factor /= 2.; + if (histo_size > 256) combine_cost_factor /= 2.f; + if (histo_size > 512) combine_cost_factor /= 2.f; + if (histo_size > 1024) combine_cost_factor /= 2.f; + if (quality <= 50) combine_cost_factor /= 2.f; } return combine_cost_factor; } @@ -1169,15 +1161,17 @@ static void RemoveEmptyHistograms(VP8LHistogramSet* const image_histo) { } int VP8LGetHistoImageSymbols(int xsize, int ysize, - const VP8LBackwardRefs* const refs, - int quality, int low_effort, - int histo_bits, int cache_bits, + const VP8LBackwardRefs* const refs, int quality, + int low_effort, int histogram_bits, int cache_bits, VP8LHistogramSet* const image_histo, VP8LHistogram* const tmp_histo, - uint16_t* const histogram_symbols) { - int ok = 0; - const int histo_xsize = histo_bits ? VP8LSubSampleSize(xsize, histo_bits) : 1; - const int histo_ysize = histo_bits ? VP8LSubSampleSize(ysize, histo_bits) : 1; + uint16_t* const histogram_symbols, + const WebPPicture* const pic, int percent_range, + int* const percent) { + const int histo_xsize = + histogram_bits ? VP8LSubSampleSize(xsize, histogram_bits) : 1; + const int histo_ysize = + histogram_bits ? VP8LSubSampleSize(ysize, histogram_bits) : 1; const int image_histo_raw_size = histo_xsize * histo_ysize; VP8LHistogramSet* const orig_histo = VP8LAllocateHistogramSet(image_histo_raw_size, cache_bits); @@ -1187,13 +1181,16 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, const int entropy_combine_num_bins = low_effort ? NUM_PARTITIONS : BIN_SIZE; int entropy_combine; uint16_t* const map_tmp = - WebPSafeMalloc(2 * image_histo_raw_size, sizeof(map_tmp)); + WebPSafeMalloc(2 * image_histo_raw_size, sizeof(*map_tmp)); uint16_t* const cluster_mappings = map_tmp + image_histo_raw_size; int num_used = image_histo_raw_size; - if (orig_histo == NULL || map_tmp == NULL) goto Error; + if (orig_histo == NULL || map_tmp == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } // Construct the histograms from backward references. - HistogramBuild(xsize, histo_bits, refs, orig_histo); + HistogramBuild(xsize, histogram_bits, refs, orig_histo); // Copies the histograms and computes its bit_cost. // histogram_symbols is optimized HistogramCopyAndAnalyze(orig_histo, image_histo, &num_used, @@ -1204,16 +1201,15 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, if (entropy_combine) { uint16_t* const bin_map = map_tmp; - const double combine_cost_factor = + const float combine_cost_factor = GetCombineCostFactor(image_histo_raw_size, quality); const uint32_t num_clusters = num_used; HistogramAnalyzeEntropyBin(image_histo, bin_map, low_effort); // Collapse histograms with similar entropy. - HistogramCombineEntropyBin(image_histo, &num_used, histogram_symbols, - cluster_mappings, tmp_histo, bin_map, - entropy_combine_num_bins, combine_cost_factor, - low_effort); + HistogramCombineEntropyBin( + image_histo, &num_used, histogram_symbols, cluster_mappings, tmp_histo, + bin_map, entropy_combine_num_bins, combine_cost_factor, low_effort); OptimizeHistogramSymbols(image_histo, cluster_mappings, num_clusters, map_tmp, histogram_symbols); } @@ -1227,11 +1223,13 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, int do_greedy; if (!HistogramCombineStochastic(image_histo, &num_used, threshold_size, &do_greedy)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } if (do_greedy) { RemoveEmptyHistograms(image_histo); if (!HistogramCombineGreedy(image_histo, &num_used)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } } @@ -1241,10 +1239,12 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, RemoveEmptyHistograms(image_histo); HistogramRemap(orig_histo, image_histo, histogram_symbols); - ok = 1; + if (!WebPReportProgress(pic, *percent + percent_range, percent)) { + goto Error; + } Error: VP8LFreeHistogramSet(orig_histo); WebPSafeFree(map_tmp); - return ok; + return (pic->error_code == VP8_ENC_OK); } diff --git a/3rdparty/libwebp/src/enc/histogram_enc.h b/3rdparty/libwebp/src/enc/histogram_enc.h index 54c2d21783..4c0bb97464 100644 --- a/3rdparty/libwebp/src/enc/histogram_enc.h +++ b/3rdparty/libwebp/src/enc/histogram_enc.h @@ -40,10 +40,10 @@ typedef struct { int palette_code_bits_; uint32_t trivial_symbol_; // True, if histograms for Red, Blue & Alpha // literal symbols are single valued. - double bit_cost_; // cached value of bit cost. - double literal_cost_; // Cached values of dominant entropy costs: - double red_cost_; // literal, red & blue. - double blue_cost_; + float bit_cost_; // cached value of bit cost. + float literal_cost_; // Cached values of dominant entropy costs: + float red_cost_; // literal, red & blue. + float blue_cost_; uint8_t is_used_[5]; // 5 for literal, red, blue, alpha, distance } VP8LHistogram; @@ -64,8 +64,8 @@ void VP8LHistogramCreate(VP8LHistogram* const p, const VP8LBackwardRefs* const refs, int palette_code_bits); -// Return the size of the histogram for a given palette_code_bits. -int VP8LGetHistogramSize(int palette_code_bits); +// Return the size of the histogram for a given cache_bits. +int VP8LGetHistogramSize(int cache_bits); // Set the palette_code_bits and reset the stats. // If init_arrays is true, the arrays are also filled with 0's. @@ -105,21 +105,23 @@ static WEBP_INLINE int VP8LHistogramNumCodes(int palette_code_bits) { ((palette_code_bits > 0) ? (1 << palette_code_bits) : 0); } -// Builds the histogram image. +// Builds the histogram image. pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). int VP8LGetHistoImageSymbols(int xsize, int ysize, - const VP8LBackwardRefs* const refs, - int quality, int low_effort, - int histogram_bits, int cache_bits, - VP8LHistogramSet* const image_in, + const VP8LBackwardRefs* const refs, int quality, + int low_effort, int histogram_bits, int cache_bits, + VP8LHistogramSet* const image_histo, VP8LHistogram* const tmp_histo, - uint16_t* const histogram_symbols); + uint16_t* const histogram_symbols, + const WebPPicture* const pic, int percent_range, + int* const percent); // Returns the entropy for the symbols in the input array. -double VP8LBitsEntropy(const uint32_t* const array, int n); +float VP8LBitsEntropy(const uint32_t* const array, int n); // Estimate how many bits the combined entropy of literals and distance // approximately maps to. -double VP8LHistogramEstimateBits(VP8LHistogram* const p); +float VP8LHistogramEstimateBits(VP8LHistogram* const p); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/enc/picture_csp_enc.c b/3rdparty/libwebp/src/enc/picture_csp_enc.c index 35eede9635..a9280e6c30 100644 --- a/3rdparty/libwebp/src/enc/picture_csp_enc.c +++ b/3rdparty/libwebp/src/enc/picture_csp_enc.c @@ -15,12 +15,19 @@ #include #include +#include "sharpyuv/sharpyuv.h" +#include "sharpyuv/sharpyuv_csp.h" #include "src/enc/vp8i_enc.h" #include "src/utils/random_utils.h" #include "src/utils/utils.h" #include "src/dsp/dsp.h" #include "src/dsp/lossless.h" #include "src/dsp/yuv.h" +#include "src/dsp/cpu.h" + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include +#endif // Uncomment to disable gamma-compression during RGB->U/V averaging #define USE_GAMMA_COMPRESSION @@ -62,10 +69,12 @@ static int CheckNonOpaque(const uint8_t* alpha, int width, int height, int WebPPictureHasTransparency(const WebPPicture* picture) { if (picture == NULL) return 0; if (picture->use_argb) { - const int alpha_offset = ALPHA_OFFSET; - return CheckNonOpaque((const uint8_t*)picture->argb + alpha_offset, - picture->width, picture->height, - 4, picture->argb_stride * sizeof(*picture->argb)); + if (picture->argb != NULL) { + return CheckNonOpaque((const uint8_t*)picture->argb + ALPHA_OFFSET, + picture->width, picture->height, + 4, picture->argb_stride * sizeof(*picture->argb)); + } + return 0; } return CheckNonOpaque(picture->a, picture->width, picture->height, 1, picture->a_stride); @@ -76,30 +85,31 @@ int WebPPictureHasTransparency(const WebPPicture* picture) { #if defined(USE_GAMMA_COMPRESSION) -// gamma-compensates loss of resolution during chroma subsampling -#define kGamma 0.80 // for now we use a different gamma value than kGammaF -#define kGammaFix 12 // fixed-point precision for linear values -#define kGammaScale ((1 << kGammaFix) - 1) -#define kGammaTabFix 7 // fixed-point fractional bits precision -#define kGammaTabScale (1 << kGammaTabFix) -#define kGammaTabRounder (kGammaTabScale >> 1) -#define kGammaTabSize (1 << (kGammaFix - kGammaTabFix)) +// Gamma correction compensates loss of resolution during chroma subsampling. +#define GAMMA_FIX 12 // fixed-point precision for linear values +#define GAMMA_TAB_FIX 7 // fixed-point fractional bits precision +#define GAMMA_TAB_SIZE (1 << (GAMMA_FIX - GAMMA_TAB_FIX)) +static const double kGamma = 0.80; +static const int kGammaScale = ((1 << GAMMA_FIX) - 1); +static const int kGammaTabScale = (1 << GAMMA_TAB_FIX); +static const int kGammaTabRounder = (1 << GAMMA_TAB_FIX >> 1); -static int kLinearToGammaTab[kGammaTabSize + 1]; +static int kLinearToGammaTab[GAMMA_TAB_SIZE + 1]; static uint16_t kGammaToLinearTab[256]; static volatile int kGammaTablesOk = 0; static void InitGammaTables(void); +extern VP8CPUInfo VP8GetCPUInfo; WEBP_DSP_INIT_FUNC(InitGammaTables) { if (!kGammaTablesOk) { int v; - const double scale = (double)(1 << kGammaTabFix) / kGammaScale; + const double scale = (double)(1 << GAMMA_TAB_FIX) / kGammaScale; const double norm = 1. / 255.; for (v = 0; v <= 255; ++v) { kGammaToLinearTab[v] = (uint16_t)(pow(norm * v, kGamma) * kGammaScale + .5); } - for (v = 0; v <= kGammaTabSize; ++v) { + for (v = 0; v <= GAMMA_TAB_SIZE; ++v) { kLinearToGammaTab[v] = (int)(255. * pow(scale * v, 1. / kGamma) + .5); } kGammaTablesOk = 1; @@ -111,12 +121,12 @@ static WEBP_INLINE uint32_t GammaToLinear(uint8_t v) { } static WEBP_INLINE int Interpolate(int v) { - const int tab_pos = v >> (kGammaTabFix + 2); // integer part + const int tab_pos = v >> (GAMMA_TAB_FIX + 2); // integer part const int x = v & ((kGammaTabScale << 2) - 1); // fractional part const int v0 = kLinearToGammaTab[tab_pos]; const int v1 = kLinearToGammaTab[tab_pos + 1]; const int y = v1 * x + v0 * ((kGammaTabScale << 2) - x); // interpolate - assert(tab_pos + 1 < kGammaTabSize + 1); + assert(tab_pos + 1 < GAMMA_TAB_SIZE + 1); return y; } @@ -124,7 +134,7 @@ static WEBP_INLINE int Interpolate(int v) { // U/V value, suitable for RGBToU/V calls. static WEBP_INLINE int LinearToGamma(uint32_t base_value, int shift) { const int y = Interpolate(base_value << shift); // final uplifted value - return (y + kGammaTabRounder) >> kGammaTabFix; // descale + return (y + kGammaTabRounder) >> GAMMA_TAB_FIX; // descale } #else @@ -158,415 +168,26 @@ static int RGBToV(int r, int g, int b, VP8Random* const rg) { //------------------------------------------------------------------------------ // Sharp RGB->YUV conversion -static const int kNumIterations = 4; static const int kMinDimensionIterativeConversion = 4; -// We could use SFIX=0 and only uint8_t for fixed_y_t, but it produces some -// banding sometimes. Better use extra precision. -#define SFIX 2 // fixed-point precision of RGB and Y/W -typedef int16_t fixed_t; // signed type with extra SFIX precision for UV -typedef uint16_t fixed_y_t; // unsigned type with extra SFIX precision for W - -#define SHALF (1 << SFIX >> 1) -#define MAX_Y_T ((256 << SFIX) - 1) -#define SROUNDER (1 << (YUV_FIX + SFIX - 1)) - -#if defined(USE_GAMMA_COMPRESSION) - -// We use tables of different size and precision for the Rec709 / BT2020 -// transfer function. -#define kGammaF (1./0.45) -static uint32_t kLinearToGammaTabS[kGammaTabSize + 2]; -#define GAMMA_TO_LINEAR_BITS 14 -static uint32_t kGammaToLinearTabS[MAX_Y_T + 1]; // size scales with Y_FIX -static volatile int kGammaTablesSOk = 0; -static void InitGammaTablesS(void); - -WEBP_DSP_INIT_FUNC(InitGammaTablesS) { - assert(2 * GAMMA_TO_LINEAR_BITS < 32); // we use uint32_t intermediate values - if (!kGammaTablesSOk) { - int v; - const double norm = 1. / MAX_Y_T; - const double scale = 1. / kGammaTabSize; - const double a = 0.09929682680944; - const double thresh = 0.018053968510807; - const double final_scale = 1 << GAMMA_TO_LINEAR_BITS; - for (v = 0; v <= MAX_Y_T; ++v) { - const double g = norm * v; - double value; - if (g <= thresh * 4.5) { - value = g / 4.5; - } else { - const double a_rec = 1. / (1. + a); - value = pow(a_rec * (g + a), kGammaF); - } - kGammaToLinearTabS[v] = (uint32_t)(value * final_scale + .5); - } - for (v = 0; v <= kGammaTabSize; ++v) { - const double g = scale * v; - double value; - if (g <= thresh) { - value = 4.5 * g; - } else { - value = (1. + a) * pow(g, 1. / kGammaF) - a; - } - // we already incorporate the 1/2 rounding constant here - kLinearToGammaTabS[v] = - (uint32_t)(MAX_Y_T * value) + (1 << GAMMA_TO_LINEAR_BITS >> 1); - } - // to prevent small rounding errors to cause read-overflow: - kLinearToGammaTabS[kGammaTabSize + 1] = kLinearToGammaTabS[kGammaTabSize]; - kGammaTablesSOk = 1; - } -} - -// return value has a fixed-point precision of GAMMA_TO_LINEAR_BITS -static WEBP_INLINE uint32_t GammaToLinearS(int v) { - return kGammaToLinearTabS[v]; -} - -static WEBP_INLINE uint32_t LinearToGammaS(uint32_t value) { - // 'value' is in GAMMA_TO_LINEAR_BITS fractional precision - const uint32_t v = value * kGammaTabSize; - const uint32_t tab_pos = v >> GAMMA_TO_LINEAR_BITS; - // fractional part, in GAMMA_TO_LINEAR_BITS fixed-point precision - const uint32_t x = v - (tab_pos << GAMMA_TO_LINEAR_BITS); // fractional part - // v0 / v1 are in GAMMA_TO_LINEAR_BITS fixed-point precision (range [0..1]) - const uint32_t v0 = kLinearToGammaTabS[tab_pos + 0]; - const uint32_t v1 = kLinearToGammaTabS[tab_pos + 1]; - // Final interpolation. Note that rounding is already included. - const uint32_t v2 = (v1 - v0) * x; // note: v1 >= v0. - const uint32_t result = v0 + (v2 >> GAMMA_TO_LINEAR_BITS); - return result; -} - -#else - -static void InitGammaTablesS(void) {} -static WEBP_INLINE uint32_t GammaToLinearS(int v) { - return (v << GAMMA_TO_LINEAR_BITS) / MAX_Y_T; -} -static WEBP_INLINE uint32_t LinearToGammaS(uint32_t value) { - return (MAX_Y_T * value) >> GAMMA_TO_LINEAR_BITS; -} - -#endif // USE_GAMMA_COMPRESSION - -//------------------------------------------------------------------------------ - -static uint8_t clip_8b(fixed_t v) { - return (!(v & ~0xff)) ? (uint8_t)v : (v < 0) ? 0u : 255u; -} - -static fixed_y_t clip_y(int y) { - return (!(y & ~MAX_Y_T)) ? (fixed_y_t)y : (y < 0) ? 0 : MAX_Y_T; -} - -//------------------------------------------------------------------------------ - -static int RGBToGray(int r, int g, int b) { - const int luma = 13933 * r + 46871 * g + 4732 * b + YUV_HALF; - return (luma >> YUV_FIX); -} - -static uint32_t ScaleDown(int a, int b, int c, int d) { - const uint32_t A = GammaToLinearS(a); - const uint32_t B = GammaToLinearS(b); - const uint32_t C = GammaToLinearS(c); - const uint32_t D = GammaToLinearS(d); - return LinearToGammaS((A + B + C + D + 2) >> 2); -} - -static WEBP_INLINE void UpdateW(const fixed_y_t* src, fixed_y_t* dst, int w) { - int i; - for (i = 0; i < w; ++i) { - const uint32_t R = GammaToLinearS(src[0 * w + i]); - const uint32_t G = GammaToLinearS(src[1 * w + i]); - const uint32_t B = GammaToLinearS(src[2 * w + i]); - const uint32_t Y = RGBToGray(R, G, B); - dst[i] = (fixed_y_t)LinearToGammaS(Y); - } -} - -static void UpdateChroma(const fixed_y_t* src1, const fixed_y_t* src2, - fixed_t* dst, int uv_w) { - int i; - for (i = 0; i < uv_w; ++i) { - const int r = ScaleDown(src1[0 * uv_w + 0], src1[0 * uv_w + 1], - src2[0 * uv_w + 0], src2[0 * uv_w + 1]); - const int g = ScaleDown(src1[2 * uv_w + 0], src1[2 * uv_w + 1], - src2[2 * uv_w + 0], src2[2 * uv_w + 1]); - const int b = ScaleDown(src1[4 * uv_w + 0], src1[4 * uv_w + 1], - src2[4 * uv_w + 0], src2[4 * uv_w + 1]); - const int W = RGBToGray(r, g, b); - dst[0 * uv_w] = (fixed_t)(r - W); - dst[1 * uv_w] = (fixed_t)(g - W); - dst[2 * uv_w] = (fixed_t)(b - W); - dst += 1; - src1 += 2; - src2 += 2; - } -} - -static void StoreGray(const fixed_y_t* rgb, fixed_y_t* y, int w) { - int i; - for (i = 0; i < w; ++i) { - y[i] = RGBToGray(rgb[0 * w + i], rgb[1 * w + i], rgb[2 * w + i]); - } -} - -//------------------------------------------------------------------------------ - -static WEBP_INLINE fixed_y_t Filter2(int A, int B, int W0) { - const int v0 = (A * 3 + B + 2) >> 2; - return clip_y(v0 + W0); -} - -//------------------------------------------------------------------------------ - -static WEBP_INLINE fixed_y_t UpLift(uint8_t a) { // 8bit -> SFIX - return ((fixed_y_t)a << SFIX) | SHALF; -} - -static void ImportOneRow(const uint8_t* const r_ptr, - const uint8_t* const g_ptr, - const uint8_t* const b_ptr, - int step, - int pic_width, - fixed_y_t* const dst) { - int i; - const int w = (pic_width + 1) & ~1; - for (i = 0; i < pic_width; ++i) { - const int off = i * step; - dst[i + 0 * w] = UpLift(r_ptr[off]); - dst[i + 1 * w] = UpLift(g_ptr[off]); - dst[i + 2 * w] = UpLift(b_ptr[off]); - } - if (pic_width & 1) { // replicate rightmost pixel - dst[pic_width + 0 * w] = dst[pic_width + 0 * w - 1]; - dst[pic_width + 1 * w] = dst[pic_width + 1 * w - 1]; - dst[pic_width + 2 * w] = dst[pic_width + 2 * w - 1]; - } -} - -static void InterpolateTwoRows(const fixed_y_t* const best_y, - const fixed_t* prev_uv, - const fixed_t* cur_uv, - const fixed_t* next_uv, - int w, - fixed_y_t* out1, - fixed_y_t* out2) { - const int uv_w = w >> 1; - const int len = (w - 1) >> 1; // length to filter - int k = 3; - while (k-- > 0) { // process each R/G/B segments in turn - // special boundary case for i==0 - out1[0] = Filter2(cur_uv[0], prev_uv[0], best_y[0]); - out2[0] = Filter2(cur_uv[0], next_uv[0], best_y[w]); - - WebPSharpYUVFilterRow(cur_uv, prev_uv, len, best_y + 0 + 1, out1 + 1); - WebPSharpYUVFilterRow(cur_uv, next_uv, len, best_y + w + 1, out2 + 1); - - // special boundary case for i == w - 1 when w is even - if (!(w & 1)) { - out1[w - 1] = Filter2(cur_uv[uv_w - 1], prev_uv[uv_w - 1], - best_y[w - 1 + 0]); - out2[w - 1] = Filter2(cur_uv[uv_w - 1], next_uv[uv_w - 1], - best_y[w - 1 + w]); - } - out1 += w; - out2 += w; - prev_uv += uv_w; - cur_uv += uv_w; - next_uv += uv_w; - } -} - -static WEBP_INLINE uint8_t ConvertRGBToY(int r, int g, int b) { - const int luma = 16839 * r + 33059 * g + 6420 * b + SROUNDER; - return clip_8b(16 + (luma >> (YUV_FIX + SFIX))); -} - -static WEBP_INLINE uint8_t ConvertRGBToU(int r, int g, int b) { - const int u = -9719 * r - 19081 * g + 28800 * b + SROUNDER; - return clip_8b(128 + (u >> (YUV_FIX + SFIX))); -} - -static WEBP_INLINE uint8_t ConvertRGBToV(int r, int g, int b) { - const int v = +28800 * r - 24116 * g - 4684 * b + SROUNDER; - return clip_8b(128 + (v >> (YUV_FIX + SFIX))); -} - -static int ConvertWRGBToYUV(const fixed_y_t* best_y, const fixed_t* best_uv, - WebPPicture* const picture) { - int i, j; - uint8_t* dst_y = picture->y; - uint8_t* dst_u = picture->u; - uint8_t* dst_v = picture->v; - const fixed_t* const best_uv_base = best_uv; - const int w = (picture->width + 1) & ~1; - const int h = (picture->height + 1) & ~1; - const int uv_w = w >> 1; - const int uv_h = h >> 1; - for (best_uv = best_uv_base, j = 0; j < picture->height; ++j) { - for (i = 0; i < picture->width; ++i) { - const int off = (i >> 1); - const int W = best_y[i]; - const int r = best_uv[off + 0 * uv_w] + W; - const int g = best_uv[off + 1 * uv_w] + W; - const int b = best_uv[off + 2 * uv_w] + W; - dst_y[i] = ConvertRGBToY(r, g, b); - } - best_y += w; - best_uv += (j & 1) * 3 * uv_w; - dst_y += picture->y_stride; - } - for (best_uv = best_uv_base, j = 0; j < uv_h; ++j) { - for (i = 0; i < uv_w; ++i) { - const int off = i; - const int r = best_uv[off + 0 * uv_w]; - const int g = best_uv[off + 1 * uv_w]; - const int b = best_uv[off + 2 * uv_w]; - dst_u[i] = ConvertRGBToU(r, g, b); - dst_v[i] = ConvertRGBToV(r, g, b); - } - best_uv += 3 * uv_w; - dst_u += picture->uv_stride; - dst_v += picture->uv_stride; - } - return 1; -} - //------------------------------------------------------------------------------ // Main function -#define SAFE_ALLOC(W, H, T) ((T*)WebPSafeMalloc((W) * (H), sizeof(T))) - static int PreprocessARGB(const uint8_t* r_ptr, const uint8_t* g_ptr, const uint8_t* b_ptr, int step, int rgb_stride, WebPPicture* const picture) { - // we expand the right/bottom border if needed - const int w = (picture->width + 1) & ~1; - const int h = (picture->height + 1) & ~1; - const int uv_w = w >> 1; - const int uv_h = h >> 1; - uint64_t prev_diff_y_sum = ~0; - int j, iter; - - // TODO(skal): allocate one big memory chunk. But for now, it's easier - // for valgrind debugging to have several chunks. - fixed_y_t* const tmp_buffer = SAFE_ALLOC(w * 3, 2, fixed_y_t); // scratch - fixed_y_t* const best_y_base = SAFE_ALLOC(w, h, fixed_y_t); - fixed_y_t* const target_y_base = SAFE_ALLOC(w, h, fixed_y_t); - fixed_y_t* const best_rgb_y = SAFE_ALLOC(w, 2, fixed_y_t); - fixed_t* const best_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); - fixed_t* const target_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); - fixed_t* const best_rgb_uv = SAFE_ALLOC(uv_w * 3, 1, fixed_t); - fixed_y_t* best_y = best_y_base; - fixed_y_t* target_y = target_y_base; - fixed_t* best_uv = best_uv_base; - fixed_t* target_uv = target_uv_base; - const uint64_t diff_y_threshold = (uint64_t)(3.0 * w * h); - int ok; - - if (best_y_base == NULL || best_uv_base == NULL || - target_y_base == NULL || target_uv_base == NULL || - best_rgb_y == NULL || best_rgb_uv == NULL || - tmp_buffer == NULL) { - ok = WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); - goto End; + const int ok = SharpYuvConvert( + r_ptr, g_ptr, b_ptr, step, rgb_stride, /*rgb_bit_depth=*/8, + picture->y, picture->y_stride, picture->u, picture->uv_stride, picture->v, + picture->uv_stride, /*yuv_bit_depth=*/8, picture->width, + picture->height, SharpYuvGetConversionMatrix(kSharpYuvMatrixWebp)); + if (!ok) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } - assert(picture->width >= kMinDimensionIterativeConversion); - assert(picture->height >= kMinDimensionIterativeConversion); - - WebPInitConvertARGBToYUV(); - - // Import RGB samples to W/RGB representation. - for (j = 0; j < picture->height; j += 2) { - const int is_last_row = (j == picture->height - 1); - fixed_y_t* const src1 = tmp_buffer + 0 * w; - fixed_y_t* const src2 = tmp_buffer + 3 * w; - - // prepare two rows of input - ImportOneRow(r_ptr, g_ptr, b_ptr, step, picture->width, src1); - if (!is_last_row) { - ImportOneRow(r_ptr + rgb_stride, g_ptr + rgb_stride, b_ptr + rgb_stride, - step, picture->width, src2); - } else { - memcpy(src2, src1, 3 * w * sizeof(*src2)); - } - StoreGray(src1, best_y + 0, w); - StoreGray(src2, best_y + w, w); - - UpdateW(src1, target_y, w); - UpdateW(src2, target_y + w, w); - UpdateChroma(src1, src2, target_uv, uv_w); - memcpy(best_uv, target_uv, 3 * uv_w * sizeof(*best_uv)); - best_y += 2 * w; - best_uv += 3 * uv_w; - target_y += 2 * w; - target_uv += 3 * uv_w; - r_ptr += 2 * rgb_stride; - g_ptr += 2 * rgb_stride; - b_ptr += 2 * rgb_stride; - } - - // Iterate and resolve clipping conflicts. - for (iter = 0; iter < kNumIterations; ++iter) { - const fixed_t* cur_uv = best_uv_base; - const fixed_t* prev_uv = best_uv_base; - uint64_t diff_y_sum = 0; - - best_y = best_y_base; - best_uv = best_uv_base; - target_y = target_y_base; - target_uv = target_uv_base; - for (j = 0; j < h; j += 2) { - fixed_y_t* const src1 = tmp_buffer + 0 * w; - fixed_y_t* const src2 = tmp_buffer + 3 * w; - { - const fixed_t* const next_uv = cur_uv + ((j < h - 2) ? 3 * uv_w : 0); - InterpolateTwoRows(best_y, prev_uv, cur_uv, next_uv, w, src1, src2); - prev_uv = cur_uv; - cur_uv = next_uv; - } - - UpdateW(src1, best_rgb_y + 0 * w, w); - UpdateW(src2, best_rgb_y + 1 * w, w); - UpdateChroma(src1, src2, best_rgb_uv, uv_w); - - // update two rows of Y and one row of RGB - diff_y_sum += WebPSharpYUVUpdateY(target_y, best_rgb_y, best_y, 2 * w); - WebPSharpYUVUpdateRGB(target_uv, best_rgb_uv, best_uv, 3 * uv_w); - - best_y += 2 * w; - best_uv += 3 * uv_w; - target_y += 2 * w; - target_uv += 3 * uv_w; - } - // test exit condition - if (iter > 0) { - if (diff_y_sum < diff_y_threshold) break; - if (diff_y_sum > prev_diff_y_sum) break; - } - prev_diff_y_sum = diff_y_sum; - } - // final reconstruction - ok = ConvertWRGBToYUV(best_y_base, best_uv_base, picture); - - End: - WebPSafeFree(best_y_base); - WebPSafeFree(best_uv_base); - WebPSafeFree(target_y_base); - WebPSafeFree(target_uv_base); - WebPSafeFree(best_rgb_y); - WebPSafeFree(best_rgb_uv); - WebPSafeFree(tmp_buffer); return ok; } -#undef SAFE_ALLOC //------------------------------------------------------------------------------ // "Fast" regular RGB->YUV @@ -591,8 +212,8 @@ static const int kAlphaFix = 19; // and constant are adjusted very tightly to fit 32b arithmetic. // In particular, they use the fact that the operands for 'v / a' are actually // derived as v = (a0.p0 + a1.p1 + a2.p2 + a3.p3) and a = a0 + a1 + a2 + a3 -// with ai in [0..255] and pi in [0..1<> 1); ++y) { @@ -1044,7 +678,7 @@ int WebPPictureYUVAToARGB(WebPPicture* picture) { return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); } // Allocate a new argb buffer (discarding the previous one). - if (!WebPPictureAllocARGB(picture, picture->width, picture->height)) return 0; + if (!WebPPictureAllocARGB(picture)) return 0; picture->use_argb = 1; // Convert @@ -1106,6 +740,8 @@ static int Import(WebPPicture* const picture, const int width = picture->width; const int height = picture->height; + if (abs(rgb_stride) < (import_alpha ? 4 : 3) * width) return 0; + if (!picture->use_argb) { const uint8_t* a_ptr = import_alpha ? rgb + 3 : NULL; return ImportYUVAFromRGBA(r_ptr, g_ptr, b_ptr, a_ptr, step, rgb_stride, @@ -1163,24 +799,24 @@ static int Import(WebPPicture* const picture, #if !defined(WEBP_REDUCE_CSP) int WebPPictureImportBGR(WebPPicture* picture, - const uint8_t* rgb, int rgb_stride) { - return (picture != NULL && rgb != NULL) - ? Import(picture, rgb, rgb_stride, 3, 1, 0) + const uint8_t* bgr, int bgr_stride) { + return (picture != NULL && bgr != NULL) + ? Import(picture, bgr, bgr_stride, 3, 1, 0) : 0; } int WebPPictureImportBGRA(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 1, 1) + const uint8_t* bgra, int bgra_stride) { + return (picture != NULL && bgra != NULL) + ? Import(picture, bgra, bgra_stride, 4, 1, 1) : 0; } int WebPPictureImportBGRX(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 1, 0) + const uint8_t* bgrx, int bgrx_stride) { + return (picture != NULL && bgrx != NULL) + ? Import(picture, bgrx, bgrx_stride, 4, 1, 0) : 0; } @@ -1201,9 +837,9 @@ int WebPPictureImportRGBA(WebPPicture* picture, } int WebPPictureImportRGBX(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 0, 0) + const uint8_t* rgbx, int rgbx_stride) { + return (picture != NULL && rgbx != NULL) + ? Import(picture, rgbx, rgbx_stride, 4, 0, 0) : 0; } diff --git a/3rdparty/libwebp/src/enc/picture_enc.c b/3rdparty/libwebp/src/enc/picture_enc.c index c691622d03..5a2703541f 100644 --- a/3rdparty/libwebp/src/enc/picture_enc.c +++ b/3rdparty/libwebp/src/enc/picture_enc.c @@ -12,10 +12,10 @@ // Author: Skal (pascal.massimino@gmail.com) #include +#include #include #include "src/enc/vp8i_enc.h" -#include "src/dsp/dsp.h" #include "src/utils/utils.h" //------------------------------------------------------------------------------ @@ -45,6 +45,22 @@ int WebPPictureInitInternal(WebPPicture* picture, int version) { //------------------------------------------------------------------------------ +int WebPValidatePicture(const WebPPicture* const picture) { + if (picture == NULL) return 0; + if (picture->width <= 0 || picture->height <= 0) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } + if (picture->width <= 0 || picture->width / 4 > INT_MAX / 4 || + picture->height <= 0 || picture->height / 4 > INT_MAX / 4) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } + if (picture->colorspace != WEBP_YUV420 && + picture->colorspace != WEBP_YUV420A) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); + } + return 1; +} + static void WebPPictureResetBufferARGB(WebPPicture* const picture) { picture->memory_argb_ = NULL; picture->argb = NULL; @@ -63,18 +79,17 @@ void WebPPictureResetBuffers(WebPPicture* const picture) { WebPPictureResetBufferYUVA(picture); } -int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height) { +int WebPPictureAllocARGB(WebPPicture* const picture) { void* memory; + const int width = picture->width; + const int height = picture->height; const uint64_t argb_size = (uint64_t)width * height; - assert(picture != NULL); + if (!WebPValidatePicture(picture)) return 0; WebPSafeFree(picture->memory_argb_); WebPPictureResetBufferARGB(picture); - if (width <= 0 || height <= 0) { - return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); - } // allocate a new buffer. memory = WebPSafeMalloc(argb_size + WEBP_ALIGN_CST, sizeof(*picture->argb)); if (memory == NULL) { @@ -86,10 +101,10 @@ int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height) { return 1; } -int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { - const WebPEncCSP uv_csp = - (WebPEncCSP)((int)picture->colorspace & WEBP_CSP_UV_MASK); +int WebPPictureAllocYUVA(WebPPicture* const picture) { const int has_alpha = (int)picture->colorspace & WEBP_CSP_ALPHA_BIT; + const int width = picture->width; + const int height = picture->height; const int y_stride = width; const int uv_width = (int)(((int64_t)width + 1) >> 1); const int uv_height = (int)(((int64_t)height + 1) >> 1); @@ -98,15 +113,11 @@ int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { uint64_t y_size, uv_size, a_size, total_size; uint8_t* mem; - assert(picture != NULL); + if (!WebPValidatePicture(picture)) return 0; WebPSafeFree(picture->memory_); WebPPictureResetBufferYUVA(picture); - if (uv_csp != WEBP_YUV420) { - return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); - } - // alpha a_width = has_alpha ? width : 0; a_stride = a_width; @@ -152,15 +163,12 @@ int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { int WebPPictureAlloc(WebPPicture* picture) { if (picture != NULL) { - const int width = picture->width; - const int height = picture->height; - WebPPictureFree(picture); // erase previous buffer if (!picture->use_argb) { - return WebPPictureAllocYUVA(picture, width, height); + return WebPPictureAllocYUVA(picture); } else { - return WebPPictureAllocARGB(picture, width, height); + return WebPPictureAllocARGB(picture); } } return 1; diff --git a/3rdparty/libwebp/src/enc/picture_rescale_enc.c b/3rdparty/libwebp/src/enc/picture_rescale_enc.c index 58a6ae7b9d..ea90d82548 100644 --- a/3rdparty/libwebp/src/enc/picture_rescale_enc.c +++ b/3rdparty/libwebp/src/enc/picture_rescale_enc.c @@ -13,14 +13,15 @@ #include "src/webp/encode.h" -#if !defined(WEBP_REDUCE_SIZE) - #include #include #include "src/enc/vp8i_enc.h" + +#if !defined(WEBP_REDUCE_SIZE) #include "src/utils/rescaler_utils.h" #include "src/utils/utils.h" +#endif // !defined(WEBP_REDUCE_SIZE) #define HALVE(x) (((x) + 1) >> 1) @@ -56,6 +57,7 @@ static int AdjustAndCheckRectangle(const WebPPicture* const pic, return 1; } +#if !defined(WEBP_REDUCE_SIZE) int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { if (src == NULL || dst == NULL) return 0; if (src == dst) return 1; @@ -81,6 +83,7 @@ int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { } return 1; } +#endif // !defined(WEBP_REDUCE_SIZE) int WebPPictureIsView(const WebPPicture* picture) { if (picture == NULL) return 0; @@ -120,6 +123,7 @@ int WebPPictureView(const WebPPicture* src, return 1; } +#if !defined(WEBP_REDUCE_SIZE) //------------------------------------------------------------------------------ // Picture cropping @@ -133,7 +137,9 @@ int WebPPictureCrop(WebPPicture* pic, PictureGrabSpecs(pic, &tmp); tmp.width = width; tmp.height = height; - if (!WebPPictureAlloc(&tmp)) return 0; + if (!WebPPictureAlloc(&tmp)) { + return WebPEncodingSetError(pic, tmp.error_code); + } if (!pic->use_argb) { const int y_offset = top * pic->y_stride + left; @@ -164,22 +170,25 @@ int WebPPictureCrop(WebPPicture* pic, //------------------------------------------------------------------------------ // Simple picture rescaler -static void RescalePlane(const uint8_t* src, - int src_width, int src_height, int src_stride, - uint8_t* dst, - int dst_width, int dst_height, int dst_stride, - rescaler_t* const work, - int num_channels) { +static int RescalePlane(const uint8_t* src, + int src_width, int src_height, int src_stride, + uint8_t* dst, + int dst_width, int dst_height, int dst_stride, + rescaler_t* const work, + int num_channels) { WebPRescaler rescaler; int y = 0; - WebPRescalerInit(&rescaler, src_width, src_height, - dst, dst_width, dst_height, dst_stride, - num_channels, work); + if (!WebPRescalerInit(&rescaler, src_width, src_height, + dst, dst_width, dst_height, dst_stride, + num_channels, work)) { + return 0; + } while (y < src_height) { y += WebPRescalerImport(&rescaler, src_height - y, src + y * src_stride, src_stride); WebPRescalerExport(&rescaler); } + return 1; } static void AlphaMultiplyARGB(WebPPicture* const pic, int inverse) { @@ -195,73 +204,76 @@ static void AlphaMultiplyY(WebPPicture* const pic, int inverse) { } } -int WebPPictureRescale(WebPPicture* pic, int width, int height) { +int WebPPictureRescale(WebPPicture* picture, int width, int height) { WebPPicture tmp; int prev_width, prev_height; rescaler_t* work; - if (pic == NULL) return 0; - prev_width = pic->width; - prev_height = pic->height; + if (picture == NULL) return 0; + prev_width = picture->width; + prev_height = picture->height; if (!WebPRescalerGetScaledDimensions( prev_width, prev_height, &width, &height)) { - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); } - PictureGrabSpecs(pic, &tmp); + PictureGrabSpecs(picture, &tmp); tmp.width = width; tmp.height = height; - if (!WebPPictureAlloc(&tmp)) return 0; + if (!WebPPictureAlloc(&tmp)) { + return WebPEncodingSetError(picture, tmp.error_code); + } - if (!pic->use_argb) { + if (!picture->use_argb) { work = (rescaler_t*)WebPSafeMalloc(2ULL * width, sizeof(*work)); if (work == NULL) { WebPPictureFree(&tmp); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } // If present, we need to rescale alpha first (for AlphaMultiplyY). - if (pic->a != NULL) { + if (picture->a != NULL) { WebPInitAlphaProcessing(); - RescalePlane(pic->a, prev_width, prev_height, pic->a_stride, - tmp.a, width, height, tmp.a_stride, work, 1); + if (!RescalePlane(picture->a, prev_width, prev_height, picture->a_stride, + tmp.a, width, height, tmp.a_stride, work, 1)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } } // We take transparency into account on the luma plane only. That's not // totally exact blending, but still is a good approximation. - AlphaMultiplyY(pic, 0); - RescalePlane(pic->y, prev_width, prev_height, pic->y_stride, - tmp.y, width, height, tmp.y_stride, work, 1); + AlphaMultiplyY(picture, 0); + if (!RescalePlane(picture->y, prev_width, prev_height, picture->y_stride, + tmp.y, width, height, tmp.y_stride, work, 1) || + !RescalePlane(picture->u, HALVE(prev_width), HALVE(prev_height), + picture->uv_stride, tmp.u, HALVE(width), HALVE(height), + tmp.uv_stride, work, 1) || + !RescalePlane(picture->v, HALVE(prev_width), HALVE(prev_height), + picture->uv_stride, tmp.v, HALVE(width), HALVE(height), + tmp.uv_stride, work, 1)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } AlphaMultiplyY(&tmp, 1); - - RescalePlane(pic->u, - HALVE(prev_width), HALVE(prev_height), pic->uv_stride, - tmp.u, - HALVE(width), HALVE(height), tmp.uv_stride, work, 1); - RescalePlane(pic->v, - HALVE(prev_width), HALVE(prev_height), pic->uv_stride, - tmp.v, - HALVE(width), HALVE(height), tmp.uv_stride, work, 1); } else { work = (rescaler_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work)); if (work == NULL) { WebPPictureFree(&tmp); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } // In order to correctly interpolate colors, we need to apply the alpha // weighting first (black-matting), scale the RGB values, and remove // the premultiplication afterward (while preserving the alpha channel). WebPInitAlphaProcessing(); - AlphaMultiplyARGB(pic, 0); - RescalePlane((const uint8_t*)pic->argb, prev_width, prev_height, - pic->argb_stride * 4, - (uint8_t*)tmp.argb, width, height, - tmp.argb_stride * 4, - work, 4); + AlphaMultiplyARGB(picture, 0); + if (!RescalePlane((const uint8_t*)picture->argb, prev_width, prev_height, + picture->argb_stride * 4, (uint8_t*)tmp.argb, width, + height, tmp.argb_stride * 4, work, 4)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } AlphaMultiplyARGB(&tmp, 1); } - WebPPictureFree(pic); + WebPPictureFree(picture); WebPSafeFree(work); - *pic = tmp; + *picture = tmp; return 1; } @@ -273,23 +285,6 @@ int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { return 0; } -int WebPPictureIsView(const WebPPicture* picture) { - (void)picture; - return 0; -} - -int WebPPictureView(const WebPPicture* src, - int left, int top, int width, int height, - WebPPicture* dst) { - (void)src; - (void)left; - (void)top; - (void)width; - (void)height; - (void)dst; - return 0; -} - int WebPPictureCrop(WebPPicture* pic, int left, int top, int width, int height) { (void)pic; diff --git a/3rdparty/libwebp/src/enc/picture_tools_enc.c b/3rdparty/libwebp/src/enc/picture_tools_enc.c index 38cb01534a..147cc18608 100644 --- a/3rdparty/libwebp/src/enc/picture_tools_enc.c +++ b/3rdparty/libwebp/src/enc/picture_tools_enc.c @@ -190,27 +190,28 @@ static WEBP_INLINE uint32_t MakeARGB32(int r, int g, int b) { return (0xff000000u | (r << 16) | (g << 8) | b); } -void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { +void WebPBlendAlpha(WebPPicture* picture, uint32_t background_rgb) { const int red = (background_rgb >> 16) & 0xff; const int green = (background_rgb >> 8) & 0xff; const int blue = (background_rgb >> 0) & 0xff; int x, y; - if (pic == NULL) return; - if (!pic->use_argb) { - const int uv_width = (pic->width >> 1); // omit last pixel during u/v loop + if (picture == NULL) return; + if (!picture->use_argb) { + // omit last pixel during u/v loop + const int uv_width = (picture->width >> 1); const int Y0 = VP8RGBToY(red, green, blue, YUV_HALF); // VP8RGBToU/V expects the u/v values summed over four pixels const int U0 = VP8RGBToU(4 * red, 4 * green, 4 * blue, 4 * YUV_HALF); const int V0 = VP8RGBToV(4 * red, 4 * green, 4 * blue, 4 * YUV_HALF); - const int has_alpha = pic->colorspace & WEBP_CSP_ALPHA_BIT; - uint8_t* y_ptr = pic->y; - uint8_t* u_ptr = pic->u; - uint8_t* v_ptr = pic->v; - uint8_t* a_ptr = pic->a; + const int has_alpha = picture->colorspace & WEBP_CSP_ALPHA_BIT; + uint8_t* y_ptr = picture->y; + uint8_t* u_ptr = picture->u; + uint8_t* v_ptr = picture->v; + uint8_t* a_ptr = picture->a; if (!has_alpha || a_ptr == NULL) return; // nothing to do - for (y = 0; y < pic->height; ++y) { + for (y = 0; y < picture->height; ++y) { // Luma blending - for (x = 0; x < pic->width; ++x) { + for (x = 0; x < picture->width; ++x) { const uint8_t alpha = a_ptr[x]; if (alpha < 0xff) { y_ptr[x] = BLEND(Y0, y_ptr[x], alpha); @@ -219,7 +220,7 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { // Chroma blending every even line if ((y & 1) == 0) { uint8_t* const a_ptr2 = - (y + 1 == pic->height) ? a_ptr : a_ptr + pic->a_stride; + (y + 1 == picture->height) ? a_ptr : a_ptr + picture->a_stride; for (x = 0; x < uv_width; ++x) { // Average four alpha values into a single blending weight. // TODO(skal): might lead to visible contouring. Can we do better? @@ -229,24 +230,24 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { u_ptr[x] = BLEND_10BIT(U0, u_ptr[x], alpha); v_ptr[x] = BLEND_10BIT(V0, v_ptr[x], alpha); } - if (pic->width & 1) { // rightmost pixel + if (picture->width & 1) { // rightmost pixel const uint32_t alpha = 2 * (a_ptr[2 * x + 0] + a_ptr2[2 * x + 0]); u_ptr[x] = BLEND_10BIT(U0, u_ptr[x], alpha); v_ptr[x] = BLEND_10BIT(V0, v_ptr[x], alpha); } } else { - u_ptr += pic->uv_stride; - v_ptr += pic->uv_stride; + u_ptr += picture->uv_stride; + v_ptr += picture->uv_stride; } - memset(a_ptr, 0xff, pic->width); // reset alpha value to opaque - a_ptr += pic->a_stride; - y_ptr += pic->y_stride; + memset(a_ptr, 0xff, picture->width); // reset alpha value to opaque + a_ptr += picture->a_stride; + y_ptr += picture->y_stride; } } else { - uint32_t* argb = pic->argb; + uint32_t* argb = picture->argb; const uint32_t background = MakeARGB32(red, green, blue); - for (y = 0; y < pic->height; ++y) { - for (x = 0; x < pic->width; ++x) { + for (y = 0; y < picture->height; ++y) { + for (x = 0; x < picture->width; ++x) { const int alpha = (argb[x] >> 24) & 0xff; if (alpha != 0xff) { if (alpha > 0) { @@ -262,7 +263,7 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { } } } - argb += pic->argb_stride; + argb += picture->argb_stride; } } } diff --git a/3rdparty/libwebp/src/enc/predictor_enc.c b/3rdparty/libwebp/src/enc/predictor_enc.c index 2e6762ea0d..b3d44b59d5 100644 --- a/3rdparty/libwebp/src/enc/predictor_enc.c +++ b/3rdparty/libwebp/src/enc/predictor_enc.c @@ -16,6 +16,7 @@ #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" +#include "src/enc/vp8i_enc.h" #include "src/enc/vp8li_enc.h" #define MAX_DIFF_COST (1e30f) @@ -31,10 +32,10 @@ static WEBP_INLINE int GetMin(int a, int b) { return (a > b) ? b : a; } // Methods to calculate Entropy (Shannon). static float PredictionCostSpatial(const int counts[256], int weight_0, - double exp_val) { + float exp_val) { const int significant_symbols = 256 >> 4; - const double exp_decay_factor = 0.6; - double bits = weight_0 * counts[0]; + const float exp_decay_factor = 0.6f; + float bits = (float)weight_0 * counts[0]; int i; for (i = 1; i < significant_symbols; ++i) { bits += exp_val * (counts[i] + counts[256 - i]); @@ -46,9 +47,9 @@ static float PredictionCostSpatial(const int counts[256], int weight_0, static float PredictionCostSpatialHistogram(const int accumulated[4][256], const int tile[4][256]) { int i; - double retval = 0; + float retval = 0.f; for (i = 0; i < 4; ++i) { - const double kExpValue = 0.94; + const float kExpValue = 0.94f; retval += PredictionCostSpatial(tile[i], 1, kExpValue); retval += VP8LCombinedShannonEntropy(tile[i], accumulated[i]); } @@ -249,7 +250,7 @@ static WEBP_INLINE void GetResidual( } else if (x == 0) { predict = upper_row[x]; // Top. } else { - predict = pred_func(current_row[x - 1], upper_row + x); + predict = pred_func(¤t_row[x - 1], upper_row + x); } #if (WEBP_NEAR_LOSSLESS == 1) if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 || @@ -472,12 +473,15 @@ static void CopyImageWithPrediction(int width, int height, // with respect to predictions. If near_lossless_quality < 100, applies // near lossless processing, shaving off more bits of residuals for lower // qualities. -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless_quality, - int exact, int used_subtract_green) { +int VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless_quality, + int exact, int used_subtract_green, + const WebPPicture* const pic, int percent_range, + int* const percent) { const int tiles_per_row = VP8LSubSampleSize(width, bits); const int tiles_per_col = VP8LSubSampleSize(height, bits); + int percent_start = *percent; int tile_y; int histo[4][256]; const int max_quantization = 1 << VP8LNearLosslessBits(near_lossless_quality); @@ -491,17 +495,24 @@ void VP8LResidualImage(int width, int height, int bits, int low_effort, for (tile_y = 0; tile_y < tiles_per_col; ++tile_y) { int tile_x; for (tile_x = 0; tile_x < tiles_per_row; ++tile_x) { - const int pred = GetBestPredictorForTile(width, height, tile_x, tile_y, - bits, histo, argb_scratch, argb, max_quantization, exact, - used_subtract_green, image); + const int pred = GetBestPredictorForTile( + width, height, tile_x, tile_y, bits, histo, argb_scratch, argb, + max_quantization, exact, used_subtract_green, image); image[tile_y * tiles_per_row + tile_x] = ARGB_BLACK | (pred << 8); } + + if (!WebPReportProgress( + pic, percent_start + percent_range * tile_y / tiles_per_col, + percent)) { + return 0; + } } } CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb, low_effort, max_quantization, exact, used_subtract_green); + return WebPReportProgress(pic, percent_start + percent_range, percent); } //------------------------------------------------------------------------------ @@ -532,7 +543,7 @@ static float PredictionCostCrossColor(const int accumulated[256], const int counts[256]) { // Favor low entropy, locally and globally. // Favor small absolute values for PredictionCostSpatial - static const double kExpValue = 2.4; + static const float kExpValue = 2.4f; return VP8LCombinedShannonEntropy(counts, accumulated) + PredictionCostSpatial(counts, 3, kExpValue); } @@ -714,11 +725,14 @@ static void CopyTileWithColorTransform(int xsize, int ysize, } } -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image) { +int VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image, + const WebPPicture* const pic, int percent_range, + int* const percent) { const int max_tile_size = 1 << bits; const int tile_xsize = VP8LSubSampleSize(width, bits); const int tile_ysize = VP8LSubSampleSize(height, bits); + int percent_start = *percent; int accumulated_red_histo[256] = { 0 }; int accumulated_blue_histo[256] = { 0 }; int tile_x, tile_y; @@ -768,5 +782,11 @@ void VP8LColorSpaceTransform(int width, int height, int bits, int quality, } } } + if (!WebPReportProgress( + pic, percent_start + percent_range * tile_y / tile_ysize, + percent)) { + return 0; + } } + return 1; } diff --git a/3rdparty/libwebp/src/enc/quant_enc.c b/3rdparty/libwebp/src/enc/quant_enc.c index 01eb565c7f..6d8202d277 100644 --- a/3rdparty/libwebp/src/enc/quant_enc.c +++ b/3rdparty/libwebp/src/enc/quant_enc.c @@ -533,7 +533,8 @@ static void InitScore(VP8ModeScore* const rd) { rd->score = MAX_COST; } -static void CopyScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { +static void CopyScore(VP8ModeScore* WEBP_RESTRICT const dst, + const VP8ModeScore* WEBP_RESTRICT const src) { dst->D = src->D; dst->SD = src->SD; dst->R = src->R; @@ -542,7 +543,8 @@ static void CopyScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { dst->score = src->score; } -static void AddScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { +static void AddScore(VP8ModeScore* WEBP_RESTRICT const dst, + const VP8ModeScore* WEBP_RESTRICT const src) { dst->D += src->D; dst->SD += src->SD; dst->R += src->R; @@ -585,15 +587,18 @@ static WEBP_INLINE score_t RDScoreTrellis(int lambda, score_t rate, return rate * lambda + RD_DISTO_MULT * distortion; } -static int TrellisQuantizeBlock(const VP8Encoder* const enc, +// Coefficient type. +enum { TYPE_I16_AC = 0, TYPE_I16_DC = 1, TYPE_CHROMA_A = 2, TYPE_I4_AC = 3 }; + +static int TrellisQuantizeBlock(const VP8Encoder* WEBP_RESTRICT const enc, int16_t in[16], int16_t out[16], int ctx0, int coeff_type, - const VP8Matrix* const mtx, + const VP8Matrix* WEBP_RESTRICT const mtx, int lambda) { const ProbaArray* const probas = enc->proba_.coeffs_[coeff_type]; CostArrayPtr const costs = (CostArrayPtr)enc->proba_.remapped_costs_[coeff_type]; - const int first = (coeff_type == 0) ? 1 : 0; + const int first = (coeff_type == TYPE_I16_AC) ? 1 : 0; Node nodes[16][NUM_NODES]; ScoreState score_states[2][NUM_NODES]; ScoreState* ss_cur = &SCORE_STATE(0, MIN_DELTA); @@ -657,16 +662,17 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, // test all alternate level values around level0. for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) { Node* const cur = &NODE(n, m); - int level = level0 + m; + const int level = level0 + m; const int ctx = (level > 2) ? 2 : level; const int band = VP8EncBands[n + 1]; score_t base_score; - score_t best_cur_score = MAX_COST; - int best_prev = 0; // default, in case + score_t best_cur_score; + int best_prev; + score_t cost, score; - ss_cur[m].score = MAX_COST; ss_cur[m].costs = costs[n + 1][ctx]; if (level < 0 || level > thresh_level) { + ss_cur[m].score = MAX_COST; // Node is dead. continue; } @@ -682,18 +688,24 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, } // Inspect all possible non-dead predecessors. Retain only the best one. - for (p = -MIN_DELTA; p <= MAX_DELTA; ++p) { + // The base_score is added to all scores so it is only added for the final + // value after the loop. + cost = VP8LevelCost(ss_prev[-MIN_DELTA].costs, level); + best_cur_score = + ss_prev[-MIN_DELTA].score + RDScoreTrellis(lambda, cost, 0); + best_prev = -MIN_DELTA; + for (p = -MIN_DELTA + 1; p <= MAX_DELTA; ++p) { // Dead nodes (with ss_prev[p].score >= MAX_COST) are automatically // eliminated since their score can't be better than the current best. - const score_t cost = VP8LevelCost(ss_prev[p].costs, level); + cost = VP8LevelCost(ss_prev[p].costs, level); // Examine node assuming it's a non-terminal one. - const score_t score = - base_score + ss_prev[p].score + RDScoreTrellis(lambda, cost, 0); + score = ss_prev[p].score + RDScoreTrellis(lambda, cost, 0); if (score < best_cur_score) { best_cur_score = score; best_prev = p; } } + best_cur_score += base_score; // Store best finding in current node. cur->sign = sign; cur->level = level; @@ -701,11 +713,11 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, ss_cur[m].score = best_cur_score; // Now, record best terminal node (and thus best entry in the graph). - if (level != 0) { + if (level != 0 && best_cur_score < best_score) { const score_t last_pos_cost = (n < 15) ? VP8BitCost(0, probas[band][ctx][0]) : 0; const score_t last_pos_score = RDScoreTrellis(lambda, last_pos_cost, 0); - const score_t score = best_cur_score + last_pos_score; + score = best_cur_score + last_pos_score; if (score < best_score) { best_score = score; best_path[0] = n; // best eob position @@ -717,10 +729,16 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, } // Fresh start - memset(in + first, 0, (16 - first) * sizeof(*in)); - memset(out + first, 0, (16 - first) * sizeof(*out)); + // Beware! We must preserve in[0]/out[0] value for TYPE_I16_AC case. + if (coeff_type == TYPE_I16_AC) { + memset(in + 1, 0, 15 * sizeof(*in)); + memset(out + 1, 0, 15 * sizeof(*out)); + } else { + memset(in, 0, 16 * sizeof(*in)); + memset(out, 0, 16 * sizeof(*out)); + } if (best_path[0] == -1) { - return 0; // skip! + return 0; // skip! } { @@ -751,9 +769,9 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, // all at once. Output is the reconstructed block in *yuv_out, and the // quantized levels in *levels. -static int ReconstructIntra16(VP8EncIterator* const it, - VP8ModeScore* const rd, - uint8_t* const yuv_out, +static int ReconstructIntra16(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8I16ModeOffsets[mode]; @@ -775,9 +793,9 @@ static int ReconstructIntra16(VP8EncIterator* const it, for (y = 0, n = 0; y < 4; ++y) { for (x = 0; x < 4; ++x, ++n) { const int ctx = it->top_nz_[x] + it->left_nz_[y]; - const int non_zero = - TrellisQuantizeBlock(enc, tmp[n], rd->y_ac_levels[n], ctx, 0, - &dqm->y1_, dqm->lambda_trellis_i16_); + const int non_zero = TrellisQuantizeBlock( + enc, tmp[n], rd->y_ac_levels[n], ctx, TYPE_I16_AC, &dqm->y1_, + dqm->lambda_trellis_i16_); it->top_nz_[x] = it->left_nz_[y] = non_zero; rd->y_ac_levels[n][0] = 0; nz |= non_zero << n; @@ -803,10 +821,10 @@ static int ReconstructIntra16(VP8EncIterator* const it, return nz; } -static int ReconstructIntra4(VP8EncIterator* const it, +static int ReconstructIntra4(VP8EncIterator* WEBP_RESTRICT const it, int16_t levels[16], - const uint8_t* const src, - uint8_t* const yuv_out, + const uint8_t* WEBP_RESTRICT const src, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8I4ModeOffsets[mode]; @@ -818,7 +836,7 @@ static int ReconstructIntra4(VP8EncIterator* const it, if (DO_TRELLIS_I4 && it->do_trellis_) { const int x = it->i4_ & 3, y = it->i4_ >> 2; const int ctx = it->top_nz_[x] + it->left_nz_[y]; - nz = TrellisQuantizeBlock(enc, tmp, levels, ctx, 3, &dqm->y1_, + nz = TrellisQuantizeBlock(enc, tmp, levels, ctx, TYPE_I4_AC, &dqm->y1_, dqm->lambda_trellis_i4_); } else { nz = VP8EncQuantizeBlock(tmp, levels, &dqm->y1_); @@ -839,7 +857,8 @@ static int ReconstructIntra4(VP8EncIterator* const it, // Quantize as usual, but also compute and return the quantization error. // Error is already divided by DSHIFT. -static int QuantizeSingle(int16_t* const v, const VP8Matrix* const mtx) { +static int QuantizeSingle(int16_t* WEBP_RESTRICT const v, + const VP8Matrix* WEBP_RESTRICT const mtx) { int V = *v; const int sign = (V < 0); if (sign) V = -V; @@ -853,9 +872,10 @@ static int QuantizeSingle(int16_t* const v, const VP8Matrix* const mtx) { return (sign ? -V : V) >> DSCALE; } -static void CorrectDCValues(const VP8EncIterator* const it, - const VP8Matrix* const mtx, - int16_t tmp[][16], VP8ModeScore* const rd) { +static void CorrectDCValues(const VP8EncIterator* WEBP_RESTRICT const it, + const VP8Matrix* WEBP_RESTRICT const mtx, + int16_t tmp[][16], + VP8ModeScore* WEBP_RESTRICT const rd) { // | top[0] | top[1] // --------+--------+--------- // left[0] | tmp[0] tmp[1] <-> err0 err1 @@ -886,8 +906,8 @@ static void CorrectDCValues(const VP8EncIterator* const it, } } -static void StoreDiffusionErrors(VP8EncIterator* const it, - const VP8ModeScore* const rd) { +static void StoreDiffusionErrors(VP8EncIterator* WEBP_RESTRICT const it, + const VP8ModeScore* WEBP_RESTRICT const rd) { int ch; for (ch = 0; ch <= 1; ++ch) { int8_t* const top = it->top_derr_[it->x_][ch]; @@ -906,8 +926,9 @@ static void StoreDiffusionErrors(VP8EncIterator* const it, //------------------------------------------------------------------------------ -static int ReconstructUV(VP8EncIterator* const it, VP8ModeScore* const rd, - uint8_t* const yuv_out, int mode) { +static int ReconstructUV(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8UVModeOffsets[mode]; const uint8_t* const src = it->yuv_in_ + U_OFF_ENC; @@ -927,9 +948,9 @@ static int ReconstructUV(VP8EncIterator* const it, VP8ModeScore* const rd, for (y = 0; y < 2; ++y) { for (x = 0; x < 2; ++x, ++n) { const int ctx = it->top_nz_[4 + ch + x] + it->left_nz_[4 + ch + y]; - const int non_zero = - TrellisQuantizeBlock(enc, tmp[n], rd->uv_levels[n], ctx, 2, - &dqm->uv_, dqm->lambda_trellis_uv_); + const int non_zero = TrellisQuantizeBlock( + enc, tmp[n], rd->uv_levels[n], ctx, TYPE_CHROMA_A, &dqm->uv_, + dqm->lambda_trellis_uv_); it->top_nz_[4 + ch + x] = it->left_nz_[4 + ch + y] = non_zero; nz |= non_zero << n; } @@ -978,7 +999,8 @@ static void SwapOut(VP8EncIterator* const it) { SwapPtr(&it->yuv_out_, &it->yuv_out2_); } -static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { +static void PickBestIntra16(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT rd) { const int kNumBlocks = 16; VP8SegmentInfo* const dqm = &it->enc_->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_i16_; @@ -1038,7 +1060,7 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { //------------------------------------------------------------------------------ // return the cost array corresponding to the surrounding prediction modes. -static const uint16_t* GetCostModeI4(VP8EncIterator* const it, +static const uint16_t* GetCostModeI4(VP8EncIterator* WEBP_RESTRICT const it, const uint8_t modes[16]) { const int preds_w = it->enc_->preds_w_; const int x = (it->i4_ & 3), y = it->i4_ >> 2; @@ -1047,7 +1069,8 @@ static const uint16_t* GetCostModeI4(VP8EncIterator* const it, return VP8FixedCostsI4[top][left]; } -static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) { +static int PickBestIntra4(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const VP8Encoder* const enc = it->enc_; const VP8SegmentInfo* const dqm = &enc->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_i4_; @@ -1143,7 +1166,8 @@ static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) { //------------------------------------------------------------------------------ -static void PickBestUV(VP8EncIterator* const it, VP8ModeScore* const rd) { +static void PickBestUV(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const int kNumBlocks = 8; const VP8SegmentInfo* const dqm = &it->enc_->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_uv_; @@ -1195,7 +1219,8 @@ static void PickBestUV(VP8EncIterator* const it, VP8ModeScore* const rd) { //------------------------------------------------------------------------------ // Final reconstruction and quantization. -static void SimpleQuantize(VP8EncIterator* const it, VP8ModeScore* const rd) { +static void SimpleQuantize(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const VP8Encoder* const enc = it->enc_; const int is_i16 = (it->mb_->type_ == 1); int nz = 0; @@ -1220,9 +1245,9 @@ static void SimpleQuantize(VP8EncIterator* const it, VP8ModeScore* const rd) { } // Refine intra16/intra4 sub-modes based on distortion only (not rate). -static void RefineUsingDistortion(VP8EncIterator* const it, +static void RefineUsingDistortion(VP8EncIterator* WEBP_RESTRICT const it, int try_both_modes, int refine_uv_mode, - VP8ModeScore* const rd) { + VP8ModeScore* WEBP_RESTRICT const rd) { score_t best_score = MAX_COST; int nz = 0; int mode; @@ -1336,7 +1361,8 @@ static void RefineUsingDistortion(VP8EncIterator* const it, //------------------------------------------------------------------------------ // Entry point -int VP8Decimate(VP8EncIterator* const it, VP8ModeScore* const rd, +int VP8Decimate(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, VP8RDLevel rd_opt) { int is_skipped; const int method = it->enc_->method_; diff --git a/3rdparty/libwebp/src/enc/syntax_enc.c b/3rdparty/libwebp/src/enc/syntax_enc.c index a9e5a6cf0f..9b8f524d69 100644 --- a/3rdparty/libwebp/src/enc/syntax_enc.c +++ b/3rdparty/libwebp/src/enc/syntax_enc.c @@ -258,7 +258,10 @@ static int EmitPartitionsSize(const VP8Encoder* const enc, buf[3 * p + 1] = (part_size >> 8) & 0xff; buf[3 * p + 2] = (part_size >> 16) & 0xff; } - return p ? pic->writer(buf, 3 * p, pic) : 1; + if (p && !pic->writer(buf, 3 * p, pic)) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); + } + return 1; } //------------------------------------------------------------------------------ @@ -349,7 +352,7 @@ int VP8EncWrite(VP8Encoder* const enc) { (enc->alpha_data_size_ & 1); riff_size += CHUNK_HEADER_SIZE + padded_alpha_size; } - // Sanity check. + // RIFF size should fit in 32-bits. if (riff_size > 0xfffffffeU) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_FILE_TOO_BIG); } @@ -381,6 +384,7 @@ int VP8EncWrite(VP8Encoder* const enc) { enc->coded_size_ = (int)(CHUNK_HEADER_SIZE + riff_size); ok = ok && WebPReportProgress(pic, final_percent, &enc->percent_); + if (!ok) WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); return ok; } diff --git a/3rdparty/libwebp/src/enc/vp8i_enc.h b/3rdparty/libwebp/src/enc/vp8i_enc.h index 0e35562a8c..19d9a6edb7 100644 --- a/3rdparty/libwebp/src/enc/vp8i_enc.h +++ b/3rdparty/libwebp/src/enc/vp8i_enc.h @@ -31,8 +31,8 @@ extern "C" { // version numbers #define ENC_MAJ_VERSION 1 -#define ENC_MIN_VERSION 2 -#define ENC_REV_VERSION 0 +#define ENC_MIN_VERSION 3 +#define ENC_REV_VERSION 1 enum { MAX_LF_LEVELS = 64, // Maximum loop filter level MAX_VARIABLE_LEVEL = 67, // last (inclusive) level with variable cost @@ -286,8 +286,7 @@ int VP8IteratorNext(VP8EncIterator* const it); // save the yuv_out_ boundary values to top_/left_ arrays for next iterations. void VP8IteratorSaveBoundary(VP8EncIterator* const it); // Report progression based on macroblock rows. Return 0 for user-abort request. -int VP8IteratorProgress(const VP8EncIterator* const it, - int final_delta_percent); +int VP8IteratorProgress(const VP8EncIterator* const it, int delta); // Intra4x4 iterations void VP8IteratorStartI4(VP8EncIterator* const it); // returns true if not done. @@ -471,7 +470,8 @@ int VP8EncAnalyze(VP8Encoder* const enc); // Sets up segment's quantization values, base_quant_ and filter strengths. void VP8SetSegmentParams(VP8Encoder* const enc, float quality); // Pick best modes and fills the levels. Returns true if skipped. -int VP8Decimate(VP8EncIterator* const it, VP8ModeScore* const rd, +int VP8Decimate(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, VP8RDLevel rd_opt); // in alpha.c @@ -491,19 +491,24 @@ int VP8FilterStrengthFromDelta(int sharpness, int delta); // misc utils for picture_*.c: +// Returns true if 'picture' is non-NULL and dimensions/colorspace are within +// their valid ranges. If returning false, the 'error_code' in 'picture' is +// updated. +int WebPValidatePicture(const WebPPicture* const picture); + // Remove reference to the ARGB/YUVA buffer (doesn't free anything). void WebPPictureResetBuffers(WebPPicture* const picture); -// Allocates ARGB buffer of given dimension (previous one is always free'd). -// Preserves the YUV(A) buffer. Returns false in case of error (invalid param, -// out-of-memory). -int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height); +// Allocates ARGB buffer according to set width/height (previous one is +// always free'd). Preserves the YUV(A) buffer. Returns false in case of error +// (invalid param, out-of-memory). +int WebPPictureAllocARGB(WebPPicture* const picture); -// Allocates YUVA buffer of given dimension (previous one is always free'd). -// Uses picture->csp to determine whether an alpha buffer is needed. +// Allocates YUVA buffer according to set width/height (previous one is always +// free'd). Uses picture->csp to determine whether an alpha buffer is needed. // Preserves the ARGB buffer. // Returns false in case of error (invalid param, out-of-memory). -int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height); +int WebPPictureAllocYUVA(WebPPicture* const picture); // Replace samples that are fully transparent by 'color' to help compressibility // (no guarantee, though). Assumes pic->use_argb is true. diff --git a/3rdparty/libwebp/src/enc/vp8l_enc.c b/3rdparty/libwebp/src/enc/vp8l_enc.c index 0b44ebe46e..c43d990d17 100644 --- a/3rdparty/libwebp/src/enc/vp8l_enc.c +++ b/3rdparty/libwebp/src/enc/vp8l_enc.c @@ -15,128 +15,25 @@ #include #include +#include "src/dsp/lossless.h" +#include "src/dsp/lossless_common.h" #include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" #include "src/enc/vp8i_enc.h" #include "src/enc/vp8li_enc.h" -#include "src/dsp/lossless.h" -#include "src/dsp/lossless_common.h" #include "src/utils/bit_writer_utils.h" #include "src/utils/huffman_encode_utils.h" +#include "src/utils/palette.h" #include "src/utils/utils.h" +#include "src/webp/encode.h" #include "src/webp/format_constants.h" // Maximum number of histogram images (sub-blocks). #define MAX_HUFF_IMAGE_SIZE 2600 -// Palette reordering for smaller sum of deltas (and for smaller storage). - -static int PaletteCompareColorsForQsort(const void* p1, const void* p2) { - const uint32_t a = WebPMemToUint32((uint8_t*)p1); - const uint32_t b = WebPMemToUint32((uint8_t*)p2); - assert(a != b); - return (a < b) ? -1 : 1; -} - -static WEBP_INLINE uint32_t PaletteComponentDistance(uint32_t v) { - return (v <= 128) ? v : (256 - v); -} - -// Computes a value that is related to the entropy created by the -// palette entry diff. -// -// Note that the last & 0xff is a no-operation in the next statement, but -// removed by most compilers and is here only for regularity of the code. -static WEBP_INLINE uint32_t PaletteColorDistance(uint32_t col1, uint32_t col2) { - const uint32_t diff = VP8LSubPixels(col1, col2); - const int kMoreWeightForRGBThanForAlpha = 9; - uint32_t score; - score = PaletteComponentDistance((diff >> 0) & 0xff); - score += PaletteComponentDistance((diff >> 8) & 0xff); - score += PaletteComponentDistance((diff >> 16) & 0xff); - score *= kMoreWeightForRGBThanForAlpha; - score += PaletteComponentDistance((diff >> 24) & 0xff); - return score; -} - -static WEBP_INLINE void SwapColor(uint32_t* const col1, uint32_t* const col2) { - const uint32_t tmp = *col1; - *col1 = *col2; - *col2 = tmp; -} - -static void GreedyMinimizeDeltas(uint32_t palette[], int num_colors) { - // Find greedily always the closest color of the predicted color to minimize - // deltas in the palette. This reduces storage needs since the - // palette is stored with delta encoding. - uint32_t predict = 0x00000000; - int i, k; - for (i = 0; i < num_colors; ++i) { - int best_ix = i; - uint32_t best_score = ~0U; - for (k = i; k < num_colors; ++k) { - const uint32_t cur_score = PaletteColorDistance(palette[k], predict); - if (best_score > cur_score) { - best_score = cur_score; - best_ix = k; - } - } - SwapColor(&palette[best_ix], &palette[i]); - predict = palette[i]; - } -} - -// The palette has been sorted by alpha. This function checks if the other -// components of the palette have a monotonic development with regards to -// position in the palette. If all have monotonic development, there is -// no benefit to re-organize them greedily. A monotonic development -// would be spotted in green-only situations (like lossy alpha) or gray-scale -// images. -static int PaletteHasNonMonotonousDeltas(uint32_t palette[], int num_colors) { - uint32_t predict = 0x000000; - int i; - uint8_t sign_found = 0x00; - for (i = 0; i < num_colors; ++i) { - const uint32_t diff = VP8LSubPixels(palette[i], predict); - const uint8_t rd = (diff >> 16) & 0xff; - const uint8_t gd = (diff >> 8) & 0xff; - const uint8_t bd = (diff >> 0) & 0xff; - if (rd != 0x00) { - sign_found |= (rd < 0x80) ? 1 : 2; - } - if (gd != 0x00) { - sign_found |= (gd < 0x80) ? 8 : 16; - } - if (bd != 0x00) { - sign_found |= (bd < 0x80) ? 64 : 128; - } - predict = palette[i]; - } - return (sign_found & (sign_found << 1)) != 0; // two consequent signs. -} - // ----------------------------------------------------------------------------- // Palette -// If number of colors in the image is less than or equal to MAX_PALETTE_SIZE, -// creates a palette and returns true, else returns false. -static int AnalyzeAndCreatePalette(const WebPPicture* const pic, - int low_effort, - uint32_t palette[MAX_PALETTE_SIZE], - int* const palette_size) { - const int num_colors = WebPGetColorPalette(pic, palette); - if (num_colors > MAX_PALETTE_SIZE) { - *palette_size = 0; - return 0; - } - *palette_size = num_colors; - qsort(palette, num_colors, sizeof(*palette), PaletteCompareColorsForQsort); - if (!low_effort && PaletteHasNonMonotonousDeltas(palette, num_colors)) { - GreedyMinimizeDeltas(palette, num_colors); - } - return 1; -} - // These five modes are evaluated and their respective entropy is computed. typedef enum { kDirect = 0, @@ -165,10 +62,11 @@ typedef enum { kHistoTotal // Must be last. } HistoIx; -static void AddSingleSubGreen(int p, uint32_t* const r, uint32_t* const b) { - const int green = p >> 8; // The upper bits are masked away later. - ++r[((p >> 16) - green) & 0xff]; - ++b[((p >> 0) - green) & 0xff]; +static void AddSingleSubGreen(uint32_t p, + uint32_t* const r, uint32_t* const b) { + const int green = (int)p >> 8; // The upper bits are masked away later. + ++r[(((int)p >> 16) - green) & 0xff]; + ++b[(((int)p >> 0) - green) & 0xff]; } static void AddSingle(uint32_t p, @@ -242,8 +140,8 @@ static int AnalyzeEntropy(const uint32_t* argb, curr_row += argb_stride; } { - double entropy_comp[kHistoTotal]; - double entropy[kNumEntropyIx]; + float entropy_comp[kHistoTotal]; + float entropy[kNumEntropyIx]; int k; int last_mode_to_analyze = use_palette ? kPalette : kSpatialSubGreen; int j; @@ -362,11 +260,14 @@ typedef struct { } CrunchSubConfig; typedef struct { int entropy_idx_; + PaletteSorting palette_sorting_type_; CrunchSubConfig sub_configs_[CRUNCH_SUBCONFIGS_MAX]; int sub_configs_size_; } CrunchConfig; -#define CRUNCH_CONFIGS_MAX kNumEntropyIx +// +2 because we add a palette sorting configuration for kPalette and +// kPaletteAndSpatial. +#define CRUNCH_CONFIGS_MAX (kNumEntropyIx + 2 * kPaletteSortingNum) static int EncoderAnalyze(VP8LEncoder* const enc, CrunchConfig crunch_configs[CRUNCH_CONFIGS_MAX], @@ -386,9 +287,12 @@ static int EncoderAnalyze(VP8LEncoder* const enc, int do_no_cache = 0; assert(pic != NULL && pic->argb != NULL); - use_palette = - AnalyzeAndCreatePalette(pic, low_effort, - enc->palette_, &enc->palette_size_); + // Check whether a palette is possible. + enc->palette_size_ = GetColorPalette(pic, enc->palette_sorted_); + use_palette = (enc->palette_size_ <= MAX_PALETTE_SIZE); + if (!use_palette) { + enc->palette_size_ = 0; + } // Empirical bit sizes. enc->histo_bits_ = GetHistoBits(method, use_palette, @@ -398,6 +302,8 @@ static int EncoderAnalyze(VP8LEncoder* const enc, if (low_effort) { // AnalyzeEntropy is somewhat slow. crunch_configs[0].entropy_idx_ = use_palette ? kPalette : kSpatialSubGreen; + crunch_configs[0].palette_sorting_type_ = + use_palette ? kSortedDefault : kUnusedPalette; n_lz77s = 1; *crunch_configs_size = 1; } else { @@ -418,13 +324,37 @@ static int EncoderAnalyze(VP8LEncoder* const enc, // a palette. if ((i != kPalette && i != kPaletteAndSpatial) || use_palette) { assert(*crunch_configs_size < CRUNCH_CONFIGS_MAX); - crunch_configs[(*crunch_configs_size)++].entropy_idx_ = i; + if (use_palette && (i == kPalette || i == kPaletteAndSpatial)) { + int sorting_method; + for (sorting_method = 0; sorting_method < kPaletteSortingNum; + ++sorting_method) { + const PaletteSorting typed_sorting_method = + (PaletteSorting)sorting_method; + // TODO(vrabaud) kSortedDefault should be tested. It is omitted + // for now for backward compatibility. + if (typed_sorting_method == kUnusedPalette || + typed_sorting_method == kSortedDefault) { + continue; + } + crunch_configs[(*crunch_configs_size)].entropy_idx_ = i; + crunch_configs[(*crunch_configs_size)].palette_sorting_type_ = + typed_sorting_method; + ++*crunch_configs_size; + } + } else { + crunch_configs[(*crunch_configs_size)].entropy_idx_ = i; + crunch_configs[(*crunch_configs_size)].palette_sorting_type_ = + kUnusedPalette; + ++*crunch_configs_size; + } } } } else { // Only choose the guessed best transform. *crunch_configs_size = 1; crunch_configs[0].entropy_idx_ = min_entropy_ix; + crunch_configs[0].palette_sorting_type_ = + use_palette ? kMinimizeDelta : kUnusedPalette; if (config->quality >= 75 && method == 5) { // Test with and without color cache. do_no_cache = 1; @@ -432,6 +362,7 @@ static int EncoderAnalyze(VP8LEncoder* const enc, if (min_entropy_ix == kPalette) { *crunch_configs_size = 2; crunch_configs[1].entropy_idx_ = kPaletteAndSpatial; + crunch_configs[1].palette_sorting_type_ = kMinimizeDelta; } } } @@ -730,11 +661,11 @@ static WEBP_INLINE void WriteHuffmanCodeWithExtraBits( VP8LPutBits(bw, (bits << depth) | symbol, depth + n_bits); } -static WebPEncodingError StoreImageToBitMask( +static int StoreImageToBitMask( VP8LBitWriter* const bw, int width, int histo_bits, const VP8LBackwardRefs* const refs, const uint16_t* histogram_symbols, - const HuffmanTreeCode* const huffman_codes) { + const HuffmanTreeCode* const huffman_codes, const WebPPicture* const pic) { const int histo_xsize = histo_bits ? VP8LSubSampleSize(width, histo_bits) : 1; const int tile_mask = (histo_bits == 0) ? 0 : -(1 << histo_bits); // x and y trace the position in the image. @@ -787,44 +718,52 @@ static WebPEncodingError StoreImageToBitMask( } VP8LRefsCursorNext(&c); } - return bw->error_ ? VP8_ENC_ERROR_OUT_OF_MEMORY : VP8_ENC_OK; + if (bw->error_) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + return 1; } -// Special case of EncodeImageInternal() for cache-bits=0, histo_bits=31 -static WebPEncodingError EncodeImageNoHuffman( - VP8LBitWriter* const bw, const uint32_t* const argb, - VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs_array, - int width, int height, int quality, int low_effort) { +// Special case of EncodeImageInternal() for cache-bits=0, histo_bits=31. +// pic and percent are for progress. +static int EncodeImageNoHuffman(VP8LBitWriter* const bw, + const uint32_t* const argb, + VP8LHashChain* const hash_chain, + VP8LBackwardRefs* const refs_array, int width, + int height, int quality, int low_effort, + const WebPPicture* const pic, int percent_range, + int* const percent) { int i; int max_tokens = 0; - WebPEncodingError err = VP8_ENC_OK; VP8LBackwardRefs* refs; HuffmanTreeToken* tokens = NULL; - HuffmanTreeCode huffman_codes[5] = { { 0, NULL, NULL } }; - const uint16_t histogram_symbols[1] = { 0 }; // only one tree, one symbol + HuffmanTreeCode huffman_codes[5] = {{0, NULL, NULL}}; + const uint16_t histogram_symbols[1] = {0}; // only one tree, one symbol int cache_bits = 0; VP8LHistogramSet* histogram_image = NULL; HuffmanTree* const huff_tree = (HuffmanTree*)WebPSafeMalloc( - 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); + 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); if (huff_tree == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Calculate backward references from ARGB image. - if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, - low_effort)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, low_effort, + pic, percent_range / 2, percent)) { + goto Error; + } + if (!VP8LGetBackwardReferences(width, height, argb, quality, /*low_effort=*/0, + kLZ77Standard | kLZ77RLE, cache_bits, + /*do_no_cache=*/0, hash_chain, refs_array, + &cache_bits, pic, + percent_range - percent_range / 2, percent)) { goto Error; } - err = VP8LGetBackwardReferences( - width, height, argb, quality, /*low_effort=*/0, kLZ77Standard | kLZ77RLE, - cache_bits, /*do_no_cache=*/0, hash_chain, refs_array, &cache_bits); - if (err != VP8_ENC_OK) goto Error; refs = &refs_array[0]; histogram_image = VP8LAllocateHistogramSet(1, cache_bits); if (histogram_image == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } VP8LHistogramSetClear(histogram_image); @@ -835,7 +774,7 @@ static WebPEncodingError EncodeImageNoHuffman( // Create Huffman bit lengths and codes for each histogram image. assert(histogram_image->size == 1); if (!GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -852,7 +791,7 @@ static WebPEncodingError EncodeImageNoHuffman( tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens)); if (tokens == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -864,27 +803,32 @@ static WebPEncodingError EncodeImageNoHuffman( } // Store actual literals. - err = StoreImageToBitMask(bw, width, 0, refs, histogram_symbols, - huffman_codes); + if (!StoreImageToBitMask(bw, width, 0, refs, histogram_symbols, huffman_codes, + pic)) { + goto Error; + } Error: WebPSafeFree(tokens); WebPSafeFree(huff_tree); VP8LFreeHistogramSet(histogram_image); WebPSafeFree(huffman_codes[0].codes); - return err; + return (pic->error_code == VP8_ENC_OK); } -static WebPEncodingError EncodeImageInternal( +// pic and percent are for progress. +static int EncodeImageInternal( VP8LBitWriter* const bw, const uint32_t* const argb, VP8LHashChain* const hash_chain, VP8LBackwardRefs refs_array[4], int width, int height, int quality, int low_effort, int use_cache, const CrunchConfig* const config, int* cache_bits, int histogram_bits, - size_t init_byte_position, int* const hdr_size, int* const data_size) { - WebPEncodingError err = VP8_ENC_ERROR_OUT_OF_MEMORY; + size_t init_byte_position, int* const hdr_size, int* const data_size, + const WebPPicture* const pic, int percent_range, int* const percent) { const uint32_t histogram_image_xysize = VP8LSubSampleSize(width, histogram_bits) * VP8LSubSampleSize(height, histogram_bits); + int remaining_percent = percent_range; + int percent_start = *percent; VP8LHistogramSet* histogram_image = NULL; VP8LHistogram* tmp_histo = NULL; int histogram_image_size = 0; @@ -893,9 +837,8 @@ static WebPEncodingError EncodeImageInternal( 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); HuffmanTreeToken* tokens = NULL; HuffmanTreeCode* huffman_codes = NULL; - uint16_t* const histogram_symbols = - (uint16_t*)WebPSafeMalloc(histogram_image_xysize, - sizeof(*histogram_symbols)); + uint16_t* const histogram_symbols = (uint16_t*)WebPSafeMalloc( + histogram_image_xysize, sizeof(*histogram_symbols)); int sub_configs_idx; int cache_bits_init, write_histogram_image; VP8LBitWriter bw_init = *bw, bw_best; @@ -907,14 +850,27 @@ static WebPEncodingError EncodeImageInternal( assert(hdr_size != NULL); assert(data_size != NULL); - // Make sure we can allocate the different objects. memset(&hash_chain_histogram, 0, sizeof(hash_chain_histogram)); - if (huff_tree == NULL || histogram_symbols == NULL || - !VP8LHashChainInit(&hash_chain_histogram, histogram_image_xysize) || - !VP8LHashChainFill(hash_chain, quality, argb, width, height, - low_effort)) { + if (!VP8LBitWriterInit(&bw_best, 0)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } + + // Make sure we can allocate the different objects. + if (huff_tree == NULL || histogram_symbols == NULL || + !VP8LHashChainInit(&hash_chain_histogram, histogram_image_xysize)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + + percent_range = remaining_percent / 5; + if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, + low_effort, pic, percent_range, percent)) { + goto Error; + } + percent_start += percent_range; + remaining_percent -= percent_range; + if (use_cache) { // If the value is different from zero, it has been set during the // palette analysis. @@ -923,22 +879,27 @@ static WebPEncodingError EncodeImageInternal( cache_bits_init = 0; } // If several iterations will happen, clone into bw_best. - if (!VP8LBitWriterInit(&bw_best, 0) || - ((config->sub_configs_size_ > 1 || - config->sub_configs_[0].do_no_cache_) && - !VP8LBitWriterClone(bw, &bw_best))) { + if ((config->sub_configs_size_ > 1 || config->sub_configs_[0].do_no_cache_) && + !VP8LBitWriterClone(bw, &bw_best)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } + for (sub_configs_idx = 0; sub_configs_idx < config->sub_configs_size_; ++sub_configs_idx) { const CrunchSubConfig* const sub_config = &config->sub_configs_[sub_configs_idx]; int cache_bits_best, i_cache; - err = VP8LGetBackwardReferences(width, height, argb, quality, low_effort, - sub_config->lz77_, cache_bits_init, - sub_config->do_no_cache_, hash_chain, - &refs_array[0], &cache_bits_best); - if (err != VP8_ENC_OK) goto Error; + int i_remaining_percent = remaining_percent / config->sub_configs_size_; + int i_percent_range = i_remaining_percent / 4; + i_remaining_percent -= i_percent_range; + + if (!VP8LGetBackwardReferences( + width, height, argb, quality, low_effort, sub_config->lz77_, + cache_bits_init, sub_config->do_no_cache_, hash_chain, + &refs_array[0], &cache_bits_best, pic, i_percent_range, percent)) { + goto Error; + } for (i_cache = 0; i_cache < (sub_config->do_no_cache_ ? 2 : 1); ++i_cache) { const int cache_bits_tmp = (i_cache == 0) ? cache_bits_best : 0; @@ -953,11 +914,17 @@ static WebPEncodingError EncodeImageInternal( histogram_image = VP8LAllocateHistogramSet(histogram_image_xysize, cache_bits_tmp); tmp_histo = VP8LAllocateHistogram(cache_bits_tmp); - if (histogram_image == NULL || tmp_histo == NULL || - !VP8LGetHistoImageSymbols(width, height, &refs_array[i_cache], - quality, low_effort, histogram_bits, - cache_bits_tmp, histogram_image, tmp_histo, - histogram_symbols)) { + if (histogram_image == NULL || tmp_histo == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + + i_percent_range = i_remaining_percent / 3; + i_remaining_percent -= i_percent_range; + if (!VP8LGetHistoImageSymbols( + width, height, &refs_array[i_cache], quality, low_effort, + histogram_bits, cache_bits_tmp, histogram_image, tmp_histo, + histogram_symbols, pic, i_percent_range, percent)) { goto Error; } // Create Huffman bit lengths and codes for each histogram image. @@ -970,6 +937,7 @@ static WebPEncodingError EncodeImageInternal( // GetHuffBitLengthsAndCodes(). if (huffman_codes == NULL || !GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Free combined histograms. @@ -992,12 +960,14 @@ static WebPEncodingError EncodeImageInternal( write_histogram_image = (histogram_image_size > 1); VP8LPutBits(bw, write_histogram_image, 1); if (write_histogram_image) { - uint32_t* const histogram_argb = - (uint32_t*)WebPSafeMalloc(histogram_image_xysize, - sizeof(*histogram_argb)); + uint32_t* const histogram_argb = (uint32_t*)WebPSafeMalloc( + histogram_image_xysize, sizeof(*histogram_argb)); int max_index = 0; uint32_t i; - if (histogram_argb == NULL) goto Error; + if (histogram_argb == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } for (i = 0; i < histogram_image_xysize; ++i) { const int symbol_index = histogram_symbols[i] & 0xffff; histogram_argb[i] = (symbol_index << 8); @@ -1008,12 +978,17 @@ static WebPEncodingError EncodeImageInternal( histogram_image_size = max_index; VP8LPutBits(bw, histogram_bits - 2, 3); - err = EncodeImageNoHuffman( - bw, histogram_argb, &hash_chain_histogram, &refs_array[2], - VP8LSubSampleSize(width, histogram_bits), - VP8LSubSampleSize(height, histogram_bits), quality, low_effort); + i_percent_range = i_remaining_percent / 2; + i_remaining_percent -= i_percent_range; + if (!EncodeImageNoHuffman( + bw, histogram_argb, &hash_chain_histogram, &refs_array[2], + VP8LSubSampleSize(width, histogram_bits), + VP8LSubSampleSize(height, histogram_bits), quality, low_effort, + pic, i_percent_range, percent)) { + WebPSafeFree(histogram_argb); + goto Error; + } WebPSafeFree(histogram_argb); - if (err != VP8_ENC_OK) goto Error; } // Store Huffman codes. @@ -1028,7 +1003,10 @@ static WebPEncodingError EncodeImageInternal( } } tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens)); - if (tokens == NULL) goto Error; + if (tokens == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } for (i = 0; i < 5 * histogram_image_size; ++i) { HuffmanTreeCode* const codes = &huffman_codes[i]; StoreHuffmanCode(bw, huff_tree, tokens, codes); @@ -1037,9 +1015,10 @@ static WebPEncodingError EncodeImageInternal( } // Store actual literals. hdr_size_tmp = (int)(VP8LBitWriterNumBytes(bw) - init_byte_position); - err = StoreImageToBitMask(bw, width, histogram_bits, &refs_array[i_cache], - histogram_symbols, huffman_codes); - if (err != VP8_ENC_OK) goto Error; + if (!StoreImageToBitMask(bw, width, histogram_bits, &refs_array[i_cache], + histogram_symbols, huffman_codes, pic)) { + goto Error; + } // Keep track of the smallest image so far. if (VP8LBitWriterNumBytes(bw) < bw_size_best) { bw_size_best = VP8LBitWriterNumBytes(bw); @@ -1059,7 +1038,10 @@ static WebPEncodingError EncodeImageInternal( } } VP8LBitWriterSwap(bw, &bw_best); - err = VP8_ENC_OK; + + if (!WebPReportProgress(pic, percent_start + remaining_percent, percent)) { + goto Error; + } Error: WebPSafeFree(tokens); @@ -1073,7 +1055,7 @@ static WebPEncodingError EncodeImageInternal( } WebPSafeFree(histogram_symbols); VP8LBitWriterWipeOut(&bw_best); - return err; + return (pic->error_code == VP8_ENC_OK); } // ----------------------------------------------------------------------------- @@ -1082,26 +1064,27 @@ static WebPEncodingError EncodeImageInternal( static void ApplySubtractGreen(VP8LEncoder* const enc, int width, int height, VP8LBitWriter* const bw) { VP8LPutBits(bw, TRANSFORM_PRESENT, 1); - VP8LPutBits(bw, SUBTRACT_GREEN, 2); + VP8LPutBits(bw, SUBTRACT_GREEN_TRANSFORM, 2); VP8LSubtractGreenFromBlueAndRed(enc->argb_, width * height); } -static WebPEncodingError ApplyPredictFilter(const VP8LEncoder* const enc, - int width, int height, - int quality, int low_effort, - int used_subtract_green, - VP8LBitWriter* const bw) { +static int ApplyPredictFilter(const VP8LEncoder* const enc, int width, + int height, int quality, int low_effort, + int used_subtract_green, VP8LBitWriter* const bw, + int percent_range, int* const percent) { const int pred_bits = enc->transform_bits_; const int transform_width = VP8LSubSampleSize(width, pred_bits); const int transform_height = VP8LSubSampleSize(height, pred_bits); // we disable near-lossless quantization if palette is used. - const int near_lossless_strength = enc->use_palette_ ? 100 - : enc->config_->near_lossless; + const int near_lossless_strength = + enc->use_palette_ ? 100 : enc->config_->near_lossless; - VP8LResidualImage(width, height, pred_bits, low_effort, enc->argb_, - enc->argb_scratch_, enc->transform_data_, - near_lossless_strength, enc->config_->exact, - used_subtract_green); + if (!VP8LResidualImage( + width, height, pred_bits, low_effort, enc->argb_, enc->argb_scratch_, + enc->transform_data_, near_lossless_strength, enc->config_->exact, + used_subtract_green, enc->pic_, percent_range / 2, percent)) { + return 0; + } VP8LPutBits(bw, TRANSFORM_PRESENT, 1); VP8LPutBits(bw, PREDICTOR_TRANSFORM, 2); assert(pred_bits >= 2); @@ -1109,19 +1092,23 @@ static WebPEncodingError ApplyPredictFilter(const VP8LEncoder* const enc, return EncodeImageNoHuffman( bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_, (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height, - quality, low_effort); + quality, low_effort, enc->pic_, percent_range - percent_range / 2, + percent); } -static WebPEncodingError ApplyCrossColorFilter(const VP8LEncoder* const enc, - int width, int height, - int quality, int low_effort, - VP8LBitWriter* const bw) { +static int ApplyCrossColorFilter(const VP8LEncoder* const enc, int width, + int height, int quality, int low_effort, + VP8LBitWriter* const bw, int percent_range, + int* const percent) { const int ccolor_transform_bits = enc->transform_bits_; const int transform_width = VP8LSubSampleSize(width, ccolor_transform_bits); const int transform_height = VP8LSubSampleSize(height, ccolor_transform_bits); - VP8LColorSpaceTransform(width, height, ccolor_transform_bits, quality, - enc->argb_, enc->transform_data_); + if (!VP8LColorSpaceTransform(width, height, ccolor_transform_bits, quality, + enc->argb_, enc->transform_data_, enc->pic_, + percent_range / 2, percent)) { + return 0; + } VP8LPutBits(bw, TRANSFORM_PRESENT, 1); VP8LPutBits(bw, CROSS_COLOR_TRANSFORM, 2); assert(ccolor_transform_bits >= 2); @@ -1129,23 +1116,21 @@ static WebPEncodingError ApplyCrossColorFilter(const VP8LEncoder* const enc, return EncodeImageNoHuffman( bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_, (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height, - quality, low_effort); + quality, low_effort, enc->pic_, percent_range - percent_range / 2, + percent); } // ----------------------------------------------------------------------------- -static WebPEncodingError WriteRiffHeader(const WebPPicture* const pic, - size_t riff_size, size_t vp8l_size) { +static int WriteRiffHeader(const WebPPicture* const pic, size_t riff_size, + size_t vp8l_size) { uint8_t riff[RIFF_HEADER_SIZE + CHUNK_HEADER_SIZE + VP8L_SIGNATURE_SIZE] = { 'R', 'I', 'F', 'F', 0, 0, 0, 0, 'W', 'E', 'B', 'P', 'V', 'P', '8', 'L', 0, 0, 0, 0, VP8L_MAGIC_BYTE, }; PutLE32(riff + TAG_SIZE, (uint32_t)riff_size); PutLE32(riff + RIFF_HEADER_SIZE + TAG_SIZE, (uint32_t)vp8l_size); - if (!pic->writer(riff, sizeof(riff), pic)) { - return VP8_ENC_ERROR_BAD_WRITE; - } - return VP8_ENC_OK; + return pic->writer(riff, sizeof(riff), pic); } static int WriteImageSize(const WebPPicture* const pic, @@ -1165,36 +1150,32 @@ static int WriteRealAlphaAndVersion(VP8LBitWriter* const bw, int has_alpha) { return !bw->error_; } -static WebPEncodingError WriteImage(const WebPPicture* const pic, - VP8LBitWriter* const bw, - size_t* const coded_size) { - WebPEncodingError err = VP8_ENC_OK; +static int WriteImage(const WebPPicture* const pic, VP8LBitWriter* const bw, + size_t* const coded_size) { const uint8_t* const webpll_data = VP8LBitWriterFinish(bw); const size_t webpll_size = VP8LBitWriterNumBytes(bw); const size_t vp8l_size = VP8L_SIGNATURE_SIZE + webpll_size; const size_t pad = vp8l_size & 1; const size_t riff_size = TAG_SIZE + CHUNK_HEADER_SIZE + vp8l_size + pad; + *coded_size = 0; - err = WriteRiffHeader(pic, riff_size, vp8l_size); - if (err != VP8_ENC_OK) goto Error; + if (bw->error_) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } - if (!pic->writer(webpll_data, webpll_size, pic)) { - err = VP8_ENC_ERROR_BAD_WRITE; - goto Error; + if (!WriteRiffHeader(pic, riff_size, vp8l_size) || + !pic->writer(webpll_data, webpll_size, pic)) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); } if (pad) { const uint8_t pad_byte[1] = { 0 }; if (!pic->writer(pad_byte, 1, pic)) { - err = VP8_ENC_ERROR_BAD_WRITE; - goto Error; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); } } *coded_size = CHUNK_HEADER_SIZE + riff_size; - return VP8_ENC_OK; - - Error: - return err; + return 1; } // ----------------------------------------------------------------------------- @@ -1210,36 +1191,32 @@ static void ClearTransformBuffer(VP8LEncoder* const enc) { // Flags influencing the memory allocated: // enc->transform_bits_ // enc->use_predict_, enc->use_cross_color_ -static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc, - int width, int height) { - WebPEncodingError err = VP8_ENC_OK; - const uint64_t image_size = width * height; +static int AllocateTransformBuffer(VP8LEncoder* const enc, int width, + int height) { + const uint64_t image_size = (uint64_t)width * height; // VP8LResidualImage needs room for 2 scanlines of uint32 pixels with an extra // pixel in each, plus 2 regular scanlines of bytes. // TODO(skal): Clean up by using arithmetic in bytes instead of words. const uint64_t argb_scratch_size = - enc->use_predict_ - ? (width + 1) * 2 + - (width * 2 + sizeof(uint32_t) - 1) / sizeof(uint32_t) - : 0; + enc->use_predict_ ? (width + 1) * 2 + (width * 2 + sizeof(uint32_t) - 1) / + sizeof(uint32_t) + : 0; const uint64_t transform_data_size = (enc->use_predict_ || enc->use_cross_color_) - ? VP8LSubSampleSize(width, enc->transform_bits_) * + ? (uint64_t)VP8LSubSampleSize(width, enc->transform_bits_) * VP8LSubSampleSize(height, enc->transform_bits_) : 0; const uint64_t max_alignment_in_words = (WEBP_ALIGN_CST + sizeof(uint32_t) - 1) / sizeof(uint32_t); - const uint64_t mem_size = - image_size + max_alignment_in_words + - argb_scratch_size + max_alignment_in_words + - transform_data_size; + const uint64_t mem_size = image_size + max_alignment_in_words + + argb_scratch_size + max_alignment_in_words + + transform_data_size; uint32_t* mem = enc->transform_mem_; if (mem == NULL || mem_size > enc->transform_mem_size_) { ClearTransformBuffer(enc); mem = (uint32_t*)WebPSafeMalloc(mem_size, sizeof(*mem)); if (mem == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; - goto Error; + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } enc->transform_mem_ = mem; enc->transform_mem_size_ = (size_t)mem_size; @@ -1252,19 +1229,16 @@ static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc, enc->transform_data_ = mem; enc->current_width_ = width; - Error: - return err; + return 1; } -static WebPEncodingError MakeInputImageCopy(VP8LEncoder* const enc) { - WebPEncodingError err = VP8_ENC_OK; +static int MakeInputImageCopy(VP8LEncoder* const enc) { const WebPPicture* const picture = enc->pic_; const int width = picture->width; const int height = picture->height; - err = AllocateTransformBuffer(enc, width, height); - if (err != VP8_ENC_OK) return err; - if (enc->argb_content_ == kEncoderARGB) return VP8_ENC_OK; + if (!AllocateTransformBuffer(enc, width, height)) return 0; + if (enc->argb_content_ == kEncoderARGB) return 1; { uint32_t* dst = enc->argb_; @@ -1278,27 +1252,11 @@ static WebPEncodingError MakeInputImageCopy(VP8LEncoder* const enc) { } enc->argb_content_ = kEncoderARGB; assert(enc->current_width_ == width); - return VP8_ENC_OK; + return 1; } // ----------------------------------------------------------------------------- -static WEBP_INLINE int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, - int hi) { - int low = 0; - if (sorted[low] == color) return low; // loop invariant: sorted[low] != color - while (1) { - const int mid = (low + hi) >> 1; - if (sorted[mid] == color) { - return mid; - } else if (sorted[mid] < color) { - low = mid; - } else { - hi = mid; - } - } -} - #define APPLY_PALETTE_GREEDY_MAX 4 static WEBP_INLINE uint32_t SearchColorGreedy(const uint32_t palette[], @@ -1333,17 +1291,6 @@ static WEBP_INLINE uint32_t ApplyPaletteHash2(uint32_t color) { (32 - PALETTE_INV_SIZE_BITS); } -// Sort palette in increasing order and prepare an inverse mapping array. -static void PrepareMapToPalette(const uint32_t palette[], int num_colors, - uint32_t sorted[], uint32_t idx_map[]) { - int i; - memcpy(sorted, palette, num_colors * sizeof(*sorted)); - qsort(sorted, num_colors, sizeof(*sorted), PaletteCompareColorsForQsort); - for (i = 0; i < num_colors; ++i) { - idx_map[SearchColorNoIdx(sorted, palette[i], num_colors)] = i; - } -} - // Use 1 pixel cache for ARGB pixels. #define APPLY_PALETTE_FOR(COLOR_INDEX) do { \ uint32_t prev_pix = palette[0]; \ @@ -1367,16 +1314,18 @@ static void PrepareMapToPalette(const uint32_t palette[], int num_colors, // using 'row' as a temporary buffer of size 'width'. // We assume that all src[] values have a corresponding entry in the palette. // Note: src[] can be the same as dst[] -static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, - uint32_t* dst, uint32_t dst_stride, - const uint32_t* palette, int palette_size, - int width, int height, int xbits) { +static int ApplyPalette(const uint32_t* src, uint32_t src_stride, uint32_t* dst, + uint32_t dst_stride, const uint32_t* palette, + int palette_size, int width, int height, int xbits, + const WebPPicture* const pic) { // TODO(skal): this tmp buffer is not needed if VP8LBundleColorMap() can be // made to work in-place. uint8_t* const tmp_row = (uint8_t*)WebPSafeMalloc(width, sizeof(*tmp_row)); int x, y; - if (tmp_row == NULL) return VP8_ENC_ERROR_OUT_OF_MEMORY; + if (tmp_row == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } if (palette_size < APPLY_PALETTE_GREEDY_MAX) { APPLY_PALETTE_FOR(SearchColorGreedy(palette, palette_size, pix)); @@ -1421,7 +1370,7 @@ static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, } } WebPSafeFree(tmp_row); - return VP8_ENC_OK; + return 1; } #undef APPLY_PALETTE_FOR #undef PALETTE_INV_SIZE_BITS @@ -1429,9 +1378,7 @@ static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, #undef APPLY_PALETTE_GREEDY_MAX // Note: Expects "enc->palette_" to be set properly. -static WebPEncodingError MapImageFromPalette(VP8LEncoder* const enc, - int in_place) { - WebPEncodingError err = VP8_ENC_OK; +static int MapImageFromPalette(VP8LEncoder* const enc, int in_place) { const WebPPicture* const pic = enc->pic_; const int width = pic->width; const int height = pic->height; @@ -1449,19 +1396,22 @@ static WebPEncodingError MapImageFromPalette(VP8LEncoder* const enc, xbits = (palette_size <= 16) ? 1 : 0; } - err = AllocateTransformBuffer(enc, VP8LSubSampleSize(width, xbits), height); - if (err != VP8_ENC_OK) return err; - - err = ApplyPalette(src, src_stride, + if (!AllocateTransformBuffer(enc, VP8LSubSampleSize(width, xbits), height)) { + return 0; + } + if (!ApplyPalette(src, src_stride, enc->argb_, enc->current_width_, - palette, palette_size, width, height, xbits); + palette, palette_size, width, height, xbits, pic)) { + return 0; + } enc->argb_content_ = kEncoderPalette; - return err; + return 1; } // Save palette_[] to bitstream. static WebPEncodingError EncodePalette(VP8LBitWriter* const bw, int low_effort, - VP8LEncoder* const enc) { + VP8LEncoder* const enc, + int percent_range, int* const percent) { int i; uint32_t tmp_palette[MAX_PALETTE_SIZE]; const int palette_size = enc->palette_size_; @@ -1476,7 +1426,7 @@ static WebPEncodingError EncodePalette(VP8LBitWriter* const bw, int low_effort, tmp_palette[0] = palette[0]; return EncodeImageNoHuffman(bw, tmp_palette, &enc->hash_chain_, &enc->refs_[0], palette_size, 1, /*quality=*/20, - low_effort); + low_effort, enc->pic_, percent_range, percent); } // ----------------------------------------------------------------------------- @@ -1520,7 +1470,6 @@ typedef struct { CrunchConfig crunch_configs_[CRUNCH_CONFIGS_MAX]; int num_crunch_configs_; int red_and_blue_always_zero_; - WebPEncodingError err_; WebPAuxStats* stats_; } StreamEncodeContext; @@ -1537,7 +1486,6 @@ static int EncodeStreamHook(void* input, void* data2) { #if !defined(WEBP_DISABLE_STATS) WebPAuxStats* const stats = params->stats_; #endif - WebPEncodingError err = VP8_ENC_OK; const int quality = (int)config->quality; const int low_effort = (config->method == 0); #if (WEBP_NEAR_LOSSLESS == 1) @@ -1545,6 +1493,7 @@ static int EncodeStreamHook(void* input, void* data2) { #endif const int height = picture->height; const size_t byte_position = VP8LBitWriterNumBytes(bw); + int percent = 2; // for WebPProgressHook #if (WEBP_NEAR_LOSSLESS == 1) int use_near_lossless = 0; #endif @@ -1558,12 +1507,13 @@ static int EncodeStreamHook(void* input, void* data2) { if (!VP8LBitWriterInit(&bw_best, 0) || (num_crunch_configs > 1 && !VP8LBitWriterClone(bw, &bw_best))) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } for (idx = 0; idx < num_crunch_configs; ++idx) { const int entropy_idx = crunch_configs[idx].entropy_idx_; + int remaining_percent = 97 / num_crunch_configs, percent_range; enc->use_palette_ = (entropy_idx == kPalette) || (entropy_idx == kPaletteAndSpatial); enc->use_subtract_green_ = @@ -1571,7 +1521,8 @@ static int EncodeStreamHook(void* input, void* data2) { enc->use_predict_ = (entropy_idx == kSpatial) || (entropy_idx == kSpatialSubGreen) || (entropy_idx == kPaletteAndSpatial); - if (low_effort) { + // When using a palette, R/B==0, hence no need to test for cross-color. + if (low_effort || enc->use_palette_) { enc->use_cross_color_ = 0; } else { enc->use_cross_color_ = red_and_blue_always_zero ? 0 : enc->use_predict_; @@ -1586,11 +1537,10 @@ static int EncodeStreamHook(void* input, void* data2) { use_near_lossless = (config->near_lossless < 100) && !enc->use_palette_ && !enc->use_predict_; if (use_near_lossless) { - err = AllocateTransformBuffer(enc, width, height); - if (err != VP8_ENC_OK) goto Error; + if (!AllocateTransformBuffer(enc, width, height)) goto Error; if ((enc->argb_content_ != kEncoderNearLossless) && !VP8ApplyNearLossless(picture, config->near_lossless, enc->argb_)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } enc->argb_content_ = kEncoderNearLossless; @@ -1603,10 +1553,18 @@ static int EncodeStreamHook(void* input, void* data2) { // Encode palette if (enc->use_palette_) { - err = EncodePalette(bw, low_effort, enc); - if (err != VP8_ENC_OK) goto Error; - err = MapImageFromPalette(enc, use_delta_palette); - if (err != VP8_ENC_OK) goto Error; + if (!PaletteSort(crunch_configs[idx].palette_sorting_type_, enc->pic_, + enc->palette_sorted_, enc->palette_size_, + enc->palette_)) { + WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + percent_range = remaining_percent / 4; + if (!EncodePalette(bw, low_effort, enc, percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; + if (!MapImageFromPalette(enc, use_delta_palette)) goto Error; // If using a color cache, do not have it bigger than the number of // colors. if (use_cache && enc->palette_size_ < (1 << MAX_COLOR_CACHE_BITS)) { @@ -1617,8 +1575,7 @@ static int EncodeStreamHook(void* input, void* data2) { // In case image is not packed. if (enc->argb_content_ != kEncoderNearLossless && enc->argb_content_ != kEncoderPalette) { - err = MakeInputImageCopy(enc); - if (err != VP8_ENC_OK) goto Error; + if (!MakeInputImageCopy(enc)) goto Error; } // ----------------------------------------------------------------------- @@ -1629,15 +1586,22 @@ static int EncodeStreamHook(void* input, void* data2) { } if (enc->use_predict_) { - err = ApplyPredictFilter(enc, enc->current_width_, height, quality, - low_effort, enc->use_subtract_green_, bw); - if (err != VP8_ENC_OK) goto Error; + percent_range = remaining_percent / 3; + if (!ApplyPredictFilter(enc, enc->current_width_, height, quality, + low_effort, enc->use_subtract_green_, bw, + percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; } if (enc->use_cross_color_) { - err = ApplyCrossColorFilter(enc, enc->current_width_, height, quality, - low_effort, bw); - if (err != VP8_ENC_OK) goto Error; + percent_range = remaining_percent / 2; + if (!ApplyCrossColorFilter(enc, enc->current_width_, height, quality, + low_effort, bw, percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; } } @@ -1645,12 +1609,13 @@ static int EncodeStreamHook(void* input, void* data2) { // ------------------------------------------------------------------------- // Encode and write the transformed image. - err = EncodeImageInternal(bw, enc->argb_, &enc->hash_chain_, enc->refs_, - enc->current_width_, height, quality, low_effort, - use_cache, &crunch_configs[idx], - &enc->cache_bits_, enc->histo_bits_, - byte_position, &hdr_size, &data_size); - if (err != VP8_ENC_OK) goto Error; + if (!EncodeImageInternal( + bw, enc->argb_, &enc->hash_chain_, enc->refs_, enc->current_width_, + height, quality, low_effort, use_cache, &crunch_configs[idx], + &enc->cache_bits_, enc->histo_bits_, byte_position, &hdr_size, + &data_size, picture, remaining_percent, &percent)) { + goto Error; + } // If we are better than what we already have. if (VP8LBitWriterNumBytes(bw) < best_size) { @@ -1680,18 +1645,15 @@ static int EncodeStreamHook(void* input, void* data2) { } VP8LBitWriterSwap(&bw_best, bw); -Error: + Error: VP8LBitWriterWipeOut(&bw_best); - params->err_ = err; // The hook should return false in case of error. - return (err == VP8_ENC_OK); + return (params->picture_->error_code == VP8_ENC_OK); } -WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, - const WebPPicture* const picture, - VP8LBitWriter* const bw_main, - int use_cache) { - WebPEncodingError err = VP8_ENC_OK; +int VP8LEncodeStream(const WebPConfig* const config, + const WebPPicture* const picture, + VP8LBitWriter* const bw_main, int use_cache) { VP8LEncoder* const enc_main = VP8LEncoderNew(config, picture); VP8LEncoder* enc_side = NULL; CrunchConfig crunch_configs[CRUNCH_CONFIGS_MAX]; @@ -1703,15 +1665,23 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, // The main thread uses picture->stats, the side thread uses stats_side. WebPAuxStats stats_side; VP8LBitWriter bw_side; + WebPPicture picture_side; const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface(); int ok_main; + if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) { + VP8LEncoderDelete(enc_main); + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + + // Avoid "garbage value" error from Clang's static analysis tool. + WebPPictureInit(&picture_side); + // Analyze image (entropy, num_palettes etc) - if (enc_main == NULL || - !EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, + if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, &red_and_blue_always_zero) || - !EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + !EncoderInit(enc_main)) { + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -1740,25 +1710,32 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, StreamEncodeContext* const param = (idx == 0) ? ¶ms_main : ¶ms_side; param->config_ = config; - param->picture_ = picture; param->use_cache_ = use_cache; param->red_and_blue_always_zero_ = red_and_blue_always_zero; if (idx == 0) { + param->picture_ = picture; param->stats_ = picture->stats; param->bw_ = bw_main; param->enc_ = enc_main; } else { + // Create a side picture (error_code is not thread-safe). + if (!WebPPictureView(picture, /*left=*/0, /*top=*/0, picture->width, + picture->height, &picture_side)) { + assert(0); + } + picture_side.progress_hook = NULL; // Progress hook is not thread-safe. + param->picture_ = &picture_side; // No need to free a view afterwards. param->stats_ = (picture->stats == NULL) ? NULL : &stats_side; // Create a side bit writer. if (!VP8LBitWriterClone(bw_main, &bw_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } param->bw_ = &bw_side; // Create a side encoder. - enc_side = VP8LEncoderNew(config, picture); + enc_side = VP8LEncoderNew(config, &picture_side); if (enc_side == NULL || !EncoderInit(enc_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Copy the values that were computed for the main encoder. @@ -1767,6 +1744,8 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, enc_side->palette_size_ = enc_main->palette_size_; memcpy(enc_side->palette_, enc_main->palette_, sizeof(enc_main->palette_)); + memcpy(enc_side->palette_sorted_, enc_main->palette_sorted_, + sizeof(enc_main->palette_sorted_)); param->enc_ = enc_side; } // Create the workers. @@ -1780,7 +1759,7 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, // Start the second thread if needed. if (num_crunch_configs_side != 0) { if (!worker_interface->Reset(&worker_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } #if !defined(WEBP_DISABLE_STATS) @@ -1790,8 +1769,6 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, memcpy(&stats_side, picture->stats, sizeof(stats_side)); } #endif - // This line is only useful to remove a Clang static analyzer warning. - params_side.err_ = VP8_ENC_OK; worker_interface->Launch(&worker_side); } // Execute the main thread. @@ -1803,7 +1780,10 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, const int ok_side = worker_interface->Sync(&worker_side); worker_interface->End(&worker_side); if (!ok_main || !ok_side) { - err = ok_main ? params_side.err_ : params_main.err_; + if (picture->error_code == VP8_ENC_OK) { + assert(picture_side.error_code != VP8_ENC_OK); + WebPEncodingSetError(picture, picture_side.error_code); + } goto Error; } if (VP8LBitWriterNumBytes(&bw_side) < VP8LBitWriterNumBytes(bw_main)) { @@ -1814,18 +1794,13 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, } #endif } - } else { - if (!ok_main) { - err = params_main.err_; - goto Error; - } } -Error: + Error: VP8LBitWriterWipeOut(&bw_side); VP8LEncoderDelete(enc_main); VP8LEncoderDelete(enc_side); - return err; + return (picture->error_code == VP8_ENC_OK); } #undef CRUNCH_CONFIGS_MAX @@ -1838,15 +1813,12 @@ int VP8LEncodeImage(const WebPConfig* const config, size_t coded_size; int percent = 0; int initial_size; - WebPEncodingError err = VP8_ENC_OK; VP8LBitWriter bw; if (picture == NULL) return 0; if (config == NULL || picture->argb == NULL) { - err = VP8_ENC_ERROR_NULL_PARAMETER; - WebPEncodingSetError(picture, err); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_NULL_PARAMETER); } width = picture->width; @@ -1856,13 +1828,13 @@ int VP8LEncodeImage(const WebPConfig* const config, initial_size = (config->image_hint == WEBP_HINT_GRAPH) ? width * height : width * height * 2; if (!VP8LBitWriterInit(&bw, initial_size)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } if (!WebPReportProgress(picture, 1, &percent)) { UserAbort: - err = VP8_ENC_ERROR_USER_ABORT; + WebPEncodingSetError(picture, VP8_ENC_ERROR_USER_ABORT); goto Error; } // Reset stats (for pure lossless coding) @@ -1878,28 +1850,26 @@ int VP8LEncodeImage(const WebPConfig* const config, // Write image size. if (!WriteImageSize(picture, &bw)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } has_alpha = WebPPictureHasTransparency(picture); // Write the non-trivial Alpha flag and lossless version. if (!WriteRealAlphaAndVersion(&bw, has_alpha)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } - if (!WebPReportProgress(picture, 5, &percent)) goto UserAbort; + if (!WebPReportProgress(picture, 2, &percent)) goto UserAbort; // Encode main image stream. - err = VP8LEncodeStream(config, picture, &bw, 1 /*use_cache*/); - if (err != VP8_ENC_OK) goto Error; + if (!VP8LEncodeStream(config, picture, &bw, 1 /*use_cache*/)) goto Error; - if (!WebPReportProgress(picture, 90, &percent)) goto UserAbort; + if (!WebPReportProgress(picture, 99, &percent)) goto UserAbort; // Finish the RIFF chunk. - err = WriteImage(picture, &bw, &coded_size); - if (err != VP8_ENC_OK) goto Error; + if (!WriteImage(picture, &bw, &coded_size)) goto Error; if (!WebPReportProgress(picture, 100, &percent)) goto UserAbort; @@ -1918,13 +1888,11 @@ int VP8LEncodeImage(const WebPConfig* const config, } Error: - if (bw.error_) err = VP8_ENC_ERROR_OUT_OF_MEMORY; - VP8LBitWriterWipeOut(&bw); - if (err != VP8_ENC_OK) { - WebPEncodingSetError(picture, err); - return 0; + if (bw.error_) { + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } - return 1; + VP8LBitWriterWipeOut(&bw); + return (picture->error_code == VP8_ENC_OK); } //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/enc/vp8li_enc.h b/3rdparty/libwebp/src/enc/vp8li_enc.h index 94210ce9f3..3d35e1612d 100644 --- a/3rdparty/libwebp/src/enc/vp8li_enc.h +++ b/3rdparty/libwebp/src/enc/vp8li_enc.h @@ -69,6 +69,8 @@ typedef struct { int use_palette_; int palette_size_; uint32_t palette_[MAX_PALETTE_SIZE]; + // Sorted version of palette_ for cache purposes. + uint32_t palette_sorted_[MAX_PALETTE_SIZE]; // Some 'scratch' (potentially large) objects. struct VP8LBackwardRefs refs_[4]; // Backward Refs array for temporaries. @@ -87,9 +89,10 @@ int VP8LEncodeImage(const WebPConfig* const config, // Encodes the main image stream using the supplied bit writer. // If 'use_cache' is false, disables the use of color cache. -WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, - const WebPPicture* const picture, - VP8LBitWriter* const bw, int use_cache); +// Returns false in case of error (stored in picture->error_code). +int VP8LEncodeStream(const WebPConfig* const config, + const WebPPicture* const picture, VP8LBitWriter* const bw, + int use_cache); #if (WEBP_NEAR_LOSSLESS == 1) // in near_lossless.c @@ -101,13 +104,18 @@ int VP8ApplyNearLossless(const WebPPicture* const picture, int quality, //------------------------------------------------------------------------------ // Image transforms in predictor.c. -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless, int exact, - int used_subtract_green); +// pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). +int VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless, int exact, + int used_subtract_green, const WebPPicture* const pic, + int percent_range, int* const percent); -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image); +int VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image, + const WebPPicture* const pic, int percent_range, + int* const percent); //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/enc/webp_enc.c b/3rdparty/libwebp/src/enc/webp_enc.c index ce2db2e94b..583fe6a8bb 100644 --- a/3rdparty/libwebp/src/enc/webp_enc.c +++ b/3rdparty/libwebp/src/enc/webp_enc.c @@ -307,7 +307,10 @@ int WebPEncodingSetError(const WebPPicture* const pic, WebPEncodingError error) { assert((int)error < VP8_ENC_ERROR_LAST); assert((int)error >= VP8_ENC_OK); - ((WebPPicture*)pic)->error_code = error; + // The oldest error reported takes precedence over the new one. + if (pic->error_code == VP8_ENC_OK) { + ((WebPPicture*)pic)->error_code = error; + } return 0; } @@ -317,8 +320,7 @@ int WebPReportProgress(const WebPPicture* const pic, *percent_store = percent; if (pic->progress_hook && !pic->progress_hook(percent, pic)) { // user abort requested - WebPEncodingSetError(pic, VP8_ENC_ERROR_USER_ABORT); - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_USER_ABORT); } } return 1; // ok @@ -329,16 +331,14 @@ int WebPEncode(const WebPConfig* config, WebPPicture* pic) { int ok = 0; if (pic == NULL) return 0; - WebPEncodingSetError(pic, VP8_ENC_OK); // all ok so far + pic->error_code = VP8_ENC_OK; // all ok so far if (config == NULL) { // bad params return WebPEncodingSetError(pic, VP8_ENC_ERROR_NULL_PARAMETER); } if (!WebPValidateConfig(config)) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } - if (pic->width <= 0 || pic->height <= 0) { - return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_DIMENSION); - } + if (!WebPValidatePicture(pic)) return 0; if (pic->width > WEBP_MAX_DIMENSION || pic->height > WEBP_MAX_DIMENSION) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_DIMENSION); } diff --git a/3rdparty/libwebp/src/mux/anim_encode.c b/3rdparty/libwebp/src/mux/anim_encode.c index 7be99068f6..d1c61a2f1e 100644 --- a/3rdparty/libwebp/src/mux/anim_encode.c +++ b/3rdparty/libwebp/src/mux/anim_encode.c @@ -248,9 +248,6 @@ WebPAnimEncoder* WebPAnimEncoderNewInternal( enc = (WebPAnimEncoder*)WebPSafeCalloc(1, sizeof(*enc)); if (enc == NULL) return NULL; - // sanity inits, so we can call WebPAnimEncoderDelete(): - enc->encoded_frames_ = NULL; - enc->mux_ = NULL; MarkNoError(enc); // Dimensions and options. @@ -421,7 +418,7 @@ static void MinimizeChangeRectangle(const WebPPicture* const src, const int max_allowed_diff_lossy = QualityToMaxDiff(quality); const int max_allowed_diff = is_lossless ? 0 : max_allowed_diff_lossy; - // Sanity checks. + // Assumption/correctness checks. assert(src->width == dst->width && src->height == dst->height); assert(rect->x_offset_ + rect->width_ <= dst->width); assert(rect->y_offset_ + rect->height_ <= dst->height); @@ -596,16 +593,17 @@ int WebPAnimEncoderRefineRect( int is_lossless, float quality, int* const x_offset, int* const y_offset, int* const width, int* const height) { FrameRectangle rect; - const int right = clip(*x_offset + *width, 0, curr_canvas->width); - const int left = clip(*x_offset, 0, curr_canvas->width - 1); - const int bottom = clip(*y_offset + *height, 0, curr_canvas->height); - const int top = clip(*y_offset, 0, curr_canvas->height - 1); + int right, left, bottom, top; if (prev_canvas == NULL || curr_canvas == NULL || prev_canvas->width != curr_canvas->width || prev_canvas->height != curr_canvas->height || !prev_canvas->use_argb || !curr_canvas->use_argb) { return 0; } + right = clip(*x_offset + *width, 0, curr_canvas->width); + left = clip(*x_offset, 0, curr_canvas->width - 1); + bottom = clip(*y_offset + *height, 0, curr_canvas->height); + top = clip(*y_offset, 0, curr_canvas->height - 1); rect.x_offset_ = left; rect.y_offset_ = top; rect.width_ = clip(right - left, 0, curr_canvas->width - rect.x_offset_); @@ -949,7 +947,8 @@ static int IncreasePreviousDuration(WebPAnimEncoder* const enc, int duration) { int new_duration; assert(enc->count_ >= 1); - assert(prev_enc_frame->sub_frame_.duration == + assert(!prev_enc_frame->is_key_frame_ || + prev_enc_frame->sub_frame_.duration == prev_enc_frame->key_frame_.duration); assert(prev_enc_frame->sub_frame_.duration == (prev_enc_frame->sub_frame_.duration & (MAX_DURATION - 1))); @@ -966,7 +965,7 @@ static int IncreasePreviousDuration(WebPAnimEncoder* const enc, int duration) { 0x10, 0x88, 0x88, 0x08 }; const WebPData lossless_1x1 = { - lossless_1x1_bytes, sizeof(lossless_1x1_bytes) + lossless_1x1_bytes, sizeof(lossless_1x1_bytes) }; const uint8_t lossy_1x1_bytes[] = { 0x52, 0x49, 0x46, 0x46, 0x40, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, @@ -1358,6 +1357,12 @@ int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int timestamp, if (!IncreasePreviousDuration(enc, (int)prev_frame_duration)) { return 0; } + // IncreasePreviousDuration() may add a frame to avoid exceeding + // MAX_DURATION which could cause CacheFrame() to over read encoded_frames_ + // before the next flush. + if (enc->count_ == enc->size_ && !FlushFrames(enc)) { + return 0; + } } else { enc->first_timestamp_ = timestamp; } diff --git a/3rdparty/libwebp/src/mux/muxedit.c b/3rdparty/libwebp/src/mux/muxedit.c index ccf14b2a0c..aab479cc6c 100644 --- a/3rdparty/libwebp/src/mux/muxedit.c +++ b/3rdparty/libwebp/src/mux/muxedit.c @@ -70,6 +70,7 @@ void WebPMuxDelete(WebPMux* mux) { err = ChunkAssignData(&chunk, data, copy_data, tag); \ if (err == WEBP_MUX_OK) { \ err = ChunkSetHead(&chunk, (LIST)); \ + if (err != WEBP_MUX_OK) ChunkRelease(&chunk); \ } \ return err; \ } @@ -235,7 +236,6 @@ WebPMuxError WebPMuxSetImage(WebPMux* mux, const WebPData* bitstream, WebPMuxImage wpi; WebPMuxError err; - // Sanity checks. if (mux == NULL || bitstream == NULL || bitstream->bytes == NULL || bitstream->size > MAX_CHUNK_PAYLOAD) { return WEBP_MUX_INVALID_ARGUMENT; @@ -267,7 +267,6 @@ WebPMuxError WebPMuxPushFrame(WebPMux* mux, const WebPMuxFrameInfo* info, WebPMuxImage wpi; WebPMuxError err; - // Sanity checks. if (mux == NULL || info == NULL) return WEBP_MUX_INVALID_ARGUMENT; if (info->id != WEBP_CHUNK_ANMF) return WEBP_MUX_INVALID_ARGUMENT; @@ -556,7 +555,8 @@ static WebPMuxError MuxCleanup(WebPMux* const mux) { if (num_frames == 1) { WebPMuxImage* frame = NULL; err = MuxImageGetNth((const WebPMuxImage**)&mux->images_, 1, &frame); - assert(err == WEBP_MUX_OK); // We know that one frame does exist. + if (err != WEBP_MUX_OK) return err; + // We know that one frame does exist. assert(frame != NULL); if (frame->header_ != NULL && ((mux->canvas_width_ == 0 && mux->canvas_height_ == 0) || diff --git a/3rdparty/libwebp/src/mux/muxi.h b/3rdparty/libwebp/src/mux/muxi.h index 2289822e8f..fc44d6f2fe 100644 --- a/3rdparty/libwebp/src/mux/muxi.h +++ b/3rdparty/libwebp/src/mux/muxi.h @@ -28,8 +28,8 @@ extern "C" { // Defines and constants. #define MUX_MAJ_VERSION 1 -#define MUX_MIN_VERSION 2 -#define MUX_REV_VERSION 0 +#define MUX_MIN_VERSION 3 +#define MUX_REV_VERSION 1 // Chunk object. typedef struct WebPChunk WebPChunk; diff --git a/3rdparty/libwebp/src/mux/muxinternal.c b/3rdparty/libwebp/src/mux/muxinternal.c index b9ee6717d3..75b6b416b9 100644 --- a/3rdparty/libwebp/src/mux/muxinternal.c +++ b/3rdparty/libwebp/src/mux/muxinternal.c @@ -155,17 +155,18 @@ WebPMuxError ChunkSetHead(WebPChunk* const chunk, WebPMuxError ChunkAppend(WebPChunk* const chunk, WebPChunk*** const chunk_list) { + WebPMuxError err; assert(chunk_list != NULL && *chunk_list != NULL); if (**chunk_list == NULL) { - ChunkSetHead(chunk, *chunk_list); + err = ChunkSetHead(chunk, *chunk_list); } else { WebPChunk* last_chunk = **chunk_list; while (last_chunk->next_ != NULL) last_chunk = last_chunk->next_; - ChunkSetHead(chunk, &last_chunk->next_); - *chunk_list = &last_chunk->next_; + err = ChunkSetHead(chunk, &last_chunk->next_); + if (err == WEBP_MUX_OK) *chunk_list = &last_chunk->next_; } - return WEBP_MUX_OK; + return err; } //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/mux/muxread.c b/3rdparty/libwebp/src/mux/muxread.c index 0101fde15d..9862ec68ee 100644 --- a/3rdparty/libwebp/src/mux/muxread.c +++ b/3rdparty/libwebp/src/mux/muxread.c @@ -56,7 +56,7 @@ static WebPMuxError ChunkVerifyAndAssign(WebPChunk* chunk, uint32_t chunk_size; WebPData chunk_data; - // Sanity checks. + // Correctness checks. if (data_size < CHUNK_HEADER_SIZE) return WEBP_MUX_NOT_ENOUGH_DATA; chunk_size = GetLE32(data + TAG_SIZE); if (chunk_size > MAX_CHUNK_PAYLOAD) return WEBP_MUX_BAD_DATA; @@ -116,9 +116,12 @@ static int MuxImageParse(const WebPChunk* const chunk, int copy_data, // Each of ANMF chunk contain a header at the beginning. So, its size should // be at least 'hdr_size'. if (size < hdr_size) goto Fail; - ChunkAssignData(&subchunk, &temp, copy_data, chunk->tag_); + if (ChunkAssignData(&subchunk, &temp, copy_data, + chunk->tag_) != WEBP_MUX_OK) { + goto Fail; + } } - ChunkSetHead(&subchunk, &wpi->header_); + if (ChunkSetHead(&subchunk, &wpi->header_) != WEBP_MUX_OK) goto Fail; wpi->is_partial_ = 1; // Waiting for ALPH and/or VP8/VP8L chunks. // Rest of the chunks. @@ -186,7 +189,6 @@ WebPMux* WebPMuxCreateInternal(const WebPData* bitstream, int copy_data, WebPChunk** chunk_list_ends[WEBP_CHUNK_NIL + 1] = { NULL }; ChunkInit(&chunk); - // Sanity checks. if (WEBP_ABI_IS_INCOMPATIBLE(version, WEBP_MUX_ABI_VERSION)) { return NULL; // version mismatch } @@ -481,7 +483,6 @@ WebPMuxError WebPMuxGetFrame( WebPMuxError err; WebPMuxImage* wpi; - // Sanity checks. if (mux == NULL || frame == NULL) { return WEBP_MUX_INVALID_ARGUMENT; } diff --git a/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h b/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h index 46b3880706..24f3af7b54 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h +++ b/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h @@ -55,7 +55,7 @@ void VP8LoadFinalBytes(VP8BitReader* const br); // makes sure br->value_ has at least BITS bits worth of data static WEBP_UBSAN_IGNORE_UNDEF WEBP_INLINE -void VP8LoadNewBytes(VP8BitReader* const br) { +void VP8LoadNewBytes(VP8BitReader* WEBP_RESTRICT const br) { assert(br != NULL && br->buf_ != NULL); // Read 'BITS' bits at a time if possible. if (br->buf_ < br->buf_max_) { @@ -104,7 +104,7 @@ void VP8LoadNewBytes(VP8BitReader* const br) { } // Read a bit with proba 'prob'. Speed-critical function! -static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, +static WEBP_INLINE int VP8GetBit(VP8BitReader* WEBP_RESTRICT const br, int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't @@ -137,7 +137,8 @@ static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, // simplified version of VP8GetBit() for prob=0x80 (note shift is always 1 here) static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE -int VP8GetSigned(VP8BitReader* const br, int v, const char label[]) { +int VP8GetSigned(VP8BitReader* WEBP_RESTRICT const br, int v, + const char label[]) { if (br->bits_ < 0) { VP8LoadNewBytes(br); } @@ -147,15 +148,15 @@ int VP8GetSigned(VP8BitReader* const br, int v, const char label[]) { const range_t value = (range_t)(br->value_ >> pos); const int32_t mask = (int32_t)(split - value) >> 31; // -1 or 0 br->bits_ -= 1; - br->range_ += mask; + br->range_ += (range_t)mask; br->range_ |= 1; - br->value_ -= (bit_t)((split + 1) & mask) << pos; + br->value_ -= (bit_t)((split + 1) & (uint32_t)mask) << pos; BT_TRACK(br); return (v ^ mask) - mask; } } -static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* const br, +static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* WEBP_RESTRICT const br, int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't diff --git a/3rdparty/libwebp/src/utils/bit_reader_utils.c b/3rdparty/libwebp/src/utils/bit_reader_utils.c index 857cd60988..a26557aa49 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_utils.c +++ b/3rdparty/libwebp/src/utils/bit_reader_utils.c @@ -15,6 +15,7 @@ #include "src/webp/config.h" #endif +#include "src/dsp/cpu.h" #include "src/utils/bit_reader_inl_utils.h" #include "src/utils/utils.h" @@ -121,7 +122,7 @@ int32_t VP8GetSignedValue(VP8BitReader* const br, int bits, #define VP8L_LOG8_WBITS 4 // Number of bytes needed to store VP8L_WBITS bits. -#if defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ +#if defined(__arm__) || defined(_M_ARM) || WEBP_AARCH64 || \ defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64__) || defined(_M_X64) #define VP8L_USE_FAST_LOAD diff --git a/3rdparty/libwebp/src/utils/bit_reader_utils.h b/3rdparty/libwebp/src/utils/bit_reader_utils.h index e64156e318..25ff31e5d9 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_utils.h +++ b/3rdparty/libwebp/src/utils/bit_reader_utils.h @@ -19,6 +19,7 @@ #ifdef _MSC_VER #include // _byteswap_ulong #endif +#include "src/dsp/cpu.h" #include "src/webp/types.h" // Warning! This macro triggers quite some MACRO wizardry around func signature! @@ -64,7 +65,7 @@ extern "C" { #define BITS 56 #elif defined(__arm__) || defined(_M_ARM) // ARM #define BITS 24 -#elif defined(__aarch64__) // ARM 64bit +#elif WEBP_AARCH64 // ARM 64bit #define BITS 56 #elif defined(__mips__) // MIPS #define BITS 24 diff --git a/3rdparty/libwebp/src/utils/bit_writer_utils.c b/3rdparty/libwebp/src/utils/bit_writer_utils.c index bef0e31ca5..2f408508f1 100644 --- a/3rdparty/libwebp/src/utils/bit_writer_utils.c +++ b/3rdparty/libwebp/src/utils/bit_writer_utils.c @@ -278,7 +278,7 @@ void VP8LPutBitsFlushBits(VP8LBitWriter* const bw) { // If needed, make some room by flushing some bits out. if (bw->cur_ + VP8L_WRITER_BYTES > bw->end_) { const uint64_t extra_size = (bw->end_ - bw->buf_) + MIN_EXTRA_SIZE; - if (extra_size != (size_t)extra_size || + if (!CheckSizeOverflow(extra_size) || !VP8LBitWriterResize(bw, (size_t)extra_size)) { bw->cur_ = bw->buf_; bw->error_ = 1; @@ -314,7 +314,7 @@ void VP8LPutBitsInternal(VP8LBitWriter* const bw, uint32_t bits, int n_bits) { while (used >= VP8L_WRITER_BITS) { if (bw->cur_ + VP8L_WRITER_BYTES > bw->end_) { const uint64_t extra_size = (bw->end_ - bw->buf_) + MIN_EXTRA_SIZE; - if (extra_size != (size_t)extra_size || + if (!CheckSizeOverflow(extra_size) || !VP8LBitWriterResize(bw, (size_t)extra_size)) { bw->cur_ = bw->buf_; bw->error_ = 1; diff --git a/3rdparty/libwebp/src/utils/color_cache_utils.c b/3rdparty/libwebp/src/utils/color_cache_utils.c index b09f538e8b..7b5222b6e5 100644 --- a/3rdparty/libwebp/src/utils/color_cache_utils.c +++ b/3rdparty/libwebp/src/utils/color_cache_utils.c @@ -20,22 +20,22 @@ //------------------------------------------------------------------------------ // VP8LColorCache. -int VP8LColorCacheInit(VP8LColorCache* const cc, int hash_bits) { +int VP8LColorCacheInit(VP8LColorCache* const color_cache, int hash_bits) { const int hash_size = 1 << hash_bits; - assert(cc != NULL); + assert(color_cache != NULL); assert(hash_bits > 0); - cc->colors_ = (uint32_t*)WebPSafeCalloc((uint64_t)hash_size, - sizeof(*cc->colors_)); - if (cc->colors_ == NULL) return 0; - cc->hash_shift_ = 32 - hash_bits; - cc->hash_bits_ = hash_bits; + color_cache->colors_ = (uint32_t*)WebPSafeCalloc( + (uint64_t)hash_size, sizeof(*color_cache->colors_)); + if (color_cache->colors_ == NULL) return 0; + color_cache->hash_shift_ = 32 - hash_bits; + color_cache->hash_bits_ = hash_bits; return 1; } -void VP8LColorCacheClear(VP8LColorCache* const cc) { - if (cc != NULL) { - WebPSafeFree(cc->colors_); - cc->colors_ = NULL; +void VP8LColorCacheClear(VP8LColorCache* const color_cache) { + if (color_cache != NULL) { + WebPSafeFree(color_cache->colors_); + color_cache->colors_ = NULL; } } diff --git a/3rdparty/libwebp/src/utils/huffman_encode_utils.c b/3rdparty/libwebp/src/utils/huffman_encode_utils.c index 6f3b1bbe02..585db91951 100644 --- a/3rdparty/libwebp/src/utils/huffman_encode_utils.c +++ b/3rdparty/libwebp/src/utils/huffman_encode_utils.c @@ -161,7 +161,7 @@ static void SetBitDepths(const HuffmanTree* const tree, // especially when population counts are longer than 2**tree_limit, but // we are not planning to use this with extremely long blocks. // -// See http://en.wikipedia.org/wiki/Huffman_coding +// See https://en.wikipedia.org/wiki/Huffman_coding static void GenerateOptimalTree(const uint32_t* const histogram, int histogram_size, HuffmanTree* tree, int tree_depth_limit, @@ -404,8 +404,7 @@ static void ConvertBitDepthsToSymbols(HuffmanTreeCode* const tree) { // Main entry point void VP8LCreateHuffmanTree(uint32_t* const histogram, int tree_depth_limit, - uint8_t* const buf_rle, - HuffmanTree* const huff_tree, + uint8_t* const buf_rle, HuffmanTree* const huff_tree, HuffmanTreeCode* const huff_code) { const int num_symbols = huff_code->num_symbols; memset(buf_rle, 0, num_symbols * sizeof(*buf_rle)); diff --git a/3rdparty/libwebp/src/utils/huffman_encode_utils.h b/3rdparty/libwebp/src/utils/huffman_encode_utils.h index 3e6763ce49..3f7f1d8074 100644 --- a/3rdparty/libwebp/src/utils/huffman_encode_utils.h +++ b/3rdparty/libwebp/src/utils/huffman_encode_utils.h @@ -51,7 +51,7 @@ int VP8LCreateCompressedHuffmanTree(const HuffmanTreeCode* const tree, // huffman code tree. void VP8LCreateHuffmanTree(uint32_t* const histogram, int tree_depth_limit, uint8_t* const buf_rle, HuffmanTree* const huff_tree, - HuffmanTreeCode* const tree); + HuffmanTreeCode* const huff_code); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/utils/huffman_utils.c b/3rdparty/libwebp/src/utils/huffman_utils.c index 0cba0fbb7d..cf73abd437 100644 --- a/3rdparty/libwebp/src/utils/huffman_utils.c +++ b/3rdparty/libwebp/src/utils/huffman_utils.c @@ -142,7 +142,7 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, { int step; // step size to replicate values in current table - uint32_t low = -1; // low bits for current root entry + uint32_t low = 0xffffffffu; // low bits for current root entry uint32_t mask = total_size - 1; // mask for low bits uint32_t key = 0; // reversed prefix code int num_nodes = 1; // number of Huffman tree nodes @@ -177,21 +177,24 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, if (num_open < 0) { return 0; } - if (root_table == NULL) continue; for (; count[len] > 0; --count[len]) { HuffmanCode code; if ((key & mask) != low) { - table += table_size; + if (root_table != NULL) table += table_size; table_bits = NextTableBitSize(count, len, root_bits); table_size = 1 << table_bits; total_size += table_size; low = key & mask; - root_table[low].bits = (uint8_t)(table_bits + root_bits); - root_table[low].value = (uint16_t)((table - root_table) - low); + if (root_table != NULL) { + root_table[low].bits = (uint8_t)(table_bits + root_bits); + root_table[low].value = (uint16_t)((table - root_table) - low); + } + } + if (root_table != NULL) { + code.bits = (uint8_t)(len - root_bits); + code.value = (uint16_t)sorted[symbol++]; + ReplicateValue(&table[key >> root_bits], step, table_size, code); } - code.bits = (uint8_t)(len - root_bits); - code.value = (uint16_t)sorted[symbol++]; - ReplicateValue(&table[key >> root_bits], step, table_size, code); key = GetNextKey(key, len); } } @@ -211,25 +214,83 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, ((1 << MAX_CACHE_BITS) + NUM_LITERAL_CODES + NUM_LENGTH_CODES) // Cut-off value for switching between heap and stack allocation. #define SORTED_SIZE_CUTOFF 512 -int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, +int VP8LBuildHuffmanTable(HuffmanTables* const root_table, int root_bits, const int code_lengths[], int code_lengths_size) { - int total_size; + const int total_size = + BuildHuffmanTable(NULL, root_bits, code_lengths, code_lengths_size, NULL); assert(code_lengths_size <= MAX_CODE_LENGTHS_SIZE); - if (root_table == NULL) { - total_size = BuildHuffmanTable(NULL, root_bits, - code_lengths, code_lengths_size, NULL); - } else if (code_lengths_size <= SORTED_SIZE_CUTOFF) { + if (total_size == 0 || root_table == NULL) return total_size; + + if (root_table->curr_segment->curr_table + total_size >= + root_table->curr_segment->start + root_table->curr_segment->size) { + // If 'root_table' does not have enough memory, allocate a new segment. + // The available part of root_table->curr_segment is left unused because we + // need a contiguous buffer. + const int segment_size = root_table->curr_segment->size; + struct HuffmanTablesSegment* next = + (HuffmanTablesSegment*)WebPSafeMalloc(1, sizeof(*next)); + if (next == NULL) return 0; + // Fill the new segment. + // We need at least 'total_size' but if that value is small, it is better to + // allocate a big chunk to prevent more allocations later. 'segment_size' is + // therefore chosen (any other arbitrary value could be chosen). + next->size = total_size > segment_size ? total_size : segment_size; + next->start = + (HuffmanCode*)WebPSafeMalloc(next->size, sizeof(*next->start)); + if (next->start == NULL) { + WebPSafeFree(next); + return 0; + } + next->curr_table = next->start; + next->next = NULL; + // Point to the new segment. + root_table->curr_segment->next = next; + root_table->curr_segment = next; + } + if (code_lengths_size <= SORTED_SIZE_CUTOFF) { // use local stack-allocated array. uint16_t sorted[SORTED_SIZE_CUTOFF]; - total_size = BuildHuffmanTable(root_table, root_bits, - code_lengths, code_lengths_size, sorted); - } else { // rare case. Use heap allocation. + BuildHuffmanTable(root_table->curr_segment->curr_table, root_bits, + code_lengths, code_lengths_size, sorted); + } else { // rare case. Use heap allocation. uint16_t* const sorted = (uint16_t*)WebPSafeMalloc(code_lengths_size, sizeof(*sorted)); if (sorted == NULL) return 0; - total_size = BuildHuffmanTable(root_table, root_bits, - code_lengths, code_lengths_size, sorted); + BuildHuffmanTable(root_table->curr_segment->curr_table, root_bits, + code_lengths, code_lengths_size, sorted); WebPSafeFree(sorted); } return total_size; } + +int VP8LHuffmanTablesAllocate(int size, HuffmanTables* huffman_tables) { + // Have 'segment' point to the first segment for now, 'root'. + HuffmanTablesSegment* const root = &huffman_tables->root; + huffman_tables->curr_segment = root; + // Allocate root. + root->start = (HuffmanCode*)WebPSafeMalloc(size, sizeof(*root->start)); + if (root->start == NULL) return 0; + root->curr_table = root->start; + root->next = NULL; + root->size = size; + return 1; +} + +void VP8LHuffmanTablesDeallocate(HuffmanTables* const huffman_tables) { + HuffmanTablesSegment *current, *next; + if (huffman_tables == NULL) return; + // Free the root node. + current = &huffman_tables->root; + next = current->next; + WebPSafeFree(current->start); + current->start = NULL; + current->next = NULL; + current = next; + // Free the following nodes. + while (current != NULL) { + next = current->next; + WebPSafeFree(current->start); + WebPSafeFree(current); + current = next; + } +} diff --git a/3rdparty/libwebp/src/utils/huffman_utils.h b/3rdparty/libwebp/src/utils/huffman_utils.h index 13b7ad1ac4..98415c5328 100644 --- a/3rdparty/libwebp/src/utils/huffman_utils.h +++ b/3rdparty/libwebp/src/utils/huffman_utils.h @@ -43,6 +43,29 @@ typedef struct { // or non-literal symbol otherwise } HuffmanCode32; +// Contiguous memory segment of HuffmanCodes. +typedef struct HuffmanTablesSegment { + HuffmanCode* start; + // Pointer to where we are writing into the segment. Starts at 'start' and + // cannot go beyond 'start' + 'size'. + HuffmanCode* curr_table; + // Pointer to the next segment in the chain. + struct HuffmanTablesSegment* next; + int size; +} HuffmanTablesSegment; + +// Chained memory segments of HuffmanCodes. +typedef struct HuffmanTables { + HuffmanTablesSegment root; + // Currently processed segment. At first, this is 'root'. + HuffmanTablesSegment* curr_segment; +} HuffmanTables; + +// Allocates a HuffmanTables with 'size' contiguous HuffmanCodes. Returns 0 on +// memory allocation error, 1 otherwise. +int VP8LHuffmanTablesAllocate(int size, HuffmanTables* huffman_tables); +void VP8LHuffmanTablesDeallocate(HuffmanTables* const huffman_tables); + #define HUFFMAN_PACKED_BITS 6 #define HUFFMAN_PACKED_TABLE_SIZE (1u << HUFFMAN_PACKED_BITS) @@ -78,9 +101,7 @@ void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups); // the huffman table. // Returns built table size or 0 in case of error (invalid tree or // memory error). -// If root_table is NULL, it returns 0 if a lookup cannot be built, something -// > 0 otherwise (but not the table size). -int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, +int VP8LBuildHuffmanTable(HuffmanTables* const root_table, int root_bits, const int code_lengths[], int code_lengths_size); #ifdef __cplusplus diff --git a/3rdparty/libwebp/src/utils/palette.c b/3rdparty/libwebp/src/utils/palette.c new file mode 100644 index 0000000000..515da21019 --- /dev/null +++ b/3rdparty/libwebp/src/utils/palette.c @@ -0,0 +1,402 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for palette analysis. +// +// Author: Vincent Rabaud (vrabaud@google.com) + +#include "src/utils/palette.h" + +#include +#include + +#include "src/dsp/lossless_common.h" +#include "src/utils/color_cache_utils.h" +#include "src/utils/utils.h" +#include "src/webp/encode.h" +#include "src/webp/format_constants.h" + +// ----------------------------------------------------------------------------- + +// Palette reordering for smaller sum of deltas (and for smaller storage). + +static int PaletteCompareColorsForQsort(const void* p1, const void* p2) { + const uint32_t a = WebPMemToUint32((uint8_t*)p1); + const uint32_t b = WebPMemToUint32((uint8_t*)p2); + assert(a != b); + return (a < b) ? -1 : 1; +} + +static WEBP_INLINE uint32_t PaletteComponentDistance(uint32_t v) { + return (v <= 128) ? v : (256 - v); +} + +// Computes a value that is related to the entropy created by the +// palette entry diff. +// +// Note that the last & 0xff is a no-operation in the next statement, but +// removed by most compilers and is here only for regularity of the code. +static WEBP_INLINE uint32_t PaletteColorDistance(uint32_t col1, uint32_t col2) { + const uint32_t diff = VP8LSubPixels(col1, col2); + const int kMoreWeightForRGBThanForAlpha = 9; + uint32_t score; + score = PaletteComponentDistance((diff >> 0) & 0xff); + score += PaletteComponentDistance((diff >> 8) & 0xff); + score += PaletteComponentDistance((diff >> 16) & 0xff); + score *= kMoreWeightForRGBThanForAlpha; + score += PaletteComponentDistance((diff >> 24) & 0xff); + return score; +} + +static WEBP_INLINE void SwapColor(uint32_t* const col1, uint32_t* const col2) { + const uint32_t tmp = *col1; + *col1 = *col2; + *col2 = tmp; +} + +int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, int num_colors) { + int low = 0, hi = num_colors; + if (sorted[low] == color) return low; // loop invariant: sorted[low] != color + while (1) { + const int mid = (low + hi) >> 1; + if (sorted[mid] == color) { + return mid; + } else if (sorted[mid] < color) { + low = mid; + } else { + hi = mid; + } + } + assert(0); + return 0; +} + +void PrepareMapToPalette(const uint32_t palette[], uint32_t num_colors, + uint32_t sorted[], uint32_t idx_map[]) { + uint32_t i; + memcpy(sorted, palette, num_colors * sizeof(*sorted)); + qsort(sorted, num_colors, sizeof(*sorted), PaletteCompareColorsForQsort); + for (i = 0; i < num_colors; ++i) { + idx_map[SearchColorNoIdx(sorted, palette[i], num_colors)] = i; + } +} + +//------------------------------------------------------------------------------ + +#define COLOR_HASH_SIZE (MAX_PALETTE_SIZE * 4) +#define COLOR_HASH_RIGHT_SHIFT 22 // 32 - log2(COLOR_HASH_SIZE). + +int GetColorPalette(const WebPPicture* const pic, uint32_t* const palette) { + int i; + int x, y; + int num_colors = 0; + uint8_t in_use[COLOR_HASH_SIZE] = {0}; + uint32_t colors[COLOR_HASH_SIZE] = {0}; + const uint32_t* argb = pic->argb; + const int width = pic->width; + const int height = pic->height; + uint32_t last_pix = ~argb[0]; // so we're sure that last_pix != argb[0] + assert(pic != NULL); + assert(pic->use_argb); + + for (y = 0; y < height; ++y) { + for (x = 0; x < width; ++x) { + int key; + if (argb[x] == last_pix) { + continue; + } + last_pix = argb[x]; + key = VP8LHashPix(last_pix, COLOR_HASH_RIGHT_SHIFT); + while (1) { + if (!in_use[key]) { + colors[key] = last_pix; + in_use[key] = 1; + ++num_colors; + if (num_colors > MAX_PALETTE_SIZE) { + return MAX_PALETTE_SIZE + 1; // Exact count not needed. + } + break; + } else if (colors[key] == last_pix) { + break; // The color is already there. + } else { + // Some other color sits here, so do linear conflict resolution. + ++key; + key &= (COLOR_HASH_SIZE - 1); // Key mask. + } + } + } + argb += pic->argb_stride; + } + + if (palette != NULL) { // Fill the colors into palette. + num_colors = 0; + for (i = 0; i < COLOR_HASH_SIZE; ++i) { + if (in_use[i]) { + palette[num_colors] = colors[i]; + ++num_colors; + } + } + qsort(palette, num_colors, sizeof(*palette), PaletteCompareColorsForQsort); + } + return num_colors; +} + +#undef COLOR_HASH_SIZE +#undef COLOR_HASH_RIGHT_SHIFT + +// ----------------------------------------------------------------------------- + +// The palette has been sorted by alpha. This function checks if the other +// components of the palette have a monotonic development with regards to +// position in the palette. If all have monotonic development, there is +// no benefit to re-organize them greedily. A monotonic development +// would be spotted in green-only situations (like lossy alpha) or gray-scale +// images. +static int PaletteHasNonMonotonousDeltas(const uint32_t* const palette, + int num_colors) { + uint32_t predict = 0x000000; + int i; + uint8_t sign_found = 0x00; + for (i = 0; i < num_colors; ++i) { + const uint32_t diff = VP8LSubPixels(palette[i], predict); + const uint8_t rd = (diff >> 16) & 0xff; + const uint8_t gd = (diff >> 8) & 0xff; + const uint8_t bd = (diff >> 0) & 0xff; + if (rd != 0x00) { + sign_found |= (rd < 0x80) ? 1 : 2; + } + if (gd != 0x00) { + sign_found |= (gd < 0x80) ? 8 : 16; + } + if (bd != 0x00) { + sign_found |= (bd < 0x80) ? 64 : 128; + } + predict = palette[i]; + } + return (sign_found & (sign_found << 1)) != 0; // two consequent signs. +} + +static void PaletteSortMinimizeDeltas(const uint32_t* const palette_sorted, + int num_colors, uint32_t* const palette) { + uint32_t predict = 0x00000000; + int i, k; + memcpy(palette, palette_sorted, num_colors * sizeof(*palette)); + if (!PaletteHasNonMonotonousDeltas(palette_sorted, num_colors)) return; + // Find greedily always the closest color of the predicted color to minimize + // deltas in the palette. This reduces storage needs since the + // palette is stored with delta encoding. + for (i = 0; i < num_colors; ++i) { + int best_ix = i; + uint32_t best_score = ~0U; + for (k = i; k < num_colors; ++k) { + const uint32_t cur_score = PaletteColorDistance(palette[k], predict); + if (best_score > cur_score) { + best_score = cur_score; + best_ix = k; + } + } + SwapColor(&palette[best_ix], &palette[i]); + predict = palette[i]; + } +} + +// ----------------------------------------------------------------------------- +// Modified Zeng method from "A Survey on Palette Reordering +// Methods for Improving the Compression of Color-Indexed Images" by Armando J. +// Pinho and Antonio J. R. Neves. + +// Finds the biggest cooccurrence in the matrix. +static void CoOccurrenceFindMax(const uint32_t* const cooccurrence, + uint32_t num_colors, uint8_t* const c1, + uint8_t* const c2) { + // Find the index that is most frequently located adjacent to other + // (different) indexes. + uint32_t best_sum = 0u; + uint32_t i, j, best_cooccurrence; + *c1 = 0u; + for (i = 0; i < num_colors; ++i) { + uint32_t sum = 0; + for (j = 0; j < num_colors; ++j) sum += cooccurrence[i * num_colors + j]; + if (sum > best_sum) { + best_sum = sum; + *c1 = i; + } + } + // Find the index that is most frequently found adjacent to *c1. + *c2 = 0u; + best_cooccurrence = 0u; + for (i = 0; i < num_colors; ++i) { + if (cooccurrence[*c1 * num_colors + i] > best_cooccurrence) { + best_cooccurrence = cooccurrence[*c1 * num_colors + i]; + *c2 = i; + } + } + assert(*c1 != *c2); +} + +// Builds the cooccurrence matrix +static int CoOccurrenceBuild(const WebPPicture* const pic, + const uint32_t* const palette, uint32_t num_colors, + uint32_t* cooccurrence) { + uint32_t *lines, *line_top, *line_current, *line_tmp; + int x, y; + const uint32_t* src = pic->argb; + uint32_t prev_pix = ~src[0]; + uint32_t prev_idx = 0u; + uint32_t idx_map[MAX_PALETTE_SIZE] = {0}; + uint32_t palette_sorted[MAX_PALETTE_SIZE]; + lines = (uint32_t*)WebPSafeMalloc(2 * pic->width, sizeof(*lines)); + if (lines == NULL) { + return 0; + } + line_top = &lines[0]; + line_current = &lines[pic->width]; + PrepareMapToPalette(palette, num_colors, palette_sorted, idx_map); + for (y = 0; y < pic->height; ++y) { + for (x = 0; x < pic->width; ++x) { + const uint32_t pix = src[x]; + if (pix != prev_pix) { + prev_idx = idx_map[SearchColorNoIdx(palette_sorted, pix, num_colors)]; + prev_pix = pix; + } + line_current[x] = prev_idx; + // 4-connectivity is what works best as mentioned in "On the relation + // between Memon's and the modified Zeng's palette reordering methods". + if (x > 0 && prev_idx != line_current[x - 1]) { + const uint32_t left_idx = line_current[x - 1]; + ++cooccurrence[prev_idx * num_colors + left_idx]; + ++cooccurrence[left_idx * num_colors + prev_idx]; + } + if (y > 0 && prev_idx != line_top[x]) { + const uint32_t top_idx = line_top[x]; + ++cooccurrence[prev_idx * num_colors + top_idx]; + ++cooccurrence[top_idx * num_colors + prev_idx]; + } + } + line_tmp = line_top; + line_top = line_current; + line_current = line_tmp; + src += pic->argb_stride; + } + WebPSafeFree(lines); + return 1; +} + +struct Sum { + uint8_t index; + uint32_t sum; +}; + +static int PaletteSortModifiedZeng(const WebPPicture* const pic, + const uint32_t* const palette_in, + uint32_t num_colors, + uint32_t* const palette) { + uint32_t i, j, ind; + uint8_t remapping[MAX_PALETTE_SIZE]; + uint32_t* cooccurrence; + struct Sum sums[MAX_PALETTE_SIZE]; + uint32_t first, last; + uint32_t num_sums; + // TODO(vrabaud) check whether one color images should use palette or not. + if (num_colors <= 1) return 1; + // Build the co-occurrence matrix. + cooccurrence = + (uint32_t*)WebPSafeCalloc(num_colors * num_colors, sizeof(*cooccurrence)); + if (cooccurrence == NULL) { + return 0; + } + if (!CoOccurrenceBuild(pic, palette_in, num_colors, cooccurrence)) { + WebPSafeFree(cooccurrence); + return 0; + } + + // Initialize the mapping list with the two best indices. + CoOccurrenceFindMax(cooccurrence, num_colors, &remapping[0], &remapping[1]); + + // We need to append and prepend to the list of remapping. To this end, we + // actually define the next start/end of the list as indices in a vector (with + // a wrap around when the end is reached). + first = 0; + last = 1; + num_sums = num_colors - 2; // -2 because we know the first two values + if (num_sums > 0) { + // Initialize the sums with the first two remappings and find the best one + struct Sum* best_sum = &sums[0]; + best_sum->index = 0u; + best_sum->sum = 0u; + for (i = 0, j = 0; i < num_colors; ++i) { + if (i == remapping[0] || i == remapping[1]) continue; + sums[j].index = i; + sums[j].sum = cooccurrence[i * num_colors + remapping[0]] + + cooccurrence[i * num_colors + remapping[1]]; + if (sums[j].sum > best_sum->sum) best_sum = &sums[j]; + ++j; + } + + while (num_sums > 0) { + const uint8_t best_index = best_sum->index; + // Compute delta to know if we need to prepend or append the best index. + int32_t delta = 0; + const int32_t n = num_colors - num_sums; + for (ind = first, j = 0; (ind + j) % num_colors != last + 1; ++j) { + const uint16_t l_j = remapping[(ind + j) % num_colors]; + delta += (n - 1 - 2 * (int32_t)j) * + (int32_t)cooccurrence[best_index * num_colors + l_j]; + } + if (delta > 0) { + first = (first == 0) ? num_colors - 1 : first - 1; + remapping[first] = best_index; + } else { + ++last; + remapping[last] = best_index; + } + // Remove best_sum from sums. + *best_sum = sums[num_sums - 1]; + --num_sums; + // Update all the sums and find the best one. + best_sum = &sums[0]; + for (i = 0; i < num_sums; ++i) { + sums[i].sum += cooccurrence[best_index * num_colors + sums[i].index]; + if (sums[i].sum > best_sum->sum) best_sum = &sums[i]; + } + } + } + assert((last + 1) % num_colors == first); + WebPSafeFree(cooccurrence); + + // Re-map the palette. + for (i = 0; i < num_colors; ++i) { + palette[i] = palette_in[remapping[(first + i) % num_colors]]; + } + return 1; +} + +// ----------------------------------------------------------------------------- + +int PaletteSort(PaletteSorting method, const struct WebPPicture* const pic, + const uint32_t* const palette_sorted, uint32_t num_colors, + uint32_t* const palette) { + switch (method) { + case kSortedDefault: + // Nothing to do, we have already sorted the palette. + memcpy(palette, palette_sorted, num_colors * sizeof(*palette)); + return 1; + case kMinimizeDelta: + PaletteSortMinimizeDeltas(palette_sorted, num_colors, palette); + return 1; + case kModifiedZeng: + return PaletteSortModifiedZeng(pic, palette_sorted, num_colors, palette); + case kUnusedPalette: + case kPaletteSortingNum: + break; + } + + assert(0); + return 0; +} diff --git a/3rdparty/libwebp/src/utils/palette.h b/3rdparty/libwebp/src/utils/palette.h new file mode 100644 index 0000000000..34479e463f --- /dev/null +++ b/3rdparty/libwebp/src/utils/palette.h @@ -0,0 +1,60 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for palette analysis. +// +// Author: Vincent Rabaud (vrabaud@google.com) + +#ifndef WEBP_UTILS_PALETTE_H_ +#define WEBP_UTILS_PALETTE_H_ + +#include "src/webp/types.h" + +struct WebPPicture; + +// The different ways a palette can be sorted. +typedef enum PaletteSorting { + kSortedDefault = 0, + // Sorts by minimizing L1 deltas between consecutive colors, giving more + // weight to RGB colors. + kMinimizeDelta = 1, + // Implements the modified Zeng method from "A Survey on Palette Reordering + // Methods for Improving the Compression of Color-Indexed Images" by Armando + // J. Pinho and Antonio J. R. Neves. + kModifiedZeng = 2, + kUnusedPalette = 3, + kPaletteSortingNum = 4 +} PaletteSorting; + +// Returns the index of 'color' in the sorted palette 'sorted' of size +// 'num_colors'. +int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, int num_colors); + +// Sort palette in increasing order and prepare an inverse mapping array. +void PrepareMapToPalette(const uint32_t palette[], uint32_t num_colors, + uint32_t sorted[], uint32_t idx_map[]); + +// Returns count of unique colors in 'pic', assuming pic->use_argb is true. +// If the unique color count is more than MAX_PALETTE_SIZE, returns +// MAX_PALETTE_SIZE+1. +// If 'palette' is not NULL and the number of unique colors is less than or +// equal to MAX_PALETTE_SIZE, also outputs the actual unique colors into +// 'palette' in a sorted order. Note: 'palette' is assumed to be an array +// already allocated with at least MAX_PALETTE_SIZE elements. +int GetColorPalette(const struct WebPPicture* const pic, + uint32_t* const palette); + +// Sorts the palette according to the criterion defined by 'method'. +// 'palette_sorted' is the input palette sorted lexicographically, as done in +// PrepareMapToPalette. Returns 0 on memory allocation error. +int PaletteSort(PaletteSorting method, const struct WebPPicture* const pic, + const uint32_t* const palette_sorted, uint32_t num_colors, + uint32_t* const palette); + +#endif // WEBP_UTILS_PALETTE_H_ diff --git a/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c b/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c index f65b6cdbb6..97e7893704 100644 --- a/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c +++ b/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c @@ -30,7 +30,7 @@ #define DFIX 4 // extra precision for ordered dithering #define DSIZE 4 // dithering size (must be a power of two) -// cf. http://en.wikipedia.org/wiki/Ordered_dithering +// cf. https://en.wikipedia.org/wiki/Ordered_dithering static const uint8_t kOrderedDither[DSIZE][DSIZE] = { { 0, 8, 2, 10 }, // coefficients are in DFIX fixed-point precision { 12, 4, 14, 6 }, diff --git a/3rdparty/libwebp/src/utils/rescaler_utils.c b/3rdparty/libwebp/src/utils/rescaler_utils.c index 4bcae24af5..a0581a14b1 100644 --- a/3rdparty/libwebp/src/utils/rescaler_utils.c +++ b/3rdparty/libwebp/src/utils/rescaler_utils.c @@ -12,66 +12,74 @@ // Author: Skal (pascal.massimino@gmail.com) #include +#include #include #include #include "src/dsp/dsp.h" #include "src/utils/rescaler_utils.h" +#include "src/utils/utils.h" //------------------------------------------------------------------------------ -void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height, - uint8_t* const dst, - int dst_width, int dst_height, int dst_stride, - int num_channels, rescaler_t* const work) { +int WebPRescalerInit(WebPRescaler* const rescaler, + int src_width, int src_height, + uint8_t* const dst, + int dst_width, int dst_height, int dst_stride, + int num_channels, rescaler_t* const work) { const int x_add = src_width, x_sub = dst_width; const int y_add = src_height, y_sub = dst_height; - wrk->x_expand = (src_width < dst_width); - wrk->y_expand = (src_height < dst_height); - wrk->src_width = src_width; - wrk->src_height = src_height; - wrk->dst_width = dst_width; - wrk->dst_height = dst_height; - wrk->src_y = 0; - wrk->dst_y = 0; - wrk->dst = dst; - wrk->dst_stride = dst_stride; - wrk->num_channels = num_channels; + const uint64_t total_size = 2ull * dst_width * num_channels * sizeof(*work); + if (!CheckSizeOverflow(total_size)) return 0; + + rescaler->x_expand = (src_width < dst_width); + rescaler->y_expand = (src_height < dst_height); + rescaler->src_width = src_width; + rescaler->src_height = src_height; + rescaler->dst_width = dst_width; + rescaler->dst_height = dst_height; + rescaler->src_y = 0; + rescaler->dst_y = 0; + rescaler->dst = dst; + rescaler->dst_stride = dst_stride; + rescaler->num_channels = num_channels; // for 'x_expand', we use bilinear interpolation - wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add; - wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub; - if (!wrk->x_expand) { // fx_scale is not used otherwise - wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub); + rescaler->x_add = rescaler->x_expand ? (x_sub - 1) : x_add; + rescaler->x_sub = rescaler->x_expand ? (x_add - 1) : x_sub; + if (!rescaler->x_expand) { // fx_scale is not used otherwise + rescaler->fx_scale = WEBP_RESCALER_FRAC(1, rescaler->x_sub); } // vertical scaling parameters - wrk->y_add = wrk->y_expand ? y_add - 1 : y_add; - wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub; - wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add; - if (!wrk->y_expand) { + rescaler->y_add = rescaler->y_expand ? y_add - 1 : y_add; + rescaler->y_sub = rescaler->y_expand ? y_sub - 1 : y_sub; + rescaler->y_accum = rescaler->y_expand ? rescaler->y_sub : rescaler->y_add; + if (!rescaler->y_expand) { // This is WEBP_RESCALER_FRAC(dst_height, x_add * y_add) without the cast. - // Its value is <= WEBP_RESCALER_ONE, because dst_height <= wrk->y_add, and - // wrk->x_add >= 1; - const uint64_t ratio = - (uint64_t)dst_height * WEBP_RESCALER_ONE / (wrk->x_add * wrk->y_add); + // Its value is <= WEBP_RESCALER_ONE, because dst_height <= rescaler->y_add + // and rescaler->x_add >= 1; + const uint64_t num = (uint64_t)dst_height * WEBP_RESCALER_ONE; + const uint64_t den = (uint64_t)rescaler->x_add * rescaler->y_add; + const uint64_t ratio = num / den; if (ratio != (uint32_t)ratio) { // When ratio == WEBP_RESCALER_ONE, we can't represent the ratio with the // current fixed-point precision. This happens when src_height == - // wrk->y_add (which == src_height), and wrk->x_add == 1. + // rescaler->y_add (which == src_height), and rescaler->x_add == 1. // => We special-case fxy_scale = 0, in WebPRescalerExportRow(). - wrk->fxy_scale = 0; + rescaler->fxy_scale = 0; } else { - wrk->fxy_scale = (uint32_t)ratio; + rescaler->fxy_scale = (uint32_t)ratio; } - wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub); + rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->y_sub); } else { - wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add); - // wrk->fxy_scale is unused here. + rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->x_add); + // rescaler->fxy_scale is unused here. } - wrk->irow = work; - wrk->frow = work + num_channels * dst_width; - memset(work, 0, 2 * dst_width * num_channels * sizeof(*work)); + rescaler->irow = work; + rescaler->frow = work + num_channels * dst_width; + memset(work, 0, (size_t)total_size); WebPRescalerDspInit(); + return 1; } int WebPRescalerGetScaledDimensions(int src_width, int src_height, @@ -82,6 +90,7 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, { int width = *scaled_width; int height = *scaled_height; + const int max_size = INT_MAX / 2; // if width is unspecified, scale original proportionally to height ratio. if (width == 0 && src_height > 0) { @@ -94,7 +103,7 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, (int)(((uint64_t)src_height * width + src_width - 1) / src_width); } // Check if the overall dimensions still make sense. - if (width <= 0 || height <= 0) { + if (width <= 0 || height <= 0 || width > max_size || height > max_size) { return 0; } @@ -107,31 +116,34 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, //------------------------------------------------------------------------------ // all-in-one calls -int WebPRescaleNeededLines(const WebPRescaler* const wrk, int max_num_lines) { - const int num_lines = (wrk->y_accum + wrk->y_sub - 1) / wrk->y_sub; +int WebPRescaleNeededLines(const WebPRescaler* const rescaler, + int max_num_lines) { + const int num_lines = + (rescaler->y_accum + rescaler->y_sub - 1) / rescaler->y_sub; return (num_lines > max_num_lines) ? max_num_lines : num_lines; } -int WebPRescalerImport(WebPRescaler* const wrk, int num_lines, +int WebPRescalerImport(WebPRescaler* const rescaler, int num_lines, const uint8_t* src, int src_stride) { int total_imported = 0; - while (total_imported < num_lines && !WebPRescalerHasPendingOutput(wrk)) { - if (wrk->y_expand) { - rescaler_t* const tmp = wrk->irow; - wrk->irow = wrk->frow; - wrk->frow = tmp; + while (total_imported < num_lines && + !WebPRescalerHasPendingOutput(rescaler)) { + if (rescaler->y_expand) { + rescaler_t* const tmp = rescaler->irow; + rescaler->irow = rescaler->frow; + rescaler->frow = tmp; } - WebPRescalerImportRow(wrk, src); - if (!wrk->y_expand) { // Accumulate the contribution of the new row. + WebPRescalerImportRow(rescaler, src); + if (!rescaler->y_expand) { // Accumulate the contribution of the new row. int x; - for (x = 0; x < wrk->num_channels * wrk->dst_width; ++x) { - wrk->irow[x] += wrk->frow[x]; + for (x = 0; x < rescaler->num_channels * rescaler->dst_width; ++x) { + rescaler->irow[x] += rescaler->frow[x]; } } - ++wrk->src_y; + ++rescaler->src_y; src += src_stride; ++total_imported; - wrk->y_accum -= wrk->y_sub; + rescaler->y_accum -= rescaler->y_sub; } return total_imported; } diff --git a/3rdparty/libwebp/src/utils/rescaler_utils.h b/3rdparty/libwebp/src/utils/rescaler_utils.h index ca41e42c4a..ef201ef86c 100644 --- a/3rdparty/libwebp/src/utils/rescaler_utils.h +++ b/3rdparty/libwebp/src/utils/rescaler_utils.h @@ -47,12 +47,13 @@ struct WebPRescaler { }; // Initialize a rescaler given scratch area 'work' and dimensions of src & dst. -void WebPRescalerInit(WebPRescaler* const rescaler, - int src_width, int src_height, - uint8_t* const dst, - int dst_width, int dst_height, int dst_stride, - int num_channels, - rescaler_t* const work); +// Returns false in case of error. +int WebPRescalerInit(WebPRescaler* const rescaler, + int src_width, int src_height, + uint8_t* const dst, + int dst_width, int dst_height, int dst_stride, + int num_channels, + rescaler_t* const work); // If either 'scaled_width' or 'scaled_height' (but not both) is 0 the value // will be calculated preserving the aspect ratio, otherwise the values are diff --git a/3rdparty/libwebp/src/utils/utils.c b/3rdparty/libwebp/src/utils/utils.c index 6080e19e21..408ce88f67 100644 --- a/3rdparty/libwebp/src/utils/utils.c +++ b/3rdparty/libwebp/src/utils/utils.c @@ -11,19 +11,19 @@ // // Author: Skal (pascal.massimino@gmail.com) +#include "src/utils/utils.h" + #include #include // for memcpy() -#include "src/webp/decode.h" + +#include "src/utils/palette.h" #include "src/webp/encode.h" -#include "src/webp/format_constants.h" // for MAX_PALETTE_SIZE -#include "src/utils/color_cache_utils.h" -#include "src/utils/utils.h" // If PRINT_MEM_INFO is defined, extra info (like total memory used, number of // alloc/free etc) is printed. For debugging/tuning purpose only (it's slow, // and not multi-thread safe!). // An interesting alternative is valgrind's 'massif' tool: -// http://valgrind.org/docs/manual/ms-manual.html +// https://valgrind.org/docs/manual/ms-manual.html // Here is an example command line: /* valgrind --tool=massif --massif-out-file=massif.out \ --stacks=yes --alloc-fn=WebPSafeMalloc --alloc-fn=WebPSafeCalloc @@ -101,6 +101,9 @@ static void Increment(int* const v) { #if defined(MALLOC_LIMIT) { const char* const malloc_limit_str = getenv("MALLOC_LIMIT"); +#if MALLOC_LIMIT > 1 + mem_limit = (size_t)MALLOC_LIMIT; +#endif if (malloc_limit_str != NULL) { mem_limit = atoi(malloc_limit_str); } @@ -169,16 +172,16 @@ static int CheckSizeArgumentsOverflow(uint64_t nmemb, size_t size) { const uint64_t total_size = nmemb * size; if (nmemb == 0) return 1; if ((uint64_t)size > WEBP_MAX_ALLOCABLE_MEMORY / nmemb) return 0; - if (total_size != (size_t)total_size) return 0; + if (!CheckSizeOverflow(total_size)) return 0; #if defined(PRINT_MEM_INFO) && defined(MALLOC_FAIL_AT) if (countdown_to_fail > 0 && --countdown_to_fail == 0) { return 0; // fake fail! } #endif -#if defined(MALLOC_LIMIT) +#if defined(PRINT_MEM_INFO) && defined(MALLOC_LIMIT) if (mem_limit > 0) { const uint64_t new_total_mem = (uint64_t)total_mem + total_size; - if (new_total_mem != (size_t)new_total_mem || + if (!CheckSizeOverflow(new_total_mem) || new_total_mem > mem_limit) { return 0; // fake fail! } @@ -249,66 +252,10 @@ void WebPCopyPixels(const WebPPicture* const src, WebPPicture* const dst) { //------------------------------------------------------------------------------ -#define COLOR_HASH_SIZE (MAX_PALETTE_SIZE * 4) -#define COLOR_HASH_RIGHT_SHIFT 22 // 32 - log2(COLOR_HASH_SIZE). - int WebPGetColorPalette(const WebPPicture* const pic, uint32_t* const palette) { - int i; - int x, y; - int num_colors = 0; - uint8_t in_use[COLOR_HASH_SIZE] = { 0 }; - uint32_t colors[COLOR_HASH_SIZE]; - const uint32_t* argb = pic->argb; - const int width = pic->width; - const int height = pic->height; - uint32_t last_pix = ~argb[0]; // so we're sure that last_pix != argb[0] - assert(pic != NULL); - assert(pic->use_argb); - - for (y = 0; y < height; ++y) { - for (x = 0; x < width; ++x) { - int key; - if (argb[x] == last_pix) { - continue; - } - last_pix = argb[x]; - key = VP8LHashPix(last_pix, COLOR_HASH_RIGHT_SHIFT); - while (1) { - if (!in_use[key]) { - colors[key] = last_pix; - in_use[key] = 1; - ++num_colors; - if (num_colors > MAX_PALETTE_SIZE) { - return MAX_PALETTE_SIZE + 1; // Exact count not needed. - } - break; - } else if (colors[key] == last_pix) { - break; // The color is already there. - } else { - // Some other color sits here, so do linear conflict resolution. - ++key; - key &= (COLOR_HASH_SIZE - 1); // Key mask. - } - } - } - argb += pic->argb_stride; - } - - if (palette != NULL) { // Fill the colors into palette. - num_colors = 0; - for (i = 0; i < COLOR_HASH_SIZE; ++i) { - if (in_use[i]) { - palette[num_colors] = colors[i]; - ++num_colors; - } - } - } - return num_colors; + return GetColorPalette(pic, palette); } -#undef COLOR_HASH_SIZE -#undef COLOR_HASH_RIGHT_SHIFT - //------------------------------------------------------------------------------ #if defined(WEBP_NEED_LOG_TABLE_8BIT) diff --git a/3rdparty/libwebp/src/utils/utils.h b/3rdparty/libwebp/src/utils/utils.h index 2a3ec92678..b2241fbf9b 100644 --- a/3rdparty/libwebp/src/utils/utils.h +++ b/3rdparty/libwebp/src/utils/utils.h @@ -20,9 +20,7 @@ #endif #include -#include -#include "src/dsp/dsp.h" #include "src/webp/types.h" #ifdef __cplusplus @@ -42,6 +40,10 @@ extern "C" { #endif #endif // WEBP_MAX_ALLOCABLE_MEMORY +static WEBP_INLINE int CheckSizeOverflow(uint64_t size) { + return size == (size_t)size; +} + // size-checking safe malloc/calloc: verify that the requested size is not too // large, or return NULL. You don't need to call these for constructs like // malloc(sizeof(foo)), but only if there's picture-dependent size involved @@ -60,7 +62,8 @@ WEBP_EXTERN void WebPSafeFree(void* const ptr); // Alignment #define WEBP_ALIGN_CST 31 -#define WEBP_ALIGN(PTR) (((uintptr_t)(PTR) + WEBP_ALIGN_CST) & ~WEBP_ALIGN_CST) +#define WEBP_ALIGN(PTR) (((uintptr_t)(PTR) + WEBP_ALIGN_CST) & \ + ~(uintptr_t)WEBP_ALIGN_CST) #include // memcpy() is the safe way of moving potentially unaligned 32b memory. @@ -69,10 +72,19 @@ static WEBP_INLINE uint32_t WebPMemToUint32(const uint8_t* const ptr) { memcpy(&A, ptr, sizeof(A)); return A; } + +static WEBP_INLINE int32_t WebPMemToInt32(const uint8_t* const ptr) { + return (int32_t)WebPMemToUint32(ptr); +} + static WEBP_INLINE void WebPUint32ToMem(uint8_t* const ptr, uint32_t val) { memcpy(ptr, &val, sizeof(val)); } +static WEBP_INLINE void WebPInt32ToMem(uint8_t* const ptr, int val) { + WebPUint32ToMem(ptr, (uint32_t)val); +} + //------------------------------------------------------------------------------ // Reading/writing data. @@ -107,24 +119,33 @@ static WEBP_INLINE void PutLE32(uint8_t* const data, uint32_t val) { PutLE16(data + 2, (int)(val >> 16)); } -// Returns (int)floor(log2(n)). n must be > 0. // use GNU builtins where available. #if defined(__GNUC__) && \ ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4) +// Returns (int)floor(log2(n)). n must be > 0. static WEBP_INLINE int BitsLog2Floor(uint32_t n) { return 31 ^ __builtin_clz(n); } +// counts the number of trailing zero +static WEBP_INLINE int BitsCtz(uint32_t n) { return __builtin_ctz(n); } #elif defined(_MSC_VER) && _MSC_VER > 1310 && \ (defined(_M_X64) || defined(_M_IX86)) #include #pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) static WEBP_INLINE int BitsLog2Floor(uint32_t n) { - unsigned long first_set_bit; + unsigned long first_set_bit; // NOLINT (runtime/int) _BitScanReverse(&first_set_bit, n); return first_set_bit; } -#else // default: use the C-version. +static WEBP_INLINE int BitsCtz(uint32_t n) { + unsigned long first_set_bit; // NOLINT (runtime/int) + _BitScanForward(&first_set_bit, n); + return first_set_bit; +} +#else // default: use the (slow) C-version. +#define WEBP_HAVE_SLOW_CLZ_CTZ // signal that the Clz/Ctz function are slow // Returns 31 ^ clz(n) = log2(n). This is the default C-implementation, either // based on table or not. Can be used as fallback if clz() is not available. #define WEBP_NEED_LOG_TABLE_8BIT @@ -139,6 +160,15 @@ static WEBP_INLINE int WebPLog2FloorC(uint32_t n) { } static WEBP_INLINE int BitsLog2Floor(uint32_t n) { return WebPLog2FloorC(n); } + +static WEBP_INLINE int BitsCtz(uint32_t n) { + int i; + for (i = 0; i < 32; ++i, n >>= 1) { + if (n & 1) return i; + } + return 32; +} + #endif //------------------------------------------------------------------------------ @@ -166,6 +196,7 @@ WEBP_EXTERN void WebPCopyPixels(const struct WebPPicture* const src, // MAX_PALETTE_SIZE, also outputs the actual unique colors into 'palette'. // Note: 'palette' is assumed to be an array already allocated with at least // MAX_PALETTE_SIZE elements. +// TODO(vrabaud) remove whenever we can break the ABI. WEBP_EXTERN int WebPGetColorPalette(const struct WebPPicture* const pic, uint32_t* const palette); diff --git a/3rdparty/libwebp/src/webp/decode.h b/3rdparty/libwebp/src/webp/decode.h index 44fcd64a84..9d968061d1 100644 --- a/3rdparty/libwebp/src/webp/decode.h +++ b/3rdparty/libwebp/src/webp/decode.h @@ -81,11 +81,12 @@ WEBP_EXTERN uint8_t* WebPDecodeBGR(const uint8_t* data, size_t data_size, // returned is the Y samples buffer. Upon return, *u and *v will point to // the U and V chroma data. These U and V buffers need NOT be passed to // WebPFree(), unlike the returned Y luma one. The dimension of the U and V -// planes are both (*width + 1) / 2 and (*height + 1)/ 2. +// planes are both (*width + 1) / 2 and (*height + 1) / 2. // Upon return, the Y buffer has a stride returned as '*stride', while U and V // have a common stride returned as '*uv_stride'. -// Return NULL in case of error. -// (*) Also named Y'CbCr. See: http://en.wikipedia.org/wiki/YCbCr +// 'width' and 'height' may be NULL, the other pointers must not be. +// Returns NULL in case of error. +// (*) Also named Y'CbCr. See: https://en.wikipedia.org/wiki/YCbCr WEBP_EXTERN uint8_t* WebPDecodeYUV(const uint8_t* data, size_t data_size, int* width, int* height, uint8_t** u, uint8_t** v, @@ -250,23 +251,24 @@ typedef enum VP8StatusCode { // WebPIDecoder object. This object can be left in a SUSPENDED state if the // picture is only partially decoded, pending additional input. // Code example: -// -// WebPInitDecBuffer(&output_buffer); -// output_buffer.colorspace = mode; -// ... -// WebPIDecoder* idec = WebPINewDecoder(&output_buffer); -// while (additional_data_is_available) { -// // ... (get additional data in some new_data[] buffer) -// status = WebPIAppend(idec, new_data, new_data_size); -// if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { -// break; // an error occurred. -// } -// -// // The above call decodes the current available buffer. -// // Part of the image can now be refreshed by calling -// // WebPIDecGetRGB()/WebPIDecGetYUVA() etc. -// } -// WebPIDelete(idec); +/* + WebPInitDecBuffer(&output_buffer); + output_buffer.colorspace = mode; + ... + WebPIDecoder* idec = WebPINewDecoder(&output_buffer); + while (additional_data_is_available) { + // ... (get additional data in some new_data[] buffer) + status = WebPIAppend(idec, new_data, new_data_size); + if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { + break; // an error occurred. + } + + // The above call decodes the current available buffer. + // Part of the image can now be refreshed by calling + // WebPIDecGetRGB()/WebPIDecGetYUVA() etc. + } + WebPIDelete(idec); +*/ // Creates a new incremental decoder with the supplied buffer parameter. // This output_buffer can be passed NULL, in which case a default output buffer @@ -388,7 +390,7 @@ WEBP_EXTERN const WebPDecBuffer* WebPIDecodedArea( CHECK(WebPGetFeatures(data, data_size, &config.input) == VP8_STATUS_OK); // C) Adjust 'config', if needed - config.no_fancy_upsampling = 1; + config.options.no_fancy_upsampling = 1; config.output.colorspace = MODE_BGRA; // etc. diff --git a/3rdparty/libwebp/src/webp/encode.h b/3rdparty/libwebp/src/webp/encode.h index b4c599df87..56b68e2f10 100644 --- a/3rdparty/libwebp/src/webp/encode.h +++ b/3rdparty/libwebp/src/webp/encode.h @@ -441,7 +441,7 @@ WEBP_EXTERN int WebPPictureCrop(WebPPicture* picture, // the original dimension will be lost). Picture 'dst' need not be initialized // with WebPPictureInit() if it is different from 'src', since its content will // be overwritten. -// Returns false in case of memory allocation error or invalid parameters. +// Returns false in case of invalid parameters. WEBP_EXTERN int WebPPictureView(const WebPPicture* src, int left, int top, int width, int height, WebPPicture* dst); @@ -455,7 +455,7 @@ WEBP_EXTERN int WebPPictureIsView(const WebPPicture* picture); // dimension will be calculated preserving the aspect ratio. // No gamma correction is applied. // Returns false in case of error (invalid parameter or insufficient memory). -WEBP_EXTERN int WebPPictureRescale(WebPPicture* pic, int width, int height); +WEBP_EXTERN int WebPPictureRescale(WebPPicture* picture, int width, int height); // Colorspace conversion function to import RGB samples. // Previous buffer will be free'd, if any. @@ -526,7 +526,7 @@ WEBP_EXTERN int WebPPictureHasTransparency(const WebPPicture* picture); // Remove the transparency information (if present) by blending the color with // the background color 'background_rgb' (specified as 24bit RGB triplet). // After this call, all alpha values are reset to 0xff. -WEBP_EXTERN void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb); +WEBP_EXTERN void WebPBlendAlpha(WebPPicture* picture, uint32_t background_rgb); //------------------------------------------------------------------------------ // Main call diff --git a/3rdparty/libwebp/src/webp/format_constants.h b/3rdparty/libwebp/src/webp/format_constants.h index eca6981a47..999035c5d2 100644 --- a/3rdparty/libwebp/src/webp/format_constants.h +++ b/3rdparty/libwebp/src/webp/format_constants.h @@ -55,7 +55,7 @@ typedef enum { PREDICTOR_TRANSFORM = 0, CROSS_COLOR_TRANSFORM = 1, - SUBTRACT_GREEN = 2, + SUBTRACT_GREEN_TRANSFORM = 2, COLOR_INDEXING_TRANSFORM = 3 } VP8LImageTransformType; diff --git a/3rdparty/libwebp/src/webp/types.h b/3rdparty/libwebp/src/webp/types.h index 47f7f2b007..f255432e41 100644 --- a/3rdparty/libwebp/src/webp/types.h +++ b/3rdparty/libwebp/src/webp/types.h @@ -42,7 +42,11 @@ typedef long long int int64_t; # if defined(__GNUC__) && __GNUC__ >= 4 # define WEBP_EXTERN extern __attribute__ ((visibility ("default"))) # else -# define WEBP_EXTERN extern +# if defined(_MSC_VER) && defined(WEBP_DLL) +# define WEBP_EXTERN __declspec(dllexport) +# else +# define WEBP_EXTERN extern +# endif # endif /* __GNUC__ >= 4 */ #endif /* WEBP_EXTERN */ From 6bc369fc56afc3d52acc0072ae43e6021ff75838 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Mon, 18 Sep 2023 10:23:24 +0300 Subject: [PATCH 13/34] Merge pull request #24250 from dkurt:ts_fixture_constructor_skip_2 Skip test on SkipTestException at fixture's constructor (version 2) #24250 ### Pull Request Readiness Checklist Another version of https://github.com/opencv/opencv/pull/24186 (reverted by https://github.com/opencv/opencv/pull/24223). Current implementation cannot handle skip exception at `static void SetUpTestCase` but works on `virtual void SetUp`. See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/test/test_misc.cpp | 12 ++++++++ modules/python/test/tests_common.py | 2 ++ modules/ts/include/opencv2/ts/ts_ext.hpp | 35 +++++++++++++++++++++--- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 8ed0afe771..53d7585277 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -917,5 +917,17 @@ REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows); typedef ::testing::Types RectTypes; INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes); +// Expected that SkipTestException thrown in the constructor should skip test but not fail +struct TestFixtureSkip: public ::testing::Test { + TestFixtureSkip(bool throwEx = true) { + if (throwEx) { + throw SkipTestException("Skip test at constructor"); + } + } +}; + +TEST_F(TestFixtureSkip, NoBodyRun) { + FAIL() << "Unreachable code called"; +} }} // namespace diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index ec49f46d0d..d673dd7b78 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -36,6 +36,8 @@ class NewOpenCVTests(unittest.TestCase): return candidate if required: self.fail('File ' + filename + ' not found') + else: + self.skipTest('File ' + filename + ' not found') return None diff --git a/modules/ts/include/opencv2/ts/ts_ext.hpp b/modules/ts/include/opencv2/ts/ts_ext.hpp index efa4860510..8395f31a4c 100644 --- a/modules/ts/include/opencv2/ts/ts_ext.hpp +++ b/modules/ts/include/opencv2/ts/ts_ext.hpp @@ -47,6 +47,15 @@ bool checkBigDataTests(); } \ } \ +struct SkipThisTest : public ::testing::Test { + SkipThisTest(const std::string& msg_) : msg(msg_) {} + + virtual void TestBody() CV_OVERRIDE { + printf("[ SKIP ] %s\n", msg.c_str()); + } + + std::string msg; +}; #undef TEST #define TEST_(test_case_name, test_name, parent_class, bodyMethodName, BODY_ATTR, BODY_IMPL) \ @@ -60,6 +69,16 @@ bool checkBigDataTests(); GTEST_DISALLOW_COPY_AND_ASSIGN_(\ GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ };\ + class test_case_name##test_name##_factory : public ::testing::internal::TestFactoryBase { \ + public:\ + virtual ::testing::Test* CreateTest() { \ + try { \ + return new GTEST_TEST_CLASS_NAME_(test_case_name, test_name); \ + } catch (const cvtest::details::SkipTestExceptionBase& e) { \ + return new SkipThisTest(e.what()); \ + } \ + } \ + };\ \ ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ ::test_info_ =\ @@ -69,8 +88,7 @@ bool checkBigDataTests(); (::testing::internal::GetTestTypeId()), \ parent_class::SetUpTestCase, \ parent_class::TearDownTestCase, \ - new ::testing::internal::TestFactoryImpl<\ - GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ + new test_case_name##test_name##_factory);\ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() BODY_IMPL( #test_case_name "_" #test_name ) \ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::bodyMethodName() @@ -113,6 +131,16 @@ bool checkBigDataTests(); GTEST_DISALLOW_COPY_AND_ASSIGN_(\ GTEST_TEST_CLASS_NAME_(test_fixture, test_name));\ };\ + class test_fixture##test_name##_factory : public ::testing::internal::TestFactoryBase { \ + public:\ + virtual ::testing::Test* CreateTest() { \ + try { \ + return new GTEST_TEST_CLASS_NAME_(test_fixture, test_name); \ + } catch (const cvtest::details::SkipTestExceptionBase& e) { \ + return new SkipThisTest(e.what()); \ + } \ + } \ + };\ \ ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_fixture, test_name)\ ::test_info_ =\ @@ -122,8 +150,7 @@ bool checkBigDataTests(); (::testing::internal::GetTypeId()), \ test_fixture::SetUpTestCase, \ test_fixture::TearDownTestCase, \ - new ::testing::internal::TestFactoryImpl<\ - GTEST_TEST_CLASS_NAME_(test_fixture, test_name)>);\ + new test_fixture##test_name##_factory);\ void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::TestBody() CV__TEST_BODY_IMPL( #test_fixture "_" #test_name ) \ void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::Body() From 9b5a719d801d8679d0937e686377a87cd9578b79 Mon Sep 17 00:00:00 2001 From: Stefan Dragnev Date: Mon, 18 Sep 2023 15:38:31 +0200 Subject: [PATCH 14/34] build fixes for emscripten 3.1.45 --- modules/core/include/opencv2/core/hal/intrin_wasm.hpp | 1 + modules/js/src/core_bindings.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp index b4178af8b7..db3cb2a9ae 100644 --- a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "opencv2/core/saturate.hpp" #define CV_SIMD128 1 diff --git a/modules/js/src/core_bindings.cpp b/modules/js/src/core_bindings.cpp index 60fe496ce3..addee2de20 100644 --- a/modules/js/src/core_bindings.cpp +++ b/modules/js/src/core_bindings.cpp @@ -89,9 +89,11 @@ using namespace cv; using namespace cv::segmentation; // FIXIT +#ifdef HAVE_OPENCV_OBJDETECT using namespace cv::aruco; typedef aruco::DetectorParameters aruco_DetectorParameters; typedef QRCodeDetectorAruco::Params QRCodeDetectorAruco_Params; +#endif #ifdef HAVE_OPENCV_DNN using namespace cv::dnn; From e16ca08b331fa18dbe81792d5933107f9560499a Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Mon, 18 Sep 2023 22:43:44 +0800 Subject: [PATCH 15/34] Fix memory leak and handle realloc failure In the previous code, there was a memory leak issue where the previously allocated memory was not freed upon a failed realloc operation. This commit addresses the problem by releasing the old memory before setting the pointer to NULL in case of a realloc failure. This ensures that memory is properly managed and avoids potential memory leaks. --- modules/dnn/src/torch/THDiskFile.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/dnn/src/torch/THDiskFile.cpp b/modules/dnn/src/torch/THDiskFile.cpp index 84b6b23e81..bede95e021 100644 --- a/modules/dnn/src/torch/THDiskFile.cpp +++ b/modules/dnn/src/torch/THDiskFile.cpp @@ -375,15 +375,21 @@ static long THDiskFile_readString(THFile *self, const char *format, char **str_) long total = TBRS_BSZ; long pos = 0L; + if (p == NULL) + THError("read error: failed to allocate buffer"); for (;;) { if(total-pos == 0) /* we need more space! */ { total += TBRS_BSZ; - p = (char*)THRealloc(p, total); + char *new_p = (char*)THRealloc(p, total); + if (new_p == NULL) + { + THFree(p); + THError("read error: failed to reallocate buffer"); + } + p = new_p; } - if (p == NULL) - THError("read error: failed to allocate buffer"); pos += fread(p+pos, 1, total-pos, dfself->handle); if (pos < total) /* eof? */ { @@ -409,15 +415,21 @@ static long THDiskFile_readString(THFile *self, const char *format, char **str_) long pos = 0L; long size; + if (p == NULL) + THError("read error: failed to allocate buffer"); for (;;) { if(total-pos <= 1) /* we can only write '\0' in there! */ { total += TBRS_BSZ; - p = (char*)THRealloc(p, total); + char *new_p = (char*)THRealloc(p, total); + if (new_p == NULL) + { + THFree(p); + THError("read error: failed to reallocate buffer"); + } + p = new_p; } - if (p == NULL) - THError("read error: failed to allocate buffer"); if (fgets(p+pos, total-pos, dfself->handle) == NULL) /* eof? */ { if(pos == 0L) From f617fbe1669c6779096a6e363bf0a6ff0d881cf3 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 19 Sep 2023 13:52:42 +0800 Subject: [PATCH 16/34] Merge pull request #24132 from hanliutong:rewrite-imgproc2 Rewrite Universal Intrinsic code by using new API: ImgProc module Part 2 #24132 The goal of this series of PRs is to modify the SIMD code blocks guarded by CV_SIMD macro in the opencv/modules/imgproc folder: rewrite them by using the new Universal Intrinsic API. This is the second part of the modification to the Imgproc module ( Part 1: #24058 ), And I tested this patch on RVV (QEMU) and AVX devices, `opencv_test_imgproc` is passed. The patch is partially auto-generated by using the [rewriter](https://github.com/hanliutong/rewriter). ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../core/include/opencv2/core/hal/intrin.hpp | 67 +- .../core/hal/intrin_rvv_compat_overloaded.hpp | 1 + .../opencv2/core/hal/intrin_rvv_scalable.hpp | 31 +- modules/imgproc/src/bilateral_filter.simd.hpp | 444 ++++---- modules/imgproc/src/color_lab.cpp | 287 +++--- modules/imgproc/src/color_yuv.simd.hpp | 452 ++++---- modules/imgproc/src/filter.simd.hpp | 969 +++++++++--------- modules/imgproc/src/hough.cpp | 56 +- modules/imgproc/src/stackblur.cpp | 328 +++--- modules/imgproc/src/thresh.cpp | 378 +++---- 10 files changed, 1589 insertions(+), 1424 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin.hpp b/modules/core/include/opencv2/core/hal/intrin.hpp index ee8310b5c5..88a002145a 100644 --- a/modules/core/include/opencv2/core/hal/intrin.hpp +++ b/modules/core/include/opencv2/core/hal/intrin.hpp @@ -745,7 +745,22 @@ namespace CV__SIMD_NAMESPACE { inline _Tpvec v_add(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \ return v_add(f1 + f2, vf...); \ } + #define OPENCV_HAL_WRAP_SHIFT_OP(_Tpvec) \ + inline _Tpvec v_shr(const _Tpvec& a, int n) \ + { \ + return a >> n; \ + } \ + inline _Tpvec v_shl(const _Tpvec& a, int n) \ + { \ + return a << n; \ + } + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint8) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint16) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint32) @@ -769,6 +784,12 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32x4) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64x2) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64x2) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64x2) #if CV_SIMD_64F OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64x2) #endif @@ -784,6 +805,12 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32x8) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64x4) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16x16) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16x16) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64x4) #if CV_SIMD_64F OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64x4) #endif @@ -801,7 +828,9 @@ namespace CV__SIMD_NAMESPACE { inline _Tpvec v_xor(const _Tpvec& a, const _Tpvec& b) \ { \ return a ^ b; \ - } \ + } + + #define OPENCV_HAL_WRAP_NOT_OP(_Tpvec) \ inline _Tpvec v_not(const _Tpvec& a) \ { \ return ~a; \ @@ -815,6 +844,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32) + OPENCV_HAL_WRAP_NOT_OP(v_uint8) + OPENCV_HAL_WRAP_NOT_OP(v_uint16) + OPENCV_HAL_WRAP_NOT_OP(v_uint32) + OPENCV_HAL_WRAP_NOT_OP(v_uint64) + OPENCV_HAL_WRAP_NOT_OP(v_int8) + OPENCV_HAL_WRAP_NOT_OP(v_int16) + OPENCV_HAL_WRAP_NOT_OP(v_int32) + OPENCV_HAL_WRAP_NOT_OP(v_int64) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64) + #endif #if CV_SIMD_WIDTH != 16/*128*/ && CV_SIMD128 OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint8x16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint16x8) @@ -824,6 +865,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16x8) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32x4) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64x2) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32x4) + OPENCV_HAL_WRAP_NOT_OP(v_uint8x16) + OPENCV_HAL_WRAP_NOT_OP(v_uint16x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint32x4) + OPENCV_HAL_WRAP_NOT_OP(v_uint64x2) + OPENCV_HAL_WRAP_NOT_OP(v_int8x16) + OPENCV_HAL_WRAP_NOT_OP(v_int16x8) + OPENCV_HAL_WRAP_NOT_OP(v_int32x4) + OPENCV_HAL_WRAP_NOT_OP(v_int64x2) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64x2) + #endif #endif #if CV_SIMD_WIDTH != 32/*256*/ && CV_SIMD256 OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint8x32) @@ -834,6 +887,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16x16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32x8) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64x4) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint8x32) + OPENCV_HAL_WRAP_NOT_OP(v_uint16x16) + OPENCV_HAL_WRAP_NOT_OP(v_uint32x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint64x4) + OPENCV_HAL_WRAP_NOT_OP(v_int8x32) + OPENCV_HAL_WRAP_NOT_OP(v_int16x16) + OPENCV_HAL_WRAP_NOT_OP(v_int32x8) + OPENCV_HAL_WRAP_NOT_OP(v_int64x4) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64x4) + #endif #endif #define OPENCV_HAL_WRAP_BIN_OP_MUL(_Tpvec) \ diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp index 7dd735f99a..914ad28978 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp @@ -45,6 +45,7 @@ OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m2_t, u8m2, vuint8m2_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m4_t, u8m4, vuint8m4_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m8_t, u8m8, vuint8m8_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vfloat32m1_t, f32m1, vuint32m1_t, i32) +OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint32m1_t, u32m1, vuint32m1_t, i32) #if CV_SIMD_SCALABLE_64F OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vfloat64m1_t, f64m1, vuint32mf2_t, i32) #endif diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 6c28b44f5b..a45c90cf90 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -475,6 +475,25 @@ OPENCV_HAL_IMPL_RVV_LUT(v_float32, float, m1) OPENCV_HAL_IMPL_RVV_LUT(v_float64, double, mf2) #endif +#define OPENCV_HAL_IMPL_RVV_LUT_VEC(_Tpvec, _Tp) \ +inline _Tpvec v_lut(const _Tp* tab, const v_int32& vidx) \ +{ \ + v_uint32 vidx_ = vmul(vreinterpret_u32m1(vidx), sizeof(_Tp), VTraits::vlanes()); \ + return vloxei32(tab, vidx_, VTraits<_Tpvec>::vlanes()); \ +} +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_float32, float) +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_int32, int) +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_uint32, unsigned) + +#if CV_SIMD_SCALABLE_64F +inline v_float64 v_lut(const double* tab, const v_int32& vidx) \ +{ \ + vuint32mf2_t vidx_ = vmul(vlmul_trunc_u32mf2(vreinterpret_u32m1(vidx)), sizeof(double), VTraits::vlanes()); \ + return vloxei32(tab, vidx_, VTraits::vlanes()); \ +} +#endif + + inline v_uint8 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } inline v_uint8 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } inline v_uint8 v_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_quads((schar*)tab, idx)); } @@ -690,23 +709,27 @@ inline v_float64 v_not (const v_float64& a) \ ////////////// Bitwise shifts ////////////// +/* Usage +1. v_shl(vec); +2. v_shl(vec, N); // instead of vec << N, when N is non-constant. +*/ #define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, vl) \ -template inline _Tpvec v_shl(const _Tpvec& a) \ +template inline _Tpvec v_shl(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsll(a, uint8_t(n), vl)); \ } \ -template inline _Tpvec v_shr(const _Tpvec& a) \ +template inline _Tpvec v_shr(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsrl(a, uint8_t(n), vl)); \ } #define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, vl) \ -template inline _Tpvec v_shl(const _Tpvec& a) \ +template inline _Tpvec v_shl(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsll(a, uint8_t(n), vl)); \ } \ -template inline _Tpvec v_shr(const _Tpvec& a) \ +template inline _Tpvec v_shr(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsra(a, uint8_t(n), vl)); \ } diff --git a/modules/imgproc/src/bilateral_filter.simd.hpp b/modules/imgproc/src/bilateral_filter.simd.hpp index 0d2c394368..332b36646c 100644 --- a/modules/imgproc/src/bilateral_filter.simd.hpp +++ b/modules/imgproc/src/bilateral_filter.simd.hpp @@ -99,33 +99,33 @@ public: const uchar* ksptr2 = sptr + space_ofs[k+2]; const uchar* ksptr3 = sptr + space_ofs[k+3]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint32 rval = vx_load_expand_q(sptr + j); v_uint32 val = vx_load_expand_q(ksptr0 + j); - v_float32 w = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; + v_float32 w = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); v_float32 v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, vx_load_aligned(sum + j)); val = vx_load_expand_q(ksptr1 + j); - w = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); val = vx_load_expand_q(ksptr2 + j); - w = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); val = vx_load_expand_q(ksptr3 + j); - w = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); v_store_aligned(wsum + j, v_wsum); @@ -172,13 +172,13 @@ public: { const uchar* ksptr = sptr + space_ofs[k]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint32 val = vx_load_expand_q(ksptr + j); - v_float32 w = kweight * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, vx_load_expand_q(sptr + j)))); - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); + v_float32 w = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, vx_load_expand_q(sptr + j))))); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); v_store_aligned(sum + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, vx_load_aligned(sum + j))); } #endif @@ -191,10 +191,10 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes) - v_pack_u_store(dptr + j, v_pack(v_round(vx_load_aligned(sum + j ) / vx_load_aligned(wsum + j )), - v_round(vx_load_aligned(sum + j + v_float32::nlanes) / vx_load_aligned(wsum + j + v_float32::nlanes)))); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) + v_pack_u_store(dptr + j, v_pack(v_round(v_div(vx_load_aligned(sum + j), vx_load_aligned(wsum + j))), + v_round(v_div(vx_load_aligned(sum + j + VTraits::vlanes()), vx_load_aligned(wsum + j + VTraits::vlanes()))))); #endif for (; j < size.width; j++) { @@ -221,13 +221,13 @@ public: const uchar* ksptr3 = sptr + space_ofs[k+3]; const uchar* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, rsptr += 3*v_uint8::nlanes, - ksptr0 += 3*v_uint8::nlanes, ksptr1 += 3*v_uint8::nlanes, ksptr2 += 3*v_uint8::nlanes, ksptr3 += 3*v_uint8::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), rsptr += 3*VTraits::vlanes(), + ksptr0 += 3*VTraits::vlanes(), ksptr1 += 3*VTraits::vlanes(), ksptr2 += 3*VTraits::vlanes(), ksptr3 += 3*VTraits::vlanes()) { v_uint8 kb, kg, kr, rb, rg, rr; v_load_deinterleave(rsptr, rb, rg, rr); @@ -236,163 +236,163 @@ public: v_uint16 val0, val1, val2, val3, val4; v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_uint32 vall, valh; v_expand(val0, vall, valh); - v_float32 w0 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - v_float32 w1 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + v_float32 w0 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(vall))); + v_float32 w1 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2*v_float32::nlanes))); - v_store_aligned(sum_r + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3*v_float32::nlanes))); + v_store_aligned(sum_r + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3*VTraits::vlanes()))); v_load_deinterleave(ksptr1, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); v_load_deinterleave(ksptr2, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); v_load_deinterleave(ksptr3, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); } #endif #if CV_SIMD128 @@ -442,9 +442,9 @@ public: const uchar* ksptr = sptr + space_ofs[k]; const uchar* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, ksptr += 3*v_uint8::nlanes, rsptr += 3*v_uint8::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), ksptr += 3*VTraits::vlanes(), rsptr += 3*VTraits::vlanes()) { v_uint8 kb, kg, kr, rb, rg, rr; v_load_deinterleave(ksptr, kb, kg, kr); @@ -456,39 +456,39 @@ public: v_expand(v_absdiff(kr, rr), r_l, r_h); v_uint32 val0, val1, val2, val3; - v_expand(b_l + g_l + r_l, val0, val1); - v_expand(b_h + g_h + r_h, val2, val3); + v_expand(v_add(v_add(b_l, g_l), r_l), val0, val1); + v_expand(v_add(v_add(b_h, g_h), r_h), val2, val3); v_expand(kb, b_l, b_h); v_expand(kg, g_l, g_h); v_expand(kr, r_l, r_h); - v_float32 w0 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val0)); - v_float32 w1 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val1)); - v_float32 w2 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val2)); - v_float32 w3 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val3)); - v_store_aligned(wsum + j , w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); - v_store_aligned(wsum + j + 2*v_float32::nlanes, w2 + vx_load_aligned(wsum + j + 2*v_float32::nlanes)); - v_store_aligned(wsum + j + 3*v_float32::nlanes, w3 + vx_load_aligned(wsum + j + 3*v_float32::nlanes)); + v_float32 w0 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val0))); + v_float32 w1 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val1))); + v_float32 w2 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val2))); + v_float32 w3 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val3))); + v_store_aligned(wsum + j , v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); + v_store_aligned(wsum + j + 2*VTraits::vlanes(), v_add(w2, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3*VTraits::vlanes(), v_add(w3, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(b_l, val0, val1); v_expand(b_h, val2, val3); v_store_aligned(sum_b + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); - v_store_aligned(sum_b + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_b + j + 2*v_float32::nlanes))); - v_store_aligned(sum_b + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_b + j + 3*v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); + v_store_aligned(sum_b + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_b + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_b + j + 3*VTraits::vlanes()))); v_expand(g_l, val0, val1); v_expand(g_h, val2, val3); v_store_aligned(sum_g + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); - v_store_aligned(sum_g + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_g + j + 2*v_float32::nlanes))); - v_store_aligned(sum_g + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_g + j + 3*v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); + v_store_aligned(sum_g + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_g + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_g + j + 3*VTraits::vlanes()))); v_expand(r_l, val0, val1); v_expand(r_h, val2, val3); v_store_aligned(sum_r + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); - v_store_aligned(sum_r + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_r + j + 2*v_float32::nlanes))); - v_store_aligned(sum_r + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_r + j + 3*v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); + v_store_aligned(sum_r + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_r + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_r + j + 3*VTraits::vlanes()))); } #endif for(; j < size.width; j++, ksptr += 3, rsptr += 3) @@ -500,27 +500,27 @@ public: } } j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); - for(; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, dptr += 3*v_uint8::nlanes) + for(; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), dptr += 3*VTraits::vlanes()) { - v_float32 w0 = v_one / vx_load_aligned(wsum + j); - v_float32 w1 = v_one / vx_load_aligned(wsum + j + v_float32::nlanes); - v_float32 w2 = v_one / vx_load_aligned(wsum + j + 2*v_float32::nlanes); - v_float32 w3 = v_one / vx_load_aligned(wsum + j + 3*v_float32::nlanes); + v_float32 w0 = v_div(v_one, vx_load_aligned(wsum + j)); + v_float32 w1 = v_div(v_one, vx_load_aligned(wsum + j + VTraits::vlanes())); + v_float32 w2 = v_div(v_one, vx_load_aligned(wsum + j + 2 * VTraits::vlanes())); + v_float32 w3 = v_div(v_one, vx_load_aligned(wsum + j + 3 * VTraits::vlanes())); - v_store_interleave(dptr, v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_b + j)), - v_round(w1 * vx_load_aligned(sum_b + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_b + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_b + j + 3*v_float32::nlanes)))), - v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_g + j)), - v_round(w1 * vx_load_aligned(sum_g + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_g + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_g + j + 3*v_float32::nlanes)))), - v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_r + j)), - v_round(w1 * vx_load_aligned(sum_r + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_r + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_r + j + 3*v_float32::nlanes))))); + v_store_interleave(dptr, v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_b + j))), + v_round(v_mul(w1, vx_load_aligned(sum_b + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))))), + v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_g + j))), + v_round(v_mul(w1, vx_load_aligned(sum_g + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))))), + v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_r + j))), + v_round(v_mul(w1, vx_load_aligned(sum_r + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes())))))); } #endif for(; j < size.width; j++) @@ -533,7 +533,7 @@ public: } } } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -589,7 +589,7 @@ public: memset(buf.data(), 0, buf.size() * sizeof(float)); float *sum = alignPtr(buf.data(), CV_SIMD_WIDTH); float *wsum = sum + alignSize(size.width, CV_SIMD_WIDTH); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); v_float32 sindex = vx_setall_f32(scale_index); #endif @@ -601,50 +601,50 @@ public: const float* ksptr2 = sptr + space_ofs[k + 2]; const float* ksptr3 = sptr + space_ofs[k + 3]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 rval = vx_load(sptr + j); v_float32 val = vx_load(ksptr0 + j); v_float32 knan = v_not_nan(val); - v_float32 alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + v_float32 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32 w = (kweight0 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one-alpha))) & knan; - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; - v_float32 v_sum = v_muladd(val & knan, w, vx_load_aligned(sum + j)); + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32 w = v_and(v_mul(kweight0, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); + v_float32 v_sum = v_muladd(v_and(val, knan), w, vx_load_aligned(sum + j)); val = vx_load(ksptr1 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight1 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight1, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); val = vx_load(ksptr2 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight2 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight2, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); val = vx_load(ksptr3 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight3 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight3, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); v_store_aligned(wsum + j, v_wsum); v_store_aligned(sum + j, v_sum); @@ -720,20 +720,20 @@ public: { const float* ksptr = sptr + space_ofs[k]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 val = vx_load(ksptr + j); v_float32 rval = vx_load(sptr + j); v_float32 knan = v_not_nan(val); - v_float32 alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + v_float32 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); + alpha = v_sub(alpha, v_cvt_f32(idx)); - v_float32 w = (kweight * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one-alpha))) & knan; - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); - v_store_aligned(sum + j, v_muladd(val & knan, w, vx_load_aligned(sum + j))); + v_float32 w = v_and(v_mul(kweight, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); + v_store_aligned(sum + j, v_muladd(v_and(val, knan), w, vx_load_aligned(sum + j))); } #endif for (; j < size.width; j++) @@ -752,11 +752,11 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_val = vx_load(sptr + j); - v_store(dptr + j, (vx_load_aligned(sum + j) + (v_val & v_not_nan(v_val))) / (vx_load_aligned(wsum + j) + (v_one & v_not_nan(v_val)))); + v_store(dptr + j, v_div(v_add(vx_load_aligned(sum + j), v_and(v_val, v_not_nan(v_val))), v_add(vx_load_aligned(wsum + j), v_and(v_one, v_not_nan(v_val))))); } #endif for (; j < size.width; j++) @@ -774,7 +774,7 @@ public: float *sum_g = sum_b + alignSize(size.width, CV_SIMD_WIDTH); float *sum_r = sum_g + alignSize(size.width, CV_SIMD_WIDTH); float *wsum = sum_r + alignSize(size.width, CV_SIMD_WIDTH); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); v_float32 sindex = vx_setall_f32(scale_index); #endif @@ -787,60 +787,60 @@ public: const float* ksptr3 = sptr + space_ofs[k+3]; const float* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, rsptr += 3 * v_float32::nlanes, - ksptr0 += 3 * v_float32::nlanes, ksptr1 += 3 * v_float32::nlanes, ksptr2 += 3 * v_float32::nlanes, ksptr3 += 3 * v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), rsptr += 3 * VTraits::vlanes(), + ksptr0 += 3 * VTraits::vlanes(), ksptr1 += 3 * VTraits::vlanes(), ksptr2 += 3 * VTraits::vlanes(), ksptr3 += 3 * VTraits::vlanes()) { v_float32 kb, kg, kr, rb, rg, rr; v_load_deinterleave(rsptr, rb, rg, rr); v_load_deinterleave(ksptr0, kb, kg, kr); - v_float32 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - v_float32 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + v_float32 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + v_float32 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32 w = (kweight0 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; - v_float32 v_sum_b = v_muladd(kb & knan, w, vx_load_aligned(sum_b + j)); - v_float32 v_sum_g = v_muladd(kg & knan, w, vx_load_aligned(sum_g + j)); - v_float32 v_sum_r = v_muladd(kr & knan, w, vx_load_aligned(sum_r + j)); + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32 w = v_and(v_mul(kweight0, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); + v_float32 v_sum_b = v_muladd(v_and(kb, knan), w, vx_load_aligned(sum_b + j)); + v_float32 v_sum_g = v_muladd(v_and(kg, knan), w, vx_load_aligned(sum_g + j)); + v_float32 v_sum_r = v_muladd(v_and(kr, knan), w, vx_load_aligned(sum_r + j)); v_load_deinterleave(ksptr1, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight1 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight1, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_load_deinterleave(ksptr2, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight2 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight2, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_load_deinterleave(ksptr3, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight3 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight3, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_store_aligned(wsum + j, v_wsum); v_store_aligned(sum_b + j, v_sum_b); @@ -938,24 +938,24 @@ public: const float* ksptr = sptr + space_ofs[k]; const float* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, ksptr += 3*v_float32::nlanes, rsptr += 3*v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), ksptr += 3*VTraits::vlanes(), rsptr += 3*VTraits::vlanes()) { v_float32 kb, kg, kr, rb, rg, rr; v_load_deinterleave(ksptr, kb, kg, kr); v_load_deinterleave(rsptr, rb, rg, rr); - v_float32 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - v_float32 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + v_float32 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + v_float32 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); + alpha = v_sub(alpha, v_cvt_f32(idx)); - v_float32 w = (kweight * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); - v_store_aligned(sum_b + j, v_muladd(kb & knan, w, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_g + j, v_muladd(kg & knan, w, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_r + j, v_muladd(kr & knan, w, vx_load_aligned(sum_r + j))); + v_float32 w = v_and(v_mul(kweight, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); + v_store_aligned(sum_b + j, v_muladd(v_and(kb, knan), w, vx_load_aligned(sum_b + j))); + v_store_aligned(sum_g + j, v_muladd(v_and(kg, knan), w, vx_load_aligned(sum_g + j))); + v_store_aligned(sum_r + j, v_muladd(v_and(kr, knan), w, vx_load_aligned(sum_r + j))); } #endif for (; j < size.width; j++, ksptr += 3, rsptr += 3) @@ -978,14 +978,14 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, sptr += 3*v_float32::nlanes, dptr += 3*v_float32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), sptr += 3*VTraits::vlanes(), dptr += 3*VTraits::vlanes()) { v_float32 b, g, r; v_load_deinterleave(sptr, b, g, r); - v_float32 mask = v_not_nan(b) & v_not_nan(g) & v_not_nan(r); - v_float32 w = v_one / (vx_load_aligned(wsum + j) + (v_one & mask)); - v_store_interleave(dptr, (vx_load_aligned(sum_b + j) + (b & mask)) * w, (vx_load_aligned(sum_g + j) + (g & mask)) * w, (vx_load_aligned(sum_r + j) + (r & mask)) * w); + v_float32 mask = v_and(v_and(v_not_nan(b), v_not_nan(g)), v_not_nan(r)); + v_float32 w = v_div(v_one, v_add(vx_load_aligned(wsum + j), v_and(v_one, mask))); + v_store_interleave(dptr, v_mul(v_add(vx_load_aligned(sum_b + j), v_and(b, mask)), w), v_mul(v_add(vx_load_aligned(sum_g + j), v_and(g, mask)), w), v_mul(v_add(vx_load_aligned(sum_r + j), v_and(r, mask)), w)); } #endif for (; j < size.width; j++) @@ -1011,7 +1011,7 @@ public: } } } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index 3b18944a0c..d111efdc47 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -56,40 +56,38 @@ template static inline _Tp splineInterpolate(_Tp x, const _Tp* tab return ((tab[3]*x + tab[2])*x + tab[1])*x + tab[0]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template static inline cv::v_float32 splineInterpolate(const cv::v_float32& x, const _Tp* tab, int n) { using namespace cv; v_int32 ix = v_min(v_max(v_trunc(x), vx_setzero_s32()), vx_setall_s32(n-1)); - cv::v_float32 xx = x - v_cvt_f32(ix); - ix = ix << 2; + cv::v_float32 xx = v_sub(x, v_cvt_f32(ix)); + ix = v_shl<2>(ix); - v_float32 t[4]; + v_float32 t0, t1, t2, t3; // assume that v_float32::nlanes == v_int32::nlanes - if(v_float32::nlanes == 4) + if(VTraits::vlanes() == 4) { -#if CV_SIMD_WIDTH == 16 int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx[4]; v_store_aligned(idx, ix); - v_float32x4 tt[4]; - tt[0] = v_load(tab + idx[0]); - tt[1] = v_load(tab + idx[1]); - tt[2] = v_load(tab + idx[2]); - tt[3] = v_load(tab + idx[3]); - v_transpose4x4(tt[0], tt[1], tt[2], tt[3], - t[0], t[1], t[2], t[3]); -#endif + v_float32 tt0, tt1, tt2, tt3; + tt0 = vx_load(tab + idx[0]); + tt1 = vx_load(tab + idx[1]); + tt2 = vx_load(tab + idx[2]); + tt3 = vx_load(tab + idx[3]); + v_transpose4x4(tt0, tt1, tt2, tt3, + t0, t1, t2, t3); } else { - t[0] = v_lut(tab + 0, ix); - t[1] = v_lut(tab + 1, ix); - t[2] = v_lut(tab + 2, ix); - t[3] = v_lut(tab + 3, ix); + t0 = v_lut(tab + 0, ix); + t1 = v_lut(tab + 1, ix); + t2 = v_lut(tab + 2, ix); + t3 = v_lut(tab + 3, ix); } - return v_fma(v_fma(v_fma(t[3], xx, t[2]), xx, t[1]), xx, t[0]); + return v_fma(v_fma(v_fma(t3, xx, t2), xx, t1), xx, t0); } #endif @@ -207,8 +205,8 @@ struct RGB2XYZ_f C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; int i = 0; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5); v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8); @@ -226,9 +224,9 @@ struct RGB2XYZ_f } v_float32 x, y, z; - x = v_fma(b, vc0, v_fma(g, vc1, r*vc2)); - y = v_fma(b, vc3, v_fma(g, vc4, r*vc5)); - z = v_fma(b, vc6, v_fma(g, vc7, r*vc8)); + x = v_fma(b, vc0, v_fma(g, vc1, v_mul(r, vc2))); + y = v_fma(b, vc3, v_fma(g, vc4, v_mul(r, vc5))); + z = v_fma(b, vc6, v_fma(g, vc7, v_mul(r, vc8))); v_store_interleave(dst, x, y, z); } @@ -313,8 +311,8 @@ struct RGB2XYZ_i C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); int descaleShift = 1 << (shift-1); v_int16 vdescale = vx_setall_s16((short)descaleShift); v_int16 cxbg, cxr1, cybg, cyr1, czbg, czr1; @@ -349,27 +347,36 @@ struct RGB2XYZ_i sg0 = v_reinterpret_as_s16(g0); sg1 = v_reinterpret_as_s16(g1); sb0 = v_reinterpret_as_s16(b0); sb1 = v_reinterpret_as_s16(b1); - v_int16 bg[4], rd[4]; - v_zip(sb0, sg0, bg[0], bg[1]); - v_zip(sb1, sg1, bg[2], bg[3]); - v_zip(sr0, vdescale, rd[0], rd[1]); - v_zip(sr1, vdescale, rd[2], rd[3]); + v_int16 bg0, bg1, bg2, bg3, rd0, rd1, rd2, rd3; + v_zip(sb0, sg0, bg0, bg1); + v_zip(sb1, sg1, bg2, bg3); + v_zip(sr0, vdescale, rd0, rd1); + v_zip(sr1, vdescale, rd2, rd3); - v_uint32 vx[4], vy[4], vz[4]; - for(int j = 0; j < 4; j++) - { - vx[j] = v_reinterpret_as_u32(v_dotprod(bg[j], cxbg) + v_dotprod(rd[j], cxr1)) >> shift; - vy[j] = v_reinterpret_as_u32(v_dotprod(bg[j], cybg) + v_dotprod(rd[j], cyr1)) >> shift; - vz[j] = v_reinterpret_as_u32(v_dotprod(bg[j], czbg) + v_dotprod(rd[j], czr1)) >> shift; - } + v_uint32 vx0, vx1, vx2, vx3; + v_uint32 vy0, vy1, vy2, vy3; + v_uint32 vz0, vz1, vz2, vz3; + + vx0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, cxbg), v_dotprod(rd0, cxr1)))); + vy0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, cybg), v_dotprod(rd0, cyr1)))); + vz0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, czbg), v_dotprod(rd0, czr1)))); + vx1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, cxbg), v_dotprod(rd1, cxr1)))); + vy1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, cybg), v_dotprod(rd1, cyr1)))); + vz1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, czbg), v_dotprod(rd1, czr1)))); + vx2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, cxbg), v_dotprod(rd2, cxr1)))); + vy2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, cybg), v_dotprod(rd2, cyr1)))); + vz2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, czbg), v_dotprod(rd2, czr1)))); + vx3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, cxbg), v_dotprod(rd3, cxr1)))); + vy3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, cybg), v_dotprod(rd3, cyr1)))); + vz3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, czbg), v_dotprod(rd3, czr1)))); v_uint16 x0, x1, y0, y1, z0, z1; - x0 = v_pack(vx[0], vx[1]); - x1 = v_pack(vx[2], vx[3]); - y0 = v_pack(vy[0], vy[1]); - y1 = v_pack(vy[2], vy[3]); - z0 = v_pack(vz[0], vz[1]); - z1 = v_pack(vz[2], vz[3]); + x0 = v_pack(vx0, vx1); + x1 = v_pack(vx2, vx3); + y0 = v_pack(vy0, vy1); + y1 = v_pack(vy2, vy3); + z0 = v_pack(vz0, vz1); + z1 = v_pack(vz2, vz3); v_uint8 x, y, z; x = v_pack(x0, x1); @@ -424,8 +431,8 @@ struct RGB2XYZ_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_int16 vdescale = vx_setall_s16(descaleShift); v_int16 vc0 = vx_setall_s16((short)C0), vc1 = vx_setall_s16((short)C1), vc2 = vx_setall_s16((short)C2); @@ -464,29 +471,29 @@ struct RGB2XYZ_i v_int16 ymr, ymg, ymb; v_int16 zmr, zmg, zmb; - v_int16 mr = sr < zero, mg = sg < zero, mb = sb < zero; + v_int16 mr = v_lt(sr, zero), mg = v_lt(sg, zero), mb = v_lt(sb, zero); - xmb = mb & vc0; - xmg = mg & vc1; - xmr = mr & vc2; - ymb = mb & vc3; - ymg = mg & vc4; - ymr = mr & vc5; - zmb = mb & vc6; - zmg = mg & vc7; - zmr = mr & vc8; + xmb = v_and(mb, vc0); + xmg = v_and(mg, vc1); + xmr = v_and(mr, vc2); + ymb = v_and(mb, vc3); + ymg = v_and(mg, vc4); + ymr = v_and(mr, vc5); + zmb = v_and(mb, vc6); + zmg = v_and(mg, vc7); + zmr = v_and(mr, vc8); v_int32 xfix0, xfix1, yfix0, yfix1, zfix0, zfix1; - v_expand(xmr + xmg + xmb, xfix0, xfix1); - v_expand(ymr + ymg + ymb, yfix0, yfix1); - v_expand(zmr + zmg + zmb, zfix0, zfix1); + v_expand(v_add(v_add(xmr, xmg), xmb), xfix0, xfix1); + v_expand(v_add(v_add(ymr, ymg), ymb), yfix0, yfix1); + v_expand(v_add(v_add(zmr, zmg), zmb), zfix0, zfix1); - xfix0 = xfix0 << 16; - xfix1 = xfix1 << 16; - yfix0 = yfix0 << 16; - yfix1 = yfix1 << 16; - zfix0 = zfix0 << 16; - zfix1 = zfix1 << 16; + xfix0 = v_shl<16>(xfix0); + xfix1 = v_shl<16>(xfix1); + yfix0 = v_shl<16>(yfix0); + yfix1 = v_shl<16>(yfix1); + zfix0 = v_shl<16>(zfix0); + zfix1 = v_shl<16>(zfix1); v_int16 bg0, bg1, rd0, rd1; v_zip(sb, sg, bg0, bg1); @@ -494,12 +501,12 @@ struct RGB2XYZ_i v_uint32 x0, x1, y0, y1, z0, z1; - x0 = v_reinterpret_as_u32(v_dotprod(bg0, cxbg) + v_dotprod(rd0, cxr1) + xfix0) >> shift; - x1 = v_reinterpret_as_u32(v_dotprod(bg1, cxbg) + v_dotprod(rd1, cxr1) + xfix1) >> shift; - y0 = v_reinterpret_as_u32(v_dotprod(bg0, cybg) + v_dotprod(rd0, cyr1) + yfix0) >> shift; - y1 = v_reinterpret_as_u32(v_dotprod(bg1, cybg) + v_dotprod(rd1, cyr1) + yfix1) >> shift; - z0 = v_reinterpret_as_u32(v_dotprod(bg0, czbg) + v_dotprod(rd0, czr1) + zfix0) >> shift; - z1 = v_reinterpret_as_u32(v_dotprod(bg1, czbg) + v_dotprod(rd1, czr1) + zfix1) >> shift; + x0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, cxbg), v_dotprod(rd0, cxr1)), xfix0))); + x1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, cxbg), v_dotprod(rd1, cxr1)), xfix1))); + y0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, cybg), v_dotprod(rd0, cyr1)), yfix0))); + y1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, cybg), v_dotprod(rd1, cyr1)), yfix1))); + z0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, czbg), v_dotprod(rd0, czr1)), zfix0))); + z1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, czbg), v_dotprod(rd1, czr1)), zfix1))); v_uint16 x, y, z; x = v_pack(x0, x1); @@ -593,8 +600,8 @@ struct XYZ2RGB_f C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; int i = 0; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 valpha = vx_setall_f32(alpha); v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5); @@ -606,9 +613,9 @@ struct XYZ2RGB_f v_load_deinterleave(src, x, y, z); v_float32 b, g, r; - b = v_fma(x, vc0, v_fma(y, vc1, z*vc2)); - g = v_fma(x, vc3, v_fma(y, vc4, z*vc5)); - r = v_fma(x, vc6, v_fma(y, vc7, z*vc8)); + b = v_fma(x, vc0, v_fma(y, vc1, v_mul(z, vc2))); + g = v_fma(x, vc3, v_fma(y, vc4, v_mul(z, vc5))); + r = v_fma(x, vc6, v_fma(y, vc7, v_mul(z, vc8))); if(dcn == 4) { @@ -707,8 +714,8 @@ struct XYZ2RGB_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift - 1); v_uint8 valpha = vx_setall_u8(alpha); v_int16 vdescale = vx_setall_s16(descaleShift); @@ -739,25 +746,35 @@ struct XYZ2RGB_i z0 = v_reinterpret_as_s16(uz0); z1 = v_reinterpret_as_s16(uz1); - v_int32 b[4], g[4], r[4]; + v_int32 bb0, bb1, bb2, bb3, + gg0, gg1, gg2, gg3, + rr0, rr1, rr2, rr3; - v_int16 xy[4], zd[4]; - v_zip(x0, y0, xy[0], xy[1]); - v_zip(x1, y1, xy[2], xy[3]); - v_zip(z0, vdescale, zd[0], zd[1]); - v_zip(z1, vdescale, zd[2], zd[3]); + v_int16 xy0, xy1, xy2, xy3; + v_int16 zd0, zd1, zd2, zd3; - for(int j = 0; j < 4; j++) - { - b[j] = (v_dotprod(xy[j], cbxy) + v_dotprod(zd[j], cbz1)) >> shift; - g[j] = (v_dotprod(xy[j], cgxy) + v_dotprod(zd[j], cgz1)) >> shift; - r[j] = (v_dotprod(xy[j], crxy) + v_dotprod(zd[j], crz1)) >> shift; - } + v_zip(x0, y0, xy0, xy1); + v_zip(x1, y1, xy2, xy3); + v_zip(z0, vdescale, zd0, zd1); + v_zip(z1, vdescale, zd2, zd3); + + bb0 = v_shr(v_add(v_dotprod(xy0, cbxy), v_dotprod(zd0, cbz1))); + gg0 = v_shr(v_add(v_dotprod(xy0, cgxy), v_dotprod(zd0, cgz1))); + rr0 = v_shr(v_add(v_dotprod(xy0, crxy), v_dotprod(zd0, crz1))); + bb1 = v_shr(v_add(v_dotprod(xy1, cbxy), v_dotprod(zd1, cbz1))); + gg1 = v_shr(v_add(v_dotprod(xy1, cgxy), v_dotprod(zd1, cgz1))); + rr1 = v_shr(v_add(v_dotprod(xy1, crxy), v_dotprod(zd1, crz1))); + bb2 = v_shr(v_add(v_dotprod(xy2, cbxy), v_dotprod(zd2, cbz1))); + gg2 = v_shr(v_add(v_dotprod(xy2, cgxy), v_dotprod(zd2, cgz1))); + rr2 = v_shr(v_add(v_dotprod(xy2, crxy), v_dotprod(zd2, crz1))); + bb3 = v_shr(v_add(v_dotprod(xy3, cbxy), v_dotprod(zd3, cbz1))); + gg3 = v_shr(v_add(v_dotprod(xy3, cgxy), v_dotprod(zd3, cgz1))); + rr3 = v_shr(v_add(v_dotprod(xy3, crxy), v_dotprod(zd3, crz1))); v_uint16 b0, b1, g0, g1, r0, r1; - b0 = v_pack_u(b[0], b[1]); b1 = v_pack_u(b[2], b[3]); - g0 = v_pack_u(g[0], g[1]); g1 = v_pack_u(g[2], g[3]); - r0 = v_pack_u(r[0], r[1]); r1 = v_pack_u(r[2], r[3]); + b0 = v_pack_u(bb0, bb1); b1 = v_pack_u(bb2, bb3); + g0 = v_pack_u(gg0, gg1); g1 = v_pack_u(gg2, gg3); + r0 = v_pack_u(rr0, rr1); r1 = v_pack_u(rr2, rr3); v_uint8 bb, gg, rr; bb = v_pack(b0, b1); @@ -820,8 +837,8 @@ struct XYZ2RGB_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_uint16 valpha = vx_setall_u16(alpha); v_int16 vdescale = vx_setall_s16(descaleShift); @@ -850,30 +867,30 @@ struct XYZ2RGB_i sz = v_reinterpret_as_s16(z); // fixing 16bit signed multiplication - v_int16 mx = sx < zero, my = sy < zero, mz = sz < zero; + v_int16 mx = v_lt(sx, zero), my = v_lt(sy, zero), mz = v_lt(sz, zero); v_int16 bmx, bmy, bmz; v_int16 gmx, gmy, gmz; v_int16 rmx, rmy, rmz; - bmx = mx & vc0; - bmy = my & vc1; - bmz = mz & vc2; - gmx = mx & vc3; - gmy = my & vc4; - gmz = mz & vc5; - rmx = mx & vc6; - rmy = my & vc7; - rmz = mz & vc8; + bmx = v_and(mx, vc0); + bmy = v_and(my, vc1); + bmz = v_and(mz, vc2); + gmx = v_and(mx, vc3); + gmy = v_and(my, vc4); + gmz = v_and(mz, vc5); + rmx = v_and(mx, vc6); + rmy = v_and(my, vc7); + rmz = v_and(mz, vc8); v_int32 bfix0, bfix1, gfix0, gfix1, rfix0, rfix1; - v_expand(bmx + bmy + bmz, bfix0, bfix1); - v_expand(gmx + gmy + gmz, gfix0, gfix1); - v_expand(rmx + rmy + rmz, rfix0, rfix1); + v_expand(v_add(v_add(bmx, bmy), bmz), bfix0, bfix1); + v_expand(v_add(v_add(gmx, gmy), gmz), gfix0, gfix1); + v_expand(v_add(v_add(rmx, rmy), rmz), rfix0, rfix1); - bfix0 = bfix0 << 16; bfix1 = bfix1 << 16; - gfix0 = gfix0 << 16; gfix1 = gfix1 << 16; - rfix0 = rfix0 << 16; rfix1 = rfix1 << 16; + bfix0 = v_shl<16>(bfix0); bfix1 = v_shl<16>(bfix1); + gfix0 = v_shl<16>(gfix0); gfix1 = v_shl<16>(gfix1); + rfix0 = v_shl<16>(rfix0); rfix1 = v_shl<16>(rfix1); v_int16 xy0, xy1, zd0, zd1; v_zip(sx, sy, xy0, xy1); @@ -881,12 +898,12 @@ struct XYZ2RGB_i v_int32 b0, b1, g0, g1, r0, r1; - b0 = (v_dotprod(xy0, cbxy) + v_dotprod(zd0, cbz1) + bfix0) >> shift; - b1 = (v_dotprod(xy1, cbxy) + v_dotprod(zd1, cbz1) + bfix1) >> shift; - g0 = (v_dotprod(xy0, cgxy) + v_dotprod(zd0, cgz1) + gfix0) >> shift; - g1 = (v_dotprod(xy1, cgxy) + v_dotprod(zd1, cgz1) + gfix1) >> shift; - r0 = (v_dotprod(xy0, crxy) + v_dotprod(zd0, crz1) + rfix0) >> shift; - r1 = (v_dotprod(xy1, crxy) + v_dotprod(zd1, crz1) + rfix1) >> shift; + b0 = v_shr(v_add(v_add(v_dotprod(xy0, cbxy), v_dotprod(zd0, cbz1)), bfix0)); + b1 = v_shr(v_add(v_add(v_dotprod(xy1, cbxy), v_dotprod(zd1, cbz1)), bfix1)); + g0 = v_shr(v_add(v_add(v_dotprod(xy0, cgxy), v_dotprod(zd0, cgz1)), gfix0)); + g1 = v_shr(v_add(v_add(v_dotprod(xy1, cgxy), v_dotprod(zd1, cgz1)), gfix1)); + r0 = v_shr(v_add(v_add(v_dotprod(xy0, crxy), v_dotprod(zd0, crz1)), rfix0)); + r1 = v_shr(v_add(v_add(v_dotprod(xy1, crxy), v_dotprod(zd1, crz1)), rfix1)); v_uint16 b, g, r; b = v_pack_u(b0, b1); g = v_pack_u(g0, g1); r = v_pack_u(r0, r1); @@ -1452,19 +1469,19 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin #undef DOT_SHIFT_PACK } -#elif CV_SIMD +#elif CV_SIMD // Fixed size v_int16x8 used below, CV_SIMD_SCALABLE is disabled. // inValues are in [0; LAB_BASE] static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint16& inY, const v_uint16& inZ, const int16_t* LUT, v_uint16& outA, v_uint16& outB, v_uint16& outC) { - const int vsize = v_uint16::nlanes; + const int vsize = VTraits::max_nlanes; // LUT idx of origin pt of cube - v_uint16 tx = inX >> (lab_base_shift - lab_lut_shift); - v_uint16 ty = inY >> (lab_base_shift - lab_lut_shift); - v_uint16 tz = inZ >> (lab_base_shift - lab_lut_shift); + v_uint16 tx = v_shr(inX); + v_uint16 ty = v_shr(inY); + v_uint16 tz = v_shr(inZ); v_uint32 btmp00, btmp01, btmp10, btmp11, btmp20, btmp21; v_uint32 baseIdx0, baseIdx1; @@ -1472,8 +1489,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 v_mul_expand(tx, vx_setall_u16(3*8), btmp00, btmp01); v_mul_expand(ty, vx_setall_u16(3*8*LAB_LUT_DIM), btmp10, btmp11); v_mul_expand(tz, vx_setall_u16(3*8*LAB_LUT_DIM*LAB_LUT_DIM), btmp20, btmp21); - baseIdx0 = btmp00 + btmp10 + btmp20; - baseIdx1 = btmp01 + btmp11 + btmp21; + baseIdx0 = v_add(v_add(btmp00, btmp10), btmp20); + baseIdx1 = v_add(v_add(btmp01, btmp11), btmp21); uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize]; v_store_aligned(vbaseIdx + 0*vsize/2, baseIdx0); @@ -1482,9 +1499,9 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 // fracX, fracY, fracZ are [0; TRILINEAR_BASE) const uint16_t bitMask = (1 << trilinear_shift) - 1; v_uint16 bitMaskReg = vx_setall_u16(bitMask); - v_uint16 fracX = (inX >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16 fracY = (inY >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16 fracZ = (inZ >> (lab_base_shift - 8 - 1)) & bitMaskReg; + v_uint16 fracX = v_and(v_shr(inX), bitMaskReg); + v_uint16 fracY = v_and(v_shr(inY), bitMaskReg); + v_uint16 fracZ = v_and(v_shr(inZ), bitMaskReg); // trilinearIdx = 8*x + 8*TRILINEAR_BASE*y + 8*TRILINEAR_BASE*TRILINEAR_BASE*z v_uint32 trilinearIdx0, trilinearIdx1; @@ -1493,8 +1510,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 v_expand(fracY, fracY0, fracY1); v_expand(fracZ, fracZ0, fracZ1); - trilinearIdx0 = (fracX0 << 3) + (fracY0 << (3+trilinear_shift)) + (fracZ0 << (3+trilinear_shift*2)); - trilinearIdx1 = (fracX1 << 3) + (fracY1 << (3+trilinear_shift)) + (fracZ1 << (3+trilinear_shift*2)); + trilinearIdx0 = v_add(v_add(v_shl<3>(fracX0), v_shl<3 + trilinear_shift>(fracY0)), v_shl<3 + trilinear_shift * 2>(fracZ0)); + trilinearIdx1 = v_add(v_add(v_shl<3>(fracX1), v_shl<3 + trilinear_shift>(fracY1)), v_shl<3 + trilinear_shift * 2>(fracZ1)); uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize]; v_store_aligned(vtrilinearIdx + 0*vsize/2, trilinearIdx0); @@ -1528,12 +1545,12 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 // CV_DESCALE const v_uint32 descaleShift = vx_setall_u32(1 << (trilinear_shift*3 - 1)); - a0 = (a0 + descaleShift) >> (trilinear_shift*3); - a1 = (a1 + descaleShift) >> (trilinear_shift*3); - b0 = (b0 + descaleShift) >> (trilinear_shift*3); - b1 = (b1 + descaleShift) >> (trilinear_shift*3); - c0 = (c0 + descaleShift) >> (trilinear_shift*3); - c1 = (c1 + descaleShift) >> (trilinear_shift*3); + a0 = v_shr(v_add(a0, descaleShift)); + a1 = v_shr(v_add(a1, descaleShift)); + b0 = v_shr(v_add(b0, descaleShift)); + b1 = v_shr(v_add(b1, descaleShift)); + c0 = v_shr(v_add(c0, descaleShift)); + c1 = v_shr(v_add(c1, descaleShift)); outA = v_pack(a0, a1); outB = v_pack(b0, b1); outC = v_pack(c0, c1); } diff --git a/modules/imgproc/src/color_yuv.simd.hpp b/modules/imgproc/src/color_yuv.simd.hpp index b5f73d873a..580329f660 100644 --- a/modules/imgproc/src/color_yuv.simd.hpp +++ b/modules/imgproc/src/color_yuv.simd.hpp @@ -49,6 +49,15 @@ void cvtOnePlaneYUVtoBGR(const uchar * src_data, size_t src_step, namespace { //constants for conversion from/to RGB and YUV, YCrCb according to BT.601 +#if CV_SIMD_SCALABLE +template +static void swap(T&a, T&b) { + T t = a; + a = b; + b = t; +} +#endif + //to YCbCr static const float YCBF = 0.564f; // == 1/2/(1-B2YF) static const float YCRF = 0.713f; // == 1/2/(1-R2YF) @@ -143,11 +152,11 @@ struct RGB2YCrCb_f float C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4); v_float32 vdelta = vx_setall_f32(delta); - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); for( ; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize*3) { @@ -162,13 +171,13 @@ struct RGB2YCrCb_f } v_float32 y, cr, cb; - y = v_fma(b, vc0, v_fma(g, vc1, r*vc2)); + y = v_fma(b, vc0, v_fma(g, vc1, v_mul(r, vc2))); if(bidx) - std::swap(r, b); + swap(r, b); - cr = v_fma(r - y, vc3, vdelta); - cb = v_fma(b - y, vc4, vdelta); + cr = v_fma(v_sub(r, y), vc3, vdelta); + cb = v_fma(v_sub(b, y), vc4, vdelta); if(yuvOrder) { @@ -266,8 +275,8 @@ struct RGB2YCrCb_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int sdelta = ColorChannel::half()*(1 << shift); int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descale = 1 << (shift-1); v_int16 b2y = vx_setall_s16((short)C0); @@ -312,13 +321,13 @@ struct RGB2YCrCb_i // fixing 16bit signed multiplication v_int16 mr, mg, mb; - mr = (sr < z) & r2y; - mg = (sg < z) & g2y; - mb = (sb < z) & b2y; - v_int16 fixmul = v_add_wrap(mr, v_add_wrap(mg, mb)) << fix_shift; + mr = v_and(v_lt(sr, z), r2y); + mg = v_and(v_lt(sg, z), g2y); + mb = v_and(v_lt(sb, z), b2y); + v_int16 fixmul = v_shl(v_add_wrap(mr, v_add_wrap(mg, mb)), fix_shift); - v_int32 ssy0 = (v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)) >> shift; - v_int32 ssy1 = (v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)) >> shift; + v_int32 ssy0 = v_shr(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y)), shift); + v_int32 ssy1 = v_shr(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y)), shift); y = v_reinterpret_as_u16(v_add_wrap(v_pack(ssy0, ssy1), fixmul)); @@ -340,15 +349,15 @@ struct RGB2YCrCb_i v_int32 sy0 = v_reinterpret_as_s32(uy0); v_int32 sy1 = v_reinterpret_as_s32(uy1); - sr0 = sr0 - sy0; sr1 = sr1 - sy1; - sb0 = sb0 - sy0; sb1 = sb1 - sy1; + sr0 = v_sub(sr0, sy0); sr1 = v_sub(sr1, sy1); + sb0 = v_sub(sb0, sy0); sb1 = v_sub(sb1, sy1); v_int32 v_scr0, v_scr1, v_scb0, v_scb1; - v_scr0 = (sr0*vc3 + vdd) >> shift; - v_scr1 = (sr1*vc3 + vdd) >> shift; - v_scb0 = (sb0*vc4 + vdd) >> shift; - v_scb1 = (sb1*vc4 + vdd) >> shift; + v_scr0 = v_shr(v_add(v_mul(sr0, vc3), vdd), shift); + v_scr1 = v_shr(v_add(v_mul(sr1, vc3), vdd), shift); + v_scb0 = v_shr(v_add(v_mul(sb0, vc4), vdd), shift); + v_scb1 = v_shr(v_add(v_mul(sb1, vc4), vdd), shift); // saturate and pack cr = v_pack_u(v_scr0, v_scr1); @@ -407,8 +416,8 @@ struct RGB2YCrCb_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int delta = ColorChannel::half()*(1 << shift); -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_int16 bg2y; v_int16 r12y; @@ -458,10 +467,10 @@ struct RGB2YCrCb_i v_zip(sr0, vdescale, rd00, rd01); v_zip(sr1, vdescale, rd10, rd11); - y00 = v_reinterpret_as_u32(v_dotprod(bg00, bg2y) + v_dotprod(rd00, r12y)) >> shift; - y01 = v_reinterpret_as_u32(v_dotprod(bg01, bg2y) + v_dotprod(rd01, r12y)) >> shift; - y10 = v_reinterpret_as_u32(v_dotprod(bg10, bg2y) + v_dotprod(rd10, r12y)) >> shift; - y11 = v_reinterpret_as_u32(v_dotprod(bg11, bg2y) + v_dotprod(rd11, r12y)) >> shift; + y00 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg00, bg2y), v_dotprod(rd00, r12y))), shift); + y01 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg01, bg2y), v_dotprod(rd01, r12y))), shift); + y10 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg10, bg2y), v_dotprod(rd10, r12y))), shift); + y11 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg11, bg2y), v_dotprod(rd11, r12y))), shift); } v_uint16 y0, y1; @@ -512,15 +521,15 @@ struct RGB2YCrCb_i v_uint8 cr, cb; - cr00 = cr00 >> shift; - cr01 = cr01 >> shift; - cr10 = cr10 >> shift; - cr11 = cr11 >> shift; + cr00 = v_shr(cr00, shift); + cr01 = v_shr(cr01, shift); + cr10 = v_shr(cr10, shift); + cr11 = v_shr(cr11, shift); - cb00 = cb00 >> shift; - cb01 = cb01 >> shift; - cb10 = cb10 >> shift; - cb11 = cb11 >> shift; + cb00 = v_shr(cb00, shift); + cb01 = v_shr(cb01, shift); + cb10 = v_shr(cb10, shift); + cb11 = v_shr(cb11, shift); v_int16 cr0, cr1, cb0, cb1; cr0 = v_pack(cr00, cr01); cr1 = v_pack(cr10, cr11); @@ -623,12 +632,12 @@ struct YCrCb2RGB_f float C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1); v_float32 vc2 = vx_setall_f32(C2), vc3 = vx_setall_f32(C3); v_float32 vdelta = vx_setall_f32(delta); v_float32 valpha = vx_setall_f32(alpha); - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); for( ; i <= n-vsize; i += vsize, src += vsize*3, dst += vsize*dcn) { @@ -640,7 +649,7 @@ struct YCrCb2RGB_f v_float32 b, g, r; - cb -= vdelta; cr -= vdelta; + cb = v_sub(cb, vdelta); cr = v_sub(cr, vdelta); b = v_fma(cb, vc3, y); g = v_fma(cr, vc1, v_fma(cb, vc2, y)); r = v_fma(cr, vc0, y); @@ -746,8 +755,8 @@ struct YCrCb2RGB_i const uchar delta = ColorChannel::half(), alpha = ColorChannel::max(); int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 valpha = vx_setall_u8(alpha); v_uint8 vdelta = vx_setall_u8(delta); const int descaleShift = 1 << (shift - 1); @@ -794,8 +803,8 @@ struct YCrCb2RGB_i v_int32 cb00, cb01, cb10, cb11; v_expand(v_scb0, cb00, cb01); v_expand(v_scb1, cb10, cb11); - b00 += cb00 << 15; b01 += cb01 << 15; - b10 += cb10 << 15; b11 += cb11 << 15; + b00 = v_add(b00, v_shl<15>(cb00)); b01 = v_add(b01, v_shl<15>(cb01)); + b10 = v_add(b10, v_shl<15>(cb10)); b11 = v_add(b11, v_shl<15>(cb11)); } v_int32 t00, t01, t10, t11; @@ -803,17 +812,17 @@ struct YCrCb2RGB_i v_mul_expand(v_scb1, vc2, t10, t11); v_mul_expand(v_scr0, vc1, g00, g01); v_mul_expand(v_scr1, vc1, g10, g11); - g00 += t00; g01 += t01; - g10 += t10; g11 += t11; + g00 = v_add(g00, t00); g01 = v_add(g01, t01); + g10 = v_add(g10, t10); g11 = v_add(g11, t11); v_mul_expand(v_scr0, vc0, r00, r01); v_mul_expand(v_scr1, vc0, r10, r11); - b00 = (b00 + vdescale) >> shift; b01 = (b01 + vdescale) >> shift; - b10 = (b10 + vdescale) >> shift; b11 = (b11 + vdescale) >> shift; - g00 = (g00 + vdescale) >> shift; g01 = (g01 + vdescale) >> shift; - g10 = (g10 + vdescale) >> shift; g11 = (g11 + vdescale) >> shift; - r00 = (r00 + vdescale) >> shift; r01 = (r01 + vdescale) >> shift; - r10 = (r10 + vdescale) >> shift; r11 = (r11 + vdescale) >> shift; + b00 = v_shr(v_add(b00, vdescale), shift); b01 = v_shr(v_add(b01, vdescale), shift); + b10 = v_shr(v_add(b10, vdescale), shift); b11 = v_shr(v_add(b11, vdescale), shift); + g00 = v_shr(v_add(g00, vdescale), shift); g01 = v_shr(v_add(g01, vdescale), shift); + g10 = v_shr(v_add(g10, vdescale), shift); g11 = v_shr(v_add(g11, vdescale), shift); + r00 = v_shr(v_add(r00, vdescale), shift); r01 = v_shr(v_add(r01, vdescale), shift); + r10 = v_shr(v_add(r10, vdescale), shift); r11 = v_shr(v_add(r11, vdescale), shift); v_int16 b0, b1, g0, g1, r0, r1; b0 = v_pack(b00, b01); b1 = v_pack(b10, b11); @@ -897,8 +906,8 @@ struct YCrCb2RGB_i const ushort delta = ColorChannel::half(), alpha = ColorChannel::max(); int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_uint16 valpha = vx_setall_u16(alpha); v_uint16 vdelta = vx_setall_u16(delta); @@ -939,22 +948,22 @@ struct YCrCb2RGB_i // so we fix the multiplication v_int32 cb0, cb1; v_expand(scb, cb0, cb1); - b0 += cb0 << 15; - b1 += cb1 << 15; + b0 = v_add(b0, v_shl<15>(cb0)); + b1 = v_add(b1, v_shl<15>(cb1)); } v_int32 t0, t1; v_mul_expand(scb, vc2, t0, t1); v_mul_expand(scr, vc1, g0, g1); - g0 += t0; g1 += t1; + g0 = v_add(g0, t0); g1 = v_add(g1, t1); v_mul_expand(scr, vc0, r0, r1); // shifted term doesn't fit into 16 bits, addition is to be done in 32 bits - b0 = ((b0 + vdescale) >> shift) + y0; - b1 = ((b1 + vdescale) >> shift) + y1; - g0 = ((g0 + vdescale) >> shift) + y0; - g1 = ((g1 + vdescale) >> shift) + y1; - r0 = ((r0 + vdescale) >> shift) + y0; - r1 = ((r1 + vdescale) >> shift) + y1; + b0 = v_add(v_shr(v_add(b0, vdescale), shift), y0); + b1 = v_add(v_shr(v_add(b1, vdescale), shift), y1); + g0 = v_add(v_shr(v_add(g0, vdescale), shift), y0); + g1 = v_add(v_shr(v_add(g1, vdescale), shift), y1); + r0 = v_add(v_shr(v_add(r0, vdescale), shift), y0); + r1 = v_add(v_shr(v_add(r1, vdescale), shift), y1); // saturate and pack v_uint16 b, g, r; @@ -1038,11 +1047,11 @@ static inline void uvToRGBuv(const uchar u, const uchar v, int& ruv, int& guv, i buv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CUB * uu; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, - v_int32 (&ruv)[4], - v_int32 (&guv)[4], - v_int32 (&buv)[4]) + v_int32 &ruv0, v_int32 &ruv1, v_int32 &ruv2, v_int32 &ruv3, + v_int32 &guv0, v_int32 &guv1, v_int32 &guv2, v_int32 &guv3, + v_int32 &buv0, v_int32 &buv1, v_int32 &buv2, v_int32 &buv3) { v_uint8 v128 = vx_setall_u8(128); v_int8 su = v_reinterpret_as_s8(v_sub_wrap(u, v128)); @@ -1051,9 +1060,10 @@ static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, v_int16 uu0, uu1, vv0, vv1; v_expand(su, uu0, uu1); v_expand(sv, vv0, vv1); - v_int32 uu[4], vv[4]; - v_expand(uu0, uu[0], uu[1]); v_expand(uu1, uu[2], uu[3]); - v_expand(vv0, vv[0], vv[1]); v_expand(vv1, vv[2], vv[3]); + v_int32 uuu0, uuu1, uuu2, uuu3; + v_int32 vvv0, vvv1, vvv2, vvv3; + v_expand(uu0, uuu0, uuu1); v_expand(uu1, uuu2, uuu3); + v_expand(vv0, vvv0, vvv1); v_expand(vv1, vvv2, vvv3); v_int32 vshift = vx_setall_s32(1 << (ITUR_BT_601_SHIFT - 1)); v_int32 vr = vx_setall_s32(ITUR_BT_601_CVR); @@ -1061,12 +1071,15 @@ static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, v_int32 ug = vx_setall_s32(ITUR_BT_601_CUG); v_int32 ub = vx_setall_s32(ITUR_BT_601_CUB); - for (int k = 0; k < 4; k++) - { - ruv[k] = vshift + vr * vv[k]; - guv[k] = vshift + vg * vv[k] + ug * uu[k]; - buv[k] = vshift + ub * uu[k]; - } + auto process_uv = [&](v_int32& ruv, v_int32& guv, v_int32& buv, const v_int32& vv, const v_int32& uu) { + ruv = v_add(vshift, v_mul(vr, vv)); + guv = v_add(v_add(vshift, v_mul(vg, vv)), v_mul(ug, uu)); + buv = v_add(vshift, v_mul(ub, uu)); + }; + process_uv(ruv0, guv0, buv0, vvv0, uuu0); + process_uv(ruv1, guv1, buv1, vvv1, uuu1); + process_uv(ruv2, guv2, buv2, vvv2, uuu2); + process_uv(ruv3, guv3, buv3, vvv3, uuu3); } #endif @@ -1081,44 +1094,48 @@ static inline void yRGBuvToRGBA(const uchar vy, const int ruv, const int guv, co a = uchar(0xff); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void yRGBuvToRGBA(const v_uint8& vy, - const v_int32 (&ruv)[4], - const v_int32 (&guv)[4], - const v_int32 (&buv)[4], + const v_int32 &ruv0, const v_int32 &ruv1, const v_int32 &ruv2, const v_int32 &ruv3, + const v_int32 &guv0, const v_int32 &guv1, const v_int32 &guv2, const v_int32 &guv3, + const v_int32 &buv0, const v_int32 &buv1, const v_int32 &buv2, const v_int32 &buv3, v_uint8& rr, v_uint8& gg, v_uint8& bb) { v_uint8 v16 = vx_setall_u8(16); - v_uint8 posY = vy - v16; + v_uint8 posY = v_sub(vy, v16); v_uint16 yy0, yy1; v_expand(posY, yy0, yy1); - v_int32 yy[4]; - v_int32 yy00, yy01, yy10, yy11; - v_expand(v_reinterpret_as_s16(yy0), yy[0], yy[1]); - v_expand(v_reinterpret_as_s16(yy1), yy[2], yy[3]); + v_int32 yyy0, yyy1, yyy2, yyy3; + v_expand(v_reinterpret_as_s16(yy0), yyy0, yyy1); + v_expand(v_reinterpret_as_s16(yy1), yyy2, yyy3); v_int32 vcy = vx_setall_s32(ITUR_BT_601_CY); - v_int32 y[4], r[4], g[4], b[4]; - for(int k = 0; k < 4; k++) - { - y[k] = yy[k]*vcy; - r[k] = (y[k] + ruv[k]) >> ITUR_BT_601_SHIFT; - g[k] = (y[k] + guv[k]) >> ITUR_BT_601_SHIFT; - b[k] = (y[k] + buv[k]) >> ITUR_BT_601_SHIFT; - } + v_int32 y0, y1, y2, y3, r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - v_int16 r0, r1, g0, g1, b0, b1; - r0 = v_pack(r[0], r[1]); - r1 = v_pack(r[2], r[3]); - g0 = v_pack(g[0], g[1]); - g1 = v_pack(g[2], g[3]); - b0 = v_pack(b[0], b[1]); - b1 = v_pack(b[2], b[3]); + auto process_yrgb = [&](const v_int32& yy, v_int32& y, v_int32& r, v_int32& g, v_int32& b, + const v_int32& ruv, const v_int32& guv, const v_int32& buv) { + y = v_mul(yy, vcy); + r = v_shr(v_add(y, ruv), ITUR_BT_601_SHIFT); + g = v_shr(v_add(y, guv), ITUR_BT_601_SHIFT); + b = v_shr(v_add(y, buv), ITUR_BT_601_SHIFT); + }; + process_yrgb(yyy0, y0, r0, g0, b0, ruv0, guv0, buv0); + process_yrgb(yyy1, y1, r1, g1, b1, ruv1, guv1, buv1); + process_yrgb(yyy2, y2, r2, g2, b2, ruv2, guv2, buv2); + process_yrgb(yyy3, y3, r3, g3, b3, ruv3, guv3, buv3); - rr = v_pack_u(r0, r1); - gg = v_pack_u(g0, g1); - bb = v_pack_u(b0, b1); + v_int16 _r0, _r1, _g0, _g1, _b0, _b1; + _r0 = v_pack(r0, r1); + _r1 = v_pack(r2, r3); + _g0 = v_pack(g0, g1); + _g1 = v_pack(g2, g3); + _b0 = v_pack(b0, b1); + _b1 = v_pack(b2, b3); + + rr = v_pack_u(_r0, _r1); + gg = v_pack_u(_g0, _g1); + bb = v_pack_u(_b0, _b1); } #endif @@ -1201,8 +1218,8 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody const uchar* y2 = y1 + my1_step; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for( ; i <= width - 2*vsize; i += 2*vsize, row1 += vsize*dcn*2, row2 += vsize*dcn*2) @@ -1215,36 +1232,50 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody swap(u, v); } - v_uint8 vy[4]; - v_load_deinterleave(y1 + i, vy[0], vy[1]); - v_load_deinterleave(y2 + i, vy[2], vy[3]); + v_uint8 vy0, vy1, vy2, vy3; + v_load_deinterleave(y1 + i, vy0, vy1); + v_load_deinterleave(y2 + i, vy2, vy3); - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[4], g[4], b[4]; + v_uint8 r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - for(int k = 0; k < 4; k++) - { - yRGBuvToRGBA(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } + auto call_yRGBuvToRGBA = [&](const v_uint8& vy, v_uint8& r, v_uint8& g, v_uint8& b) { + yRGBuvToRGBA(vy, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r, g, b); + }; + call_yRGBuvToRGBA(vy0, r0, g0, b0); + call_yRGBuvToRGBA(vy1, r1, g1, b1); + call_yRGBuvToRGBA(vy2, r2, g2, b2); + call_yRGBuvToRGBA(vy3, r3, g3, b3); if(bIdx) { - for(int k = 0; k < 4; k++) - swap(r[k], b[k]); + swap(r0, b0); + swap(r1, b1); + swap(r2, b2); + swap(r3, b3); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); + v_zip(r0, r1, r0_0, r0_1); + v_zip(r2, r3, r1_0, r1_1); v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); + v_zip(g0, g1, g0_0, g0_1); + v_zip(g2, g3, g1_0, g1_1); v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); + v_zip(b0, b1, b0_0, b0_1); + v_zip(b2, b3, b1_0, b1_1); if(dcn == 4) { @@ -1319,8 +1350,8 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody const uchar* y2 = y1 + stride; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for( ; i <= width/2 - vsize; i += vsize, row1 += vsize*dcn*2, row2 += vsize*dcn*2) @@ -1329,36 +1360,50 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody u = vx_load(u1 + i); v = vx_load(v1 + i); - v_uint8 vy[4]; - v_load_deinterleave(y1 + 2*i, vy[0], vy[1]); - v_load_deinterleave(y2 + 2*i, vy[2], vy[3]); + v_uint8 vy0, vy1, vy2, vy3; + v_load_deinterleave(y1 + 2*i, vy0, vy1); + v_load_deinterleave(y2 + 2*i, vy2, vy3); - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[4], g[4], b[4]; + v_uint8 r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - for(int k = 0; k < 4; k++) - { - yRGBuvToRGBA(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } + auto call_yRGBuvToRGBA = [&](const v_uint8& vy, v_uint8& r, v_uint8& g, v_uint8& b) { + yRGBuvToRGBA(vy, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r, g, b); + }; + call_yRGBuvToRGBA(vy0, r0, g0, b0); + call_yRGBuvToRGBA(vy1, r1, g1, b1); + call_yRGBuvToRGBA(vy2, r2, g2, b2); + call_yRGBuvToRGBA(vy3, r3, g3, b3); if(bIdx) { - for(int k = 0; k < 4; k++) - swap(r[k], b[k]); + swap(r0, b0); + swap(r1, b1); + swap(r2, b2); + swap(r3, b3); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); + v_zip(r0, r1, r0_0, r0_1); + v_zip(r2, r3, r1_0, r1_1); v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); + v_zip(g0, g1, g0_0, g0_1); + v_zip(g2, g3, g1_0, g1_1); v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); + v_zip(b0, b1, b0_0, b0_1); + v_zip(b2, b3, b1_0, b1_1); if(dcn == 4) { @@ -1430,7 +1475,7 @@ static inline uchar rgbToY42x(uchar r, uchar g, uchar b) return saturate_cast(yy >> ITUR_BT_601_SHIFT); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline v_uint8 rgbToY42x(const v_uint8& r, const v_uint8& g, const v_uint8& b) { const int shifted16 = (16 << ITUR_BT_601_SHIFT); @@ -1440,25 +1485,25 @@ static inline v_uint8 rgbToY42x(const v_uint8& r, const v_uint8& g, const v_uint v_expand(g, g0, g1); v_expand(b, b0, b1); - v_uint32 rq[4], gq[4], bq[4]; - v_expand(r0, rq[0], rq[1]); v_expand(r1, rq[2], rq[3]); - v_expand(g0, gq[0], gq[1]); v_expand(g1, gq[2], gq[3]); - v_expand(b0, bq[0], bq[1]); v_expand(b1, bq[2], bq[3]); + v_uint32 rq0, rq1, rq2, rq3, gq0, gq1, gq2, gq3, bq0, bq1, bq2, bq3; + v_expand(r0, rq0, rq1); v_expand(r1, rq2, rq3); + v_expand(g0, gq0, gq1); v_expand(g1, gq2, gq3); + v_expand(b0, bq0, bq1); v_expand(b1, bq2, bq3); v_uint32 ry = vx_setall_u32(ITUR_BT_601_CRY), gy = vx_setall_u32(ITUR_BT_601_CGY); v_uint32 by = vx_setall_u32(ITUR_BT_601_CBY), shift = vx_setall_u32(halfShift + shifted16); - v_uint32 y[4]; - for(int k = 0; k < 4; k++) - { - y[k] = (rq[k]*ry + gq[k]*gy + bq[k]*by + shift) >> ITUR_BT_601_SHIFT; - } + v_uint32 y0, y1, y2, y3; + y0 = v_shr(v_add(v_add(v_add(v_mul(rq0, ry), v_mul(gq0, gy)), v_mul(bq0, by)), shift)); + y1 = v_shr(v_add(v_add(v_add(v_mul(rq1, ry), v_mul(gq1, gy)), v_mul(bq1, by)), shift)); + y2 = v_shr(v_add(v_add(v_add(v_mul(rq2, ry), v_mul(gq2, gy)), v_mul(bq2, by)), shift)); + y3 = v_shr(v_add(v_add(v_add(v_mul(rq3, ry), v_mul(gq3, gy)), v_mul(bq3, by)), shift)); - v_uint16 y0, y1; - y0 = v_pack(y[0], y[1]); - y1 = v_pack(y[2], y[3]); + v_uint16 _y0, _y1; + _y0 = v_pack(y0, y1); + _y1 = v_pack(y2, y3); - return v_pack(y0, y1); + return v_pack(_y0, _y1); } #endif @@ -1473,27 +1518,27 @@ static inline void rgbToUV42x(uchar r, uchar g, uchar b, uchar& u, uchar& v) v = saturate_cast(vv >> ITUR_BT_601_SHIFT); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void rgbToUV42x(const v_uint8& r0, const v_uint8& r1, const v_uint8& g0, const v_uint8& g1, const v_uint8& b0, const v_uint8& b1, v_uint8& u, v_uint8& v) { // [r0, r1, r2, r3,..] => [r0, 0, r2, 0,..] v_int16 vlowByte = vx_setall_s16(0x00ff); v_int16 rd0, rd1, gd0, gd1, bd0, bd1; - rd0 = v_reinterpret_as_s16(r0) & vlowByte; - rd1 = v_reinterpret_as_s16(r1) & vlowByte; - gd0 = v_reinterpret_as_s16(g0) & vlowByte; - gd1 = v_reinterpret_as_s16(g1) & vlowByte; - bd0 = v_reinterpret_as_s16(b0) & vlowByte; - bd1 = v_reinterpret_as_s16(b1) & vlowByte; + rd0 = v_and(v_reinterpret_as_s16(r0), vlowByte); + rd1 = v_and(v_reinterpret_as_s16(r1), vlowByte); + gd0 = v_and(v_reinterpret_as_s16(g0), vlowByte); + gd1 = v_and(v_reinterpret_as_s16(g1), vlowByte); + bd0 = v_and(v_reinterpret_as_s16(b0), vlowByte); + bd1 = v_and(v_reinterpret_as_s16(b1), vlowByte); - v_int32 rq[4], gq[4], bq[4]; - v_expand(rd0, rq[0], rq[1]); - v_expand(rd1, rq[2], rq[3]); - v_expand(gd0, gq[0], gq[1]); - v_expand(gd1, gq[2], gq[3]); - v_expand(bd0, bq[0], bq[1]); - v_expand(bd1, bq[2], bq[3]); + v_int32 rq0, rq1, rq2, rq3, gq0, gq1, gq2, gq3, bq0, bq1, bq2, bq3; + v_expand(rd0, rq0, rq1); + v_expand(rd1, rq2, rq3); + v_expand(gd0, gq0, gq1); + v_expand(gd1, gq2, gq3); + v_expand(bd0, bq0, bq1); + v_expand(bd1, bq2, bq3); const int halfShift = (1 << (ITUR_BT_601_SHIFT - 1)); const int shifted128 = (128 << ITUR_BT_601_SHIFT); @@ -1505,18 +1550,21 @@ static inline void rgbToUV42x(const v_uint8& r0, const v_uint8& r1, const v_uint bu = vx_setall_s32(ITUR_BT_601_CBU); bv = vx_setall_s32(ITUR_BT_601_CBV); - v_int32 uq[4], vq[4]; - for(int k = 0; k < 4; k++) - { - uq[k] = (ru*rq[k] + gu*gq[k] + bu*bq[k] + shift) >> ITUR_BT_601_SHIFT; - vq[k] = (bu*rq[k] + gv*gq[k] + bv*bq[k] + shift) >> ITUR_BT_601_SHIFT; - } + v_int32 uq0, uq1, uq2, uq3, vq0, vq1, vq2, vq3; + uq0 = v_shr(v_add(v_add(v_add(v_mul(ru, rq0), v_mul(gu, gq0)), v_mul(bu, bq0)), shift)); + vq0 = v_shr(v_add(v_add(v_add(v_mul(bu, rq0), v_mul(gv, gq0)), v_mul(bv, bq0)), shift)); + uq1 = v_shr(v_add(v_add(v_add(v_mul(ru, rq1), v_mul(gu, gq1)), v_mul(bu, bq1)), shift)); + vq1 = v_shr(v_add(v_add(v_add(v_mul(bu, rq1), v_mul(gv, gq1)), v_mul(bv, bq1)), shift)); + uq2 = v_shr(v_add(v_add(v_add(v_mul(ru, rq2), v_mul(gu, gq2)), v_mul(bu, bq2)), shift)); + vq2 = v_shr(v_add(v_add(v_add(v_mul(bu, rq2), v_mul(gv, gq2)), v_mul(bv, bq2)), shift)); + uq3 = v_shr(v_add(v_add(v_add(v_mul(ru, rq3), v_mul(gu, gq3)), v_mul(bu, bq3)), shift)); + vq3 = v_shr(v_add(v_add(v_add(v_mul(bu, rq3), v_mul(gv, gq3)), v_mul(bv, bq3)), shift)); v_int16 u0, u1, v0, v1; - u0 = v_pack(uq[0], uq[1]); - u1 = v_pack(uq[2], uq[3]); - v0 = v_pack(vq[0], vq[1]); - v1 = v_pack(vq[2], vq[3]); + u0 = v_pack(uq0, uq1); + u1 = v_pack(uq2, uq3); + v0 = v_pack(vq0, vq1); + v1 = v_pack(vq2, vq3); u = v_pack_u(u0, u1); v = v_pack_u(v0, v1); @@ -1559,8 +1607,8 @@ struct RGB8toYUV420pInvoker: public ParallelLoopBody } } int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); for( ; i <= w/2 - vsize; i += vsize) @@ -1708,47 +1756,61 @@ struct YUV422toRGB8Invoker : ParallelLoopBody { uchar* row = dst_data + dst_step * j; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for(; i <= 2*width - 4*vsize; i += 4*vsize, row += vsize*dcn*2) { - v_uint8 u, v, vy[2]; + v_uint8 u, v, vy0, vy1; if(yIdx == 1) // UYVY { - v_load_deinterleave(yuv_src + i, u, vy[0], v, vy[1]); + v_load_deinterleave(yuv_src + i, u, vy0, v, vy1); } else // YUYV or YVYU { - v_load_deinterleave(yuv_src + i, vy[0], u, vy[1], v); + v_load_deinterleave(yuv_src + i, vy0, u, vy1, v); if(uIdx == 1) // YVYU { swap(u, v); } } - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[2], g[2], b[2]; + v_uint8 r0, r1, g0, g1, b0, b1; - yRGBuvToRGBA(vy[0], ruv, guv, buv, r[0], g[0], b[0]); - yRGBuvToRGBA(vy[1], ruv, guv, buv, r[1], g[1], b[1]); + + yRGBuvToRGBA(vy0, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r0, g0, b0); + yRGBuvToRGBA(vy1, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r1, g1, b1); if(bIdx) { - swap(r[0], b[0]); - swap(r[1], b[1]); + swap(r0, b0); + swap(r1, b1); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1; - v_zip(r[0], r[1], r0_0, r0_1); + v_zip(r0, r1, r0_0, r0_1); v_uint8 g0_0, g0_1; - v_zip(g[0], g[1], g0_0, g0_1); + v_zip(g0, g1, g0_0, g0_1); v_uint8 b0_0, b0_1; - v_zip(b[0], b[1], b0_0, b0_1); + v_zip(b0, b1, b0_0, b0_1); if(dcn == 4) { diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index 8dcf5235af..06053e63fe 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -349,7 +349,7 @@ struct FilterNoVec }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) ///////////////////////////////////// 8u-16s & 8u-8u ////////////////////////////////// @@ -383,7 +383,7 @@ struct RowVec_8u32s if( smallValues ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const uchar* src = _src + i; v_int32 s0 = vx_setzero_s32(); @@ -396,27 +396,27 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint8 x0, x1; v_zip(vx_load(src), vx_load(src + cn), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)); - s2 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)); - s3 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f))); + s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f))); + s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f))); } if (k < _ksize) { v_int32 f = vx_setall_s32(_kx[k]); v_uint16 x0, x1; v_expand(vx_load(src), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)); - s2 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)); - s3 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f))); + s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f))); + s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f))); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const uchar* src = _src + i; v_int32 s0 = vx_setzero_s32(); @@ -427,22 +427,22 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint16 x0, x1; v_zip(vx_load_expand(src), vx_load_expand(src + cn), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f))); } if( k < _ksize ) { v_int32 f = vx_setall_s32(_kx[k]); v_uint32 x0, x1; v_expand(vx_load_expand(src), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f))); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 d = vx_setzero_s32(); k = 0; @@ -452,12 +452,12 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint32 x0, x1; v_zip(vx_load_expand_q(src), vx_load_expand_q(src + cn), x0, x1); - d += v_dotprod(v_pack(v_reinterpret_as_s32(x0), v_reinterpret_as_s32(x1)), v_reinterpret_as_s16(f)); + d = v_add(d, v_dotprod(v_pack(v_reinterpret_as_s32(x0), v_reinterpret_as_s32(x1)), v_reinterpret_as_s16(f))); } if (k < _ksize) - d += v_dotprod(v_reinterpret_as_s16(vx_load_expand_q(src)), v_reinterpret_as_s16(vx_setall_s32(_kx[k]))); + d = v_add(d, v_dotprod(v_reinterpret_as_s16(vx_load_expand_q(src)), v_reinterpret_as_s16(vx_setall_s32(_kx[k])))); v_store(dst + i, d); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } return i; @@ -480,7 +480,7 @@ struct RowVec_8u32f float* dst = (float*)_dst; const float* _kx = kernel.ptr(); width *= cn; - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 s0 = vx_setzero_f32(); v_float32 s1 = vx_setzero_f32(); @@ -492,18 +492,18 @@ struct RowVec_8u32f v_float32 f = vx_setall_f32(_kx[k]); const uchar* src = (const uchar*)_src + i + k * cn; v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src))); - v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + v_float32::nlanes))); - v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*v_float32::nlanes))); - v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*v_float32::nlanes))); + v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + VTraits::vlanes()))); + v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*VTraits::vlanes()))); + v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*VTraits::vlanes()))); s0 = v_muladd(vs_ll, f, s0); s1 = v_muladd(vs_lh, f, s1); s2 = v_muladd(vs_hl, f, s2); s3 = v_muladd(vs_hh, f, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } return i; } @@ -553,7 +553,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == 2 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -562,29 +562,29 @@ struct SymmRowSmallVec_8u32s x1l = v_add_wrap(v_add_wrap(x1l, x1l), v_add_wrap(x0l, x2l)); x1h = v_add_wrap(v_add_wrap(x1h, x1h), v_add_wrap(x0h, x2h)); v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x1l))); - v_store(dst + i + v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_reinterpret_as_s32(v_expand_low(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x1h))); + v_store(dst + i + VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_reinterpret_as_s32(v_expand_low(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_add_wrap(v_add_wrap(x, x), v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn))); v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x))); - v_store(dst + i + v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint32 x = vx_load_expand_q(src); - x = (x + x) + vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn); + x = v_add(v_add(v_add(x, x), vx_load_expand_q(src - cn)), vx_load_expand_q(src + cn)); v_store(dst + i, v_reinterpret_as_s32(x)); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else if( kx[0] == -2 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -593,31 +593,31 @@ struct SymmRowSmallVec_8u32s x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l)); x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(v_reinterpret_as_s16(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1h))); + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_sub_wrap(v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn)), v_add_wrap(x, x)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src)); - x = v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)) - (x + x); + x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), v_add(x, x)); v_store(dst + i, x); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)kx[0]); v_int16 k1 = vx_setall_s16((short)kx[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -628,34 +628,34 @@ struct SymmRowSmallVec_8u32s v_int16 x0, x1; v_mul_expand(v_reinterpret_as_s16(x1l), k0, dl, dh); v_zip(v_reinterpret_as_s16(x0l), v_reinterpret_as_s16(x2l), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); v_store(dst + i, dl); - v_store(dst + i + v_int32::nlanes, dh); + v_store(dst + i + VTraits::vlanes(), dh); v_mul_expand(v_reinterpret_as_s16(x1h), k0, dl, dh); v_zip(v_reinterpret_as_s16(x0h), v_reinterpret_as_s16(x2h), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); - v_store(dst + i + 2*v_int32::nlanes, dl); - v_store(dst + i + 3*v_int32::nlanes, dh); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); + v_store(dst + i + 2*VTraits::vlanes(), dl); + v_store(dst + i + 3*VTraits::vlanes(), dh); } - if ( i <= width - v_uint16::nlanes ) + if ( i <= width - VTraits::vlanes() ) { v_int32 dl, dh; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, dl, dh); v_int16 x0, x1; v_zip(v_reinterpret_as_s16(vx_load_expand(src - cn)), v_reinterpret_as_s16(vx_load_expand(src + cn)), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); v_store(dst + i, dl); - v_store(dst + i + v_int32::nlanes, dh); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), dh); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if ( i <= width - v_uint32::nlanes ) + if ( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)) * vx_setall_s32(kx[1]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1])))); + i += VTraits::vlanes(); } } } @@ -663,7 +663,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == -2 && kx[1] == 0 && kx[2] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - 2*cn), x0l, x0h); @@ -672,31 +672,31 @@ struct SymmRowSmallVec_8u32s x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l)); x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(v_reinterpret_as_s16(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1h))); + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_sub_wrap(v_add_wrap(vx_load_expand(src - 2*cn), vx_load_expand(src + 2*cn)), v_add_wrap(x, x)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src)); - x = v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn) + vx_load_expand_q(src + 2*cn)) - (x + x); + x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), v_add(x, x)); v_store(dst + i, x); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_int32 x0, x1, x2, x3; v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h; @@ -710,45 +710,45 @@ struct SymmRowSmallVec_8u32s v_expand(vx_load(src + cn), x1l, x1h); v_expand(vx_load(src - 2*cn), x2l, x2h); v_expand(vx_load(src + 2*cn), x3l, x3h); - v_zip(v_reinterpret_as_s16(x0l + x1l), v_reinterpret_as_s16(x2l + x3l), xl, xh); - x0 += v_dotprod(xl, k12); - x1 += v_dotprod(xh, k12); - v_zip(v_reinterpret_as_s16(x0h + x1h), v_reinterpret_as_s16(x2h + x3h), xl, xh); - x2 += v_dotprod(xl, k12); - x3 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(x0l, x1l)), v_reinterpret_as_s16(v_add(x2l, x3l)), xl, xh); + x0 = v_add(x0, v_dotprod(xl, k12)); + x1 = v_add(x1, v_dotprod(xh, k12)); + v_zip(v_reinterpret_as_s16(v_add(x0h, x1h)), v_reinterpret_as_s16(v_add(x2h, x3h)), xl, xh); + x2 = v_add(x2, v_dotprod(xl, k12)); + x3 = v_add(x3, v_dotprod(xh, k12)); v_store(dst + i, x0); - v_store(dst + i + v_int32::nlanes, x1); - v_store(dst + i + 2*v_int32::nlanes, x2); - v_store(dst + i + 3*v_int32::nlanes, x3); + v_store(dst + i + VTraits::vlanes(), x1); + v_store(dst + i + 2*VTraits::vlanes(), x2); + v_store(dst + i + 3*VTraits::vlanes(), x3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x1, x2; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, x1, x2); v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(vx_load_expand(src - cn) + vx_load_expand(src + cn)), v_reinterpret_as_s16(vx_load_expand(src - 2*cn) + vx_load_expand(src + 2*cn)), xl, xh); - x1 += v_dotprod(xl, k12); - x2 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn))), v_reinterpret_as_s16(v_add(vx_load_expand(src - 2 * cn), vx_load_expand(src + 2 * cn))), xl, xh); + x1 = v_add(x1, v_dotprod(xl, k12)); + x2 = v_add(x2, v_dotprod(xh, k12)); v_store(dst + i, x1); - v_store(dst + i + v_int32::nlanes, x2); - i += v_uint16::nlanes, src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), x2); + i += VTraits::vlanes(), src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), - v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), - v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn) + vx_load_expand_q(src + 2*cn)) * vx_setall_s32(kx[2])))); - i += v_uint32::nlanes; + v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1]), + v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), vx_setall_s32(kx[2]))))); + i += VTraits::vlanes(); } } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint8 v_src = vx_load(src); v_int32 s0, s1, s2, s3; @@ -764,12 +764,12 @@ struct SymmRowSmallVec_8u32s v_uint8 v_src3 = vx_load(src + j + cn); v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(v_expand_low(v_src0) + v_expand_low(v_src2)), v_reinterpret_as_s16(v_expand_low(v_src1) + v_expand_low(v_src3)), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); - v_zip(v_reinterpret_as_s16(v_expand_high(v_src0) + v_expand_high(v_src2)), v_reinterpret_as_s16(v_expand_high(v_src1) + v_expand_high(v_src3)), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(v_expand_low(v_src0), v_expand_low(v_src2))), v_reinterpret_as_s16(v_add(v_expand_low(v_src1), v_expand_low(v_src3))), xl, xh); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); + v_zip(v_reinterpret_as_s16(v_add(v_expand_high(v_src0), v_expand_high(v_src2))), v_reinterpret_as_s16(v_add(v_expand_high(v_src1), v_expand_high(v_src3))), xl, xh); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { @@ -780,48 +780,48 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_expand_low(v_src0)), v_reinterpret_as_s16(v_expand_low(v_src1)), xl, xh); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); v_zip(v_reinterpret_as_s16(v_expand_high(v_src0)), v_reinterpret_as_s16(v_expand_high(v_src1)), xl, xh); - s2 += v_dotprod(xl, k1); - s3 += v_dotprod(xh, k1); + s2 = v_add(s2, v_dotprod(xl, k1)); + s3 = v_add(s3, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s0, s1; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1); for (k = 1, j = cn; k <= _ksize / 2 - 1; k+=2, j += 2*cn) { v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(vx_load_expand(src - j) + vx_load_expand(src + j)), v_reinterpret_as_s16(vx_load_expand(src - j - cn) + vx_load_expand(src + j + cn)), xl, xh); + v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - j), vx_load_expand(src + j))), v_reinterpret_as_s16(v_add(vx_load_expand(src - j - cn), vx_load_expand(src + j + cn))), xl, xh); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k+1] << 16))); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); } if ( k < _ksize / 2 + 1 ) { v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src - j)), v_reinterpret_as_s16(vx_load_expand(src + j)), xl, xh); v_int16 k1 = vx_setall_s16((short)(kx[k])); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_int32 s0 = v_reinterpret_as_s32(vx_load_expand_q(src)) * vx_setall_s32(kx[0]); + v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0])); for( k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn ) - s0 = v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src - j) + vx_load_expand_q(src + j)), vx_setall_s32(kx[k]), s0); + s0 = v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - j), vx_load_expand_q(src + j))), vx_setall_s32(kx[k]), s0); v_store(dst + i, s0); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } } @@ -831,7 +831,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == 0 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -839,27 +839,27 @@ struct SymmRowSmallVec_8u32s v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(x2l, x0l)); v_int16 dh = v_reinterpret_as_s16(v_sub_wrap(x2h, x0h)); v_store(dst + i, v_expand_low(dl)); - v_store(dst + i + v_int32::nlanes, v_expand_high(dl)); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(dh)); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(dh)); + v_store(dst + i + VTraits::vlanes(), v_expand_high(dl)); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(dh)); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(dh)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn))); v_store(dst + i, v_expand_low(dl)); - v_store(dst + i + v_int32::nlanes, v_expand_high(dl)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(dl)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if (i <= width - v_uint32::nlanes) + if (i <= width - VTraits::vlanes()) { - v_store(dst + i, v_reinterpret_as_s32(vx_load_expand_q(src + cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - cn))); - i += v_uint32::nlanes; + v_store(dst + i, v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn)))); + i += VTraits::vlanes(); } } else { v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (-kx[1] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -867,30 +867,30 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(x2l), v_reinterpret_as_s16(x0l), xl, xh); v_store(dst + i, v_dotprod(xl, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(xh, k0)); + v_store(dst + i + VTraits::vlanes(), v_dotprod(xh, k0)); v_zip(v_reinterpret_as_s16(x2h), v_reinterpret_as_s16(x0h), xl, xh); - v_store(dst + i + 2*v_int32::nlanes, v_dotprod(xl, k0)); - v_store(dst + i + 3*v_int32::nlanes, v_dotprod(xh, k0)); + v_store(dst + i + 2*VTraits::vlanes(), v_dotprod(xl, k0)); + v_store(dst + i + 3*VTraits::vlanes(), v_dotprod(xh, k0)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src + cn)), v_reinterpret_as_s16(vx_load_expand(src - cn)), xl, xh); v_store(dst + i, v_dotprod(xl, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(xh, k0)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_dotprod(xh, k0)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if (i <= width - v_uint32::nlanes) + if (i <= width - VTraits::vlanes()) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), v_reinterpret_as_s32(vx_load_expand_q(src - cn)) * vx_setall_s32(-kx[1]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), v_mul(v_reinterpret_as_s32(vx_load_expand_q(src - cn)), vx_setall_s32(-kx[1])))); + i += VTraits::vlanes(); } } } else if( _ksize == 5 ) { v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h; v_expand(vx_load(src - cn), x0l, x0h); @@ -900,31 +900,31 @@ struct SymmRowSmallVec_8u32s v_int16 x0, x1; v_zip(v_reinterpret_as_s16(v_sub_wrap(x2l, x0l)), v_reinterpret_as_s16(v_sub_wrap(x3l, x1l)), x0, x1); v_store(dst + i, v_dotprod(x0, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(x1, k0)); + v_store(dst + i + VTraits::vlanes(), v_dotprod(x1, k0)); v_zip(v_reinterpret_as_s16(v_sub_wrap(x2h, x0h)), v_reinterpret_as_s16(v_sub_wrap(x3h, x1h)), x0, x1); - v_store(dst + i + 2*v_int32::nlanes, v_dotprod(x0, k0)); - v_store(dst + i + 3*v_int32::nlanes, v_dotprod(x1, k0)); + v_store(dst + i + 2*VTraits::vlanes(), v_dotprod(x0, k0)); + v_store(dst + i + 3*VTraits::vlanes(), v_dotprod(x1, k0)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 x0, x1; v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn))), v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + 2*cn), vx_load_expand(src - 2*cn))), x0, x1); v_store(dst + i, v_dotprod(x0, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(x1, k0)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_dotprod(x1, k0)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - cn)), vx_setall_s32(kx[1]), - (v_reinterpret_as_s32(vx_load_expand_q(src + 2*cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn))) * vx_setall_s32(kx[2]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn))), vx_setall_s32(kx[1]), + v_mul(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + 2 * cn)), v_reinterpret_as_s32(vx_load_expand_q(src - 2 * cn))), vx_setall_s32(kx[2])))); + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint8 v_src = vx_load(src); v_int32 s0, s1, s2, s3; @@ -941,11 +941,11 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src2), v_expand_low(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src3), v_expand_low(v_src1))), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src2), v_expand_high(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src3), v_expand_high(v_src1))), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { @@ -955,18 +955,18 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_expand_low(v_src1)), v_reinterpret_as_s16(v_expand_low(v_src0)), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); v_zip(v_reinterpret_as_s16(v_expand_high(v_src1)), v_reinterpret_as_s16(v_expand_high(v_src0)), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s0, s1; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1); @@ -975,28 +975,28 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j), vx_load_expand(src - j))), v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j + cn), vx_load_expand(src - j - cn))), xl, xh); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k + 1] << 16))); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { v_int16 k1 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (-kx[k] << 16))); v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src + j)), v_reinterpret_as_s16(vx_load_expand(src - j)), xl, xh); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_int32 s0 = v_reinterpret_as_s32(vx_load_expand_q(src)) * vx_setall_s32(kx[0]); + v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0])); for (k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn) - s0 = v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + j)) - v_reinterpret_as_s32(vx_load_expand_q(src - j)), vx_setall_s32(kx[k]), s0); + s0 = v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + j)), v_reinterpret_as_s32(vx_load_expand_q(src - j))), vx_setall_s32(kx[k]), s0); v_store(dst + i, s0); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } } @@ -1038,120 +1038,120 @@ struct SymmColumnVec_32s8u { v_float32 f0 = vx_setall_f32(ky[0]); v_float32 f1 = vx_setall_f32(ky[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const int* S = src[0] + i; v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + v_int32::nlanes)), f0, d4); - v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S + 2*v_int32::nlanes)), f0, d4); - v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S + 3*v_int32::nlanes)), f0, d4); + v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits::vlanes())), f0, d4); + v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S + 2*VTraits::vlanes())), f0, d4); + v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S + 3*VTraits::vlanes())), f0, d4); const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f1, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f1, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2 * v_int32::nlanes) + vx_load(S1 + 2 * v_int32::nlanes)), f1, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3 * v_int32::nlanes) + vx_load(S1 + 3 * v_int32::nlanes)), f1, s3); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, s1); + s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f1, s2); + s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f1, s3); for( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2*v_int32::nlanes) + vx_load(S1 + 2*v_int32::nlanes)), f, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3*v_int32::nlanes) + vx_load(S1 + 3*v_int32::nlanes)), f, s3); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); + s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f, s2); + s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f, s3); } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const int* S = src[0] + i; v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + v_int32::nlanes)), f0, d4); + v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits::vlanes())), f0, d4); const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f1, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f1, s1); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, s1); for( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f, s1); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_load(src[0] + i)), v_setall_f32(ky[0]), v_setall_f32(delta)); - s0 = v_muladd(v_cvt_f32(v_load(src[1] + i) + v_load(src[-1] + i)), v_setall_f32(ky[1]), s0); + v_float32 s0 = v_muladd(v_cvt_f32(vx_load(src[0] + i)), vx_setall_f32(ky[0]), vx_setall_f32(delta)); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), s0); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(v_cvt_f32(v_load(src[k] + i) + v_load(src[-k] + i)), v_setall_f32(ky[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_add(vx_load(src[k] + i), vx_load(src[-k] + i))), vx_setall_f32(ky[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } } else { v_float32 f1 = vx_setall_f32(ky[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f1, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f1, d4); - v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2 * v_int32::nlanes) - vx_load(S1 + 2 * v_int32::nlanes)), f1, d4); - v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3 * v_int32::nlanes) - vx_load(S1 + 3 * v_int32::nlanes)), f1, d4); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4); + v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, d4); + v_float32 s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f1, d4); + v_float32 s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f1, d4); for ( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2*v_int32::nlanes) - vx_load(S1 + 2*v_int32::nlanes)), f, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3*v_int32::nlanes) - vx_load(S1 + 3*v_int32::nlanes)), f, s3); + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); + s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f, s2); + s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f, s3); } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f1, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f1, d4); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4); + v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, d4); for ( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f, s1); + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_load(src[1] + i) - v_load(src[-1] + i)), v_setall_f32(ky[1]), v_setall_f32(delta)); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), vx_setall_f32(delta)); for (k = 2; k <= ksize2; k++) - s0 = v_muladd(v_cvt_f32(v_load(src[k] + i) - v_load(src[-k] + i)), v_setall_f32(ky[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i))), vx_setall_f32(ky[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } } return i; @@ -1187,31 +1187,31 @@ struct SymmColumnVec_32f8u if( symmetrical ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 v_ky0 = vx_setall_f32(ky[0]); v_float32 v32_delta = vx_setall_f32(delta); const float* S = src[0] + i; v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta); - v_float32 s1 = v_muladd(v_ky0, vx_load(S + v_float32::nlanes), v32_delta); - v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*v_float32::nlanes), v32_delta); - v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*v_float32::nlanes), v32_delta); + v_float32 s1 = v_muladd(v_ky0, vx_load(S + VTraits::vlanes()), v32_delta); + v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*VTraits::vlanes()), v32_delta); + v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*VTraits::vlanes()), v32_delta); for( k = 1; k <= ksize2; k++ ) { v_float32 v_kyk = vx_setall_f32(ky[k]); const float* S0 = src[k] + i; const float* S1 = src[-k] + i; - s0 = v_muladd(v_kyk, vx_load(S0) + vx_load(S1), s0); - s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) + vx_load(S1 + v_float32::nlanes), s1); - s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) + vx_load(S1 + 2*v_float32::nlanes), s2); - s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) + vx_load(S1 + 3*v_float32::nlanes), s3); + s0 = v_muladd(v_kyk, v_add(vx_load(S0), vx_load(S1)), s0); + s1 = v_muladd(v_kyk, v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes())), s1); + s2 = v_muladd(v_kyk, v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes())), s2); + s3 = v_muladd(v_kyk, v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes())), s3); } v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } } else { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 s0 = vx_setall_f32(delta); v_float32 s1 = vx_setall_f32(delta); @@ -1222,10 +1222,10 @@ struct SymmColumnVec_32f8u v_float32 v_kyk = vx_setall_f32(ky[k]); const float* S0 = src[k] + i; const float* S1 = src[-k] + i; - s0 = v_muladd(v_kyk, vx_load(S0) - vx_load(S1), s0); - s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) - vx_load(S1 + v_float32::nlanes), s1); - s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) - vx_load(S1 + 2*v_float32::nlanes), s2); - s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) - vx_load(S1 + 3*v_float32::nlanes), s3); + s0 = v_muladd(v_kyk, v_sub(vx_load(S0), vx_load(S1)), s0); + s1 = v_muladd(v_kyk, v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes())), s1); + s2 = v_muladd(v_kyk, v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes())), s2); + s3 = v_muladd(v_kyk, v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes())), s3); } v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } @@ -1268,55 +1268,52 @@ struct SymmColumnSmallVec_32s16s { if( ky[0] == 2 && ky[1] == 1 ) { - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_int32 s0 = vx_load(S1 + i); - v_int32 s1 = vx_load(S1 + i + v_int32::nlanes); - v_int32 s2 = vx_load(S1 + i + 2*v_int32::nlanes); - v_int32 s3 = vx_load(S1 + i + 3*v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) + (s0 + s0), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) + (s1 + s1)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes) + (s2 + s2), - vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes) + (s3 + s3)) + d8); + v_int32 s1 = vx_load(S1 + i + VTraits::vlanes()); + v_int32 s2 = vx_load(S1 + i + 2*VTraits::vlanes()); + v_int32 s3 = vx_load(S1 + i + 3*VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_add(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(s1, s1))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_add(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes())), v_add(s2, s2)), v_add(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes())), v_add(s3, s3))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 sl = vx_load(S1 + i); - v_int32 sh = vx_load(S1 + i + v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) + (sl + sl), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) + (sh + sh)) + d8); - i += v_int16::nlanes; + v_int32 sh = vx_load(S1 + i + VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_add(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(sh, sh))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s = vx_load(S1 + i); - v_pack_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + vx_setall_s32(d) + (s + s)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_add(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s))); + i += VTraits::vlanes(); } } else if( ky[0] == -2 && ky[1] == 1 ) { - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_int32 s0 = vx_load(S1 + i); - v_int32 s1 = vx_load(S1 + i + v_int32::nlanes); - v_int32 s2 = vx_load(S1 + i + 2*v_int32::nlanes); - v_int32 s3 = vx_load(S1 + i + 3*v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) - (s0 + s0), - vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) - (s1 + s1)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes) - (s2 + s2), - vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes) - (s3 + s3)) + d8); + v_int32 s1 = vx_load(S1 + i + VTraits::vlanes()); + v_int32 s2 = vx_load(S1 + i + 2*VTraits::vlanes()); + v_int32 s3 = vx_load(S1 + i + 3*VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_sub(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(s1, s1))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_sub(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes())), v_add(s2, s2)), v_sub(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes())), v_add(s3, s3))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 sl = vx_load(S1 + i); - v_int32 sh = vx_load(S1 + i + v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) - (sl + sl), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) - (sh + sh)) + d8); - i += v_int16::nlanes; + v_int32 sh = vx_load(S1 + i + VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_sub(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(sh, sh))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s = vx_load(S1 + i); - v_pack_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + vx_setall_s32(d) - (s + s)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_sub(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s))); + i += VTraits::vlanes(); } } #if CV_NEON @@ -1347,23 +1344,23 @@ struct SymmColumnSmallVec_32s16s else { v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + v_int32::nlanes)), k0, df4))))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 2*v_int32::nlanes)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 3*v_int32::nlanes)), k0, df4))))); + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits::vlanes())), k0, df4))))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 2*VTraits::vlanes())), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 3*VTraits::vlanes())), k0, df4))))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + v_int32::nlanes)), k0, df4))))); - i += v_int16::nlanes; + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits::vlanes())), k0, df4))))); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4)))); - i += v_int32::nlanes; + v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4)))); + i += VTraits::vlanes(); } } } @@ -1373,42 +1370,42 @@ struct SymmColumnSmallVec_32s16s { if( ky[1] < 0 ) std::swap(S0, S2); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(vx_load(S2 + i) - vx_load(S0 + i), vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S2 + i + 2*v_int32::nlanes) - vx_load(S0 + i + 2*v_int32::nlanes), vx_load(S2 + i + 3*v_int32::nlanes) - vx_load(S0 + i + 3*v_int32::nlanes)) + d8); + v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_sub(vx_load(S2 + i + 2 * VTraits::vlanes()), vx_load(S0 + i + 2 * VTraits::vlanes())), v_sub(vx_load(S2 + i + 3 * VTraits::vlanes()), vx_load(S0 + i + 3 * VTraits::vlanes()))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(vx_load(S2 + i) - vx_load(S0 + i), vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)) + d8); - i += v_int16::nlanes; + v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, vx_load(S2 + i) - vx_load(S0 + i) + vx_setall_s32(d)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), vx_setall_s32(d))); + i += VTraits::vlanes(); } } else { v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)), k1, df4)))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + 2*v_int32::nlanes) - vx_load(S0 + i + 2*v_int32::nlanes)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + 3*v_int32::nlanes) - vx_load(S0 + i + 3*v_int32::nlanes)), k1, df4)))); + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), k1, df4)))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 2 * VTraits::vlanes()), vx_load(S0 + i + 2 * VTraits::vlanes()))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 3 * VTraits::vlanes()), vx_load(S0 + i + 3 * VTraits::vlanes()))), k1, df4)))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)), k1, df4)))); - i += v_int16::nlanes; + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), k1, df4)))); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4))); - i += v_int32::nlanes; + v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4))); + i += VTraits::vlanes(); } } } @@ -1440,7 +1437,7 @@ struct RowVec_16s32f const float* _kx = kernel.ptr(); width *= cn; - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); @@ -1451,18 +1448,18 @@ struct RowVec_16s32f { v_float32 f = vx_setall_f32(_kx[k]); v_int16 xl = vx_load(src); - v_int16 xh = vx_load(src + v_int16::nlanes); + v_int16 xh = vx_load(src + VTraits::vlanes()); s0 = v_muladd(v_cvt_f32(v_expand_low(xl)), f, s0); s1 = v_muladd(v_cvt_f32(v_expand_high(xl)), f, s1); s2 = v_muladd(v_cvt_f32(v_expand_low(xh)), f, s2); s3 = v_muladd(v_cvt_f32(v_expand_high(xh)), f, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); @@ -1475,17 +1472,17 @@ struct RowVec_16s32f s1 = v_muladd(v_cvt_f32(v_expand_high(x)), f, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += v_int16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); for( k = 0; k < _ksize; k++, src += cn ) s0 = v_muladd(v_cvt_f32(vx_load_expand(src)), vx_setall_f32(_kx[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -1524,92 +1521,92 @@ struct SymmColumnVec_32f16s { v_float32 k0 = vx_setall_f32(ky[0]); v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), k0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); - s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) + vx_load(src[-1] + i + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) + vx_load(src[-1] + i + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) + vx_load(src[-1] + i + 3*v_float32::nlanes), k1, s3); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), k0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), k0, d4); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, s1); + s2 = v_muladd(v_add(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, s2); + s3 = v_muladd(v_add(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, s3); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) + vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) + vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); - s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) + vx_load(src[-1] + i + v_float32::nlanes), k1, s1); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, s1); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_int16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } else { v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); - v_float32 s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) - vx_load(src[-1] + i + 2*v_float32::nlanes), k1, d4); - v_float32 s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) - vx_load(src[-1] + i + 3*v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); + v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, d4); + v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) - vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) - vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_int16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } @@ -1682,52 +1679,52 @@ struct RowVec_32f } #endif v_float32 k0 = vx_setall_f32(_kx[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; - v_float32 s1 = vx_load(src + v_float32::nlanes) * k0; - v_float32 s2 = vx_load(src + 2*v_float32::nlanes) * k0; - v_float32 s3 = vx_load(src + 3*v_float32::nlanes) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); + v_float32 s1 = v_mul(vx_load(src + VTraits::vlanes()), k0); + v_float32 s2 = v_mul(vx_load(src + 2 * VTraits::vlanes()), k0); + v_float32 s3 = v_mul(vx_load(src + 3 * VTraits::vlanes()), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) { v_float32 k1 = vx_setall_f32(_kx[k]); s0 = v_muladd(vx_load(src), k1, s0); - s1 = v_muladd(vx_load(src + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src + 3*v_float32::nlanes), k1, s3); + s1 = v_muladd(vx_load(src + VTraits::vlanes()), k1, s1); + s2 = v_muladd(vx_load(src + 2*VTraits::vlanes()), k1, s2); + s3 = v_muladd(vx_load(src + 3*VTraits::vlanes()), k1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; - v_float32 s1 = vx_load(src + v_float32::nlanes) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); + v_float32 s1 = v_mul(vx_load(src + VTraits::vlanes()), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) { v_float32 k1 = vx_setall_f32(_kx[k]); s0 = v_muladd(vx_load(src), k1, s0); - s1 = v_muladd(vx_load(src + v_float32::nlanes), k1, s1); + s1 = v_muladd(vx_load(src + VTraits::vlanes()), k1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) s0 = v_muladd(vx_load(src), vx_setall_f32(_kx[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -1806,28 +1803,28 @@ struct SymmRowSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(kx[0]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, vx_load(src - cn) + vx_load(src + cn))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - cn), vx_load(src + cn)))); #else if( kx[0] > 0 ) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - cn) + vx_load(src + cn) + (x + x)); + v_store(dst + i, v_add(vx_load(src - cn), vx_load(src + cn), x , x)); } else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - cn) + vx_load(src + cn) - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(src - cn), vx_load(src + cn)), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, (vx_load(src - cn) + vx_load(src + cn)) * k1)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1))); } } else if( _ksize == 5 ) @@ -1836,21 +1833,21 @@ struct SymmRowSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(-2); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, vx_load(src - 2*cn) + vx_load(src + 2*cn))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - 2 * cn), vx_load(src + 2 * cn)))); #else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - 2*cn) + vx_load(src + 2*cn) - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(src - 2*cn), vx_load(src + 2*cn)), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src + 2*cn) + vx_load(src - 2*cn), k2, v_muladd(vx_load(src), k0, (vx_load(src - cn) + vx_load(src + cn)) * k1))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_add(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1)))); } } } @@ -1859,20 +1856,20 @@ struct SymmRowSmallVec_32f if( _ksize == 3 ) { if( kx[0] == 0 && kx[1] == 1 ) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, vx_load(src + cn) - vx_load(src - cn)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_sub(vx_load(src + cn), vx_load(src - cn))); else { v_float32 k1 = vx_setall_f32(kx[1]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, (vx_load(src + cn) - vx_load(src - cn)) * k1); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1)); } } else if( _ksize == 5 ) { v_float32 k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src + 2*cn) - vx_load(src - 2*cn), k2, (vx_load(src + cn) - vx_load(src - cn)) * k1)); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_sub(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1))); } } return i; @@ -1961,46 +1958,46 @@ struct SymmColumnVec_32f #endif const v_float32 d4 = vx_setall_f32(delta); const v_float32 k0 = vx_setall_f32(ky[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), k0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), k0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), k0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), k0, d4); for( k = 1; k <= ksize2; k++ ) { v_float32 k1 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) + vx_load(src[-k] + i + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) + vx_load(src[-k] + i + 3*v_float32::nlanes), k1, s3); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k1, s1); + s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k1, s2); + s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); for( k = 1; k <= ksize2; k++ ) { v_float32 k1 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k1, s1); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); for( k = 1; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } else @@ -2042,46 +2039,46 @@ struct SymmColumnVec_32f #endif const v_float32 d4 = vx_setall_f32(delta); const v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); - v_float32 s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) - vx_load(src[-1] + i + 2*v_float32::nlanes), k1, d4); - v_float32 s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) - vx_load(src[-1] + i + 3*v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); + v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, d4); + v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) - vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) - vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } return i; @@ -2123,28 +2120,28 @@ struct SymmColumnSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(ky[0]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S1 + i), k0, vx_load(S0 + i) + vx_load(S2 + i) + d4)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(S1 + i), k0, v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), d4))); #else if(ky[0] > 0) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 x = vx_load(S1 + i); - v_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + d4 + (x + x)); + v_store(dst + i, v_add(vx_load(S0 + i), vx_load(S2 + i), d4, x, x)); } else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 x = vx_load(S1 + i); - v_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + d4 - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i), d4), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4))); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4))); } } else @@ -2153,14 +2150,14 @@ struct SymmColumnSmallVec_32f { if( ky[1] < 0 ) std::swap(S0, S2); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, vx_load(S2 + i) - vx_load(S0 + i) + d4); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), d4)); } else { v_float32 k1 = vx_setall_f32(ky[1]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S2 + i) - vx_load(S0 + i), k1, d4)); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_sub(vx_load(S2 + i), vx_load(S0 + i)), k1, d4)); } } return i; @@ -2199,7 +2196,7 @@ struct FilterVec_8u v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_uint16 xl, xh; v_expand(vx_load(src[0] + i), xl, xh); @@ -2223,7 +2220,7 @@ struct FilterVec_8u } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint32 x0, x1; v_expand(vx_load_expand(src[0] + i), x0, x1); @@ -2237,21 +2234,21 @@ struct FilterVec_8u s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand_q(src[0] + i))), v_setall_f32(kf[0]), v_setall_f32(delta)); + v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), vx_setall_f32(kf[0]), vx_setall_f32(delta)); for( k = 1; k < nz; k++ ) - s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand_q(src[k] + i))), v_setall_f32(kf[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[k] + i))), vx_setall_f32(kf[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } return i; } @@ -2286,7 +2283,7 @@ struct FilterVec_8u16s v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_uint16 xl, xh; v_expand(vx_load(src[0] + i), xl, xh); @@ -2304,9 +2301,9 @@ struct FilterVec_8u16s s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xh))), f, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src[0] + i); v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(x))), f0, d4); @@ -2319,15 +2316,15 @@ struct FilterVec_8u16s s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(x))), f, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), f0, d4); for( k = 1; k < nz; k++ ) s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[k] + i))), vx_setall_f32(kf[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_int32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -2360,46 +2357,46 @@ struct FilterVec_32f v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), f0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), f0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), f0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), f0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), f0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), f0, d4); for( k = 1; k < nz; k++ ) { v_float32 f1 = vx_setall_f32(kf[k]); s0 = v_muladd(vx_load(src[k] + i), f1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes), f1, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes), f1, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes), f1, s3); + s1 = v_muladd(vx_load(src[k] + i + VTraits::vlanes()), f1, s1); + s2 = v_muladd(vx_load(src[k] + i + 2*VTraits::vlanes()), f1, s2); + s3 = v_muladd(vx_load(src[k] + i + 3*VTraits::vlanes()), f1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), f0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), f0, d4); for( k = 1; k < nz; k++ ) { v_float32 f1 = vx_setall_f32(kf[k]); s0 = v_muladd(vx_load(src[k] + i), f1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes), f1, s1); + s1 = v_muladd(vx_load(src[k] + i + VTraits::vlanes()), f1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); for( k = 1; k < nz; k++ ) s0 = v_muladd(vx_load(src[k] + i), vx_setall_f32(kf[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 96ba338f30..9961e9aace 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -1156,13 +1156,13 @@ public: for(; x < numCols; ++x ) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_uint8 v_zero = vx_setzero_u8(); - for(; x <= numCols - 2*v_uint8::nlanes; x += 2*v_uint8::nlanes) { - v_uint8 v_edge1 = (vx_load(edgeData + x ) != v_zero); - v_uint8 v_edge2 = (vx_load(edgeData + x + v_uint8::nlanes) != v_zero); + for(; x <= numCols - 2*VTraits::vlanes(); x += 2*VTraits::vlanes()) { + v_uint8 v_edge1 = (v_ne(vx_load(edgeData + x), v_zero)); + v_uint8 v_edge2 = (v_ne(vx_load(edgeData + x + VTraits::vlanes()), v_zero)); if(v_check_any(v_edge1)) { @@ -1172,7 +1172,7 @@ public: if(v_check_any(v_edge2)) { - x += v_uint8::nlanes + v_scan_forward(v_edge2); + x += VTraits::vlanes() + v_scan_forward(v_edge2); goto _next_step; } } @@ -1183,7 +1183,7 @@ public: if(x == numCols) continue; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) _next_step: #endif float vx, vy; @@ -1514,7 +1514,7 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po int nzCount = 0; const Point* nz_ = &nz[0]; int j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_float32 v_minRadius2 = vx_setall_f32(minRadius2); const v_float32 v_maxRadius2 = vx_setall_f32(maxRadius2); @@ -1522,9 +1522,9 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po v_float32 v_curCenterX = vx_setall_f32(curCenter.x); v_float32 v_curCenterY = vx_setall_f32(curCenter.y); - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[v_float32::nlanes]; - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[v_int32::nlanes]; - for(; j <= nzSz - v_float32::nlanes; j += v_float32::nlanes) + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[VTraits::max_nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[VTraits::max_nlanes]; + for(; j <= nzSz - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_nzX, v_nzY; v_load_deinterleave((const float*)&nz_[j], v_nzX, v_nzY); // FIXIT use proper datatype @@ -1532,16 +1532,16 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po v_float32 v_x = v_cvt_f32(v_reinterpret_as_s32(v_nzX)); v_float32 v_y = v_cvt_f32(v_reinterpret_as_s32(v_nzY)); - v_float32 v_dx = v_x - v_curCenterX; - v_float32 v_dy = v_y - v_curCenterY; + v_float32 v_dx = v_sub(v_x, v_curCenterX); + v_float32 v_dy = v_sub(v_y, v_curCenterY); - v_float32 v_r2 = (v_dx * v_dx) + (v_dy * v_dy); - v_float32 vmask = (v_minRadius2 <= v_r2) & (v_r2 <= v_maxRadius2); + v_float32 v_r2 = v_add(v_mul(v_dx, v_dx), v_mul(v_dy, v_dy)); + v_float32 vmask = v_and(v_le(v_minRadius2, v_r2), v_le(v_r2, v_maxRadius2)); if (v_check_any(vmask)) { v_store_aligned(rmask, v_reinterpret_as_s32(vmask)); v_store_aligned(rbuf, v_r2); - for (int i = 0; i < v_int32::nlanes; ++i) + for (int i = 0; i < VTraits::vlanes(); ++i) if (rmask[i]) ddata[nzCount++] = rbuf[i]; } } @@ -1573,13 +1573,13 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Poi const Range xOuter = Range(std::max(int(curCenter.x - rOuter), 0), std::min(int(curCenter.x + rOuter), positions.cols)); const Range yOuter = Range(std::max(int(curCenter.y - rOuter), 0), std::min(int(curCenter.y + rOuter), positions.rows)); -#if CV_SIMD - float v_seq[v_float32::nlanes]; - for (int i = 0; i < v_float32::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + float v_seq[VTraits::max_nlanes]; + for (int i = 0; i < VTraits::vlanes(); ++i) v_seq[i] = (float)i; const v_float32 v_minRadius2 = vx_setall_f32(minRadius2); const v_float32 v_maxRadius2 = vx_setall_f32(maxRadius2); - const v_float32 v_curCenterX_0123 = vx_setall_f32(curCenter.x) - vx_load(v_seq); + const v_float32 v_curCenterX_0123 = v_sub(vx_setall_f32(curCenter.x), vx_load(v_seq)); #endif for (int y = yOuter.start; y < yOuter.end; y++) @@ -1589,27 +1589,27 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Poi float dy2 = dy * dy; int x = xOuter.start; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_float32 v_dy2 = vx_setall_f32(dy2); const v_uint32 v_zero_u32 = vx_setall_u32(0); - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[v_float32::nlanes]; - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[v_int32::nlanes]; - for (; x <= xOuter.end - v_float32::nlanes; x += v_float32::nlanes) + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[VTraits::max_nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[VTraits::max_nlanes]; + for (; x <= xOuter.end - VTraits::vlanes(); x += VTraits::vlanes()) { v_uint32 v_mask = vx_load_expand_q(ptr + x); - v_mask = v_mask != v_zero_u32; + v_mask = v_ne(v_mask, v_zero_u32); v_float32 v_x = v_cvt_f32(vx_setall_s32(x)); - v_float32 v_dx = v_x - v_curCenterX_0123; + v_float32 v_dx = v_sub(v_x, v_curCenterX_0123); - v_float32 v_r2 = (v_dx * v_dx) + v_dy2; - v_float32 vmask = (v_minRadius2 <= v_r2) & (v_r2 <= v_maxRadius2) & v_reinterpret_as_f32(v_mask); + v_float32 v_r2 = v_add(v_mul(v_dx, v_dx), v_dy2); + v_float32 vmask = v_and(v_and(v_le(v_minRadius2, v_r2), v_le(v_r2, v_maxRadius2)), v_reinterpret_as_f32(v_mask)); if (v_check_any(vmask)) { v_store_aligned(rmask, v_reinterpret_as_s32(vmask)); v_store_aligned(rbuf, v_r2); - for (int i = 0; i < v_int32::nlanes; ++i) + for (int i = 0; i < VTraits::vlanes(); ++i) if (rmask[i]) ddata[nzCount++] = rbuf[i]; } } diff --git a/modules/imgproc/src/stackblur.cpp b/modules/imgproc/src/stackblur.cpp index 5d60a1d365..6becbe5c41 100644 --- a/modules/imgproc/src/stackblur.cpp +++ b/modules/imgproc/src/stackblur.cpp @@ -88,7 +88,7 @@ static unsigned char const stackblurShr[255] = namespace cv{ -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template inline int opRow(const T* , T* , const std::vector& , const float , const int radius, const int CN, const int ) { @@ -107,7 +107,7 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector::vlanes(); if (kernelSize == 3) { @@ -126,10 +126,10 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector>shrValTab; - y01 = (y01 * v_mulVal)>>shrValTab; - y10 = (y10 * v_mulVal)>>shrValTab; - y11 = (y11 * v_mulVal)>>shrValTab; + y00 = v_shr(v_mul(y00, v_mulVal), shrValTab); + y01 = v_shr(v_mul(y01, v_mulVal), shrValTab); + y10 = v_shr(v_mul(y10, v_mulVal), shrValTab); + y11 = v_shr(v_mul(y11, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(v_pack(y00, y01), v_pack(y10, y11))); } @@ -159,12 +159,12 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector(const uchar* srcPtr, uchar* dstPtr, const std::vector>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; - s2 = (s2 * v_mulVal)>>shrValTab; - s3 = (s3 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); + s2 = v_shr(v_mul(s2, v_mulVal), shrValTab); + s3 = v_shr(v_mul(s3, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(v_reinterpret_as_u16(v_pack(s0, s1)), v_reinterpret_as_u16(v_pack(s2, s3)))); } @@ -205,7 +205,7 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector const int mulValTab= stackblurMul[radius]; const int shrValTab= stackblurShr[radius]; - const int VEC_LINE = v_uint16::nlanes; + const int VEC_LINE = VTraits::vlanes(); v_uint32 v_mulVal = vx_setall_u32(mulValTab); if (kernelSize == 3) @@ -220,7 +220,7 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector x1l = v_add(v_add(x1l, x1l), v_add(x0l, x2l)); x1h = v_add(v_add(x1h, x1h), v_add(x0h, x2h)); - v_store(dstPtr + i, v_pack((x1l * v_mulVal)>>shrValTab, (x1h * v_mulVal)>>shrValTab)); + v_store(dstPtr + i, v_pack(v_shr(v_mul(x1l, v_mulVal), shrValTab), v_shr(v_mul(x1h, v_mulVal), shrValTab))); } } else @@ -243,25 +243,25 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector v_uint16 k2 = vx_setall_u16(kx[k + 1]); v_uint32 y0, y1; - v_mul_expand(vx_load(srcPtr - j) + vx_load(srcPtr + j), k1, y0, y1); - s0 += y0; - s1 += y1; - v_mul_expand(vx_load(srcPtr - j - CN) + vx_load(srcPtr + j + CN), k2, y0, y1); - s0 += y0; - s1 += y1; + v_mul_expand(v_add(vx_load(srcPtr - j), vx_load(srcPtr + j)), k1, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); + v_mul_expand(v_add(vx_load(srcPtr - j - CN), vx_load(srcPtr + j + CN)), k2, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); } if( k < kernelSize / 2 + 1 ) { v_uint16 k1 = vx_setall_u16(kx[k]); v_uint32 y0, y1; - v_mul_expand(vx_load(srcPtr - j) + vx_load(srcPtr + j), k1, y0, y1); - s0 += y0; - s1 += y1; + v_mul_expand(v_add(vx_load(srcPtr - j), vx_load(srcPtr + j)), k1, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); } - s0 = (s0 * v_mulVal)>>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(s0, s1)); } @@ -282,7 +282,7 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); if (kernelSize == 3) @@ -297,7 +297,7 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector>shrValTab, (x1h * v_mulVal)>>shrValTab)); + v_store(dstPtr + i, v_pack(v_shr(v_mul(x1l, v_mulVal), shrValTab), v_shr(v_mul(x1h, v_mulVal), shrValTab))); } } else @@ -320,24 +320,24 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(s0, s1)); } @@ -352,7 +352,7 @@ inline int opRow(const float* srcPtr, float* dstPtr, const std::vector::vlanes(); const int VEC_LINE4 = VEC_LINE * 4; if (kernelSize == 3) @@ -364,22 +364,22 @@ inline int opRow(const float* srcPtr, float* dstPtr, const std::vector(const float* srcPtr, float* dstPtr, const std::vector(const float* srcPtr, float* dstPtr, const std::vector inline int opComputeDiff(const uchar*& srcPtr, int*& diff0, const int w, const int CNR1) { int index = 0; - const int VEC_LINE_8 = v_uint8::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_8 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); for (; index <= w - VEC_LINE_8; index += VEC_LINE_8, diff0+=VEC_LINE_8, srcPtr+=VEC_LINE_8) { v_uint16 x0l, x0h, x1l, x1h; @@ -435,8 +435,8 @@ inline int opComputeDiff(const uchar*& srcPtr, int*& diff0, const in v_expand(vx_load(srcPtr), x1l, x1h); v_int32 y0, y1, y2, y3; - v_expand(v_reinterpret_as_s16(x0l) - v_reinterpret_as_s16(x1l), y0, y1); - v_expand(v_reinterpret_as_s16(x0h) - v_reinterpret_as_s16(x1h), y2, y3); + v_expand(v_sub(v_reinterpret_as_s16(x0l), v_reinterpret_as_s16(x1l)), y0, y1); + v_expand(v_sub(v_reinterpret_as_s16(x0h), v_reinterpret_as_s16(x1h)), y2, y3); v_store(diff0, y0); v_store(diff0 + VEC_LINE_32, y1); @@ -517,7 +517,7 @@ public: // middle int wc = radius * CN; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) wc = opRow(srcPtr, dstPtr, kVec, mulVal, radius, CN, widthCN); #endif for (; wc < widthCN; wc++) @@ -586,7 +586,7 @@ public: // middle auto diff0 = diff + radius * CN; int index = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) index = opComputeDiff(srcPtr, diff0, widthCN, CNR1); #endif @@ -688,7 +688,7 @@ private: float mulVal; }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template inline int opColumn(const T* , T* , T* , TBuf* , TBuf* , TBuf* , const float , const int , const int , const int , const int , const int ) @@ -703,7 +703,7 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta { int k = 0; v_float32 v_mulVal = vx_setall_f32(mulVal); - const int VEC_LINE = v_float32::nlanes; + const int VEC_LINE = VTraits::vlanes(); const int VEC_LINE4 = 4 * VEC_LINE; auto stackStartPtr = stack + ss * widthLen; @@ -726,20 +726,20 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_float32 v_sumIn2 = vx_load(sumIn + VEC_LINE * 2 + k); v_float32 v_sumIn3 = vx_load(sumIn + VEC_LINE * 3+ k); - v_store(dstPtr + k, v_sum0 * v_mulVal); - v_store(dstPtr + VEC_LINE + k, v_sum1 * v_mulVal); - v_store(dstPtr + VEC_LINE * 2 + k, v_sum2 * v_mulVal); - v_store(dstPtr + VEC_LINE * 3 + k, v_sum3 * v_mulVal); + v_store(dstPtr + k, v_mul(v_sum0, v_mulVal)); + v_store(dstPtr + VEC_LINE + k, v_mul(v_sum1, v_mulVal)); + v_store(dstPtr + VEC_LINE * 2 + k, v_mul(v_sum2, v_mulVal)); + v_store(dstPtr + VEC_LINE * 3 + k, v_mul(v_sum3, v_mulVal)); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; - v_sum2 -= v_sumOut2; - v_sum3 -= v_sumOut3; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); + v_sum2 = v_sub(v_sum2, v_sumOut2); + v_sum3 = v_sub(v_sum3, v_sumOut3); - v_sumOut0 -= vx_load(stackStartPtr + k); - v_sumOut1 -= vx_load(stackStartPtr + VEC_LINE + k); - v_sumOut2 -= vx_load(stackStartPtr + VEC_LINE * 2 + k); - v_sumOut3 -= vx_load(stackStartPtr + VEC_LINE * 3 + k); + v_sumOut0 = v_sub(v_sumOut0, vx_load(stackStartPtr + k)); + v_sumOut1 = v_sub(v_sumOut1, vx_load(stackStartPtr + VEC_LINE + k)); + v_sumOut2 = v_sub(v_sumOut2, vx_load(stackStartPtr + VEC_LINE * 2 + k)); + v_sumOut3 = v_sub(v_sumOut3, vx_load(stackStartPtr + VEC_LINE * 3 + k)); v_float32 v_srcPtr0 = vx_load(srcPtr + k); v_float32 v_srcPtr1 = vx_load(srcPtr + VEC_LINE + k); @@ -751,35 +751,35 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_store(stackStartPtr + VEC_LINE * 2 + k, v_srcPtr2); v_store(stackStartPtr + VEC_LINE * 3 + k, v_srcPtr3); - v_sumIn0 += v_srcPtr0; - v_sumIn1 += v_srcPtr1; - v_sumIn2 += v_srcPtr2; - v_sumIn3 += v_srcPtr3; + v_sumIn0 = v_add(v_sumIn0, v_srcPtr0); + v_sumIn1 = v_add(v_sumIn1, v_srcPtr1); + v_sumIn2 = v_add(v_sumIn2, v_srcPtr2); + v_sumIn3 = v_add(v_sumIn3, v_srcPtr3); - v_store(sum + k, v_sum0 + v_sumIn0); - v_store(sum + VEC_LINE + k, v_sum1 + v_sumIn1); - v_store(sum + VEC_LINE * 2 + k, v_sum2 + v_sumIn2); - v_store(sum + VEC_LINE * 3 + k, v_sum3 + v_sumIn3); + v_store(sum + k, v_add(v_sum0, v_sumIn0)); + v_store(sum + VEC_LINE + k, v_add(v_sum1, v_sumIn1)); + v_store(sum + VEC_LINE * 2 + k, v_add(v_sum2, v_sumIn2)); + v_store(sum + VEC_LINE * 3 + k, v_add(v_sum3, v_sumIn3)); v_srcPtr0 = vx_load(stackSp1Ptr + k); v_srcPtr1 = vx_load(stackSp1Ptr + VEC_LINE + k); v_srcPtr2 = vx_load(stackSp1Ptr + VEC_LINE * 2 + k); v_srcPtr3 = vx_load(stackSp1Ptr + VEC_LINE * 3 + k); - v_sumOut0 += v_srcPtr0; - v_sumOut1 += v_srcPtr1; - v_sumOut2 += v_srcPtr2; - v_sumOut3 += v_srcPtr3; + v_sumOut0 = v_add(v_sumOut0, v_srcPtr0); + v_sumOut1 = v_add(v_sumOut1, v_srcPtr1); + v_sumOut2 = v_add(v_sumOut2, v_srcPtr2); + v_sumOut3 = v_add(v_sumOut3, v_srcPtr3); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE + k, v_sumOut1); v_store(sumOut + VEC_LINE * 2 + k, v_sumOut2); v_store(sumOut + VEC_LINE * 3 + k, v_sumOut3); - v_sumIn0 -= v_srcPtr0; - v_sumIn1 -= v_srcPtr1; - v_sumIn2 -= v_srcPtr2; - v_sumIn3 -= v_srcPtr3; + v_sumIn0 = v_sub(v_sumIn0, v_srcPtr0); + v_sumIn1 = v_sub(v_sumIn1, v_srcPtr1); + v_sumIn2 = v_sub(v_sumIn2, v_srcPtr2); + v_sumIn3 = v_sub(v_sumIn3, v_srcPtr3); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE + k, v_sumIn1); @@ -793,20 +793,20 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_float32 v_sumOut = vx_load(sumOut + k); v_float32 v_sumIn = vx_load(sumIn + k); - v_store(dstPtr + k, v_sum * v_mulVal); - v_sum -= v_sumOut; - v_sumOut -= vx_load(stackStartPtr + k); + v_store(dstPtr + k, v_mul(v_sum, v_mulVal)); + v_sum = v_sub(v_sum, v_sumOut); + v_sumOut = v_sub(v_sumOut, vx_load(stackStartPtr + k)); v_float32 v_srcPtr = vx_load(srcPtr + k); v_store(stackStartPtr + k, v_srcPtr); - v_sumIn += v_srcPtr; - v_store(sum + k, v_sum + v_sumIn); + v_sumIn = v_add(v_sumIn, v_srcPtr); + v_store(sum + k, v_add(v_sum, v_sumIn)); v_srcPtr = vx_load(stackSp1Ptr + k); - v_sumOut += v_srcPtr; + v_sumOut = v_add(v_sumOut, v_srcPtr); v_store(sumOut + k, v_sumOut); - v_sumIn -= v_srcPtr; + v_sumIn = v_sub(v_sumIn, v_srcPtr); v_store(sumIn + k, v_sumIn); } return k; @@ -820,8 +820,8 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_8 = v_uint8::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_8 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -850,13 +850,13 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack v_store(dstPtr + k, v_pack( - v_reinterpret_as_u16(v_pack((v_sum0 * v_mulVal)>>shrValTab, (v_sum1 * v_mulVal)>>shrValTab)), - v_reinterpret_as_u16(v_pack((v_sum2 * v_mulVal)>>shrValTab, (v_sum3 * v_mulVal)>>shrValTab)))); + v_reinterpret_as_u16(v_pack(v_shr(v_mul(v_sum0, v_mulVal), shrValTab), v_shr(v_mul(v_sum1, v_mulVal), shrValTab))), + v_reinterpret_as_u16(v_pack(v_shr(v_mul(v_sum2, v_mulVal), shrValTab), v_shr(v_mul(v_sum3, v_mulVal), shrValTab))))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; - v_sum2 -= v_sumOut2; - v_sum3 -= v_sumOut3; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); + v_sum2 = v_sub(v_sum2, v_sumOut2); + v_sum3 = v_sub(v_sum3, v_sumOut3); v_uint16 x0l, x0h; v_int32 v_ss0, v_ss1, v_ss2, v_ss3; @@ -865,10 +865,10 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); v_expand(v_reinterpret_as_s16(x0h), v_ss2, v_ss3); - v_sumOut0 -= v_ss0; - v_sumOut1 -= v_ss1; - v_sumOut2 -= v_ss2; - v_sumOut3 -= v_ss3; + v_sumOut0 = v_sub(v_sumOut0, v_ss0); + v_sumOut1 = v_sub(v_sumOut1, v_ss1); + v_sumOut2 = v_sub(v_sumOut2, v_ss2); + v_sumOut3 = v_sub(v_sumOut3, v_ss3); v_expand(vx_load(srcPtr + k), x0l, x0h); v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); @@ -876,34 +876,34 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_8 * sizeof (uchar)); - v_sumIn0 += v_ss0; - v_sumIn1 += v_ss1; - v_sumIn2 += v_ss2; - v_sumIn3 += v_ss3; + v_sumIn0 = v_add(v_sumIn0, v_ss0); + v_sumIn1 = v_add(v_sumIn1, v_ss1); + v_sumIn2 = v_add(v_sumIn2, v_ss2); + v_sumIn3 = v_add(v_sumIn3, v_ss3); - v_store(sum + k, v_sum0 + v_sumIn0); - v_store(sum + VEC_LINE_32 + k, v_sum1 + v_sumIn1); - v_store(sum + VEC_LINE_32 * 2 + k, v_sum2 + v_sumIn2); - v_store(sum + VEC_LINE_32 * 3 + k, v_sum3 + v_sumIn3); + v_store(sum + k, v_add(v_sum0, v_sumIn0)); + v_store(sum + VEC_LINE_32 + k, v_add(v_sum1, v_sumIn1)); + v_store(sum + VEC_LINE_32 * 2 + k, v_add(v_sum2, v_sumIn2)); + v_store(sum + VEC_LINE_32 * 3 + k, v_add(v_sum3, v_sumIn3)); v_expand(vx_load(stackSp1Ptr + k), x0l, x0h); v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); v_expand(v_reinterpret_as_s16(x0h), v_ss2, v_ss3); - v_sumOut0 += v_ss0; - v_sumOut1 += v_ss1; - v_sumOut2 += v_ss2; - v_sumOut3 += v_ss3; + v_sumOut0 = v_add(v_sumOut0, v_ss0); + v_sumOut1 = v_add(v_sumOut1, v_ss1); + v_sumOut2 = v_add(v_sumOut2, v_ss2); + v_sumOut3 = v_add(v_sumOut3, v_ss3); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); v_store(sumOut + VEC_LINE_32 * 2 + k, v_sumOut2); v_store(sumOut + VEC_LINE_32 * 3 + k, v_sumOut3); - v_sumIn0 -= v_ss0; - v_sumIn1 -= v_ss1; - v_sumIn2 -= v_ss2; - v_sumIn3 -= v_ss3; + v_sumIn0 = v_sub(v_sumIn0, v_ss0); + v_sumIn1 = v_sub(v_sumIn1, v_ss1); + v_sumIn2 = v_sub(v_sumIn2, v_ss2); + v_sumIn3 = v_sub(v_sumIn3, v_ss3); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -922,8 +922,8 @@ inline int opColumn(const short* srcPtr, short* dstPtr, short* stack int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_16 = v_int16::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_16 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -943,39 +943,39 @@ inline int opColumn(const short* srcPtr, short* dstPtr, short* stack v_sumOut0 = vx_load(sumOut + k); v_sumOut1 = vx_load(sumOut + k + VEC_LINE_32); - v_store(dstPtr + k,v_pack((v_sum0 * v_mulVal)>>shrValTab, (v_sum1 * v_mulVal)>>shrValTab)); + v_store(dstPtr + k,v_pack(v_shr(v_mul(v_sum0, v_mulVal), shrValTab), v_shr(v_mul(v_sum1, v_mulVal), shrValTab))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); v_int32 v_ss0, v_ss1; v_expand(vx_load(stackStartPtr + k), v_ss0, v_ss1); - v_sumOut0 -= v_ss0; - v_sumOut1 -= v_ss1; + v_sumOut0 = v_sub(v_sumOut0, v_ss0); + v_sumOut1 = v_sub(v_sumOut1, v_ss1); v_expand(vx_load(srcPtr + k), v_ss0, v_ss1); memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_16 * sizeof (short)); - v_sumIn0 += v_ss0; - v_sumIn1 += v_ss1; + v_sumIn0 = v_add(v_sumIn0, v_ss0); + v_sumIn1 = v_add(v_sumIn1, v_ss1); - v_sum0 += v_sumIn0; - v_sum1 += v_sumIn1; + v_sum0 = v_add(v_sum0, v_sumIn0); + v_sum1 = v_add(v_sum1, v_sumIn1); v_store(sum + k, v_sum0); v_store(sum + VEC_LINE_32 + k, v_sum1); v_expand(vx_load(stackSp1Ptr + k), v_ss0, v_ss1); - v_sumOut0 += v_ss0; - v_sumOut1 += v_ss1; + v_sumOut0 = v_add(v_sumOut0, v_ss0); + v_sumOut1 = v_add(v_sumOut1, v_ss1); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); - v_sumIn0 -= v_ss0; - v_sumIn1 -= v_ss1; + v_sumIn0 = v_sub(v_sumIn0, v_ss0); + v_sumIn1 = v_sub(v_sumIn1, v_ss1); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -992,8 +992,8 @@ inline int opColumn(const ushort* srcPtr, ushort* dstPtr, ushort* s int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_16 = v_uint16::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_16 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_uint32 v_mulVal = vx_setall_u32((uint32_t)mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -1013,40 +1013,40 @@ inline int opColumn(const ushort* srcPtr, ushort* dstPtr, ushort* s v_sumOut0 = vx_load(sumOut + k); v_sumOut1 = vx_load(sumOut + k + VEC_LINE_32); - v_store(dstPtr + k, v_pack((v_reinterpret_as_u32(v_sum0) * v_mulVal)>>shrValTab, (v_reinterpret_as_u32(v_sum1) * v_mulVal)>>shrValTab)); + v_store(dstPtr + k, v_pack(v_shr(v_mul(v_reinterpret_as_u32(v_sum0), v_mulVal), shrValTab), v_shr(v_mul(v_reinterpret_as_u32(v_sum1), v_mulVal), shrValTab))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); v_uint32 v_ss0, v_ss1; v_expand(vx_load(stackStartPtr + k), v_ss0, v_ss1); - v_sumOut0 -= v_reinterpret_as_s32(v_ss0); - v_sumOut1 -= v_reinterpret_as_s32(v_ss1); + v_sumOut0 = v_sub(v_sumOut0, v_reinterpret_as_s32(v_ss0)); + v_sumOut1 = v_sub(v_sumOut1, v_reinterpret_as_s32(v_ss1)); v_expand(vx_load(srcPtr + k), v_ss0, v_ss1); memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_16 * sizeof (ushort)); - v_sumIn0 += v_reinterpret_as_s32(v_ss0); - v_sumIn1 += v_reinterpret_as_s32(v_ss1); + v_sumIn0 = v_add(v_sumIn0, v_reinterpret_as_s32(v_ss0)); + v_sumIn1 = v_add(v_sumIn1, v_reinterpret_as_s32(v_ss1)); - v_sum0 += v_sumIn0; - v_sum1 += v_sumIn1; + v_sum0 = v_add(v_sum0, v_sumIn0); + v_sum1 = v_add(v_sum1, v_sumIn1); v_store(sum + k, v_sum0); v_store(sum + VEC_LINE_32 + k, v_sum1); v_expand(vx_load(stackSp1Ptr + k), v_ss0, v_ss1); - v_sumOut0 += v_reinterpret_as_s32(v_ss0); - v_sumOut1 += v_reinterpret_as_s32(v_ss1); + v_sumOut0 = v_add(v_sumOut0, v_reinterpret_as_s32(v_ss0)); + v_sumOut1 = v_add(v_sumOut1, v_reinterpret_as_s32(v_ss1)); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); - v_sumIn0 -= v_reinterpret_as_s32(v_ss0); - v_sumIn1 -= v_reinterpret_as_s32(v_ss1); + v_sumIn0 = v_sub(v_sumIn0, v_reinterpret_as_s32(v_ss0)); + v_sumIn1 = v_sub(v_sumIn1, v_reinterpret_as_s32(v_ss1)); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -1152,7 +1152,7 @@ public: } int k = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) k = opColumn(srcPtr, dstPtr, stack, sum, sumIn, sumOut, mulVal, mulValTab, shrValTab, widthLen, stackStart, sp1); #endif diff --git a/modules/imgproc/src/thresh.cpp b/modules/imgproc/src/thresh.cpp index bed0d37f26..4622691e68 100644 --- a/modules/imgproc/src/thresh.cpp +++ b/modules/imgproc/src/thresh.cpp @@ -190,7 +190,7 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) int j = 0; const uchar* src = _src.ptr(); uchar* dst = _dst.ptr(); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint8 thresh_u = vx_setall_u8( thresh ); v_uint8 maxval16 = vx_setall_u8( maxval ); @@ -199,12 +199,12 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_BINARY: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = thresh_u < v0; - v0 = v0 & maxval16; + v0 = v_lt(thresh_u, v0); + v0 = v_and(v0, maxval16); v_store( dst + j, v0 ); } } @@ -213,12 +213,12 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_BINARY_INV: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = v0 <= thresh_u; - v0 = v0 & maxval16; + v0 = v_le(v0, thresh_u); + v0 = v_and(v0, maxval16); v_store( dst + j, v0 ); } } @@ -227,11 +227,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TRUNC: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = v0 - ( v0 - thresh_u ); + v0 = v_sub(v0, v_sub(v0, thresh_u)); v_store( dst + j, v0 ); } } @@ -240,11 +240,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TOZERO: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = ( thresh_u < v0 ) & v0; + v0 = v_and(v_lt(thresh_u, v0), v0); v_store( dst + j, v0 ); } } @@ -253,11 +253,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TOZERO_INV: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = ( v0 <= thresh_u ) & v0; + v0 = v_and(v_le(v0, thresh_u), v0); v_store( dst + j, v0 ); } } @@ -351,7 +351,7 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) const ushort* src = _src.ptr(); ushort* dst = _dst.ptr(); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_uint16 thresh_u = vx_setall_u16(thresh); v_uint16 maxval16 = vx_setall_u16(maxval); @@ -361,25 +361,25 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) case THRESH_BINARY: for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { - for (j = 0; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (j = 0; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = thresh_u < v0; - v1 = thresh_u < v1; - v0 = v0 & maxval16; - v1 = v1 & maxval16; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_lt(thresh_u, v0); + v1 = v_lt(thresh_u, v1); + v0 = v_and(v0, maxval16); + v1 = v_and(v1, maxval16); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = thresh_u < v0; - v0 = v0 & maxval16; + v0 = v_lt(thresh_u, v0); + v0 = v_and(v0, maxval16); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -391,25 +391,25 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = v0 <= thresh_u; - v1 = v1 <= thresh_u; - v0 = v0 & maxval16; - v1 = v1 & maxval16; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_le(v0, thresh_u); + v1 = v_le(v1, thresh_u); + v0 = v_and(v0, maxval16); + v1 = v_and(v1, maxval16); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = v0 <= thresh_u; - v0 = v0 & maxval16; + v0 = v_le(v0, thresh_u); + v0 = v_and(v0, maxval16); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -421,22 +421,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); + v1 = vx_load(src + j + VTraits::vlanes()); v0 = v_min(v0, thresh_u); v1 = v_min(v1, thresh_u); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); v0 = v_min(v0, thresh_u); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -448,22 +448,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = (thresh_u < v0) & v0; - v1 = (thresh_u < v1) & v1; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_and(v_lt(thresh_u, v0), v0); + v1 = v_and(v_lt(thresh_u, v1), v1); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = (thresh_u < v0) & v0; + v0 = v_and(v_lt(thresh_u, v0), v0); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -475,22 +475,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = (v0 <= thresh_u) & v0; - v1 = (v1 <= thresh_u) & v1; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_and(v_le(v0, thresh_u), v0); + v1 = v_and(v_le(v1, thresh_u), v1); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = (v0 <= thresh_u) & v0; + v0 = v_and(v_le(v0, thresh_u), v0); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -571,7 +571,7 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) } #endif -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_int16 thresh8 = vx_setall_s16( thresh ); v_int16 maxval8 = vx_setall_s16( maxval ); @@ -582,25 +582,25 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = thresh8 < v0; - v1 = thresh8 < v1; - v0 = v0 & maxval8; - v1 = v1 & maxval8; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh8, v0); + v1 = v_lt(thresh8, v1); + v0 = v_and(v0, maxval8); + v1 = v_and(v1, maxval8); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = thresh8 < v0; - v0 = v0 & maxval8; + v0 = v_lt(thresh8, v0); + v0 = v_and(v0, maxval8); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -612,25 +612,25 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = v0 <= thresh8; - v1 = v1 <= thresh8; - v0 = v0 & maxval8; - v1 = v1 & maxval8; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh8); + v1 = v_le(v1, thresh8); + v0 = v_and(v0, maxval8); + v1 = v_and(v1, maxval8); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = v0 <= thresh8; - v0 = v0 & maxval8; + v0 = v_le(v0, thresh8); + v0 = v_and(v0, maxval8); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -642,22 +642,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh8 ); v1 = v_min( v1, thresh8 ); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); v0 = v_min( v0, thresh8 ); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -669,22 +669,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = ( thresh8 < v0 ) & v0; - v1 = ( thresh8 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh8, v0), v0); + v1 = v_and(v_lt(thresh8, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = ( thresh8 < v0 ) & v0; + v0 = v_and(v_lt(thresh8, v0), v0); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -696,22 +696,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = ( v0 <= thresh8 ) & v0; - v1 = ( v1 <= thresh8 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh8), v0); + v1 = v_and(v_le(v1, thresh8), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh8 ) & v0; + v0 = v_and(v_le(v0, thresh8), v0); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -777,7 +777,7 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) } #endif -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_float32 thresh4 = vx_setall_f32( thresh ); v_float32 maxval4 = vx_setall_f32( maxval ); @@ -788,25 +788,25 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = thresh4 < v0; - v1 = thresh4 < v1; - v0 = v0 & maxval4; - v1 = v1 & maxval4; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh4, v0); + v1 = v_lt(thresh4, v1); + v0 = v_and(v0, maxval4); + v1 = v_and(v1, maxval4); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = thresh4 < v0; - v0 = v0 & maxval4; + v0 = v_lt(thresh4, v0); + v0 = v_and(v0, maxval4); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -818,25 +818,25 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = v0 <= thresh4; - v1 = v1 <= thresh4; - v0 = v0 & maxval4; - v1 = v1 & maxval4; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh4); + v1 = v_le(v1, thresh4); + v0 = v_and(v0, maxval4); + v1 = v_and(v1, maxval4); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = v0 <= thresh4; - v0 = v0 & maxval4; + v0 = v_le(v0, thresh4); + v0 = v_and(v0, maxval4); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -848,22 +848,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh4 ); v1 = v_min( v1, thresh4 ); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); v0 = v_min( v0, thresh4 ); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -875,22 +875,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = ( thresh4 < v0 ) & v0; - v1 = ( thresh4 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh4, v0), v0); + v1 = v_and(v_lt(thresh4, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = ( thresh4 < v0 ) & v0; + v0 = v_and(v_lt(thresh4, v0), v0); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -902,22 +902,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = ( v0 <= thresh4 ) & v0; - v1 = ( v1 <= thresh4 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh4), v0); + v1 = v_and(v_le(v1, thresh4), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh4 ) & v0; + v0 = v_and(v_le(v0, thresh4), v0); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -948,7 +948,7 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) roi.height = 1; } -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) int i, j; v_float64 thresh2 = vx_setall_f64( thresh ); v_float64 maxval2 = vx_setall_f64( maxval ); @@ -959,25 +959,25 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = thresh2 < v0; - v1 = thresh2 < v1; - v0 = v0 & maxval2; - v1 = v1 & maxval2; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh2, v0); + v1 = v_lt(thresh2, v1); + v0 = v_and(v0, maxval2); + v1 = v_and(v1, maxval2); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = thresh2 < v0; - v0 = v0 & maxval2; + v0 = v_lt(thresh2, v0); + v0 = v_and(v0, maxval2); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -989,25 +989,25 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = v0 <= thresh2; - v1 = v1 <= thresh2; - v0 = v0 & maxval2; - v1 = v1 & maxval2; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh2); + v1 = v_le(v1, thresh2); + v0 = v_and(v0, maxval2); + v1 = v_and(v1, maxval2); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = v0 <= thresh2; - v0 = v0 & maxval2; + v0 = v_le(v0, thresh2); + v0 = v_and(v0, maxval2); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1019,22 +1019,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh2 ); v1 = v_min( v1, thresh2 ); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); v0 = v_min( v0, thresh2 ); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1046,22 +1046,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = ( thresh2 < v0 ) & v0; - v1 = ( thresh2 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh2, v0), v0); + v1 = v_and(v_lt(thresh2, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = ( thresh2 < v0 ) & v0; + v0 = v_and(v_lt(thresh2, v0), v0); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1073,22 +1073,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = ( v0 <= thresh2 ) & v0; - v1 = ( v1 <= thresh2 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh2), v0); + v1 = v_and(v_le(v1, thresh2), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh2 ) & v0; + v0 = v_and(v_le(v0, thresh2), v0); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) From b870ad46bf1a9bb5f4310db1e09a20f4265b823f Mon Sep 17 00:00:00 2001 From: Kumataro Date: Tue, 19 Sep 2023 16:32:47 +0900 Subject: [PATCH 17/34] Merge pull request #24074 from Kumataro/fix24057 Python: support tuple src for cv::add()/subtract()/... #24074 fix https://github.com/opencv/opencv/issues/24057 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ x The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/include/opencv2/core.hpp | 15 ++++ modules/python/src2/cv2.hpp | 10 ++- modules/python/src2/cv2_convert.cpp | 62 ++++++++++++-- modules/python/src2/cv2_convert.hpp | 4 +- modules/python/src2/gen2.py | 14 +++- modules/python/src2/hdr_parser.py | 9 ++ modules/python/test/test_misc.py | 113 ++++++++++++++++++++++++++ 7 files changed, 211 insertions(+), 16 deletions(-) diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 7b5108fcc4..3cd5901af4 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -349,6 +349,9 @@ be set to the default -1. In this case, the output array will have the same dept array, be it src1, src2 or both. @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`add(src,X)` means `add(src,(X,X,X,X))`. +`add(src,(X,))` means `add(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array that has the same size and number of channels as the input array(s); the @@ -390,6 +393,9 @@ in the first case, when src1.depth() == src2.depth(), dtype can be set to the de case the output array will have the same depth as the input array, be it src1, src2 or both. @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`subtract(src,X)` means `subtract(src,(X,X,X,X))`. +`subtract(src,(X,))` means `subtract(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array of the same size and the same number of channels as the input array. @@ -415,6 +421,9 @@ For a not-per-element matrix product, see gemm . @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`multiply(src,X)` means `multiply(src,(X,X,X,X))`. +`multiply(src,(X,))` means `multiply(src,(X,0,0,0))`. @param src1 first input array. @param src2 second input array of the same size and the same type as src1. @param dst output array of the same size and type as src1. @@ -443,6 +452,9 @@ Expect correct IEEE-754 behaviour for floating-point data (with NaN, Inf result @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`divide(src,X)` means `divide(src,(X,X,X,X))`. +`divide(src,(X,))` means `divide(src,(X,0,0,0))`. @param src1 first input array. @param src2 second input array of the same size and type as src1. @param scale scalar factor. @@ -1412,6 +1424,9 @@ The function cv::absdiff calculates: multi-channel arrays, each channel is processed independently. @note Saturation is not applied when the arrays have the depth CV_32S. You may even get a negative value in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`absdiff(src,X)` means `absdiff(src,(X,X,X,X))`. +`absdiff(src,(X,))` means `absdiff(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array that has the same size and type as input arrays. diff --git a/modules/python/src2/cv2.hpp b/modules/python/src2/cv2.hpp index 9293a593f2..b7992582ad 100644 --- a/modules/python/src2/cv2.hpp +++ b/modules/python/src2/cv2.hpp @@ -39,12 +39,20 @@ class ArgInfo { +private: + static const uint32_t arg_outputarg_flag = 0x1; + static const uint32_t arg_arithm_op_src_flag = 0x2; + public: const char* name; bool outputarg; + bool arithm_op_src; // more fields may be added if necessary - ArgInfo(const char* name_, bool outputarg_) : name(name_), outputarg(outputarg_) {} + ArgInfo(const char* name_, uint32_t arg_) : + name(name_), + outputarg((arg_ & arg_outputarg_flag) != 0), + arithm_op_src((arg_ & arg_arithm_op_src_flag) != 0) {} private: ArgInfo(const ArgInfo&) = delete; diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp index e9e1fed4fd..40e1608fae 100644 --- a/modules/python/src2/cv2_convert.cpp +++ b/modules/python/src2/cv2_convert.cpp @@ -63,20 +63,39 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) if( PyInt_Check(o) ) { double v[] = {static_cast(PyInt_AsLong((PyObject*)o)), 0., 0., 0.}; + if ( info.arithm_op_src ) + { + // Normally cv.XXX(x) means cv.XXX( (x, 0., 0., 0.) ); + // However cv.add(mat,x) means cv::add(mat, (x,x,x,x) ). + v[1] = v[0]; + v[2] = v[0]; + v[3] = v[0]; + } m = Mat(4, 1, CV_64F, v).clone(); return true; } if( PyFloat_Check(o) ) { double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.}; + + if ( info.arithm_op_src ) + { + // Normally cv.XXX(x) means cv.XXX( (x, 0., 0., 0.) ); + // However cv.add(mat,x) means cv::add(mat, (x,x,x,x) ). + v[1] = v[0]; + v[2] = v[0]; + v[3] = v[0]; + } m = Mat(4, 1, CV_64F, v).clone(); return true; } if( PyTuple_Check(o) ) { - int i, sz = (int)PyTuple_Size((PyObject*)o); - m = Mat(sz, 1, CV_64F); - for( i = 0; i < sz; i++ ) + // see https://github.com/opencv/opencv/issues/24057 + const int sz = (int)PyTuple_Size((PyObject*)o); + const int sz2 = info.arithm_op_src ? std::max(4, sz) : sz; // Scalar has 4 elements. + m = Mat::zeros(sz2, 1, CV_64F); + for( int i = 0; i < sz; i++ ) { PyObject* oi = PyTuple_GetItem(o, i); if( PyInt_Check(oi) ) @@ -241,6 +260,31 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } } + // see https://github.com/opencv/opencv/issues/24057 + if ( ( info.arithm_op_src ) && ( ndims == 1 ) && ( size[0] <= 4 ) ) + { + const int sz = size[0]; // Real Data Length(1, 2, 3 or 4) + const int sz2 = 4; // Scalar has 4 elements. + m = Mat::zeros(sz2, 1, CV_64F); + + const char *base_ptr = PyArray_BYTES(oarr); + for(int i = 0; i < sz; i++ ) + { + PyObject* oi = PyArray_GETITEM(oarr, base_ptr + step[0] * i); + if( PyInt_Check(oi) ) + m.at(i) = (double)PyInt_AsLong(oi); + else if( PyFloat_Check(oi) ) + m.at(i) = (double)PyFloat_AsDouble(oi); + else + { + failmsg("%s has some non-numerical elements", info.name); + m.release(); + return false; + } + } + return true; + } + // handle degenerate case // FIXIT: Don't force 1D for Scalars if( ndims == 0) { @@ -807,7 +851,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String centerItemName = format("'%s' center point", info.name); - const ArgInfo centerItemInfo(centerItemName.c_str(), false); + const ArgInfo centerItemInfo(centerItemName.c_str(), 0); SafeSeqItem centerItem(obj, 0); if (!pyopencv_to(centerItem.item, dst.center, centerItemInfo)) { @@ -816,7 +860,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String sizeItemName = format("'%s' size", info.name); - const ArgInfo sizeItemInfo(sizeItemName.c_str(), false); + const ArgInfo sizeItemInfo(sizeItemName.c_str(), 0); SafeSeqItem sizeItem(obj, 1); if (!pyopencv_to(sizeItem.item, dst.size, sizeItemInfo)) { @@ -825,7 +869,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String angleItemName = format("'%s' angle", info.name); - const ArgInfo angleItemInfo(angleItemName.c_str(), false); + const ArgInfo angleItemInfo(angleItemName.c_str(), 0); SafeSeqItem angleItem(obj, 2); if (!pyopencv_to(angleItem.item, dst.angle, angleItemInfo)) { @@ -1075,7 +1119,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String typeItemName = format("'%s' criteria type", info.name); - const ArgInfo typeItemInfo(typeItemName.c_str(), false); + const ArgInfo typeItemInfo(typeItemName.c_str(), 0); SafeSeqItem typeItem(obj, 0); if (!pyopencv_to(typeItem.item, dst.type, typeItemInfo)) { @@ -1084,7 +1128,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String maxCountItemName = format("'%s' max count", info.name); - const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), false); + const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), 0); SafeSeqItem maxCountItem(obj, 1); if (!pyopencv_to(maxCountItem.item, dst.maxCount, maxCountItemInfo)) { @@ -1093,7 +1137,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String epsilonItemName = format("'%s' epsilon", info.name); - const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), false); + const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), 0); SafeSeqItem epsilonItem(obj, 2); if (!pyopencv_to(epsilonItem.item, dst.epsilon, epsilonItemInfo)) { diff --git a/modules/python/src2/cv2_convert.hpp b/modules/python/src2/cv2_convert.hpp index 43ef7b2302..96a30e521f 100644 --- a/modules/python/src2/cv2_convert.hpp +++ b/modules/python/src2/cv2_convert.hpp @@ -286,13 +286,13 @@ bool pyopencv_to(PyObject *obj, std::map &map, const ArgInfo& info) while(PyDict_Next(obj, &pos, &py_key, &py_value)) { K cpp_key; - if (!pyopencv_to(py_key, cpp_key, ArgInfo("key", false))) { + if (!pyopencv_to(py_key, cpp_key, ArgInfo("key", 0))) { failmsg("Can't parse dict key. Key on position %lu has a wrong type", pos); return false; } V cpp_value; - if (!pyopencv_to(py_value, cpp_value, ArgInfo("value", false))) { + if (!pyopencv_to(py_value, cpp_value, ArgInfo("value", 0))) { failmsg("Can't parse dict value. Value on position %lu has a wrong type", pos); return false; } diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 95bd0495cf..78f9c79b06 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -109,7 +109,7 @@ gen_template_set_prop_from_map = Template(""" if( PyMapping_HasKeyString(src, (char*)"$propname") ) { tmp = PyMapping_GetItemString(src, (char*)"$propname"); - ok = tmp && pyopencv_to_safe(tmp, dst.$propname, ArgInfo("$propname", false)); + ok = tmp && pyopencv_to_safe(tmp, dst.$propname, ArgInfo("$propname", 0)); Py_DECREF(tmp); if(!ok) return false; }""") @@ -163,7 +163,7 @@ static int pyopencv_${name}_set_${member}(pyopencv_${name}_t* p, PyObject *value PyErr_SetString(PyExc_TypeError, "Cannot delete the ${member} attribute"); return -1; } - return pyopencv_to_safe(value, p->v${access}${member}, ArgInfo("value", false)) ? 0 : -1; + return pyopencv_to_safe(value, p->v${access}${member}, ArgInfo("value", 0)) ? 0 : -1; } """) @@ -181,7 +181,7 @@ static int pyopencv_${name}_set_${member}(pyopencv_${name}_t* p, PyObject *value failmsgp("Incorrect type of object (must be '${name}' or its derivative)"); return -1; } - return pyopencv_to_safe(value, _self_${access}${member}, ArgInfo("value", false)) ? 0 : -1; + return pyopencv_to_safe(value, _self_${access}${member}, ArgInfo("value", 0)) ? 0 : -1; } """) @@ -492,6 +492,10 @@ class ArgInfo(object): def inputarg(self): return '/O' not in self._modifiers + @property + def arithm_op_src_arg(self): + return '/AOS' in self._modifiers + @property def outputarg(self): return '/O' in self._modifiers or '/IO' in self._modifiers @@ -517,7 +521,9 @@ class ArgInfo(object): "UMat", "vector_UMat"] # or self.tp.startswith("vector") def crepr(self): - return "ArgInfo(\"%s\", %d)" % (self.name, self.outputarg) + arg = 0x01 if self.outputarg else 0x0 + arg += 0x02 if self.arithm_op_src_arg else 0x0 + return "ArgInfo(\"%s\", %d)" % (self.name, arg) def find_argument_class_info(argument_type, function_namespace, diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 710c792179..0dc5dd1488 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -535,6 +535,13 @@ class CppHeaderParser(object): funcname = self.get_dotted_name(funcname) + # see https://github.com/opencv/opencv/issues/24057 + is_arithm_op_func = funcname in {"cv.add", + "cv.subtract", + "cv.absdiff", + "cv.multiply", + "cv.divide"} + if not self.wrap_mode: decl = self.parse_func_decl_no_wrap(decl_str, static_method, docstring) decl[0] = funcname @@ -595,6 +602,8 @@ class CppHeaderParser(object): if arg_type == "InputArray": arg_type = mat + if is_arithm_op_func: + modlist.append("/AOS") # Arithm Ope Source elif arg_type == "InputOutputArray": arg_type = mat modlist.append("/IO") diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 9f7406587c..51ece30dc6 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -42,6 +42,93 @@ def get_conversion_error_msg(value, expected, actual): def get_no_exception_msg(value): return 'Exception is not risen for {} of type {}'.format(value, type(value).__name__) + +def rpad(src, dst_size, pad_value=0): + """Extend `src` up to `dst_size` with given value. + + Args: + src (np.ndarray | tuple | list): 1d array like object to pad. + dst_size (_type_): Desired `src` size after padding. + pad_value (int, optional): Padding value. Defaults to 0. + + Returns: + np.ndarray: 1d array with len == `dst_size`. + """ + src = np.asarray(src) + if len(src.shape) != 1: + raise ValueError("Only 1d arrays are supported") + + # Considering the meaning, it is desirable to use np.pad(). + # However, the old numpy doesn't include the following fixes and cannot work as expected. + # So an alternative fix that combines np.append() and np.fill() is used. + # https://docs.scipy.org/doc/numpy-1.13.0/release.html#support-for-returning-arrays-of-arbitrary-dimensions-in-apply-along-axis + + return np.append(src, np.full( dst_size - len(src), pad_value, dtype=src.dtype) ) + +def get_ocv_arithm_op_table(apply_saturation=False): + def saturate(func): + def wrapped_func(x, y): + dst_dtype = x.dtype + if apply_saturation: + if np.issubdtype(x.dtype, np.integer): + x = x.astype(np.int64) + # Apply padding or truncation for array-like `y` inputs + if not isinstance(y, (float, int)): + if len(y) > x.shape[-1]: + y = y[:x.shape[-1]] + else: + y = rpad(y, x.shape[-1], pad_value=0) + + dst = func(x, y) + if apply_saturation: + min_val, max_val = get_limits(dst_dtype) + dst = np.clip(dst, min_val, max_val) + return dst.astype(dst_dtype) + return wrapped_func + + @saturate + def subtract(x, y): + return x - y + + @saturate + def add(x, y): + return x + y + + @saturate + def divide(x, y): + if not isinstance(y, (int, float)): + dst_dtype = np.result_type(x, y) + y = np.array(y).astype(dst_dtype) + _, max_value = get_limits(dst_dtype) + y[y == 0] = max_value + + # to compatible between python2 and python3, it calicurates with float. + # python2: int / int = int + # python3: int / int = float + dst = 1.0 * x / y + + if np.issubdtype(x.dtype, np.integer): + dst = np.rint(dst) + return dst + + @saturate + def multiply(x, y): + return x * y + + @saturate + def absdiff(x, y): + res = np.abs(x - y) + return res + + return { + cv.subtract: subtract, + cv.add: add, + cv.multiply: multiply, + cv.divide: divide, + cv.absdiff: absdiff + } + + class Bindings(NewOpenCVTests): def test_inheritance(self): @@ -816,6 +903,32 @@ class Arguments(NewOpenCVTests): np.testing.assert_equal(dst, src_copy) self.assertEqual(arguments_dump, 'lambda=25, sigma=5.5') + def test_arithm_op_without_saturation(self): + np.random.seed(4231568) + src = np.random.randint(20, 40, 8 * 4 * 3).astype(np.uint8).reshape(8, 4, 3) + operations = get_ocv_arithm_op_table(apply_saturation=False) + for ocv_op, numpy_op in operations.items(): + for val in (2, 4, (5, ), (6, 4), (2., 4., 1.), + np.uint8([1, 2, 2]), np.float64([5, 2, 6, 3]),): + dst = ocv_op(src, val) + expected = numpy_op(src, val) + # Temporarily allows a difference of 1 for arm64 workaround. + self.assertLess(np.max(np.abs(dst - expected)), 2, + msg="Operation '{}' is failed for {}".format(ocv_op.__name__, val ) ) + + def test_arithm_op_with_saturation(self): + np.random.seed(4231568) + src = np.random.randint(20, 40, 4 * 8 * 4).astype(np.uint8).reshape(4, 8, 4) + operations = get_ocv_arithm_op_table(apply_saturation=True) + + for ocv_op, numpy_op in operations.items(): + for val in (10, 4, (40, ), (15, 12), (25., 41., 15.), + np.uint8([1, 2, 20]), np.float64([50, 21, 64, 30]),): + dst = ocv_op(src, val) + expected = numpy_op(src, val) + # Temporarily allows a difference of 1 for arm64 workaround. + self.assertLess(np.max(np.abs(dst - expected)), 2, + msg="Saturated Operation '{}' is failed for {}".format(ocv_op.__name__, val ) ) class CanUsePurePythonModuleFunction(NewOpenCVTests): def test_can_get_ocv_version(self): From 320c0bf419043e0179d47742f9f0ebd7885b5457 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 19 Sep 2023 20:12:52 +0800 Subject: [PATCH 18/34] Merge pull request #24166 from hanliutong:rewrite-remaining Rewrite Universal Intrinsic code: ImgProc (CV_SIMD_WIDTH related Part) #24166 Related PR: #24058, #24132. The goal of this series of PRs is to modify the SIMD code blocks in the opencv/modules/imgproc folder by using the new Universal Intrinsic API. The modification of this PR mainly focuses on the code that uses the `CV_SIMD_WIDTH` macro. This macro is sometimes used for loop tail processing, such as `box_filter.simd.hpp` and `morph.simd.hpp`. ```cpp #if CV_SIMD int i = 0; for (i < n - v_uint16::nlanes; i += v_uint16::nlanes) { // some universal intrinsic code // e.g. v_uint16... } #if CV_SIMD_WIDTH > 16 for (i < n - v_uint16x8::nlanes; i += v_uint16x8::nlanes) { // handle loop tail by 128 bit SIMD // e.g. v_uint16x8 } #endif //CV_SIMD_WIDTH #endif// CV_SIMD ``` The main contradiction is that the variable-length Universal Intrinsic backend cannot use 128bit fixed-length data structures. Therefore, this PR uses the scalar loop to handle the loop tail. This PR is marked as draft because the modification of the `box_filter.simd.hpp` file caused a compilation error. The cause of the error is initially believed to be due to an internal error in the GCC compiler. ```bash box_filter.simd.hpp:1162:5: internal compiler error: Segmentation fault 1162 | } | ^ 0xe03883 crash_signal /wafer/share/gcc/gcc/toplev.cc:314 0x7ff261c4251f ??? ./signal/../sysdeps/unix/sysv/linux/x86_64/libc_sigaction.c:0 0x6bde48 hash_set >::iterator::operator*() /wafer/share/gcc/gcc/hash-set.h:125 0x6bde48 extract_single_source /wafer/share/gcc/gcc/config/riscv/riscv-vsetvl.cc:1184 0x6bde48 extract_single_source /wafer/share/gcc/gcc/config/riscv/riscv-vsetvl.cc:1174 0x119ad9e pass_vsetvl::propagate_avl() const /wafer/share/gcc/gcc/config/riscv/riscv-vsetvl.cc:4087 0x119ceaf pass_vsetvl::execute(function*) /wafer/share/gcc/gcc/config/riscv/riscv-vsetvl.cc:4344 0x119ceaf pass_vsetvl::execute(function*) /wafer/share/gcc/gcc/config/riscv/riscv-vsetvl.cc:4325 Please submit a full bug report, with preprocessed source (by using -freport-bug). Please include the complete backtrace with any bug report. ``` This PR can be compiled with Clang 16, and `opencv_test_imgproc` is passed on QEMU. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/box_filter.simd.hpp | 316 ++++++++++++------------ modules/imgproc/src/morph.simd.hpp | 170 ++++++------- modules/imgproc/src/shapedescr.cpp | 52 ++-- 3 files changed, 269 insertions(+), 269 deletions(-) diff --git a/modules/imgproc/src/box_filter.simd.hpp b/modules/imgproc/src/box_filter.simd.hpp index 4a4d205216..735935c04f 100644 --- a/modules/imgproc/src/box_filter.simd.hpp +++ b/modules/imgproc/src/box_filter.simd.hpp @@ -309,15 +309,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -339,37 +339,37 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width - v_uint16::nlanes; i += v_uint16::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * _v_scale)); - v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * _v_scale)); + v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), _v_scale))); + v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), _v_scale))); v_uint16 v_dst = v_pack(v_s0d, v_s01d); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * v_scale)); - v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * v_scale)); + v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); + v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); v_uint16x8 v_dst = v_pack(v_s0d, v_s01d); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -383,29 +383,29 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_uint16 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_uint16x8 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -480,15 +480,15 @@ public BaseColumnFilter { const ushort* Sp = (const ushort*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_uint16::nlanes; i += v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_uint16x8::nlanes; i += v_uint16x8::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -510,27 +510,27 @@ public BaseColumnFilter if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint32 _ds4 = vx_setall_u32((unsigned)ds); v_uint16 _dd8 = vx_setall_u16((ushort)dd); - for( ; i <= width-v_uint8::nlanes; i+=v_uint8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_uint16 _sm0 = vx_load(Sm + i); - v_uint16 _sm1 = vx_load(Sm + i + v_uint16::nlanes); + v_uint16 _sm1 = vx_load(Sm + i + VTraits::vlanes()); v_uint16 _s0 = v_add_wrap(vx_load(SUM + i), vx_load(Sp + i)); - v_uint16 _s1 = v_add_wrap(vx_load(SUM + i + v_uint16::nlanes), vx_load(Sp + i + v_uint16::nlanes)); + v_uint16 _s1 = v_add_wrap(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_uint32 _s00, _s01, _s10, _s11; - v_expand(_s0 + _dd8, _s00, _s01); - v_expand(_s1 + _dd8, _s10, _s11); + v_expand(v_add(_s0, _dd8), _s00, _s01); + v_expand(v_add(_s1, _dd8), _s10, _s11); - _s00 = v_shr(_s00*_ds4); - _s01 = v_shr(_s01*_ds4); - _s10 = v_shr(_s10*_ds4); - _s11 = v_shr(_s11*_ds4); + _s00 = v_shr(v_mul(_s00, _ds4)); + _s01 = v_shr(v_mul(_s01, _ds4)); + _s10 = v_shr(v_mul(_s10, _ds4)); + _s11 = v_shr(v_mul(_s11, _ds4)); v_int16 r0 = v_pack(v_reinterpret_as_s32(_s00), v_reinterpret_as_s32(_s01)); v_int16 r1 = v_pack(v_reinterpret_as_s32(_s10), v_reinterpret_as_s32(_s11)); @@ -540,9 +540,9 @@ public BaseColumnFilter v_store(D + i, v_pack_u(r0, r1)); v_store(SUM + i, _s0); - v_store(SUM + i + v_uint16::nlanes, _s1); + v_store(SUM + i + VTraits::vlanes(), _s1); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_uint32x4 ds4 = v_setall_u32((unsigned)ds); v_uint16x8 dd8 = v_setall_u16((ushort)dd); @@ -556,13 +556,13 @@ public BaseColumnFilter v_uint32x4 _s00, _s01, _s10, _s11; - v_expand(_s0 + dd8, _s00, _s01); - v_expand(_s1 + dd8, _s10, _s11); + v_expand(v_add(_s0, dd8), _s00, _s01); + v_expand(v_add(_s1, dd8), _s10, _s11); - _s00 = v_shr(_s00*ds4); - _s01 = v_shr(_s01*ds4); - _s10 = v_shr(_s10*ds4); - _s11 = v_shr(_s11*ds4); + _s00 = v_shr(v_mul(_s00, ds4)); + _s01 = v_shr(v_mul(_s01, ds4)); + _s10 = v_shr(v_mul(_s10, ds4)); + _s11 = v_shr(v_mul(_s11, ds4)); v_int16x8 r0 = v_pack(v_reinterpret_as_s32(_s00), v_reinterpret_as_s32(_s01)); v_int16x8 r1 = v_pack(v_reinterpret_as_s32(_s10), v_reinterpret_as_s32(_s11)); @@ -643,15 +643,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -673,33 +673,33 @@ struct ColumnSum : if( haveScale ) { i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_int16::nlanes; i+=v_int16::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_int32 v_s0d = v_round(v_cvt_f32(v_s0) * _v_scale); - v_int32 v_s01d = v_round(v_cvt_f32(v_s01) * _v_scale); + v_int32 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), _v_scale)); + v_int32 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), _v_scale)); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_int32x4 v_s0d = v_round(v_cvt_f32(v_s0) * v_scale); - v_int32x4 v_s01d = v_round(v_cvt_f32(v_s01) * v_scale); + v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); + v_int32x4 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), v_scale)); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -713,27 +713,27 @@ struct ColumnSum : else { i = 0; -#if CV_SIMD - for( ; i <= width-v_int16::nlanes; i+=v_int16::nlanes ) +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, GCC 13 related + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_s0, v_s01)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_store(D + i, v_pack(v_s0, v_s01)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -792,15 +792,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -822,33 +822,33 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * _v_scale)); - v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * _v_scale)); + v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), _v_scale))); + v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), _v_scale))); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * v_scale)); - v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * v_scale)); + v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); + v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -862,27 +862,27 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01))); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01))); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -939,15 +939,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -969,25 +969,25 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s0d = v_round(v_cvt_f32(v_s0) * _v_scale); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), _v_scale)); v_store(D + i, v_s0d); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s0d = v_round(v_cvt_f32(v_s0) * v_scale); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); v_store(D + i, v_s0d); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1001,21 +1001,21 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); v_store(D + i, v_s0); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_s0); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1073,15 +1073,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -1105,21 +1105,21 @@ struct ColumnSum : { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_store(D + i, v_cvt_f32(v_s0) * _v_scale); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_store(D + i, v_mul(v_cvt_f32(v_s0), _v_scale)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_store(D + i, v_cvt_f32(v_s0) * v_scale); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_store(D + i, v_mul(v_cvt_f32(v_s0), v_scale)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1134,19 +1134,19 @@ struct ColumnSum : { int i = 0; -#if CV_SIMD - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); v_store(D + i, v_cvt_f32(v_s0)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_cvt_f32(v_s0)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif diff --git a/modules/imgproc/src/morph.simd.hpp b/modules/imgproc/src/morph.simd.hpp index 9b3023f8f0..ef813dccec 100644 --- a/modules/imgproc/src/morph.simd.hpp +++ b/modules/imgproc/src/morph.simd.hpp @@ -106,12 +106,12 @@ struct MorphNoVec int operator()(uchar**, int, uchar*, int) const { return 0; } }; -#if CV_SIMD +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, GCC 13 related template struct MorphRowVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; MorphRowVec(int _ksize, int _anchor) : ksize(_ksize), anchor(_anchor) {} int operator()(const uchar* src, uchar* dst, int width, int cn) const { @@ -121,52 +121,52 @@ template struct MorphRowVec width *= cn; VecUpdate updateOp; - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes ) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { vtype s0 = vx_load((const stype*)src + i); - vtype s1 = vx_load((const stype*)src + i + vtype::nlanes); - vtype s2 = vx_load((const stype*)src + i + 2*vtype::nlanes); - vtype s3 = vx_load((const stype*)src + i + 3*vtype::nlanes); + vtype s1 = vx_load((const stype*)src + i + VTraits::vlanes()); + vtype s2 = vx_load((const stype*)src + i + 2*VTraits::vlanes()); + vtype s3 = vx_load((const stype*)src + i + 3*VTraits::vlanes()); for (k = cn; k < _ksize; k += cn) { s0 = updateOp(s0, vx_load((const stype*)src + i + k)); - s1 = updateOp(s1, vx_load((const stype*)src + i + k + vtype::nlanes)); - s2 = updateOp(s2, vx_load((const stype*)src + i + k + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load((const stype*)src + i + k + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load((const stype*)src + i + k + VTraits::vlanes())); + s2 = updateOp(s2, vx_load((const stype*)src + i + k + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load((const stype*)src + i + k + 3*VTraits::vlanes())); } v_store((stype*)dst + i, s0); - v_store((stype*)dst + i + vtype::nlanes, s1); - v_store((stype*)dst + i + 2*vtype::nlanes, s2); - v_store((stype*)dst + i + 3*vtype::nlanes, s3); + v_store((stype*)dst + i + VTraits::vlanes(), s1); + v_store((stype*)dst + i + 2*VTraits::vlanes(), s2); + v_store((stype*)dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { vtype s0 = vx_load((const stype*)src + i); - vtype s1 = vx_load((const stype*)src + i + vtype::nlanes); + vtype s1 = vx_load((const stype*)src + i + VTraits::vlanes()); for( k = cn; k < _ksize; k += cn ) { s0 = updateOp(s0, vx_load((const stype*)src + i + k)); - s1 = updateOp(s1, vx_load((const stype*)src + i + k + vtype::nlanes)); + s1 = updateOp(s1, vx_load((const stype*)src + i + k + VTraits::vlanes())); } v_store((stype*)dst + i, s0); - v_store((stype*)dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store((stype*)dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s = vx_load((const stype*)src + i); for( k = cn; k < _ksize; k += cn ) s = updateOp(s, vx_load((const stype*)src + i + k)); v_store((stype*)dst + i, s); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s = vx_load_low((const stype*)src + i); for( k = cn; k < _ksize; k += cn ) s = updateOp(s, vx_load_low((const stype*)src + i + k)); v_store_low((stype*)dst + i, s); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } return i - i % cn; @@ -179,7 +179,7 @@ template struct MorphRowVec template struct MorphColumnVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; MorphColumnVec(int _ksize, int _anchor) : ksize(_ksize), anchor(_anchor) {} int operator()(const uchar** _src, uchar* _dst, int dststep, int count, int width) const { @@ -189,7 +189,7 @@ template struct MorphColumnVec VecUpdate updateOp; for( i = 0; i < count + ksize - 1; i++ ) - CV_Assert( ((size_t)_src[i] & (CV_SIMD_WIDTH-1)) == 0 ); + CV_Assert( ((size_t)_src[i] & (VTraits::vlanes()-1)) == 0 ); const stype** src = (const stype**)_src; stype* dst = (stype*)_dst; @@ -197,58 +197,58 @@ template struct MorphColumnVec for( ; _ksize > 1 && count > 1; count -= 2, dst += dststep*2, src += 2 ) { - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes()) { const stype* sptr = src[1] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); - vtype s2 = vx_load_aligned(sptr + 2*vtype::nlanes); - vtype s3 = vx_load_aligned(sptr + 3*vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); + vtype s2 = vx_load_aligned(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load_aligned(sptr + 3*VTraits::vlanes()); for( k = 2; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes())); } sptr = src[0] + i; v_store(dst + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - v_store(dst + i + 2*vtype::nlanes, updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes))); - v_store(dst + i + 3*vtype::nlanes, updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes))); + v_store(dst + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + v_store(dst + i + 2*VTraits::vlanes(), updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes()))); + v_store(dst + i + 3*VTraits::vlanes(), updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes()))); sptr = src[k] + i; v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + dststep + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - v_store(dst + dststep + i + 2*vtype::nlanes, updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes))); - v_store(dst + dststep + i + 3*vtype::nlanes, updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes))); + v_store(dst + dststep + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + v_store(dst + dststep + i + 2*VTraits::vlanes(), updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes()))); + v_store(dst + dststep + i + 3*VTraits::vlanes(), updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes()))); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[1] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); for( k = 2; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); } sptr = src[0] + i; v_store(dst + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); + v_store(dst + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); sptr = src[k] + i; v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + dststep + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - i += 2*vtype::nlanes; + v_store(dst + dststep + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load_aligned(src[1] + i); @@ -257,9 +257,9 @@ template struct MorphColumnVec v_store(dst + i, updateOp(s0, vx_load_aligned(src[0] + i))); v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(src[k] + i))); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[1] + i); @@ -268,66 +268,66 @@ template struct MorphColumnVec v_store_low(dst + i, updateOp(s0, vx_load_low(src[0] + i))); v_store_low(dst + dststep + i, updateOp(s0, vx_load_low(src[k] + i))); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } } for( ; count > 0; count--, dst += dststep, src++ ) { - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes()) { const stype* sptr = src[0] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); - vtype s2 = vx_load_aligned(sptr + 2*vtype::nlanes); - vtype s3 = vx_load_aligned(sptr + 3*vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); + vtype s2 = vx_load_aligned(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load_aligned(sptr + 3*VTraits::vlanes()); for( k = 1; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - v_store(dst + i + 2*vtype::nlanes, s2); - v_store(dst + i + 3*vtype::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); for( k = 1; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load_aligned(src[0] + i); for( k = 1; k < _ksize; k++ ) s0 = updateOp(s0, vx_load_aligned(src[k] + i)); v_store(dst + i, s0); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[0] + i); for( k = 1; k < _ksize; k++ ) s0 = updateOp(s0, vx_load_low(src[k] + i)); v_store_low(dst + i, s0); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } } @@ -341,7 +341,7 @@ template struct MorphColumnVec template struct MorphVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; int operator()(uchar** _src, int nz, uchar* _dst, int width) const { CV_INSTRUMENT_REGION(); @@ -351,56 +351,56 @@ template struct MorphVec int i, k; VecUpdate updateOp; - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes ) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load(sptr); - vtype s1 = vx_load(sptr + vtype::nlanes); - vtype s2 = vx_load(sptr + 2*vtype::nlanes); - vtype s3 = vx_load(sptr + 3*vtype::nlanes); + vtype s1 = vx_load(sptr + VTraits::vlanes()); + vtype s2 = vx_load(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load(sptr + 3*VTraits::vlanes()); for( k = 1; k < nz; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load(sptr)); - s1 = updateOp(s1, vx_load(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load(sptr + 3*VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - v_store(dst + i + 2*vtype::nlanes, s2); - v_store(dst + i + 3*vtype::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load(sptr); - vtype s1 = vx_load(sptr + vtype::nlanes); + vtype s1 = vx_load(sptr + VTraits::vlanes()); for( k = 1; k < nz; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load(sptr)); - s1 = updateOp(s1, vx_load(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load(sptr + VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load(src[0] + i); for( k = 1; k < nz; k++ ) s0 = updateOp(s0, vx_load(src[k] + i)); v_store(dst + i, s0); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[0] + i); for( k = 1; k < nz; k++ ) s0 = updateOp(s0, vx_load_low(src[k] + i)); v_store_low(dst + i, s0); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } return i; } diff --git a/modules/imgproc/src/shapedescr.cpp b/modules/imgproc/src/shapedescr.cpp index 4c73910e27..194e5a0862 100644 --- a/modules/imgproc/src/shapedescr.cpp +++ b/modules/imgproc/src/shapedescr.cpp @@ -879,14 +879,14 @@ static Rect pointSetBoundingRect( const Mat& points ) if( npoints == 0 ) return Rect(); -#if CV_SIMD +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, loop tail related. const int64_t* pts = points.ptr(); if( !is_float ) { v_int32 minval, maxval; minval = maxval = v_reinterpret_as_s32(vx_setall_s64(*pts)); //min[0]=pt.x, min[1]=pt.y, min[2]=pt.x, min[3]=pt.y - for( i = 1; i <= npoints - v_int32::nlanes/2; i+= v_int32::nlanes/2 ) + for( i = 1; i <= npoints - VTraits::vlanes()/2; i+= VTraits::vlanes()/2 ) { v_int32 ptXY2 = v_reinterpret_as_s32(vx_load(pts + i)); minval = v_min(ptXY2, minval); @@ -894,22 +894,22 @@ static Rect pointSetBoundingRect( const Mat& points ) } minval = v_min(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); - if( i <= npoints - v_int32::nlanes/4 ) + if( i <= npoints - VTraits::vlanes()/4 ) { v_int32 ptXY = v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(vx_load_low(pts + i)))); minval = v_min(ptXY, minval); maxval = v_max(ptXY, maxval); - i += v_int64::nlanes/2; + i += VTraits::vlanes()/2; } - for(int j = 16; j < CV_SIMD_WIDTH; j*=2) + for(int j = 16; j < VTraits::vlanes(); j*=2) { minval = v_min(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); } - xmin = minval.get0(); - xmax = maxval.get0(); - ymin = v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval))).get0(); - ymax = v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval))).get0(); + xmin = v_get0(minval); + xmax = v_get0(maxval); + ymin = v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); + ymax = v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); #if CV_SIMD_WIDTH > 16 if( i < npoints ) { @@ -921,18 +921,18 @@ static Rect pointSetBoundingRect( const Mat& points ) minval2 = v_min(ptXY, minval2); maxval2 = v_max(ptXY, maxval2); } - xmin = min(xmin, minval2.get0()); - xmax = max(xmax, maxval2.get0()); - ymin = min(ymin, v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval2))).get0()); - ymax = max(ymax, v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval2))).get0()); + xmin = min(xmin, v_get0(minval2)); + xmax = max(xmax, v_get0(maxval2)); + ymin = min(ymin, v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval2))))); + ymax = max(ymax, v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval2))))); } -#endif +#endif // CV_SIMD } else { v_float32 minval, maxval; minval = maxval = v_reinterpret_as_f32(vx_setall_s64(*pts)); //min[0]=pt.x, min[1]=pt.y, min[2]=pt.x, min[3]=pt.y - for( i = 1; i <= npoints - v_float32::nlanes/2; i+= v_float32::nlanes/2 ) + for( i = 1; i <= npoints - VTraits::vlanes()/2; i+= VTraits::vlanes()/2 ) { v_float32 ptXY2 = v_reinterpret_as_f32(vx_load(pts + i)); minval = v_min(ptXY2, minval); @@ -940,22 +940,22 @@ static Rect pointSetBoundingRect( const Mat& points ) } minval = v_min(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval)))); - if( i <= npoints - v_float32::nlanes/4 ) + if( i <= npoints - VTraits::vlanes()/4 ) { v_float32 ptXY = v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(vx_load_low(pts + i)))); minval = v_min(ptXY, minval); maxval = v_max(ptXY, maxval); - i += v_float32::nlanes/4; + i += VTraits::vlanes()/4; } - for(int j = 16; j < CV_SIMD_WIDTH; j*=2) + for(int j = 16; j < VTraits::vlanes(); j*=2) { minval = v_min(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval)))); } - xmin = cvFloor(minval.get0()); - xmax = cvFloor(maxval.get0()); - ymin = cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval))).get0()); - ymax = cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval))).get0()); + xmin = cvFloor(v_get0(minval)); + xmax = cvFloor(v_get0(maxval)); + ymin = cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval))))); + ymax = cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval))))); #if CV_SIMD_WIDTH > 16 if( i < npoints ) { @@ -967,10 +967,10 @@ static Rect pointSetBoundingRect( const Mat& points ) minval2 = v_min(ptXY, minval2); maxval2 = v_max(ptXY, maxval2); } - xmin = min(xmin, cvFloor(minval2.get0())); - xmax = max(xmax, cvFloor(maxval2.get0())); - ymin = min(ymin, cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval2))).get0())); - ymax = max(ymax, cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval2))).get0())); + xmin = min(xmin, cvFloor(v_get0(minval2))); + xmax = max(xmax, cvFloor(v_get0(maxval2))); + ymin = min(ymin, cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval2)))))); + ymax = max(ymax, cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval2)))))); } #endif } From 70d7e83dca7824e7e9d09e03630d1134628dc70a Mon Sep 17 00:00:00 2001 From: lpylpy0514 <67569760+lpylpy0514@users.noreply.github.com> Date: Tue, 19 Sep 2023 20:36:38 +0800 Subject: [PATCH 19/34] Merge pull request #24201 from lpylpy0514:4.x VIT track(gsoc realtime object tracking model) #24201 Vit tracker(vision transformer tracker) is a much better model for real-time object tracking. Vit tracker can achieve speeds exceeding nanotrack by 20% in single-threaded mode with ARM chip, and the advantage becomes even more pronounced in multi-threaded mode. In addition, on the dataset, vit tracker demonstrates better performance compared to nanotrack. Moreover, vit trackerprovides confidence values during the tracking process, which can be used to determine if the tracking is currently lost. opencv_zoo: https://github.com/opencv/opencv_zoo/pull/194 opencv_extra: [https://github.com/opencv/opencv_extra/pull/1088](https://github.com/opencv/opencv_extra/pull/1088) # Performance comparison is as follows: NOTE: The speed below is tested by **onnxruntime** because opencv has poor support for the transformer architecture for now. ONNX speed test on ARM platform(apple M2)(ms): | thread nums | 1| 2| 3| 4| |--------|--------|--------|--------|--------| | nanotrack| 5.25| 4.86| 4.72| 4.49| | vit tracker| 4.18| 2.41| 1.97| **1.46 (3X)**| ONNX speed test on x86 platform(intel i3 10105)(ms): | thread nums | 1| 2| 3| 4| |--------|--------|--------|--------|--------| | nanotrack|3.20|2.75|2.46|2.55| | vit tracker|3.84|2.37|2.10|2.01| opencv speed test on x86 platform(intel i3 10105)(ms): | thread nums | 1| 2| 3| 4| |--------|--------|--------|--------|--------| | vit tracker|31.3|31.4|31.4|31.4| preformance test on lasot dataset(AUC is the most important data. Higher AUC means better tracker): |LASOT | AUC| P| Pnorm| |--------|--------|--------|--------| | nanotrack| 46.8| 45.0| 43.3| | vit tracker| 48.6| 44.8| 54.7| [https://youtu.be/MJiPnu1ZQRI](https://youtu.be/MJiPnu1ZQRI) In target tracking tasks, the score is an important indicator that can indicate whether the current target is lost. In the video, vit tracker can track the target and display the current score in the upper left corner of the video. When the target is lost, the score drops significantly. While nanotrack will only return 0.9 score in any situation, so that we cannot determine whether the target is lost. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../video/include/opencv2/video/tracking.hpp | 37 +++ modules/video/src/tracking/tracker_vit.cpp | 219 ++++++++++++++++++ modules/video/test/test_trackers.cpp | 9 + samples/dnn/vit_tracker.cpp | 176 ++++++++++++++ samples/python/tracker.py | 8 +- 5 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 modules/video/src/tracking/tracker_vit.cpp create mode 100644 samples/dnn/vit_tracker.cpp diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index eb5a6c7030..7f93a79a72 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -887,6 +887,43 @@ public: //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; }; +/** @brief the VIT tracker is a super lightweight dnn-based general object tracking. + * + * VIT tracker is much faster and extremely lightweight due to special model structure, the model file is about 767KB. + * Model download link: https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack + * Author: PengyuLiu, 1872918507@qq.com + */ +class CV_EXPORTS_W TrackerVit : public Tracker +{ +protected: + TrackerVit(); // use ::create() +public: + virtual ~TrackerVit() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + CV_PROP_RW std::string net; + CV_PROP_RW int backend; + CV_PROP_RW int target; + CV_PROP_RW Scalar meanvalue; + CV_PROP_RW Scalar stdvalue; + }; + + /** @brief Constructor + @param parameters vit tracker parameters TrackerVit::Params + */ + static CV_WRAP + Ptr create(const TrackerVit::Params& parameters = TrackerVit::Params()); + + /** @brief Return tracking score + */ + CV_WRAP virtual float getTrackingScore() = 0; + + // void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + // bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; +}; + //! @} video_track } // cv diff --git a/modules/video/src/tracking/tracker_vit.cpp b/modules/video/src/tracking/tracker_vit.cpp new file mode 100644 index 0000000000..7611e184ef --- /dev/null +++ b/modules/video/src/tracking/tracker_vit.cpp @@ -0,0 +1,219 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Author, PengyuLiu, 1872918507@qq.com + +#include "../precomp.hpp" +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#endif + +namespace cv { + +TrackerVit::TrackerVit() +{ + // nothing +} + +TrackerVit::~TrackerVit() +{ + // nothing +} + +TrackerVit::Params::Params() +{ + net = "vitTracker.onnx"; + meanvalue = Scalar{0.485, 0.456, 0.406}; + stdvalue = Scalar{0.229, 0.224, 0.225}; +#ifdef HAVE_OPENCV_DNN + backend = dnn::DNN_BACKEND_DEFAULT; + target = dnn::DNN_TARGET_CPU; +#else + backend = -1; // invalid value + target = -1; // invalid value +#endif +} + +#ifdef HAVE_OPENCV_DNN + +class TrackerVitImpl : public TrackerVit +{ +public: + TrackerVitImpl(const TrackerVit::Params& parameters) + : params(parameters) + { + net = dnn::readNet(params.net); + CV_Assert(!net.empty()); + } + + void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + float getTrackingScore() CV_OVERRIDE; + + Rect rect_last; + float tracking_score; + + TrackerVit::Params params; + + +protected: + void preprocess(const Mat& src, Mat& dst, Size size); + + const Size searchSize{256, 256}; + const Size templateSize{128, 128}; + + Mat hanningWindow; + + dnn::Net net; + Mat image; +}; + +static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) +{ + int x = box.x, y = box.y, w = box.width, h = box.height; + int crop_sz = ceil(sqrt(w * h) * factor); + + int x1 = round(x + 0.5 * w - crop_sz * 0.5); + int x2 = x1 + crop_sz; + int y1 = round(y + 0.5 * h - crop_sz * 0.5); + int y2 = y1 + crop_sz; + + int x1_pad = std::max(0, -x1); + int y1_pad = std::max(0, -y1); + int x2_pad = std::max(x2 - src.size[1] + 1, 0); + int y2_pad = std::max(y2 - src.size[0] + 1, 0); + + Rect roi(x1 + x1_pad, y1 + y1_pad, x2 - x2_pad - x1 - x1_pad, y2 - y2_pad - y1 - y1_pad); + Mat im_crop = src(roi); + copyMakeBorder(im_crop, dst, y1_pad, y2_pad, x1_pad, x2_pad, BORDER_CONSTANT); +} + +void TrackerVitImpl::preprocess(const Mat& src, Mat& dst, Size size) +{ + Mat mean = Mat(size, CV_32FC3, params.meanvalue); + Mat std = Mat(size, CV_32FC3, params.stdvalue); + mean = dnn::blobFromImage(mean, 1.0, Size(), Scalar(), false); + std = dnn::blobFromImage(std, 1.0, Size(), Scalar(), false); + + Mat img; + resize(src, img, size); + + dst = dnn::blobFromImage(img, 1.0, Size(), Scalar(), false); + dst /= 255; + dst = (dst - mean) / std; +} + +static Mat hann1d(int sz, bool centered = true) { + Mat hanningWindow(sz, 1, CV_32FC1); + float* data = hanningWindow.ptr(0); + + if(centered) { + for(int i = 0; i < sz; i++) { + float val = 0.5 * (1 - std::cos((2 * M_PI / (sz + 1)) * (i + 1))); + data[i] = val; + } + } + else { + int half_sz = sz / 2; + for(int i = 0; i <= half_sz; i++) { + float val = 0.5 * (1 + std::cos((2 * M_PI / (sz + 2)) * i)); + data[i] = val; + data[sz - 1 - i] = val; + } + } + + return hanningWindow; +} + +static Mat hann2d(Size size, bool centered = true) { + int rows = size.height; + int cols = size.width; + + Mat hanningWindowRows = hann1d(rows, centered); + Mat hanningWindowCols = hann1d(cols, centered); + + Mat hanningWindow = hanningWindowRows * hanningWindowCols.t(); + + return hanningWindow; +} + +static Rect returnfromcrop(float x, float y, float w, float h, Rect res_Last) +{ + int cropwindowwh = 4 * sqrt(res_Last.width * res_Last.height); + int x0 = res_Last.x + 0.5 * res_Last.width - 0.5 * cropwindowwh; + int y0 = res_Last.y + 0.5 * res_Last.height - 0.5 * cropwindowwh; + Rect finalres; + finalres.x = x * cropwindowwh + x0; + finalres.y = y * cropwindowwh + y0; + finalres.width = w * cropwindowwh; + finalres.height = h * cropwindowwh; + return finalres; +} + +void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) +{ + image = image_.getMat().clone(); + Mat crop; + crop_image(image, crop, boundingBox_, 2); + Mat blob; + preprocess(crop, blob, templateSize); + net.setInput(blob, "template"); + Size size(16, 16); + hanningWindow = hann2d(size, false); + rect_last = boundingBox_; +} + +bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) +{ + image = image_.getMat().clone(); + Mat crop; + crop_image(image, crop, rect_last, 4); + Mat blob; + preprocess(crop, blob, searchSize); + net.setInput(blob, "search"); + std::vector outputName = {"output1", "output2", "output3"}; + std::vector outs; + net.forward(outs, outputName); + CV_Assert(outs.size() == 3); + + Mat conf_map = outs[0].reshape(0, {16, 16}); + Mat size_map = outs[1].reshape(0, {2, 16, 16}); + Mat offset_map = outs[2].reshape(0, {2, 16, 16}); + + multiply(conf_map, (1.0 - hanningWindow), conf_map); + + double maxVal; + Point maxLoc; + minMaxLoc(conf_map, nullptr, &maxVal, nullptr, &maxLoc); + tracking_score = maxVal; + + float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; + float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; + float w = size_map.at(0, maxLoc.y, maxLoc.x); + float h = size_map.at(1, maxLoc.y, maxLoc.x); + + Rect finalres = returnfromcrop(cx - w / 2, cy - h / 2, w, h, rect_last); + rect_last = finalres; + boundingBoxRes = finalres; + return true; +} + +float TrackerVitImpl::getTrackingScore() +{ + return tracking_score; +} + +Ptr TrackerVit::create(const TrackerVit::Params& parameters) +{ + return makePtr(parameters); +} + +#else // OPENCV_HAVE_DNN +Ptr TrackerVit::create(const TrackerVit::Params& parameters) +{ + CV_UNUSED(parameters); + CV_Error(Error::StsNotImplemented, "to use vittrack, the tracking module needs to be built with opencv_dnn !"); +} +#endif // OPENCV_HAVE_DNN +} diff --git a/modules/video/test/test_trackers.cpp b/modules/video/test/test_trackers.cpp index 6ede40896c..4970343099 100644 --- a/modules/video/test/test_trackers.cpp +++ b/modules/video/test/test_trackers.cpp @@ -160,4 +160,13 @@ TEST(NanoTrack, accuracy_NanoTrack_V2) checkTrackingAccuracy(tracker, 0.69); } +TEST(vittrack, accuracy_vittrack) +{ + std::string model = cvtest::findDataFile("dnn/onnx/models/vitTracker.onnx", false); + cv::TrackerVit::Params params; + params.net = model; + cv::Ptr tracker = TrackerVit::create(params); + checkTrackingAccuracy(tracker, 0.67); +} + }} // namespace opencv_test:: diff --git a/samples/dnn/vit_tracker.cpp b/samples/dnn/vit_tracker.cpp new file mode 100644 index 0000000000..02e5cea83f --- /dev/null +++ b/samples/dnn/vit_tracker.cpp @@ -0,0 +1,176 @@ +// VitTracker +// model: https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack + +#include +#include + +#include +#include +#include +#include + +using namespace cv; +using namespace cv::dnn; + +const char *keys = + "{ help h | | Print help message }" + "{ input i | | Full path to input video folder, the specific camera index. (empty for camera 0) }" + "{ net | vitTracker.onnx | Path to onnx model of vitTracker.onnx}" + "{ backend | 0 | Choose one of computation backends: " + "0: automatically (by default), " + "1: Halide language (http://halide-lang.org/), " + "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "3: OpenCV implementation, " + "4: VKCOM, " + "5: CUDA }," + "{ target | 0 | Choose one of target computation devices: " + "0: CPU target (by default), " + "1: OpenCL, " + "2: OpenCL fp16 (half-float precision), " + "3: VPU, " + "4: Vulkan, " + "6: CUDA, " + "7: CUDA fp16 (half-float preprocess) }" +; + +static +int run(int argc, char** argv) +{ + // Parse command line arguments. + CommandLineParser parser(argc, argv, keys); + + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + + std::string inputName = parser.get("input"); + std::string net = parser.get("net"); + int backend = parser.get("backend"); + int target = parser.get("target"); + + Ptr tracker; + try + { + TrackerVit::Params params; + params.net = samples::findFile(net); + params.backend = backend; + params.target = target; + tracker = TrackerVit::create(params); + } + catch (const cv::Exception& ee) + { + std::cerr << "Exception: " << ee.what() << std::endl; + std::cout << "Can't load the network by using the following files:" << std::endl; + std::cout << "net : " << net << std::endl; + return 2; + } + + const std::string winName = "vitTracker"; + namedWindow(winName, WINDOW_AUTOSIZE); + + // Open a video file or an image file or a camera stream. + VideoCapture cap; + + if (inputName.empty() || (isdigit(inputName[0]) && inputName.size() == 1)) + { + int c = inputName.empty() ? 0 : inputName[0] - '0'; + std::cout << "Trying to open camera #" << c << " ..." << std::endl; + if (!cap.open(c)) + { + std::cout << "Capture from camera #" << c << " didn't work. Specify -i=