build: added DEBUG build guard

To prevent linkage of binary incompatible DEBUG/RELEASE binaries/runtimes

.gitattributes

@@ -31,6 +31,8 @@
 *.py           text
 *.qrc          text
 *.qss          text
+*.rc           text
+*.rc.in        text
 *.S            text
 *.sbt          text
 *.scala        text

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!--
+If you have a question rather than reporting a bug please go to http://answers.opencv.org where you get much faster responses.
+If you need further assistance please read [How To Contribute](https://github.com/opencv/opencv/wiki/How_to_contribute).
+
+This is a template helping you to create an issue which can be processed as quickly as possible. This is the bug reporting section for the OpenCV library.
+-->
+
+##### System information (version)
+<!-- Example
+- OpenCV => 3.1
+- Operating System / Platform => Windows 64 Bit
+- Compiler => Visual Studio 2015
+-->
+
+- OpenCV => :grey_question:
+- Operating System / Platform => :grey_question:
+- Compiler => :grey_question:
+
+##### Detailed description
+
+<!-- your description -->
+
+##### Steps to reproduce
+
+<!-- to add code example fence it with triple backticks and optional file extension
+    ```.cpp
+    // C++ code example
+    ```
+ or attach as .txt or .zip file
+-->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+<!-- Please use this line to close one or multiple issues when this pullrequest gets merged
+You can add another line right under the first one:
+resolves #1234
+resolves #1235
+-->
+
+### This pullrequest changes
+
+<!-- Please describe what your pullrequest is changing -->

.gitignore

@@ -17,9 +17,8 @@ bin/
 *.rule
 *.tmp
 */debug
-*/CMakeFiles
-CMakeCache.txt
 *.suo
 *.log
 *.tlog
 build
+.cache

3rdparty/carotene/.gitignore

@@ -0,0 +1,8 @@
+# Gedit temp files
+*~
+
+# Qt Creator file
+*.user
+
+# MacOS-specific (Desktop Services Store)
+.DS_Store

3rdparty/carotene/CMakeLists.txt

@@ -0,0 +1,42 @@
+cmake_minimum_required(VERSION 2.8.11 FATAL_ERROR)
+
+project(Carotene)
+
+set(CAROTENE_NS "carotene" CACHE STRING "Namespace for Carotene definitions")
+
+set(CAROTENE_INCLUDE_DIR include)
+set(CAROTENE_SOURCE_DIR src)
+
+file(GLOB_RECURSE carotene_headers RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "${CAROTENE_INCLUDE_DIR}/*.hpp")
+file(GLOB_RECURSE carotene_sources RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "${CAROTENE_SOURCE_DIR}/*.cpp"
+                                                                        "${CAROTENE_SOURCE_DIR}/*.hpp")
+
+include_directories(${CAROTENE_INCLUDE_DIR})
+
+if(CMAKE_COMPILER_IS_GNUCC)
+    set(CMAKE_CXX_FLAGS "-fvisibility=hidden ${CMAKE_CXX_FLAGS}")
+
+    # allow more inlines - these parameters improve performance for:
+    # - matchTemplate about 5-10%
+    # - goodFeaturesToTrack 10-20%
+    # - cornerHarris 30% for some cases
+
+    set_source_files_properties(${carotene_sources} COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000")
+endif()
+
+add_library(carotene_objs OBJECT
+  ${carotene_headers}
+  ${carotene_sources}
+)
+
+if(NOT CAROTENE_NS STREQUAL "carotene")
+    target_compile_definitions(carotene_objs PUBLIC "-DCAROTENE_NS=${CAROTENE_NS}")
+endif()
+
+if(WITH_NEON)
+    target_compile_definitions(carotene_objs PRIVATE "-DWITH_NEON")
+endif()
+
+set_target_properties(carotene_objs PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+
+add_library(carotene STATIC EXCLUDE_FROM_ALL "$<TARGET_OBJECTS:carotene_objs>")

3rdparty/carotene/README.md

@@ -0,0 +1,2 @@
+This is Carotene, a low-level library containing optimized CPU routines
+that are useful for computer vision algorithms.

3rdparty/carotene/hal/CMakeLists.txt

@@ -0,0 +1,114 @@
+cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)
+
+include(CheckCCompilerFlag)
+include(CheckCXXCompilerFlag)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(TEGRA_HAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(CAROTENE_DIR "${TEGRA_HAL_DIR}/../")
+
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
+  set(ARM TRUE)
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64.*|AARCH64.*")
+  set(AARCH64 TRUE)
+endif()
+
+set(TEGRA_COMPILER_FLAGS "")
+
+if(CMAKE_COMPILER_IS_GNUCXX)
+  # Generate unwind information even for functions that can't throw/propagate exceptions.
+  # This lets debuggers and such get non-broken backtraces for such functions, even without debugging symbols.
+  list(APPEND TEGRA_COMPILER_FLAGS -funwind-tables)
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX)
+  if(X86 OR ARMEABI_V6 OR (MIPS AND ANDROID_COMPILER_VERSION VERSION_LESS "4.6"))
+    list(APPEND TEGRA_COMPILER_FLAGS -fweb -fwrapv -frename-registers -fsched-stalled-insns-dep=100 -fsched-stalled-insns=2)
+  elseif(CMAKE_COMPILER_IS_CLANGCXX)
+    list(APPEND TEGRA_COMPILER_FLAGS -fwrapv)
+  else()
+    list(APPEND TEGRA_COMPILER_FLAGS -fweb -fwrapv -frename-registers -fsched2-use-superblocks -fsched2-use-traces
+                                     -fsched-stalled-insns-dep=100 -fsched-stalled-insns=2)
+  endif()
+  if((ANDROID_COMPILER_IS_CLANG OR NOT ANDROID_COMPILER_VERSION VERSION_LESS "4.7") AND ANDROID_NDK_RELEASE STRGREATER "r8d" )
+    list(APPEND TEGRA_COMPILER_FLAGS -fgraphite -fgraphite-identity -floop-block -floop-flatten -floop-interchange
+                                     -floop-strip-mine -floop-parallelize-all -ftree-loop-linear)
+  endif()
+endif()
+
+string(REPLACE ";" " " TEGRA_COMPILER_FLAGS "${TEGRA_COMPILER_FLAGS}")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TEGRA_COMPILER_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TEGRA_COMPILER_FLAGS}")
+
+if(ARMEABI_V7A)
+  if (CMAKE_COMPILER_IS_GNUCXX)
+    set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-tree-vectorize" )
+    set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-tree-vectorize" )
+  endif()
+endif()
+
+if(WITH_LOGS)
+  add_definitions(-DHAVE_LOGS)
+endif()
+
+set(CAROTENE_NS "carotene_o4t" CACHE STRING "" FORCE)
+
+function(compile_carotene)
+  if(ENABLE_NEON)
+    set(WITH_NEON ON)
+  endif()
+
+  add_subdirectory("${CAROTENE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/carotene")
+
+  if(ARM OR AARCH64)
+    if(CMAKE_BUILD_TYPE)
+      set(CMAKE_TRY_COMPILE_CONFIGURATION ${CMAKE_BUILD_TYPE})
+    endif()
+    check_cxx_compiler_flag("-mfpu=neon" CXX_HAS_MFPU_NEON)
+    check_c_compiler_flag("-mfpu=neon" C_HAS_MFPU_NEON)
+    if(${CXX_HAS_MFPU_NEON} AND ${C_HAS_MFPU_NEON} AND NOT "${CMAKE_CXX_FLAGS} " MATCHES "-mfpu=neon[^ ]*")
+      get_target_property(old_flags "carotene_objs" COMPILE_FLAGS)
+      if(old_flags)
+        set_target_properties("carotene_objs" PROPERTIES COMPILE_FLAGS "${old_flags} -mfpu=neon")
+      else()
+        set_target_properties("carotene_objs" PROPERTIES COMPILE_FLAGS "-mfpu=neon")
+      endif()
+    endif()
+  endif()
+endfunction()
+
+compile_carotene()
+
+include_directories("${CAROTENE_DIR}/include")
+
+get_target_property(carotene_defs carotene_objs INTERFACE_COMPILE_DEFINITIONS)
+set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS ${carotene_defs})
+
+  if (CMAKE_COMPILER_IS_GNUCXX)
+    # allow more inlines - these parameters improve performance for:
+    #   matchTemplate about 5-10%
+    #   goodFeaturesToTrack 10-20%
+    #   cornerHarris 30% for some cases
+    set_source_files_properties(impl.cpp $<TARGET_OBJECTS:carotene_objs> COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000")
+#    set_source_files_properties(impl.cpp $<TARGET_OBJECTS:carotene_objs> COMPILE_FLAGS "--param ipcp-unit-growth=100000 --param inline-unit-growth=100000 --param large-stack-frame-growth=5000")
+  endif()
+
+add_library(tegra_hal STATIC $<TARGET_OBJECTS:carotene_objs>)
+set_target_properties(tegra_hal PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+set_target_properties(tegra_hal PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${3P_LIBRARY_OUTPUT_PATH})
+set(OPENCV_SRC_DIR "${CMAKE_SOURCE_DIR}")
+if(NOT BUILD_SHARED_LIBS)
+  ocv_install_target(tegra_hal EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+endif()
+target_include_directories(tegra_hal PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${OPENCV_SRC_DIR}/modules/core/include)
+
+set(CAROTENE_HAL_VERSION "0.0.1" PARENT_SCOPE)
+set(CAROTENE_HAL_LIBRARIES "tegra_hal" PARENT_SCOPE)
+set(CAROTENE_HAL_HEADERS "carotene/tegra_hal.hpp" PARENT_SCOPE)
+set(CAROTENE_HAL_INCLUDE_DIRS "${CMAKE_BINARY_DIR}" PARENT_SCOPE)
+
+configure_file("tegra_hal.hpp" "${CMAKE_BINARY_DIR}/carotene/tegra_hal.hpp" COPYONLY)
+configure_file("${CAROTENE_DIR}/include/carotene/definitions.hpp" "${CMAKE_BINARY_DIR}/carotene/definitions.hpp" COPYONLY)
+configure_file("${CAROTENE_DIR}/include/carotene/functions.hpp" "${CMAKE_BINARY_DIR}/carotene/functions.hpp" COPYONLY)
+configure_file("${CAROTENE_DIR}/include/carotene/types.hpp" "${CMAKE_BINARY_DIR}/carotene/types.hpp" COPYONLY)

3rdparty/carotene/include/carotene/definitions.hpp

@@ -0,0 +1,47 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_DEFINITIONS_HPP
+#define CAROTENE_DEFINITIONS_HPP
+
+#ifndef CAROTENE_NS
+#define CAROTENE_NS carotene
+#endif
+
+#endif

3rdparty/carotene/include/carotene/types.hpp

@@ -0,0 +1,125 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_TYPES_HPP
+#define CAROTENE_TYPES_HPP
+
+#include <carotene/definitions.hpp>
+#include <stdint.h>
+#include <cstddef>
+
+#ifndef UINT32_MAX
+    #define UINT32_MAX (4294967295U)
+#endif
+
+namespace CAROTENE_NS {
+    using std::size_t;
+    using std::ptrdiff_t;
+
+    typedef int8_t   s8;
+    typedef uint8_t  u8;
+    typedef int16_t  s16;
+    typedef uint16_t u16;
+    typedef int32_t  s32;
+    typedef uint32_t u32;
+    typedef float    f32;
+    typedef int64_t  s64;
+    typedef uint64_t u64;
+    typedef double   f64;
+
+    typedef ptrdiff_t  stride_t;
+
+    enum CONVERT_POLICY
+    {
+        CONVERT_POLICY_WRAP,
+        CONVERT_POLICY_SATURATE
+    };
+
+    enum BORDER_MODE
+    {
+        BORDER_MODE_UNDEFINED,
+        BORDER_MODE_CONSTANT,
+        BORDER_MODE_REPLICATE,
+        BORDER_MODE_REFLECT,
+        BORDER_MODE_REFLECT101,
+        BORDER_MODE_WRAP
+    };
+
+    enum FLIP_MODE
+    {
+        FLIP_HORIZONTAL_MODE = 1,
+        FLIP_VERTICAL_MODE = 2,
+        FLIP_BOTH_MODE = FLIP_HORIZONTAL_MODE | FLIP_VERTICAL_MODE
+    };
+
+    enum COLOR_SPACE
+    {
+        COLOR_SPACE_BT601,
+        COLOR_SPACE_BT709
+    };
+
+    struct Size2D {
+        Size2D() : width(0), height(0) {}
+        Size2D(size_t width_, size_t height_) : width(width_), height(height_) {}
+
+        size_t width;
+        size_t height;
+
+        inline size_t total() const
+        {
+            return width * height;
+        }
+    };
+
+    struct Margin {
+        Margin() : left(0), right(0), top(0), bottom(0) {}
+        Margin(size_t left_, size_t right_, size_t top_, size_t bottom_)
+            : left(left_), right(right_), top(top_), bottom(bottom_) {}
+
+        // these are measured in elements
+        size_t left, right, top, bottom;
+    };
+
+    struct KeypointStore {
+        virtual void push(f32 kpX, f32 kpY, f32 kpSize, f32 kpAngle=-1, f32 kpResponse=0, s32 kpOctave=0, s32 kpClass_id=-1) = 0;
+        virtual ~KeypointStore() {};
+    };
+}
+
+#endif

3rdparty/carotene/src/absdiff.cpp

@@ -0,0 +1,241 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <algorithm>
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T>
+struct AbsDiff
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vabdq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vabd(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = src0[0] >= src1[0] ? src0[0] - src1[0] : src1[0] - src0[0];
+    }
+};
+
+template <typename T>
+struct AbsDiffSigned
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        typename internal::VecTraits<T>::vec128 v_min = internal::vminq(v_src0, v_src1);
+        typename internal::VecTraits<T>::vec128 v_max = internal::vmaxq(v_src0, v_src1);
+        v_dst = internal::vqsubq(v_max, v_min);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        typename internal::VecTraits<T>::vec64 v_min = internal::vmin(v_src0, v_src1);
+        typename internal::VecTraits<T>::vec64 v_max = internal::vmax(v_src0, v_src1);
+        v_dst = internal::vqsub(v_max, v_min);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = internal::saturate_cast<T>(src0[0] >= src1[0] ? (s64)src0[0] - src1[0] : (s64)src1[0] - src0[0]);
+    }
+};
+
+} // namespace
+
+#endif
+
+void absDiff(const Size2D &size,
+             const u8 *src0Base, ptrdiff_t src0Stride,
+             const u8 *src1Base, ptrdiff_t src1Stride,
+             u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiff<u8>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void absDiff(const Size2D &size,
+             const u16 *src0Base, ptrdiff_t src0Stride,
+             const u16 *src1Base, ptrdiff_t src1Stride,
+             u16 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiff<u16>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void absDiff(const Size2D &size,
+             const s8 *src0Base, ptrdiff_t src0Stride,
+             const s8 *src1Base, ptrdiff_t src1Stride,
+             s8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiffSigned<s8>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void absDiff(const Size2D &size,
+             const s16 *src0Base, ptrdiff_t src0Stride,
+             const s16 *src1Base, ptrdiff_t src1Stride,
+             s16 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiffSigned<s16>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void absDiff(const Size2D &size,
+             const s32 *src0Base, ptrdiff_t src0Stride,
+             const s32 *src1Base, ptrdiff_t src1Stride,
+             s32 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiffSigned<s32>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void absDiff(const Size2D &size,
+             const f32 * src0Base, ptrdiff_t src0Stride,
+             const f32 * src1Base, ptrdiff_t src1Stride,
+             f32 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, AbsDiff<f32>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/accumulate.cpp

@@ -0,0 +1,408 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+void accumulate(const Size2D &size,
+                const u8 *srcBase, ptrdiff_t srcStride,
+                s16 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8* src = internal::getRowPtr(srcBase, srcStride, i);
+        s16* dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            internal::prefetch(dst + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+            int16x8_t v_dst0 = vld1q_s16(dst + j);
+            int16x8_t v_dst1 = vld1q_s16(dst + j + 8);
+            int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src)));
+            int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src)));
+            v_dst0 = vqaddq_s16(v_dst0, v_src0);
+            v_dst1 = vqaddq_s16(v_dst1, v_src1);
+            vst1q_s16(dst + j, v_dst0);
+            vst1q_s16(dst + j + 8, v_dst1);
+        }
+        for (; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src = vld1_u8(src + j);
+            int16x8_t v_src16 = vreinterpretq_s16_u16(vmovl_u8(v_src));
+            int16x8_t v_dst = vld1q_s16(dst + j);
+            v_dst = vqaddq_s16(v_dst, v_src16);
+            vst1q_s16(dst + j, v_dst);
+        }
+
+        for (; j < size.width; j++)
+            dst[j] = internal::saturate_cast<s16>(src[j] + dst[j]);
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <int shift>
+void accumulateSquareConst(const Size2D &size,
+                           const u8 *srcBase, ptrdiff_t srcStride,
+                           s16 *dstBase, ptrdiff_t dstStride)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8* src = internal::getRowPtr(srcBase, srcStride, i);
+        s16* dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            internal::prefetch(dst + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+            int16x8_t v_dst0 = vld1q_s16(dst + j), v_dst1 = vld1q_s16(dst + j + 8);
+            int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src)));
+            int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src)));
+
+            int16x4_t v_srclo = vget_low_s16(v_src0), v_srchi = vget_high_s16(v_src0);
+            v_dst0 = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst0))),
+                                  vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst0))));
+
+            v_srclo = vget_low_s16(v_src1);
+            v_srchi = vget_high_s16(v_src1);
+            v_dst1 = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst1))),
+                                  vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst1))));
+
+            vst1q_s16(dst + j, v_dst0);
+            vst1q_s16(dst + j + 8, v_dst1);
+        }
+        for (; j < roiw8; j += 8)
+        {
+            int16x8_t v_src = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j)));
+            int16x8_t v_dst = vld1q_s16(dst + j);
+            int16x4_t v_srclo = vget_low_s16(v_src), v_srchi = vget_high_s16(v_src);
+            v_dst = vcombine_s16(vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srclo, v_srclo), shift), vget_low_s16(v_dst))),
+                                 vqmovn_s32(vaddw_s16(vshrq_n_s32(vmull_s16(v_srchi, v_srchi), shift), vget_high_s16(v_dst))));
+            vst1q_s16(dst + j, v_dst);
+        }
+
+        for (; j < size.width; j++)
+        {
+            s32 srcVal = src[j];
+            dst[j] = internal::saturate_cast<s16>(dst[j] + ((srcVal * srcVal) >> shift));
+        }
+    }
+}
+
+template <>
+void accumulateSquareConst<0>(const Size2D &size,
+                              const u8 *srcBase, ptrdiff_t srcStride,
+                              s16 *dstBase, ptrdiff_t dstStride)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8* src = internal::getRowPtr(srcBase, srcStride, i);
+        s16* dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            internal::prefetch(dst + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+            int16x8_t v_dst0 = vld1q_s16(dst + j), v_dst1 = vld1q_s16(dst + j + 8);
+            int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src)));
+            int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src)));
+
+            int16x4_t v_srclo = vget_low_s16(v_src0), v_srchi = vget_high_s16(v_src0);
+            v_dst0 = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst0))),
+                                  vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst0))));
+
+            v_srclo = vget_low_s16(v_src1);
+            v_srchi = vget_high_s16(v_src1);
+            v_dst1 = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst1))),
+                                  vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst1))));
+
+            vst1q_s16(dst + j, v_dst0);
+            vst1q_s16(dst + j + 8, v_dst1);
+        }
+        for (; j < roiw8; j += 8)
+        {
+            int16x8_t v_src = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j)));
+            int16x8_t v_dst = vld1q_s16(dst + j);
+            int16x4_t v_srclo = vget_low_s16(v_src), v_srchi = vget_high_s16(v_src);
+            v_dst = vcombine_s16(vqmovn_s32(vaddw_s16(vmull_s16(v_srclo, v_srclo), vget_low_s16(v_dst))),
+                                 vqmovn_s32(vaddw_s16(vmull_s16(v_srchi, v_srchi), vget_high_s16(v_dst))));
+            vst1q_s16(dst + j, v_dst);
+        }
+
+        for (; j < size.width; j++)
+        {
+            s32 srcVal = src[j];
+            dst[j] = internal::saturate_cast<s16>(dst[j] + srcVal * srcVal);
+        }
+    }
+}
+
+typedef void (* accumulateSquareConstFunc)(const Size2D &size,
+                                           const u8 *srcBase, ptrdiff_t srcStride,
+                                           s16 *dstBase, ptrdiff_t dstStride);
+
+} // namespace
+
+#endif
+
+void accumulateSquare(const Size2D &size,
+                      const u8 *srcBase, ptrdiff_t srcStride,
+                      s16 *dstBase, ptrdiff_t dstStride,
+                      u32 shift)
+{
+    if (shift >= 16)
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            std::memset(dst, 0, sizeof(s16) * size.width);
+        }
+        return;
+    }
+
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    // this ugly contruction is needed to avoid:
+    // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant
+    // return (int16x8_t)__builtin_neon_vshr_nv8hi (__a, __b, 1);
+
+    accumulateSquareConstFunc funcs[16] =
+    {
+        accumulateSquareConst<0>,
+        accumulateSquareConst<1>,
+        accumulateSquareConst<2>,
+        accumulateSquareConst<3>,
+        accumulateSquareConst<4>,
+        accumulateSquareConst<5>,
+        accumulateSquareConst<6>,
+        accumulateSquareConst<7>,
+        accumulateSquareConst<8>,
+        accumulateSquareConst<9>,
+        accumulateSquareConst<10>,
+        accumulateSquareConst<11>,
+        accumulateSquareConst<12>,
+        accumulateSquareConst<13>,
+        accumulateSquareConst<14>,
+        accumulateSquareConst<15>
+    }, func = funcs[shift];
+
+    func(size, srcBase, srcStride, dstBase, dstStride);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)shift;
+#endif
+}
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+struct AccumulateWeightedHalf
+{
+    typedef u8 type;
+
+    void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1,
+                     uint8x16_t & v_dst) const
+    {
+        v_dst = vhaddq_u8(v_src0, v_src1);
+    }
+
+    void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1,
+                     uint8x8_t & v_dst) const
+    {
+        v_dst = vhadd_u8(v_src0, v_src1);
+    }
+
+    void operator() (const u8 * src0, const u8 * src1, u8 * dst) const
+    {
+        dst[0] = ((u16)(src0[0]) + src1[0]) >> 1;
+    }
+};
+
+struct AccumulateWeighted
+{
+    typedef u8 type;
+
+    float alpha, beta;
+    float32x4_t v_alpha, v_beta;
+
+    explicit AccumulateWeighted(float _alpha) :
+        alpha(_alpha), beta(1 - _alpha)
+    {
+        v_alpha = vdupq_n_f32(alpha);
+        v_beta = vdupq_n_f32(beta);
+    }
+
+    void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1,
+                     uint8x16_t & v_dst) const
+    {
+        uint16x8_t v_src0_p = vmovl_u8(vget_low_u8(v_src0));
+        uint16x8_t v_src1_p = vmovl_u8(vget_low_u8(v_src1));
+        float32x4_t v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p))), v_beta),
+                                        v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))));
+        float32x4_t v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p))), v_beta),
+                                        v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))));
+        uint16x8_t v_dst0 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)),
+                                         vmovn_u32(vcvtq_u32_f32(v_dst1f)));
+
+        v_src0_p = vmovl_u8(vget_high_u8(v_src0));
+        v_src1_p = vmovl_u8(vget_high_u8(v_src1));
+        v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1_p))), v_beta),
+                            v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0_p))));
+        v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1_p))), v_beta),
+                            v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0_p))));
+        uint16x8_t v_dst1 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)),
+                                         vmovn_u32(vcvtq_u32_f32(v_dst1f)));
+
+        v_dst = vcombine_u8(vmovn_u16(v_dst0), vmovn_u16(v_dst1));
+    }
+
+    void operator() (const uint8x8_t & _v_src0, const uint8x8_t & _v_src1,
+                     uint8x8_t & v_dst) const
+    {
+        uint16x8_t v_src0 = vmovl_u8(_v_src0), v_src1 = vmovl_u8(_v_src1);
+
+        float32x4_t v_dst0f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src1))), v_beta),
+                                        v_alpha, vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_src0))));
+        float32x4_t v_dst1f = vmlaq_f32(vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src1))), v_beta),
+                                        v_alpha, vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_src0))));
+        uint16x8_t _v_dst = vcombine_u16(vmovn_u32(vcvtq_u32_f32(v_dst0f)),
+                                        vmovn_u32(vcvtq_u32_f32(v_dst1f)));
+
+        v_dst = vmovn_u16(_v_dst);
+    }
+
+    void operator() (const u8 * src0, const u8 * src1, u8 * dst) const
+    {
+        dst[0] = beta * src1[0] + alpha * src0[0];
+    }
+};
+
+} // namespace
+
+#endif
+
+void accumulateWeighted(const Size2D &size,
+                        const u8 *srcBase, ptrdiff_t srcStride,
+                        u8 *dstBase, ptrdiff_t dstStride,
+                        f32 alpha)
+{
+    if (alpha == 0.0f)
+        return;
+    if (alpha == 1.0f)
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+            u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            std::memcpy(dst, src, sizeof(u8) * size.width);
+        }
+        return;
+    }
+
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    // in this case we can use the following scheme:
+    // dst[p] = (src[p] + dst[p]) >> 1
+    // which is faster
+    if (alpha == 0.5f)
+    {
+        internal::vtransform(size,
+                             srcBase, srcStride,
+                             dstBase, dstStride,
+                             dstBase, dstStride,
+                             AccumulateWeightedHalf());
+
+        return;
+    }
+
+    internal::vtransform(size,
+                     srcBase, srcStride,
+                     dstBase, dstStride,
+                     dstBase, dstStride,
+                     AccumulateWeighted(alpha));
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)alpha;
+#endif
+}
+
+} //namespace CAROTENE_NS

3rdparty/carotene/src/add.cpp

@@ -0,0 +1,475 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T, typename WT>
+struct AddWrap
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vaddq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vadd(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = (T)((WT)src0[0] + (WT)src1[0]);
+    }
+};
+
+template <typename T, typename WT>
+struct AddSaturate
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vqaddq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vqadd(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = internal::saturate_cast<T>((WT)src0[0] + (WT)src1[0]);
+    }
+};
+
+} // namespace
+
+#endif
+
+void add(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         u8 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<u8, u16>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<u8, u16>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const s8 * src0Base, ptrdiff_t src0Stride,
+         const s8 * src1Base, ptrdiff_t src1Stride,
+         s8 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<s8, s16>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<s8, s16>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        u16 * dst = internal::getRowPtr((u16 *)dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw32; j += 32)
+        {
+            internal::prefetch(src0 + j);
+            internal::prefetch(src1 + j);
+            uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16);
+            uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16);
+            vst1q_u16(dst + j, vaddl_u8(vget_low_u8(v_src00), vget_low_u8(v_src10)));
+            vst1q_u16(dst + j + 8, vaddl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10)));
+            vst1q_u16(dst + j + 16, vaddl_u8(vget_low_u8(v_src01), vget_low_u8(v_src11)));
+            vst1q_u16(dst + j + 24, vaddl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11)));
+        }
+        for (; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src0 = vld1_u8(src0 + j);
+            uint8x8_t v_src1 = vld1_u8(src1 + j);
+            vst1q_u16(dst + j, vaddl_u8(v_src0, v_src1));
+        }
+
+        for (; j < size.width; j++)
+            dst[j] = (u16)src0[j] + (u16)src1[j];
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void add(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const s16 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (policy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                uint8x16_t v_src0 = vld1q_u8(src0 + j);
+                int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0)));
+                int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0)));
+                int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8);
+                int16x8_t v_dst0 = vqaddq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vqaddq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j)));
+                int16x8_t v_src1 = vld1q_s16(src1 + j);
+                int16x8_t v_dst = vqaddq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = internal::saturate_cast<s16>((s32)src0[j] + (s32)src1[j]);
+        }
+        else
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                uint8x16_t v_src0 = vld1q_u8(src0 + j);
+                int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0)));
+                int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0)));
+                int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8);
+                int16x8_t v_dst0 = vaddq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vaddq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j)));
+                int16x8_t v_src1 = vld1q_s16(src1 + j);
+                int16x8_t v_dst = vaddq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = (s16)((s32)src0[j] + (s32)src1[j]);
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const s16 * src0Base, ptrdiff_t src0Stride,
+         const s16 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+        if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<s16, s32>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<s16, s32>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const u16 * src0Base, ptrdiff_t src0Stride,
+         const u16 * src1Base, ptrdiff_t src1Stride,
+         u16 * dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+        if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<u16, u32>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<u16, u32>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const s32 * src0Base, ptrdiff_t src0Stride,
+         const s32 * src1Base, ptrdiff_t src1Stride,
+         s32 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+        if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<s32, s64>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<s32, s64>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const u32 * src0Base, ptrdiff_t src0Stride,
+         const u32 * src1Base, ptrdiff_t src1Stride,
+         u32 * dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+        if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddSaturate<u32, u64>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             AddWrap<u32, u64>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void add(const Size2D &size,
+         const f32 * src0Base, ptrdiff_t src0Stride,
+         const f32 * src1Base, ptrdiff_t src1Stride,
+         f32 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride,
+                         AddWrap<f32, f32>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/add_weighted.cpp

@@ -0,0 +1,265 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+using namespace internal;
+
+template <typename T> struct TypeTraits;
+template <> struct TypeTraits< u8> { typedef u16 wide;                     typedef  u8 unsign; typedef  uint8x16_t vec128; };
+template <> struct TypeTraits< s8> { typedef s16 wide;                     typedef  u8 unsign; typedef   int8x16_t vec128; };
+template <> struct TypeTraits<u16> { typedef u32 wide; typedef  u8 narrow; typedef u16 unsign; typedef  uint16x8_t vec128; };
+template <> struct TypeTraits<s16> { typedef s32 wide; typedef  s8 narrow; typedef u16 unsign; typedef   int16x8_t vec128; };
+template <> struct TypeTraits<u32> { typedef u64 wide; typedef u16 narrow; typedef u32 unsign; typedef  uint32x4_t vec128; };
+template <> struct TypeTraits<s32> { typedef s64 wide; typedef s16 narrow; typedef u32 unsign; typedef   int32x4_t vec128; };
+template <> struct TypeTraits<f32> { typedef f64 wide;                                         typedef float32x4_t vec128; };
+
+template <typename T> struct wAdd
+{
+    typedef T type;
+
+    f32 alpha, beta, gamma;
+    typedef typename TypeTraits<T>::wide wtype;
+    wAdd<wtype> wideAdd;
+    wAdd(f32 _alpha, f32 _beta, f32 _gamma):
+        alpha(_alpha), beta(_beta), gamma(_gamma),
+        wideAdd(_alpha, _beta, _gamma) {}
+
+    void operator() (const typename VecTraits<T>::vec128 & v_src0,
+                     const typename VecTraits<T>::vec128 & v_src1,
+                     typename VecTraits<T>::vec128 & v_dst) const
+    {
+        typename VecTraits<wtype>::vec128 vrl, vrh;
+        wideAdd(vmovl( vget_low(v_src0)), vmovl( vget_low(v_src1)), vrl);
+        wideAdd(vmovl(vget_high(v_src0)), vmovl(vget_high(v_src1)), vrh);
+
+        v_dst = vcombine(vqmovn(vrl), vqmovn(vrh));
+    }
+
+    void operator() (const typename VecTraits<T>::vec64 & v_src0,
+                     const typename VecTraits<T>::vec64 & v_src1,
+                     typename VecTraits<T>::vec64 & v_dst) const
+    {
+        typename VecTraits<wtype>::vec128 vr;
+        wideAdd(vmovl(v_src0), vmovl(v_src1), vr);
+
+        v_dst = vqmovn(vr);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = saturate_cast<T>(alpha*src0[0] + beta*src1[0] + gamma);
+    }
+};
+
+template <> struct wAdd<s32>
+{
+    typedef s32 type;
+
+    f32 alpha, beta, gamma;
+    float32x4_t valpha, vbeta, vgamma;
+    wAdd(f32 _alpha, f32 _beta, f32 _gamma):
+        alpha(_alpha), beta(_beta), gamma(_gamma)
+    {
+        valpha = vdupq_n_f32(_alpha);
+        vbeta = vdupq_n_f32(_beta);
+        vgamma = vdupq_n_f32(_gamma + 0.5);
+    }
+
+    void operator() (const typename VecTraits<s32>::vec128 & v_src0,
+                     const typename VecTraits<s32>::vec128 & v_src1,
+                     typename VecTraits<s32>::vec128 & v_dst) const
+    {
+        float32x4_t vs1 = vcvtq_f32_s32(v_src0);
+        float32x4_t vs2 = vcvtq_f32_s32(v_src1);
+
+        vs1 = vmlaq_f32(vgamma, vs1, valpha);
+        vs1 = vmlaq_f32(vs1, vs2, vbeta);
+        v_dst = vcvtq_s32_f32(vs1);
+    }
+
+    void operator() (const typename VecTraits<s32>::vec64 & v_src0,
+                     const typename VecTraits<s32>::vec64 & v_src1,
+                     typename VecTraits<s32>::vec64 & v_dst) const
+    {
+        float32x2_t vs1 = vcvt_f32_s32(v_src0);
+        float32x2_t vs2 = vcvt_f32_s32(v_src1);
+
+        vs1 = vmla_f32(vget_low(vgamma), vs1, vget_low(valpha));
+        vs1 = vmla_f32(vs1, vs2, vget_low(vbeta));
+        v_dst = vcvt_s32_f32(vs1);
+    }
+
+    void operator() (const s32 * src0, const s32 * src1, s32 * dst) const
+    {
+        dst[0] = saturate_cast<s32>(alpha*src0[0] + beta*src1[0] + gamma);
+    }
+};
+
+template <> struct wAdd<u32>
+{
+    typedef u32 type;
+
+    f32 alpha, beta, gamma;
+    float32x4_t valpha, vbeta, vgamma;
+    wAdd(f32 _alpha, f32 _beta, f32 _gamma):
+        alpha(_alpha), beta(_beta), gamma(_gamma)
+    {
+        valpha = vdupq_n_f32(_alpha);
+        vbeta = vdupq_n_f32(_beta);
+        vgamma = vdupq_n_f32(_gamma + 0.5);
+    }
+
+    void operator() (const typename VecTraits<u32>::vec128 & v_src0,
+                     const typename VecTraits<u32>::vec128 & v_src1,
+                     typename VecTraits<u32>::vec128 & v_dst) const
+    {
+        float32x4_t vs1 = vcvtq_f32_u32(v_src0);
+        float32x4_t vs2 = vcvtq_f32_u32(v_src1);
+
+        vs1 = vmlaq_f32(vgamma, vs1, valpha);
+        vs1 = vmlaq_f32(vs1, vs2, vbeta);
+        v_dst = vcvtq_u32_f32(vs1);
+    }
+
+    void operator() (const typename VecTraits<u32>::vec64 & v_src0,
+                     const typename VecTraits<u32>::vec64 & v_src1,
+                     typename VecTraits<u32>::vec64 & v_dst) const
+    {
+        float32x2_t vs1 = vcvt_f32_u32(v_src0);
+        float32x2_t vs2 = vcvt_f32_u32(v_src1);
+
+        vs1 = vmla_f32(vget_low(vgamma), vs1, vget_low(valpha));
+        vs1 = vmla_f32(vs1, vs2, vget_low(vbeta));
+        v_dst = vcvt_u32_f32(vs1);
+    }
+
+    void operator() (const u32 * src0, const u32 * src1, u32 * dst) const
+    {
+        dst[0] = saturate_cast<u32>(alpha*src0[0] + beta*src1[0] + gamma);
+    }
+};
+
+template <> struct wAdd<f32>
+{
+    typedef f32 type;
+
+    f32 alpha, beta, gamma;
+    float32x4_t valpha, vbeta, vgamma;
+    wAdd(f32 _alpha, f32 _beta, f32 _gamma):
+        alpha(_alpha), beta(_beta), gamma(_gamma)
+    {
+        valpha = vdupq_n_f32(_alpha);
+        vbeta = vdupq_n_f32(_beta);
+        vgamma = vdupq_n_f32(_gamma + 0.5);
+    }
+
+    void operator() (const typename VecTraits<f32>::vec128 & v_src0,
+                     const typename VecTraits<f32>::vec128 & v_src1,
+                     typename VecTraits<f32>::vec128 & v_dst) const
+    {
+        float32x4_t vs1 = vmlaq_f32(vgamma, v_src0, valpha);
+        v_dst = vmlaq_f32(vs1, v_src1, vbeta);
+    }
+
+    void operator() (const typename VecTraits<f32>::vec64 & v_src0,
+                     const typename VecTraits<f32>::vec64 & v_src1,
+                     typename VecTraits<f32>::vec64 & v_dst) const
+    {
+        float32x2_t vs1 = vmla_f32(vget_low(vgamma), v_src0, vget_low(valpha));
+        v_dst = vmla_f32(vs1, v_src1, vget_low(vbeta));
+
+    }
+
+    void operator() (const f32 * src0, const f32 * src1, f32 * dst) const
+    {
+        dst[0] = alpha*src0[0] + beta*src1[0] + gamma;
+    }
+};
+
+} // namespace
+
+#define IMPL_ADDWEIGHTED(type)                                \
+void addWeighted(const Size2D &size,                          \
+                 const type * src0Base, ptrdiff_t src0Stride, \
+                 const type * src1Base, ptrdiff_t src1Stride, \
+                 type * dstBase, ptrdiff_t dstStride,         \
+                 f32 alpha, f32 beta, f32 gamma)              \
+{                                                             \
+    internal::assertSupportedConfiguration();                 \
+    wAdd<type> wgtAdd(alpha,                                  \
+                      beta,                                   \
+                      gamma);                                 \
+    internal::vtransform(size,                                \
+                         src0Base, src0Stride,                \
+                         src1Base, src1Stride,                \
+                         dstBase, dstStride,                  \
+                         wgtAdd);                             \
+}
+
+#else
+
+#define IMPL_ADDWEIGHTED(type)                                \
+void addWeighted(const Size2D &,                              \
+                 const type *, ptrdiff_t,                     \
+                 const type *, ptrdiff_t,                     \
+                 type *, ptrdiff_t,                           \
+                 f32, f32, f32)                               \
+{                                                             \
+    internal::assertSupportedConfiguration();                 \
+}
+
+#endif
+
+IMPL_ADDWEIGHTED(u8)
+IMPL_ADDWEIGHTED(s8)
+IMPL_ADDWEIGHTED(u16)
+IMPL_ADDWEIGHTED(s16)
+IMPL_ADDWEIGHTED(u32)
+IMPL_ADDWEIGHTED(s32)
+IMPL_ADDWEIGHTED(f32)
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/bitwise.cpp

@@ -0,0 +1,225 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+struct BitwiseAnd
+{
+    typedef u8 type;
+
+    void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1,
+                     uint8x16_t & v_dst) const
+    {
+        v_dst = vandq_u8(v_src0, v_src1);
+    }
+
+    void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1,
+                     uint8x8_t & v_dst) const
+    {
+        v_dst = vand_u8(v_src0, v_src1);
+    }
+
+    void operator() (const u8 * src0, const u8 * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] & src1[0];
+    }
+};
+
+struct BitwiseOr
+{
+    typedef u8 type;
+
+    void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1,
+                     uint8x16_t & v_dst) const
+    {
+        v_dst = vorrq_u8(v_src0, v_src1);
+    }
+
+    void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1,
+                     uint8x8_t & v_dst) const
+    {
+        v_dst = vorr_u8(v_src0, v_src1);
+    }
+
+    void operator() (const u8 * src0, const u8 * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] | src1[0];
+    }
+};
+
+struct BitwiseXor
+{
+    typedef u8 type;
+
+    void operator() (const uint8x16_t & v_src0, const uint8x16_t & v_src1,
+                     uint8x16_t & v_dst) const
+    {
+        v_dst = veorq_u8(v_src0, v_src1);
+    }
+
+    void operator() (const uint8x8_t & v_src0, const uint8x8_t & v_src1,
+                     uint8x8_t & v_dst) const
+    {
+        v_dst = veor_u8(v_src0, v_src1);
+    }
+
+    void operator() (const u8 * src0, const u8 * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] ^ src1[0];
+    }
+};
+
+#endif
+
+void bitwiseNot(const Size2D &size,
+                const u8 *srcBase, ptrdiff_t srcStride,
+                u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8* src = internal::getRowPtr(srcBase, srcStride, i);
+        u8* dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw32; j += 32)
+        {
+            internal::prefetch(src + j);
+            uint8x16_t v_src0 = vld1q_u8(src + j), v_src1 = vld1q_u8(src + j + 16);
+            uint8x16_t v_dst0 = vmvnq_u8(v_src0), v_dst1 = vmvnq_u8(v_src1);
+            vst1q_u8(dst + j, v_dst0);
+            vst1q_u8(dst + j + 16, v_dst1);
+        }
+        for (; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src = vld1_u8(src + j);
+            uint8x8_t v_dst = vmvn_u8(v_src);
+            vst1_u8(dst + j, v_dst);
+        }
+
+        for (; j < size.width; j++)
+        {
+            dst[j] = ~src[j];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void bitwiseAnd(const Size2D &size,
+                const u8 *src0Base, ptrdiff_t src0Stride,
+                const u8 *src1Base, ptrdiff_t src1Stride,
+                u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, BitwiseAnd());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void bitwiseOr(const Size2D &size,
+               const u8 *src0Base, ptrdiff_t src0Stride,
+               const u8 *src1Base, ptrdiff_t src1Stride,
+               u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, BitwiseOr());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void bitwiseXor(const Size2D &size,
+                const u8 *src0Base, ptrdiff_t src0Stride,
+                const u8 *src1Base, ptrdiff_t src1Stride,
+                u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride, BitwiseXor());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/canny.cpp

@@ -0,0 +1,773 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include "saturate_cast.hpp"
+#include <vector>
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+namespace {
+struct RowFilter3x3Canny
+{
+    inline RowFilter3x3Canny(const ptrdiff_t borderxl, const ptrdiff_t borderxr)
+    {
+        vfmask = vreinterpret_u8_u64(vmov_n_u64(borderxl ? 0x0000FFffFFffFFffULL : 0x0100FFffFFffFFffULL));
+        vtmask = vreinterpret_u8_u64(vmov_n_u64(borderxr ? 0x0707060504030201ULL : 0x0706050403020100ULL));
+        lookLeft = offsetk - borderxl;
+        lookRight = offsetk - borderxr;
+    }
+
+    inline void operator()(const u8* src, s16* dstx, s16* dsty, ptrdiff_t width)
+    {
+        uint8x8_t l = vtbl1_u8(vld1_u8(src - lookLeft), vfmask);
+        ptrdiff_t i = 0;
+        for (; i < width - 8 + lookRight; i += 8)
+        {
+            internal::prefetch(src + i);
+            uint8x8_t l18u = vld1_u8(src + i + 1);
+
+            uint8x8_t l2 = l18u;
+            uint8x8_t l0 = vext_u8(l, l18u, 6);
+            int16x8_t l1x2 = vreinterpretq_s16_u16(vshll_n_u8(vext_u8(l, l18u, 7), 1));
+
+            l = l18u;
+
+            int16x8_t l02 = vreinterpretq_s16_u16(vaddl_u8(l2, l0));
+            int16x8_t ldx = vreinterpretq_s16_u16(vsubl_u8(l2, l0));
+            int16x8_t ldy = vaddq_s16(l02, l1x2);
+
+            vst1q_s16(dstx + i, ldx);
+            vst1q_s16(dsty + i, ldy);
+        }
+
+        //tail
+        if (lookRight == 0 || i != width)
+        {
+            uint8x8_t tail0 = vld1_u8(src + (width - 9));//can't get left 1 pixel another way if width==8*k+1
+            uint8x8_t tail2 = vtbl1_u8(vld1_u8(src + (width - 8 + lookRight)), vtmask);
+            uint8x8_t tail1 = vext_u8(vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(tail0), 8*6)), tail2, 7);
+
+            int16x8_t tail02 = vreinterpretq_s16_u16(vaddl_u8(tail2, tail0));
+            int16x8_t tail1x2 = vreinterpretq_s16_u16(vshll_n_u8(tail1, 1));
+            int16x8_t taildx = vreinterpretq_s16_u16(vsubl_u8(tail2, tail0));
+            int16x8_t taildy = vqaddq_s16(tail02, tail1x2);
+
+            vst1q_s16(dstx + (width - 8), taildx);
+            vst1q_s16(dsty + (width - 8), taildy);
+        }
+    }
+
+    uint8x8_t vfmask;
+    uint8x8_t vtmask;
+    enum { offsetk = 1};
+    ptrdiff_t lookLeft;
+    ptrdiff_t lookRight;
+};
+
+template <bool L2gradient>
+inline void ColFilter3x3Canny(const s16* src0, const s16* src1, const s16* src2, s16* dstx, s16* dsty, s32* mag, ptrdiff_t width)
+{
+    ptrdiff_t j = 0;
+    for (; j <= width - 8; j += 8)
+    {
+        ColFilter3x3CannyL1Loop:
+        int16x8_t line0x = vld1q_s16(src0 + j);
+        int16x8_t line1x = vld1q_s16(src1 + j);
+        int16x8_t line2x = vld1q_s16(src2 + j);
+        int16x8_t line0y = vld1q_s16(src0 + j + width);
+        int16x8_t line2y = vld1q_s16(src2 + j + width);
+
+        int16x8_t l02 = vaddq_s16(line0x, line2x);
+        int16x8_t l1x2 = vshlq_n_s16(line1x, 1);
+        int16x8_t dy = vsubq_s16(line2y, line0y);
+        int16x8_t dx = vaddq_s16(l1x2, l02);
+
+        int16x8_t dya = vabsq_s16(dy);
+        int16x8_t dxa = vabsq_s16(dx);
+        int16x8_t norm = vaddq_s16(dya, dxa);
+
+        int32x4_t normh = vmovl_s16(vget_high_s16(norm));
+        int32x4_t norml = vmovl_s16(vget_low_s16(norm));
+
+        vst1q_s16(dsty + j, dy);
+        vst1q_s16(dstx + j, dx);
+        vst1q_s32(mag + j + 4, normh);
+        vst1q_s32(mag + j, norml);
+    }
+    if (j != width)
+    {
+        j = width - 8;
+        goto ColFilter3x3CannyL1Loop;
+    }
+}
+template <>
+inline void ColFilter3x3Canny<true>(const s16* src0, const s16* src1, const s16* src2, s16* dstx, s16* dsty, s32* mag, ptrdiff_t width)
+{
+    ptrdiff_t j = 0;
+    for (; j <= width - 8; j += 8)
+    {
+        ColFilter3x3CannyL2Loop:
+        int16x8_t line0x = vld1q_s16(src0 + j);
+        int16x8_t line1x = vld1q_s16(src1 + j);
+        int16x8_t line2x = vld1q_s16(src2 + j);
+        int16x8_t line0y = vld1q_s16(src0 + j + width);
+        int16x8_t line2y = vld1q_s16(src2 + j + width);
+
+        int16x8_t l02 = vaddq_s16(line0x, line2x);
+        int16x8_t l1x2 = vshlq_n_s16(line1x, 1);
+        int16x8_t dy = vsubq_s16(line2y, line0y);
+        int16x8_t dx = vaddq_s16(l1x2, l02);
+
+        int32x4_t norml = vmull_s16(vget_low_s16(dx), vget_low_s16(dx));
+        int32x4_t normh = vmull_s16(vget_high_s16(dy), vget_high_s16(dy));
+
+        norml = vmlal_s16(norml, vget_low_s16(dy), vget_low_s16(dy));
+        normh = vmlal_s16(normh, vget_high_s16(dx), vget_high_s16(dx));
+
+        vst1q_s16(dsty + j, dy);
+        vst1q_s16(dstx + j, dx);
+        vst1q_s32(mag + j, norml);
+        vst1q_s32(mag + j + 4, normh);
+    }
+    if (j != width)
+    {
+        j = width - 8;
+        goto ColFilter3x3CannyL2Loop;
+    }
+}
+
+template <bool L2gradient>
+inline void NormCanny(const ptrdiff_t colscn, s16* _dx, s16* _dy, s32* _norm)
+{
+    ptrdiff_t j = 0;
+    if (colscn >= 8)
+    {
+        int16x8_t vx = vld1q_s16(_dx);
+        int16x8_t vy = vld1q_s16(_dy);
+        for (; j <= colscn - 16; j+=8)
+        {
+            internal::prefetch(_dx);
+            internal::prefetch(_dy);
+
+            int16x8_t vx2 = vld1q_s16(_dx + j + 8);
+            int16x8_t vy2 = vld1q_s16(_dy + j + 8);
+
+            int16x8_t vabsx = vabsq_s16(vx);
+            int16x8_t vabsy = vabsq_s16(vy);
+
+            int16x8_t norm = vaddq_s16(vabsx, vabsy);
+
+            int32x4_t normh = vmovl_s16(vget_high_s16(norm));
+            int32x4_t norml = vmovl_s16(vget_low_s16(norm));
+
+            vst1q_s32(_norm + j + 4, normh);
+            vst1q_s32(_norm + j + 0, norml);
+
+            vx = vx2;
+            vy = vy2;
+        }
+        int16x8_t vabsx = vabsq_s16(vx);
+        int16x8_t vabsy = vabsq_s16(vy);
+
+        int16x8_t norm = vaddq_s16(vabsx, vabsy);
+
+        int32x4_t normh = vmovl_s16(vget_high_s16(norm));
+        int32x4_t norml = vmovl_s16(vget_low_s16(norm));
+
+        vst1q_s32(_norm + j + 4, normh);
+        vst1q_s32(_norm + j + 0, norml);
+    }
+    for (; j < colscn; j++)
+        _norm[j] = std::abs(s32(_dx[j])) + std::abs(s32(_dy[j]));
+}
+
+template <>
+inline void NormCanny<true>(const ptrdiff_t colscn, s16* _dx, s16* _dy, s32* _norm)
+{
+    ptrdiff_t j = 0;
+    if (colscn >= 8)
+    {
+        int16x8_t vx = vld1q_s16(_dx);
+        int16x8_t vy = vld1q_s16(_dy);
+
+        for (; j <= colscn - 16; j+=8)
+        {
+            internal::prefetch(_dx);
+            internal::prefetch(_dy);
+
+            int16x8_t vxnext = vld1q_s16(_dx + j + 8);
+            int16x8_t vynext = vld1q_s16(_dy + j + 8);
+
+            int32x4_t norml = vmull_s16(vget_low_s16(vx), vget_low_s16(vx));
+            int32x4_t normh = vmull_s16(vget_high_s16(vy), vget_high_s16(vy));
+
+            norml = vmlal_s16(norml, vget_low_s16(vy), vget_low_s16(vy));
+            normh = vmlal_s16(normh, vget_high_s16(vx), vget_high_s16(vx));
+
+            vst1q_s32(_norm + j + 0, norml);
+            vst1q_s32(_norm + j + 4, normh);
+
+            vx = vxnext;
+            vy = vynext;
+        }
+        int32x4_t norml = vmull_s16(vget_low_s16(vx), vget_low_s16(vx));
+        int32x4_t normh = vmull_s16(vget_high_s16(vy), vget_high_s16(vy));
+
+        norml = vmlal_s16(norml, vget_low_s16(vy), vget_low_s16(vy));
+        normh = vmlal_s16(normh, vget_high_s16(vx), vget_high_s16(vx));
+
+        vst1q_s32(_norm + j + 0, norml);
+        vst1q_s32(_norm + j + 4, normh);
+    }
+    for (; j < colscn; j++)
+        _norm[j] = s32(_dx[j])*_dx[j] + s32(_dy[j])*_dy[j];
+}
+
+template <bool L2gradient>
+inline void prepareThresh(f64 low_thresh, f64 high_thresh,
+                          s32 &low, s32 &high)
+{
+    if (low_thresh > high_thresh)
+        std::swap(low_thresh, high_thresh);
+#if defined __GNUC__
+    low = (s32)low_thresh;
+    high = (s32)high_thresh;
+    low -= (low > low_thresh);
+    high -= (high > high_thresh);
+#else
+    low = internal::round(low_thresh);
+    high = internal::round(high_thresh);
+    f32 ldiff = (f32)(low_thresh - low);
+    f32 hdiff = (f32)(high_thresh - high);
+    low -= (ldiff < 0);
+    high -= (hdiff < 0);
+#endif
+}
+template <>
+inline void prepareThresh<true>(f64 low_thresh, f64 high_thresh,
+                                s32 &low, s32 &high)
+{
+    if (low_thresh > high_thresh)
+        std::swap(low_thresh, high_thresh);
+    if (low_thresh > 0) low_thresh *= low_thresh;
+    if (high_thresh > 0) high_thresh *= high_thresh;
+#if defined __GNUC__
+    low = (s32)low_thresh;
+    high = (s32)high_thresh;
+    low -= (low > low_thresh);
+    high -= (high > high_thresh);
+#else
+    low = internal::round(low_thresh);
+    high = internal::round(high_thresh);
+    f32 ldiff = (f32)(low_thresh - low);
+    f32 hdiff = (f32)(high_thresh - high);
+    low -= (ldiff < 0);
+    high -= (hdiff < 0);
+#endif
+}
+
+template <bool L2gradient, bool externalSobel>
+struct _normEstimator
+{
+    ptrdiff_t magstep;
+    ptrdiff_t dxOffset;
+    ptrdiff_t dyOffset;
+    ptrdiff_t shxOffset;
+    ptrdiff_t shyOffset;
+    std::vector<u8> buffer;
+    const ptrdiff_t offsetk;
+    ptrdiff_t borderyt, borderyb;
+    RowFilter3x3Canny sobelRow;
+
+    inline _normEstimator(const Size2D &size, s32, Margin borderMargin,
+                          ptrdiff_t &mapstep, s32** mag_buf, u8* &map):
+                          offsetk(1),
+                          sobelRow(std::max<ptrdiff_t>(0, offsetk - (ptrdiff_t)borderMargin.left),
+                                   std::max<ptrdiff_t>(0, offsetk - (ptrdiff_t)borderMargin.right))
+    {
+        mapstep = size.width + 2;
+        magstep = size.width + 2 + size.width * (4 * sizeof(s16)/sizeof(s32));
+        dxOffset = mapstep * sizeof(s32)/sizeof(s16);
+        dyOffset = dxOffset + size.width * 1;
+        shxOffset = dxOffset + size.width * 2;
+        shyOffset = dxOffset + size.width * 3;
+        buffer.resize( (size.width+2)*(size.height+2) + magstep*3*sizeof(s32) );
+        mag_buf[0] = (s32*)&buffer[0];
+        mag_buf[1] = mag_buf[0] + magstep;
+        mag_buf[2] = mag_buf[1] + magstep;
+        memset(mag_buf[0], 0, mapstep * sizeof(s32));
+
+        map = (u8*)(mag_buf[2] + magstep);
+        memset(map, 1, mapstep);
+        memset(map + mapstep*(size.height + 1), 1, mapstep);
+        borderyt = std::max<ptrdiff_t>(0, offsetk - (ptrdiff_t)borderMargin.top);
+        borderyb = std::max<ptrdiff_t>(0, offsetk - (ptrdiff_t)borderMargin.bottom);
+    }
+    inline void firstRow(const Size2D &size, s32,
+                         const u8 *srcBase, ptrdiff_t srcStride,
+                         s16*, ptrdiff_t,
+                         s16*, ptrdiff_t,
+                         s32** mag_buf)
+    {
+        //sobelH row #0
+        const u8* _src = internal::getRowPtr(srcBase, srcStride, 0);
+        sobelRow(_src, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[0]) + shyOffset, size.width);
+        //sobelH row #1
+        _src = internal::getRowPtr(srcBase, srcStride, 1);
+        sobelRow(_src, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[1]) + shyOffset, size.width);
+
+        mag_buf[1][0] = mag_buf[1][size.width+1] = 0;
+        if (borderyt == 0)
+        {
+            //sobelH row #-1
+            _src = internal::getRowPtr(srcBase, srcStride, -1);
+            sobelRow(_src, ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[2]) + shyOffset, size.width);
+
+            ColFilter3x3Canny<L2gradient>( ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset,
+                                           ((s16*)mag_buf[1]) + dxOffset,  ((s16*)mag_buf[1]) + dyOffset, mag_buf[1] + 1, size.width);
+        }
+        else
+        {
+            ColFilter3x3Canny<L2gradient>( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset,
+                                           ((s16*)mag_buf[1]) + dxOffset,  ((s16*)mag_buf[1]) + dyOffset, mag_buf[1] + 1, size.width);
+        }
+    }
+    inline void nextRow(const Size2D &size, s32,
+                        const u8 *srcBase, ptrdiff_t srcStride,
+                        s16*, ptrdiff_t,
+                        s16*, ptrdiff_t,
+                        const ptrdiff_t &mapstep, s32** mag_buf,
+                        size_t i, const s16* &_x, const s16* &_y)
+    {
+        mag_buf[2][0] = mag_buf[2][size.width+1] = 0;
+        if (i < size.height - borderyb)
+        {
+            const u8* _src = internal::getRowPtr(srcBase, srcStride, i+1);
+            //sobelH row #i+1
+            sobelRow(_src, ((s16*)mag_buf[2]) + shxOffset, ((s16*)mag_buf[2]) + shyOffset, size.width);
+
+            ColFilter3x3Canny<L2gradient>( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[2]) + shxOffset,
+                                           ((s16*)mag_buf[2]) + dxOffset,  ((s16*)mag_buf[2]) + dyOffset, mag_buf[2] + 1, size.width);
+        }
+        else if (i < size.height)
+        {
+            ColFilter3x3Canny<L2gradient>( ((s16*)mag_buf[0]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset, ((s16*)mag_buf[1]) + shxOffset,
+                                           ((s16*)mag_buf[2]) + dxOffset,  ((s16*)mag_buf[2]) + dyOffset, mag_buf[2] + 1, size.width);
+        }
+        else
+            memset(mag_buf[2], 0, mapstep*sizeof(s32));
+        _x = ((s16*)mag_buf[1]) + dxOffset;
+        _y = ((s16*)mag_buf[1]) + dyOffset;
+    }
+};
+template <bool L2gradient>
+struct _normEstimator<L2gradient, true>
+{
+    std::vector<u8> buffer;
+
+    inline _normEstimator(const Size2D &size, s32 cn, Margin,
+                          ptrdiff_t &mapstep, s32** mag_buf, u8* &map)
+    {
+        mapstep = size.width + 2;
+        buffer.resize( (size.width+2)*(size.height+2) + cn*mapstep*3*sizeof(s32) );
+        mag_buf[0] = (s32*)&buffer[0];
+        mag_buf[1] = mag_buf[0] + mapstep*cn;
+        mag_buf[2] = mag_buf[1] + mapstep*cn;
+        memset(mag_buf[0], 0, /* cn* */mapstep * sizeof(s32));
+
+        map = (u8*)(mag_buf[2] + mapstep*cn);
+        memset(map, 1, mapstep);
+        memset(map + mapstep*(size.height + 1), 1, mapstep);
+    }
+    inline void firstRow(const Size2D &size, s32 cn,
+                         const u8 *, ptrdiff_t,
+                         s16* dxBase, ptrdiff_t dxStride,
+                         s16* dyBase, ptrdiff_t dyStride,
+                         s32** mag_buf)
+    {
+        s32* _norm = mag_buf[1] + 1;
+
+        s16* _dx = internal::getRowPtr(dxBase, dxStride, 0);
+        s16* _dy = internal::getRowPtr(dyBase, dyStride, 0);
+
+        NormCanny<L2gradient>(size.width*cn, _dx, _dy, _norm);
+
+        if(cn > 1)
+        {
+            for(size_t j = 0, jn = 0; j < size.width; ++j, jn += cn)
+            {
+                size_t maxIdx = jn;
+                for(s32 k = 1; k < cn; ++k)
+                    if(_norm[jn + k] > _norm[maxIdx]) maxIdx = jn + k;
+                _norm[j] = _norm[maxIdx];
+                _dx[j] = _dx[maxIdx];
+                _dy[j] = _dy[maxIdx];
+            }
+        }
+
+        _norm[-1] = _norm[size.width] = 0;
+    }
+    inline void nextRow(const Size2D &size, s32 cn,
+                        const u8 *, ptrdiff_t,
+                        s16* dxBase, ptrdiff_t dxStride,
+                        s16* dyBase, ptrdiff_t dyStride,
+                        const ptrdiff_t &mapstep, s32** mag_buf,
+                        size_t i, const s16* &_x, const s16* &_y)
+    {
+        s32* _norm = mag_buf[(i > 0) + 1] + 1;
+        if (i < size.height)
+        {
+            s16* _dx = internal::getRowPtr(dxBase, dxStride, i);
+            s16* _dy = internal::getRowPtr(dyBase, dyStride, i);
+
+            NormCanny<L2gradient>(size.width*cn, _dx, _dy, _norm);
+
+            if(cn > 1)
+            {
+                for(size_t j = 0, jn = 0; j < size.width; ++j, jn += cn)
+                {
+                    size_t maxIdx = jn;
+                    for(s32 k = 1; k < cn; ++k)
+                        if(_norm[jn + k] > _norm[maxIdx]) maxIdx = jn + k;
+                    _norm[j] = _norm[maxIdx];
+                    _dx[j] = _dx[maxIdx];
+                    _dy[j] = _dy[maxIdx];
+                }
+            }
+
+            _norm[-1] = _norm[size.width] = 0;
+        }
+        else
+            memset(_norm-1, 0, /* cn* */mapstep*sizeof(s32));
+
+        _x = internal::getRowPtr(dxBase, dxStride, i-1);
+        _y = internal::getRowPtr(dyBase, dyStride, i-1);
+    }
+};
+
+template <bool L2gradient, bool externalSobel>
+inline void Canny3x3(const Size2D &size, s32 cn,
+                     const u8 * srcBase, ptrdiff_t srcStride,
+                     u8 * dstBase, ptrdiff_t dstStride,
+                     s16 * dxBase, ptrdiff_t dxStride,
+                     s16 * dyBase, ptrdiff_t dyStride,
+                     f64 low_thresh, f64 high_thresh,
+                     Margin borderMargin)
+{
+    s32 low, high;
+    prepareThresh<L2gradient>(low_thresh, high_thresh, low, high);
+
+    ptrdiff_t mapstep;
+    s32* mag_buf[3];
+    u8* map;
+    _normEstimator<L2gradient, externalSobel> normEstimator(size, cn, borderMargin, mapstep, mag_buf, map);
+
+    size_t maxsize = std::max<size_t>( 1u << 10, size.width * size.height / 10 );
+    std::vector<u8*> stack( maxsize );
+    u8 **stack_top = &stack[0];
+    u8 **stack_bottom = &stack[0];
+
+    /* sector numbers
+       (Top-Left Origin)
+
+        1   2   3
+         *  *  *
+          * * *
+        0*******0
+          * * *
+         *  *  *
+        3   2   1
+    */
+
+    #define CANNY_PUSH(d)    *(d) = u8(2), *stack_top++ = (d)
+    #define CANNY_POP(d)     (d) = *--stack_top
+
+    //i == 0
+    normEstimator.firstRow(size, cn, srcBase, srcStride, dxBase, dxStride, dyBase, dyStride, mag_buf);
+    // calculate magnitude and angle of gradient, perform non-maxima supression.
+    // fill the map with one of the following values:
+    //   0 - the pixel might belong to an edge
+    //   1 - the pixel can not belong to an edge
+    //   2 - the pixel does belong to an edge
+    for (size_t i = 1; i <= size.height; i++)
+    {
+        const s16 *_x, *_y;
+        normEstimator.nextRow(size, cn, srcBase, srcStride, dxBase, dxStride, dyBase, dyStride, mapstep, mag_buf, i, _x, _y);
+
+        u8* _map = map + mapstep*i + 1;
+        _map[-1] = _map[size.width] = 1;
+
+        s32* _mag = mag_buf[1] + 1; // take the central row
+        ptrdiff_t magstep1 = mag_buf[2] - mag_buf[1];
+        ptrdiff_t magstep2 = mag_buf[0] - mag_buf[1];
+
+        if ((stack_top - stack_bottom) + size.width > maxsize)
+        {
+            ptrdiff_t sz = (ptrdiff_t)(stack_top - stack_bottom);
+            maxsize = maxsize * 3/2;
+            stack.resize(maxsize);
+            stack_bottom = &stack[0];
+            stack_top = stack_bottom + sz;
+        }
+
+        s32 prev_flag = 0;
+        for (ptrdiff_t j = 0; j < (ptrdiff_t)size.width; j++)
+        {
+            #define CANNY_SHIFT 15
+            const s32 TG22 = (s32)(0.4142135623730950488016887242097*(1<<CANNY_SHIFT) + 0.5);
+
+            s32 m = _mag[j];
+
+            if (m > low)
+            {
+                s32 xs = _x[j];
+                s32 ys = _y[j];
+                s32 x = abs(xs);
+                s32 y = abs(ys) << CANNY_SHIFT;
+
+                s32 tg22x = x * TG22;
+
+                if (y < tg22x)
+                {
+                    if (m > _mag[j-1] && m >= _mag[j+1]) goto __push;
+                }
+                else
+                {
+                    s32 tg67x = tg22x + (x << (CANNY_SHIFT+1));
+                    if (y > tg67x)
+                    {
+                        if (m > _mag[j+magstep2] && m >= _mag[j+magstep1]) goto __push;
+                    }
+                    else
+                    {
+                        s32 s = (xs ^ ys) < 0 ? -1 : 1;
+                        if(m > _mag[j+magstep2-s] && m > _mag[j+magstep1+s]) goto __push;
+                    }
+                }
+            }
+            prev_flag = 0;
+            _map[j] = u8(1);
+            continue;
+            __push:
+            if (!prev_flag && m > high && _map[j-mapstep] != 2)
+            {
+                CANNY_PUSH(_map + j);
+                prev_flag = 1;
+            }
+            else
+                _map[j] = 0;
+        }
+
+        // scroll the ring buffer
+        _mag = mag_buf[0];
+        mag_buf[0] = mag_buf[1];
+        mag_buf[1] = mag_buf[2];
+        mag_buf[2] = _mag;
+    }
+
+    // now track the edges (hysteresis thresholding)
+    while (stack_top > stack_bottom)
+    {
+        u8* m;
+        if ((size_t)(stack_top - stack_bottom) + 8u > maxsize)
+        {
+            ptrdiff_t sz = (ptrdiff_t)(stack_top - stack_bottom);
+            maxsize = maxsize * 3/2;
+            stack.resize(maxsize);
+            stack_bottom = &stack[0];
+            stack_top = stack_bottom + sz;
+        }
+
+        CANNY_POP(m);
+
+        if (!m[-1])         CANNY_PUSH(m - 1);
+        if (!m[1])          CANNY_PUSH(m + 1);
+        if (!m[-mapstep-1]) CANNY_PUSH(m - mapstep - 1);
+        if (!m[-mapstep])   CANNY_PUSH(m - mapstep);
+        if (!m[-mapstep+1]) CANNY_PUSH(m - mapstep + 1);
+        if (!m[mapstep-1])  CANNY_PUSH(m + mapstep - 1);
+        if (!m[mapstep])    CANNY_PUSH(m + mapstep);
+        if (!m[mapstep+1])  CANNY_PUSH(m + mapstep + 1);
+    }
+
+    // the final pass, form the final image
+    uint8x16_t v2 = vmovq_n_u8(2);
+    const u8* ptrmap = map + mapstep + 1;
+    for (size_t i = 0; i < size.height; i++, ptrmap += mapstep)
+    {
+        u8* _dst = internal::getRowPtr(dstBase, dstStride, i);
+        ptrdiff_t j = 0;
+        for (; j < (ptrdiff_t)size.width - 16; j += 16)
+        {
+            internal::prefetch(ptrmap);
+            uint8x16_t vmap = vld1q_u8(ptrmap + j);
+            uint8x16_t vdst = vceqq_u8(vmap, v2);
+            vst1q_u8(_dst+j, vdst);
+        }
+        for (; j < (ptrdiff_t)size.width; j++)
+            _dst[j] = (u8)-(ptrmap[j] >> 1);
+    }
+}
+
+} // namespace
+#endif
+
+bool isCanny3x3Supported(const Size2D &size)
+{
+    return isSupportedConfiguration() &&
+           size.height >= 2 && size.width >= 9;
+}
+
+void Canny3x3L1(const Size2D &size,
+                const u8 * srcBase, ptrdiff_t srcStride,
+                u8 * dstBase, ptrdiff_t dstStride,
+                f64 low_thresh, f64 high_thresh,
+                Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(isCanny3x3Supported(size));
+#ifdef CAROTENE_NEON
+    Canny3x3<false, false>(size, 1,
+                           srcBase, srcStride,
+                           dstBase, dstStride,
+                           NULL, 0,
+                           NULL, 0,
+                           low_thresh, high_thresh,
+                           borderMargin);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)low_thresh;
+    (void)high_thresh;
+    (void)borderMargin;
+#endif
+}
+
+void Canny3x3L2(const Size2D &size,
+                const u8 * srcBase, ptrdiff_t srcStride,
+                u8 * dstBase, ptrdiff_t dstStride,
+                f64 low_thresh, f64 high_thresh,
+                Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(isCanny3x3Supported(size));
+#ifdef CAROTENE_NEON
+    Canny3x3<true, false>(size, 1,
+                          srcBase, srcStride,
+                          dstBase, dstStride,
+                          NULL, 0,
+                          NULL, 0,
+                          low_thresh, high_thresh,
+                          borderMargin);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)low_thresh;
+    (void)high_thresh;
+    (void)borderMargin;
+#endif
+}
+
+void Canny3x3L1(const Size2D &size, s32 cn,
+                     s16 * dxBase, ptrdiff_t dxStride,
+                     s16 * dyBase, ptrdiff_t dyStride,
+                     u8 * dstBase, ptrdiff_t dstStride,
+                     f64 low_thresh, f64 high_thresh)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Canny3x3<false, true>(size, cn,
+                          NULL, 0,
+                          dstBase, dstStride,
+                          dxBase, dxStride,
+                          dyBase, dyStride,
+                          low_thresh, high_thresh,
+                          Margin());
+#else
+    (void)size;
+    (void)cn;
+    (void)dstBase;
+    (void)dstStride;
+    (void)dxBase;
+    (void)dxStride;
+    (void)dyBase;
+    (void)dyStride;
+    (void)low_thresh;
+    (void)high_thresh;
+#endif
+}
+
+void Canny3x3L2(const Size2D &size, s32 cn,
+                     s16 * dxBase, ptrdiff_t dxStride,
+                     s16 * dyBase, ptrdiff_t dyStride,
+                     u8 * dstBase, ptrdiff_t dstStride,
+                     f64 low_thresh, f64 high_thresh)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Canny3x3<true, true>(size, cn,
+                         NULL, 0,
+                         dstBase, dstStride,
+                         dxBase, dxStride,
+                         dyBase, dyStride,
+                         low_thresh, high_thresh,
+                         Margin());
+#else
+    (void)size;
+    (void)cn;
+    (void)dstBase;
+    (void)dstStride;
+    (void)dxBase;
+    (void)dxStride;
+    (void)dyBase;
+    (void)dyStride;
+    (void)low_thresh;
+    (void)high_thresh;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/channel_extract.cpp

@@ -0,0 +1,486 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+void extract2(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u8 * dstBase, ptrdiff_t dstStride,
+              u32 coi)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+#ifndef ANDROID
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+#endif
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0u; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t sj = 0u, dj = 0u;
+
+#ifndef ANDROID
+        for (; dj < roiw32; sj += 64, dj += 32)
+        {
+            internal::prefetch(src + sj);
+
+            uint8x16x2_t v_src = vld2q_u8(src + sj);
+            vst1q_u8(dst + dj, v_src.val[coi]);
+
+            v_src = vld2q_u8(src + sj + 32);
+            vst1q_u8(dst + dj + 16, v_src.val[coi]);
+        }
+#endif
+
+        for (; dj < roiw8; sj += 16, dj += 8)
+        {
+            uint8x8x2_t v_src = vld2_u8(src + sj);
+            vst1_u8(dst + dj, v_src.val[coi]);
+        }
+
+        for (; dj < size.width; sj += 2, ++dj)
+        {
+            dst[dj] = src[sj + coi];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)coi;
+#endif
+}
+
+void extract3(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u8 * dstBase, ptrdiff_t dstStride,
+              u32 coi)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+#ifndef ANDROID
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+#endif
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0u; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t sj = 0u, dj = 0u;
+
+#ifndef ANDROID
+        for (; dj < roiw32; sj += 96, dj += 32)
+        {
+            internal::prefetch(src + sj);
+
+            uint8x16x3_t v_src = vld3q_u8(src + sj);
+            vst1q_u8(dst + dj, v_src.val[coi]);
+
+            v_src = vld3q_u8(src + sj + 48);
+            vst1q_u8(dst + dj + 16, v_src.val[coi]);
+        }
+#endif
+
+        for (; dj < roiw8; sj += 24, dj += 8)
+        {
+            uint8x8x3_t v_src = vld3_u8(src + sj);
+            vst1_u8(dst + dj, v_src.val[coi]);
+        }
+
+        for (; dj < size.width; sj += 3, ++dj)
+        {
+            dst[dj] = src[sj + coi];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)coi;
+#endif
+}
+
+void extract4(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u8 * dstBase, ptrdiff_t dstStride,
+              u32 coi)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+#ifndef ANDROID
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+#endif
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0u; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t sj = 0u, dj = 0u;
+
+#ifndef ANDROID
+        for (; dj < roiw32; sj += 128, dj += 32)
+        {
+            internal::prefetch(src + sj);
+
+            uint8x16x4_t v_src = vld4q_u8(src + sj);
+            vst1q_u8(dst + dj, v_src.val[coi]);
+
+            v_src = vld4q_u8(src + sj + 64);
+            vst1q_u8(dst + dj + 16, v_src.val[coi]);
+        }
+#endif
+
+        for (; dj < roiw8; sj += 32, dj += 8)
+        {
+            uint8x8x4_t v_src = vld4_u8(src + sj);
+            vst1_u8(dst + dj, v_src.val[coi]);
+        }
+
+        for (; dj < size.width; sj += 4, ++dj)
+        {
+            dst[dj] = src[sj + coi];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)coi;
+#endif
+}
+
+#define FILL_LINES2(macro,type) \
+            macro##_LINE(type,0) \
+            macro##_LINE(type,1)
+#define FILL_LINES3(macro,type) \
+            FILL_LINES2(macro,type) \
+            macro##_LINE(type,2)
+#define FILL_LINES4(macro,type) \
+            FILL_LINES3(macro,type) \
+            macro##_LINE(type,3)
+
+#define FARG_LINE(type, n) , type * dst##n##Base, ptrdiff_t dst##n##Stride
+
+#ifdef CAROTENE_NEON
+
+#define VROW_LINE(type, n) type * dst##n = internal::getRowPtr(dst##n##Base, dst##n##Stride, i);
+#define VST1Q_LINE(type, n) vst1q_##type(dst##n + dj, v_src.val[n]);
+#define VST1_LINE(type, n) vst1_##type(dst##n + dj, v_src.val[n]);
+#define SST_LINE(type, n) dst##n[dj] = src[sj + n];
+
+#define MUL2(val) (val << 1)
+#define MUL3(val) (MUL2(val) + val)
+#define MUL4(val) (val << 2)
+
+#define CONTDST2 srcStride == dst0Stride && \
+                 srcStride == dst1Stride &&
+#define CONTDST3 srcStride == dst0Stride && \
+                 srcStride == dst1Stride && \
+                 srcStride == dst2Stride &&
+#define CONTDST4 srcStride == dst0Stride && \
+                 srcStride == dst1Stride && \
+                 srcStride == dst2Stride && \
+                 srcStride == dst3Stride &&
+
+#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7
+
+#define SPLIT_ASM2(sgn, bits) __asm__ ( \
+                                          "vld2." #bits " {d0, d2}, [%[in0]]            \n\t" \
+                                          "vld2." #bits " {d1, d3}, [%[in1]]            \n\t" \
+                                          "vst1." #bits " {d0-d1}, [%[out0]]            \n\t" \
+                                          "vst1." #bits " {d2-d3}, [%[out1]]            \n\t" \
+                                          : \
+                                          : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), \
+                                            [in0]  "r" (src + sj), [in1]  "r" (src + sj + MUL2(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3" \
+                                      );
+#define SPLIT_ASM3(sgn, bits) __asm__ ( \
+                                          "vld3." #bits " {d0, d2, d4}, [%[in0]]        \n\t" \
+                                          "vld3." #bits " {d1, d3, d5}, [%[in1]]        \n\t" \
+                                          "vst1." #bits " {d0-d1}, [%[out0]]            \n\t" \
+                                          "vst1." #bits " {d2-d3}, [%[out1]]            \n\t" \
+                                          "vst1." #bits " {d4-d5}, [%[out2]]            \n\t" \
+                                          : \
+                                          : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), \
+                                            [in0]  "r" (src + sj), [in1]  "r" (src + sj + MUL3(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3","d4","d5" \
+                                      );
+#define SPLIT_ASM4(sgn, bits) __asm__ ( \
+                                          "vld4." #bits " {d0, d2, d4, d6}, [%[in0]]    \n\t" \
+                                          "vld4." #bits " {d1, d3, d5, d7}, [%[in1]]    \n\t" \
+                                          "vst1." #bits " {d0-d1}, [%[out0]]            \n\t" \
+                                          "vst1." #bits " {d2-d3}, [%[out1]]            \n\t" \
+                                          "vst1." #bits " {d4-d5}, [%[out2]]            \n\t" \
+                                          "vst1." #bits " {d6-d7}, [%[out3]]            \n\t" \
+                                          : \
+                                          : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), [out3] "r" (dst3 + dj), \
+                                            [in0]  "r" (src + sj), [in1]  "r" (src + sj + MUL4(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3","d4","d5","d6","d7" \
+                                      );
+
+#define SPLIT_QUAD(sgn, bits, n) { \
+                                     internal::prefetch(src + sj); \
+                                     SPLIT_ASM##n(sgn, bits) \
+                                 }
+
+#else
+
+#define SPLIT_QUAD(sgn, bits, n) { \
+                                     internal::prefetch(src + sj); \
+                                     vec128 v_src = vld##n##q_##sgn##bits(src + sj); \
+                                     FILL_LINES##n(VST1Q, sgn##bits) \
+                                 }
+
+#endif
+
+#define SPLIT(sgn,bits,n) void split##n(const Size2D &_size,                                            \
+                                    const sgn##bits * srcBase, ptrdiff_t srcStride                      \
+                                    FILL_LINES##n(FARG, sgn##bits) )                                    \
+{                                                                                                       \
+    internal::assertSupportedConfiguration();                                                           \
+    Size2D size(_size);                                                                                 \
+    if (CONTDST##n                                                                                      \
+        dst0Stride == (ptrdiff_t)(size.width))                                                          \
+    {                                                                                                   \
+        size.width *= size.height;                                                                      \
+        size.height = 1;                                                                                \
+    }                                                                                                   \
+    typedef internal::VecTraits<sgn##bits, n>::vec128 vec128;                                           \
+    size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \
+    typedef internal::VecTraits<sgn##bits, n>::vec64 vec64;                                             \
+    size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0;    \
+                                                                                                        \
+    for (size_t i = 0u; i < size.height; ++i)                                                           \
+    {                                                                                                   \
+        const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i);                             \
+        FILL_LINES##n(VROW, sgn##bits)                                                                  \
+        size_t sj = 0u, dj = 0u;                                                                        \
+                                                                                                        \
+        for (; dj < roiw16; sj += MUL##n(16)/sizeof(sgn##bits), dj += 16/sizeof(sgn##bits))             \
+            SPLIT_QUAD(sgn, bits, n)                                                                    \
+                                                                                                        \
+        if (dj < roiw8)                                                                                 \
+        {                                                                                               \
+            vec64 v_src = vld##n##_##sgn##bits(src + sj);                                               \
+            FILL_LINES##n(VST1, sgn##bits)                                                              \
+            sj += MUL##n(8)/sizeof(sgn##bits);                                                          \
+            dj += 8/sizeof(sgn##bits);                                                                  \
+        }                                                                                               \
+                                                                                                        \
+        for (; dj < size.width; sj += n, ++dj)                                                          \
+        {                                                                                               \
+            FILL_LINES##n(SST, sgn##bits)                                                               \
+        }                                                                                               \
+    }                                                                                                   \
+}
+
+#define SPLIT64(sgn,n) void split##n(const Size2D &_size,                                               \
+                                     const sgn##64 * srcBase, ptrdiff_t srcStride                       \
+                                     FILL_LINES##n(FARG, sgn##64) )                                     \
+{                                                                                                       \
+    internal::assertSupportedConfiguration();                                                           \
+    Size2D size(_size);                                                                                 \
+    if (CONTDST##n                                                                                      \
+        dst0Stride == (ptrdiff_t)(size.width))                                                          \
+    {                                                                                                   \
+        size.width *= size.height;                                                                      \
+        size.height = 1;                                                                                \
+    }                                                                                                   \
+    typedef internal::VecTraits<sgn##64, n>::vec64 vec64;                                               \
+                                                                                                        \
+    for (size_t i = 0u; i < size.height; ++i)                                                           \
+    {                                                                                                   \
+        const sgn##64 * src = internal::getRowPtr(srcBase, srcStride, i);                               \
+        FILL_LINES##n(VROW, sgn##64)                                                                    \
+        size_t sj = 0u, dj = 0u;                                                                        \
+                                                                                                        \
+        for (; dj < size.width; sj += n, ++dj)                                                          \
+        {                                                                                               \
+            vec64 v_src = vld##n##_##sgn##64(src + sj);                                                 \
+            FILL_LINES##n(VST1, sgn##64)                                                                \
+        }                                                                                               \
+    }                                                                                                   \
+}
+
+#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7
+
+#define ALPHA_QUAD(sgn, bits) { \
+                                  internal::prefetch(src + sj); \
+                                  __asm__ ( \
+                                      "vld4." #bits " {d0, d2, d4, d6}, [%[in0]]    \n\t" \
+                                      "vld4." #bits " {d1, d3, d5, d7}, [%[in1]]    \n\t" \
+                                      "vst3." #bits " {d0, d2, d4}, [%[out3_1]]     \n\t" \
+                                      "vst3." #bits " {d1, d3, d5}, [%[out3_2]]     \n\t" \
+                                      "vst1." #bits " {d6-d7}, [%[out1]]            \n\t" \
+                                      : \
+                                      : [out3_1] "r" (dst3 + d3j), [out3_2] "r" (dst3 + d3j + 24/sizeof(sgn##bits)), [out1] "r" (dst1 + d1j), \
+                                        [in0]  "r" (src + sj), [in1]  "r" (src + sj + 32/sizeof(sgn##bits)) \
+                                      : "d0","d1","d2","d3","d4","d5","d6","d7" \
+                                  ); \
+                              }
+
+#else
+
+#define ALPHA_QUAD(sgn, bits) { \
+                                  internal::prefetch(src + sj); \
+                                  union { vec128_4 v4; vec128_3 v3; } vals; \
+                                  vals.v4 = vld4q_##sgn##bits(src + sj); \
+                                  vst3q_##sgn##bits(dst3 + d3j, vals.v3); \
+                                  vst1q_##sgn##bits(dst1 + d1j, vals.v4.val[3]); \
+                              }
+
+#endif 
+
+#define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &_size,                                          \
+                                          const sgn##bits * srcBase, ptrdiff_t srcStride,               \
+                                          sgn##bits * dst3Base, ptrdiff_t dst3Stride,                   \
+                                          sgn##bits * dst1Base, ptrdiff_t dst1Stride)                   \
+{                                                                                                       \
+    internal::assertSupportedConfiguration();                                                           \
+    Size2D size(_size);                                                                                 \
+    if (srcStride == dst3Stride &&                                                                      \
+        srcStride == dst1Stride &&                                                                      \
+        srcStride == (ptrdiff_t)(size.width))                                                           \
+    {                                                                                                   \
+        size.width *= size.height;                                                                      \
+        size.height = 1;                                                                                \
+    }                                                                                                   \
+    typedef internal::VecTraits<sgn##bits, 4>::vec128 vec128_4;                                         \
+    typedef internal::VecTraits<sgn##bits, 3>::vec128 vec128_3;                                         \
+    size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \
+    typedef internal::VecTraits<sgn##bits, 4>::vec64 vec64_4;                                           \
+    typedef internal::VecTraits<sgn##bits, 3>::vec64 vec64_3;                                           \
+    size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0;    \
+                                                                                                        \
+    for (size_t i = 0u; i < size.height; ++i)                                                           \
+    {                                                                                                   \
+        const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i);                             \
+        sgn##bits * dst3 = internal::getRowPtr(dst3Base, dst3Stride, i);                                \
+        sgn##bits * dst1 = internal::getRowPtr(dst1Base, dst1Stride, i);                                \
+        size_t sj = 0u, d3j = 0u, d1j = 0u;                                                             \
+                                                                                                        \
+        for (; d1j < roiw16; sj += MUL4(16)/sizeof(sgn##bits), d3j += MUL3(16)/sizeof(sgn##bits),       \
+                                                               d1j += 16/sizeof(sgn##bits))             \
+            ALPHA_QUAD(sgn, bits)                                                                       \
+                                                                                                        \
+        if (d1j < roiw8)                                                                                \
+        {                                                                                               \
+            union { vec64_4 v4; vec64_3 v3; } vals;                                                     \
+            vals.v4 = vld4_##sgn##bits(src + sj);                                                       \
+            vst3_u8(dst3 + d3j, vals.v3);                                                               \
+            vst1_u8(dst1 + d1j, vals.v4.val[3]);                                                        \
+            sj += MUL4(8)/sizeof(sgn##bits);                                                            \
+            d3j += MUL3(8)/sizeof(sgn##bits);                                                           \
+            d1j += 8/sizeof(sgn##bits);                                                                 \
+        }                                                                                               \
+                                                                                                        \
+        for (; d1j < size.width; sj += 4, d3j += 3, ++d1j)                                              \
+        {                                                                                               \
+            dst3[d3j+0] = src[sj + 0];                                                                  \
+            dst3[d3j+1] = src[sj + 1];                                                                  \
+            dst3[d3j+2] = src[sj + 2];                                                                  \
+            dst1[d1j]   = src[sj + 3];                                                                  \
+        }                                                                                               \
+    }                                                                                                   \
+}
+
+#else
+
+#define VOID_LINE(type, n) (void)dst##n##Base; (void)dst##n##Stride;
+
+#define SPLIT(sgn,bits,n) void split##n(const Size2D &size,                                          \
+                                    const sgn##bits * srcBase, ptrdiff_t srcStride                   \
+                                    FILL_LINES##n(FARG, sgn##bits) )                                 \
+{                                                                                                    \
+    internal::assertSupportedConfiguration();                                                        \
+    (void)size;                                                                                      \
+    (void)srcBase;                                                                                   \
+    (void)srcStride;                                                                                 \
+    FILL_LINES##n(VOID, sgn##bits)                                                                   \
+}
+
+#define SPLIT64(sgn,n) SPLIT(sgn,64,n)
+
+#define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &size,                                        \
+                                          const sgn##bits * srcBase, ptrdiff_t srcStride,            \
+                                          sgn##bits * dst3Base, ptrdiff_t dst3Stride,                \
+                                          sgn##bits * dst1Base, ptrdiff_t dst1Stride)                \
+{                                                                                                    \
+    internal::assertSupportedConfiguration();                                                        \
+    (void)size;                                                                                      \
+    (void)srcBase;                                                                                   \
+    (void)srcStride;                                                                                 \
+    (void)dst3Base;                                                                                  \
+    (void)dst3Stride;                                                                                \
+    (void)dst1Base;                                                                                  \
+    (void)dst1Stride;                                                                                \
+}
+
+#endif //CAROTENE_NEON
+
+SPLIT(u, 8,2)
+SPLIT(u, 8,3)
+SPLIT(u, 8,4)
+SPLIT(u,16,2)
+SPLIT(u,16,3)
+SPLIT(u,16,4)
+SPLIT(s,32,2)
+SPLIT(s,32,3)
+SPLIT(s,32,4)
+
+SPLIT64(s, 2)
+SPLIT64(s, 3)
+SPLIT64(s, 4)
+
+SPLIT4ALPHA(u,8)
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/channels_combine.cpp

@@ -0,0 +1,389 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#define FILL_LINES2(macro,type) \
+            macro##_LINE(type,0) \
+            macro##_LINE(type,1)
+#define FILL_LINES3(macro,type) \
+            FILL_LINES2(macro,type) \
+            macro##_LINE(type,2)
+#define FILL_LINES4(macro,type) \
+            FILL_LINES3(macro,type) \
+            macro##_LINE(type,3)
+
+#define  FARG_LINE(type, n) , const type * src##n##Base, ptrdiff_t src##n##Stride
+
+#ifdef CAROTENE_NEON
+
+#define  VROW_LINE(type, n) const type * src##n = internal::getRowPtr(src##n##Base, src##n##Stride, i);
+#define  PREF_LINE(type, n) internal::prefetch(src##n + sj);
+#define VLD1Q_LINE(type, n) v_dst.val[n] = vld1q_##type(src##n + sj);
+#define  PRLD_LINE(type, n) internal::prefetch(src##n + sj); v_dst.val[n] = vld1q_##type(src##n + sj);
+#define  VLD1_LINE(type, n) v_dst.val[n] = vld1_##type(src##n + sj);
+#define   SLD_LINE(type, n) dst[dj + n] = src##n[sj];
+
+#define MUL2(val) (val << 1)
+#define MUL3(val) (MUL2(val) + val)
+#define MUL4(val) (val << 2)
+
+#define CONTSRC2 dstStride == src0Stride && \
+                 dstStride == src1Stride &&
+#define CONTSRC3 dstStride == src0Stride && \
+                 dstStride == src1Stride && \
+                 dstStride == src2Stride &&
+#define CONTSRC4 dstStride == src0Stride && \
+                 dstStride == src1Stride && \
+                 dstStride == src2Stride && \
+                 dstStride == src3Stride &&
+
+#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7
+
+#define MERGE_ASM2(sgn, bits) __asm__ ( \
+                                          "vld1." #bits " {d0-d1}, [%[in0]]             \n\t" \
+                                          "vld1." #bits " {d2-d3}, [%[in1]]             \n\t" \
+                                          "vst2." #bits " {d0, d2}, [%[out0]]           \n\t" \
+                                          "vst2." #bits " {d1, d3}, [%[out1]]           \n\t" \
+                                          : \
+                                          : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), \
+                                            [out0]  "r" (dst + dj), [out1]  "r" (dst + dj + MUL2(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3" \
+                                      );
+#define MERGE_ASM3(sgn, bits) __asm__ ( \
+                                          "vld1." #bits " {d0-d1}, [%[in0]]             \n\t" \
+                                          "vld1." #bits " {d2-d3}, [%[in1]]             \n\t" \
+                                          "vld1." #bits " {d4-d5}, [%[in2]]             \n\t" \
+                                          "vst3." #bits " {d0, d2, d4}, [%[out0]]       \n\t" \
+                                          "vst3." #bits " {d1, d3, d5}, [%[out1]]       \n\t" \
+                                          : \
+                                          : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), [in2] "r" (src2 + sj), \
+                                            [out0]  "r" (dst + dj), [out1]  "r" (dst + dj + MUL3(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3","d4","d5" \
+                                      );
+#define MERGE_ASM4(sgn, bits) __asm__ ( \
+                                          "vld1." #bits " {d0-d1}, [%[in0]]             \n\t" \
+                                          "vld1." #bits " {d2-d3}, [%[in1]]             \n\t" \
+                                          "vld1." #bits " {d4-d5}, [%[in2]]             \n\t" \
+                                          "vld1." #bits " {d6-d7}, [%[in3]]             \n\t" \
+                                          "vst4." #bits " {d0, d2, d4, d6}, [%[out0]]   \n\t" \
+                                          "vst4." #bits " {d1, d3, d5, d7}, [%[out1]]   \n\t" \
+                                          : \
+                                          : [in0] "r" (src0 + sj), [in1] "r" (src1 + sj), [in2] "r" (src2 + sj), [in3] "r" (src3 + sj), \
+                                            [out0]  "r" (dst + dj), [out1]  "r" (dst + dj + MUL4(8)/sizeof(sgn##bits)) \
+                                          : "d0","d1","d2","d3","d4","d5","d6","d7" \
+                                      );
+
+#define MERGE_QUAD(sgn, bits, n) { \
+                                     FILL_LINES##n(PREF, sgn##bits) \
+                                     MERGE_ASM##n(sgn, bits) \
+                                 }
+
+#else
+
+#define MERGE_QUAD(sgn, bits, n) { \
+                                     vec128 v_dst; \
+                                     /*FILL_LINES##n(PREF, sgn##bits) \
+                                     FILL_LINES##n(VLD1Q, sgn##bits)*/ \
+                                     FILL_LINES##n(PRLD, sgn##bits) \
+                                     vst##n##q_##sgn##bits(dst + dj, v_dst); \
+                                 }
+
+#endif
+
+#define COMBINE(sgn,bits,n) void combine##n(const Size2D &_size                                             \
+                                        FILL_LINES##n(FARG, sgn##bits),                                     \
+                                        sgn##bits * dstBase, ptrdiff_t dstStride)                           \
+{                                                                                                           \
+    internal::assertSupportedConfiguration();                                                               \
+    Size2D size(_size);                                                                                     \
+    if (CONTSRC##n                                                                                          \
+        dstStride == (ptrdiff_t)(size.width))                                                               \
+    {                                                                                                       \
+        size.width *= size.height;                                                                          \
+        size.height = 1;                                                                                    \
+    }                                                                                                       \
+    typedef internal::VecTraits<sgn##bits, n>::vec128 vec128;                                               \
+    size_t roiw16 = size.width >= (16/sizeof(sgn##bits) - 1) ? size.width - (16/sizeof(sgn##bits) - 1) : 0; \
+    typedef internal::VecTraits<sgn##bits, n>::vec64 vec64;                                                 \
+    size_t roiw8 = size.width >= (8/sizeof(sgn##bits) - 1) ? size.width - (8/sizeof(sgn##bits) - 1) : 0;    \
+                                                                                                            \
+    for (size_t i = 0u; i < size.height; ++i)                                                               \
+    {                                                                                                       \
+        FILL_LINES##n(VROW, sgn##bits)                                                                      \
+        sgn##bits * dst = internal::getRowPtr(dstBase, dstStride, i);                                       \
+        size_t sj = 0u, dj = 0u;                                                                            \
+                                                                                                            \
+        for (; sj < roiw16; sj += 16/sizeof(sgn##bits), dj += MUL##n(16)/sizeof(sgn##bits))                 \
+            MERGE_QUAD(sgn, bits, n)                                                                        \
+                                                                                                            \
+        if ( sj < roiw8 )                                                                                   \
+        {                                                                                                   \
+            vec64 v_dst;                                                                                    \
+            FILL_LINES##n(VLD1, sgn##bits)                                                                  \
+            vst##n##_##sgn##bits(dst + dj, v_dst);                                                          \
+            sj += 8/sizeof(sgn##bits); dj += MUL##n(8)/sizeof(sgn##bits);                                   \
+        }                                                                                                   \
+                                                                                                            \
+        for (; sj < size.width; ++sj, dj += n)                                                              \
+        {                                                                                                   \
+            FILL_LINES##n(SLD, sgn##bits)                                                                   \
+        }                                                                                                   \
+    }                                                                                                       \
+}
+
+#define COMBINE64(sgn,n) void combine##n(const Size2D &_size                                                \
+                                               FILL_LINES##n(FARG, sgn##64),                                \
+                                               sgn##64 * dstBase, ptrdiff_t dstStride)                      \
+{                                                                                                           \
+    internal::assertSupportedConfiguration();                                                               \
+    Size2D size(_size);                                                                                     \
+    if (CONTSRC##n                                                                                          \
+        dstStride == (ptrdiff_t)(size.width))                                                               \
+    {                                                                                                       \
+        size.width *= size.height;                                                                          \
+        size.height = 1;                                                                                    \
+    }                                                                                                       \
+    typedef internal::VecTraits<sgn##64, n>::vec64 vec64;                                                   \
+                                                                                                            \
+    for (size_t i = 0u; i < size.height; ++i)                                                               \
+    {                                                                                                       \
+        FILL_LINES##n(VROW, sgn##64)                                                                        \
+        sgn##64 * dst = internal::getRowPtr(dstBase, dstStride, i);                                         \
+        size_t sj = 0u, dj = 0u;                                                                            \
+                                                                                                            \
+        for (; sj < size.width; ++sj, dj += n)                                                              \
+        {                                                                                                   \
+            vec64 v_dst;                                                                                    \
+            FILL_LINES##n(VLD1, sgn##64)                                                                    \
+            vst##n##_##sgn##64(dst + dj, v_dst);                                                            \
+            /*FILL_LINES##n(SLD, sgn##64)*/                                                                 \
+        }                                                                                                   \
+    }                                                                                                       \
+}
+
+#else
+
+#define  VOID_LINE(type, n) (void)src##n##Base; (void)src##n##Stride;
+
+#define COMBINE(sgn,bits,n) void combine##n(const Size2D &size                                              \
+                                        FILL_LINES##n(FARG, sgn##bits),                                     \
+                                        sgn##bits * dstBase, ptrdiff_t dstStride)                           \
+{                                                                                                           \
+    internal::assertSupportedConfiguration();                                                               \
+    (void)size;                                                                                             \
+    FILL_LINES##n(VOID, sgn##bits)                                                                          \
+    (void)dstBase;                                                                                          \
+    (void)dstStride;                                                                                        \
+}
+#define COMBINE64(sgn,n) COMBINE(sgn,64,n)
+
+#endif //CAROTENE_NEON
+
+COMBINE(u, 8,2)
+COMBINE(u, 8,3)
+COMBINE(u, 8,4)
+COMBINE(u,16,2)
+COMBINE(u,16,3)
+COMBINE(u,16,4)
+COMBINE(s,32,2)
+COMBINE(s,32,3)
+COMBINE(s,32,4)
+COMBINE64(s, 2)
+COMBINE64(s, 3)
+COMBINE64(s, 4)
+
+void combineYUYV(const Size2D &size,
+                 const u8 * srcyBase, ptrdiff_t srcyStride,
+                 const u8 * srcuBase, ptrdiff_t srcuStride,
+                 const u8 * srcvBase, ptrdiff_t srcvStride,
+                 u8 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+#ifndef ANDROID
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+#endif
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0u; i < size.height; i += 1)
+    {
+        const u8 * srcy = internal::getRowPtr(srcyBase, srcyStride, i);
+        const u8 * srcu = internal::getRowPtr(srcuBase, srcuStride, i);
+        const u8 * srcv = internal::getRowPtr(srcvBase, srcvStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t syj = 0u, sj = 0u, dj = 0u;
+
+#ifndef ANDROID
+        for (; sj < roiw32; sj += 32, syj += 64, dj += 128)
+        {
+            internal::prefetch(srcy + syj);
+            internal::prefetch(srcu + sj);
+            internal::prefetch(srcv + sj);
+
+            uint8x16x2_t v_y = vld2q_u8(srcy + syj);
+            uint8x16x4_t v_dst;
+            v_dst.val[0] = v_y.val[0];
+            v_dst.val[1] = vld1q_u8(srcu + sj);
+            v_dst.val[2] = v_y.val[1];
+            v_dst.val[3] = vld1q_u8(srcv + sj);
+            vst4q_u8(dst + dj, v_dst);
+
+            v_y = vld2q_u8(srcy + syj + 32);
+            v_dst.val[0] = v_y.val[0];
+            v_dst.val[1] = vld1q_u8(srcu + sj + 16);
+            v_dst.val[2] = v_y.val[1];
+            v_dst.val[3] = vld1q_u8(srcv + sj + 16);
+            vst4q_u8(dst + dj + 64, v_dst);
+        }
+#endif
+
+        for (; sj < roiw8; sj += 8, syj += 16, dj += 32)
+        {
+            uint8x8x2_t v_y = vld2_u8(srcy + syj);
+            uint8x8x4_t v_dst;
+            v_dst.val[0] = v_y.val[0];
+            v_dst.val[1] = vld1_u8(srcu + sj);
+            v_dst.val[2] = v_y.val[1];
+            v_dst.val[3] = vld1_u8(srcv + sj);
+            vst4_u8(dst + dj, v_dst);
+        }
+
+        for (; sj < size.width; ++sj, syj += 2, dj += 4)
+        {
+            dst[dj] = srcy[syj];
+            dst[dj + 1] = srcu[sj];
+            dst[dj + 2] = srcy[syj + 1];
+            dst[dj + 3] = srcv[sj];
+        }
+    }
+#else
+    (void)size;
+    (void)srcyBase;
+    (void)srcyStride;
+    (void)srcuBase;
+    (void)srcuStride;
+    (void)srcvBase;
+    (void)srcvStride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void combineUYVY(const Size2D &size,
+                 const u8 * srcyBase, ptrdiff_t srcyStride,
+                 const u8 * srcuBase, ptrdiff_t srcuStride,
+                 const u8 * srcvBase, ptrdiff_t srcvStride,
+                 u8 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+#ifndef ANDROID
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+#endif
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0u; i < size.height; ++i)
+    {
+        const u8 * srcy = internal::getRowPtr(srcyBase, srcyStride, i);
+        const u8 * srcu = internal::getRowPtr(srcuBase, srcuStride, i);
+        const u8 * srcv = internal::getRowPtr(srcvBase, srcvStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t syj = 0u, sj = 0u, dj = 0u;
+
+#ifndef ANDROID
+        for (; sj < roiw32; sj += 32, syj += 64, dj += 128)
+        {
+            internal::prefetch(srcy + syj);
+            internal::prefetch(srcu + sj);
+            internal::prefetch(srcv + sj);
+
+            uint8x16x2_t v_y = vld2q_u8(srcy + syj);
+            uint8x16x4_t v_dst;
+            v_dst.val[0] = vld1q_u8(srcu + sj);
+            v_dst.val[1] = v_y.val[0];
+            v_dst.val[2] = vld1q_u8(srcv + sj);
+            v_dst.val[3] = v_y.val[1];
+            vst4q_u8(dst + dj, v_dst);
+
+            v_y = vld2q_u8(srcy + syj + 32);
+            v_dst.val[0] = vld1q_u8(srcu + sj + 16);
+            v_dst.val[1] = v_y.val[0];
+            v_dst.val[2] = vld1q_u8(srcv + sj + 16);
+            v_dst.val[3] = v_y.val[1];
+            vst4q_u8(dst + dj + 64, v_dst);
+        }
+#endif
+
+        for (; sj < roiw8; sj += 8, syj += 16, dj += 32)
+        {
+            uint8x8x2_t v_y = vld2_u8(srcy + syj);
+            uint8x8x4_t v_dst;
+            v_dst.val[0] = vld1_u8(srcu + sj);
+            v_dst.val[1] = v_y.val[0];
+            v_dst.val[2] = vld1_u8(srcv + sj);
+            v_dst.val[3] = v_y.val[1];
+            vst4_u8(dst + dj, v_dst);
+        }
+
+        for (; sj < size.width; ++sj, syj += 2, dj += 4)
+        {
+            dst[dj] = srcu[sj];
+            dst[dj + 1] = srcy[syj];
+            dst[dj + 2] = srcv[sj];
+            dst[dj + 3] = srcy[syj + 1];
+        }
+    }
+#else
+    (void)size;
+    (void)srcyBase;
+    (void)srcyStride;
+    (void)srcuBase;
+    (void)srcuStride;
+    (void)srcvBase;
+    (void)srcvStride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/cmp.cpp

@@ -0,0 +1,340 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+inline void vnst(u8* dst, uint8x16_t v1, uint8x16_t v2) { vst1q_u8(dst, v1); vst1q_u8(dst+16, v2); }
+inline void vnst(u8* dst, uint16x8_t v1, uint16x8_t v2) { vst1q_u8(dst, vcombine_u8(vmovn_u16(v1), vmovn_u16(v2))); }
+inline void vnst(u8* dst, uint32x4_t v1, uint32x4_t v2) { vst1_u8(dst, vmovn_u16(vcombine_u16(vmovn_u32(v1), vmovn_u32(v2)))); }
+
+template <typename Op, int elsize> struct vtail
+{
+    static inline void compare(const typename Op::type * src0, const typename Op::type * src1,
+                               u8 * dst, const Op & op,
+                               size_t &x, size_t width)
+    {
+        //do nothing since there couldn't be enough data
+        (void)src0;
+        (void)src1;
+        (void)dst;
+        (void)op;
+        (void)x;
+        (void)width;
+    }
+};
+template <typename Op> struct vtail<Op, 2>
+{
+    static inline void compare(const typename Op::type * src0, const typename Op::type * src1,
+                               u8 * dst, const Op & op,
+                               size_t &x, size_t width)
+    {
+        typedef typename Op::type type;
+        typedef typename internal::VecTraits<type>::vec128 vec128;
+        typedef typename internal::VecTraits<type>::unsign::vec128 uvec128;
+        //There no more than 15 elements in the tail, so we could handle 8 element vector only once
+        if( x + 8 < width)
+        {
+            vec128  v_src0, v_src1;
+            uvec128 v_dst;
+
+            v_src0 = internal::vld1q(src0 + x);
+            v_src1 = internal::vld1q(src1 + x);
+            op(v_src0, v_src1, v_dst);
+            internal::vst1(dst + x, internal::vmovn(v_dst));
+            x+=8;
+        }
+    }
+};
+template <typename Op> struct vtail<Op, 1>
+{
+    static inline void compare(const typename Op::type * src0, const typename Op::type * src1,
+                               u8 * dst, const Op & op,
+                               size_t &x, size_t width)
+    {
+        typedef typename Op::type type;
+        typedef typename internal::VecTraits<type>::vec128 vec128;
+        typedef typename internal::VecTraits<type>::unsign::vec128 uvec128;
+        typedef typename internal::VecTraits<type>::vec64 vec64;
+        typedef typename internal::VecTraits<type>::unsign::vec64 uvec64;
+        //There no more than 31 elements in the tail, so we could handle once 16+8 or 16 or 8 elements
+        if( x + 16 < width)
+        {
+            vec128  v_src0, v_src1;
+            uvec128 v_dst;
+
+            v_src0 = internal::vld1q(src0 + x);
+            v_src1 = internal::vld1q(src1 + x);
+            op(v_src0, v_src1, v_dst);
+            internal::vst1q(dst + x, v_dst);
+            x+=16;
+        }
+        if( x + 8 < width)
+        {
+            vec64  v_src0, v_src1;
+            uvec64 v_dst;
+
+            v_src0 = internal::vld1(src0 + x);
+            v_src1 = internal::vld1(src1 + x);
+            op(v_src0, v_src1, v_dst);
+            internal::vst1(dst + x, v_dst);
+            x+=8;
+        }
+    }
+};
+
+template <typename Op>
+void vcompare(Size2D size,
+              const typename Op::type * src0Base, ptrdiff_t src0Stride,
+              const typename Op::type * src1Base, ptrdiff_t src1Stride,
+              u8 * dstBase, ptrdiff_t dstStride, const Op & op)
+{
+    typedef typename Op::type type;
+    typedef typename internal::VecTraits<type>::vec128 vec128;
+    typedef typename internal::VecTraits<type>::unsign::vec128 uvec128;
+
+    if (src0Stride == src1Stride && src0Stride == dstStride &&
+        src0Stride == (ptrdiff_t)(size.width * sizeof(type)))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+
+    const u32 step_base = 32 / sizeof(type);
+    size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0;
+
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const type * src0 = internal::getRowPtr(src0Base, src0Stride, y);
+        const type * src1 = internal::getRowPtr(src1Base, src1Stride, y);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, y);
+        size_t x = 0;
+
+        for( ; x < roiw_base; x += step_base )
+        {
+            internal::prefetch(src0 + x);
+            internal::prefetch(src1 + x);
+
+            vec128 v_src00 = internal::vld1q(src0 + x), v_src01 = internal::vld1q(src0 + x + 16 / sizeof(type));
+            vec128 v_src10 = internal::vld1q(src1 + x), v_src11 = internal::vld1q(src1 + x + 16 / sizeof(type));
+            uvec128 v_dst0;
+            uvec128 v_dst1;
+
+            op(v_src00, v_src10, v_dst0);
+            op(v_src01, v_src11, v_dst1);
+
+            vnst(dst + x, v_dst0, v_dst1);
+        }
+
+        vtail<Op, sizeof(type)>::compare(src0, src1, dst, op, x, size.width);
+
+        for (; x < size.width; ++x)
+        {
+            op(src0 + x, src1 + x, dst + x);
+        }
+    }
+}
+
+template<typename T>
+struct OpCmpEQ
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0, const typename internal::VecTraits<T>::vec128 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec128 & v_dst) const
+    {
+        v_dst = internal::vceqq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0, const typename internal::VecTraits<T>::vec64 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec64 & v_dst) const
+    {
+        v_dst = internal::vceq(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] == src1[0] ? 255 : 0;
+    }
+};
+
+template<typename T>
+struct OpCmpNE
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0, const typename internal::VecTraits<T>::vec128 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec128 & v_dst) const
+    {
+        v_dst = internal::vmvnq(internal::vceqq(v_src0, v_src1));
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0, const typename internal::VecTraits<T>::vec64 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec64 & v_dst) const
+    {
+        v_dst = internal::vmvn(internal::vceq(v_src0, v_src1));
+    }
+
+    void operator() (const T * src0, const T * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] == src1[0] ? 0 : 255;
+    }
+};
+
+template<typename T>
+struct OpCmpGT
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0, const typename internal::VecTraits<T>::vec128 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec128 & v_dst) const
+    {
+        v_dst = internal::vcgtq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0, const typename internal::VecTraits<T>::vec64 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec64 & v_dst) const
+    {
+        v_dst = internal::vcgt(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] > src1[0] ? 255 : 0;
+    }
+};
+
+template<typename T>
+struct OpCmpGE
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0, const typename internal::VecTraits<T>::vec128 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec128 & v_dst) const
+    {
+        v_dst = internal::vcgeq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0, const typename internal::VecTraits<T>::vec64 & v_src1,
+              typename internal::VecTraits<T>::unsign::vec64 & v_dst) const
+    {
+        v_dst = internal::vcge(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, u8 * dst) const
+    {
+        dst[0] = src0[0] >= src1[0] ? 255 : 0;
+    }
+};
+
+}
+
+#define IMPL_CMPOP(op, type)                              \
+void cmp##op(const Size2D &size,                          \
+             const type * src0Base, ptrdiff_t src0Stride, \
+             const type * src1Base, ptrdiff_t src1Stride, \
+                       u8 *dstBase, ptrdiff_t dstStride)  \
+{                                                         \
+    internal::assertSupportedConfiguration();             \
+    vcompare(size,                                        \
+             src0Base, src0Stride,                        \
+             src1Base, src1Stride,                        \
+             dstBase, dstStride,                          \
+             OpCmp##op<type>());                          \
+}
+
+#else
+
+#define IMPL_CMPOP(op, type)                              \
+void cmp##op(const Size2D &size,                          \
+             const type * src0Base, ptrdiff_t src0Stride, \
+             const type * src1Base, ptrdiff_t src1Stride, \
+             u8 *dstBase, ptrdiff_t dstStride)            \
+{                                                         \
+    internal::assertSupportedConfiguration();             \
+    (void)size;                                           \
+    (void)src0Base;                                       \
+    (void)src0Stride;                                     \
+    (void)src1Base;                                       \
+    (void)src1Stride;                                     \
+    (void)dstBase;                                        \
+    (void)dstStride;                                      \
+}
+
+#endif
+
+IMPL_CMPOP(EQ, u8)
+IMPL_CMPOP(EQ, s8)
+IMPL_CMPOP(EQ, u16)
+IMPL_CMPOP(EQ, s16)
+IMPL_CMPOP(EQ, u32)
+IMPL_CMPOP(EQ, s32)
+IMPL_CMPOP(EQ, f32)
+
+IMPL_CMPOP(NE, u8)
+IMPL_CMPOP(NE, s8)
+IMPL_CMPOP(NE, u16)
+IMPL_CMPOP(NE, s16)
+IMPL_CMPOP(NE, u32)
+IMPL_CMPOP(NE, s32)
+IMPL_CMPOP(NE, f32)
+
+IMPL_CMPOP(GT, u8)
+IMPL_CMPOP(GT, s8)
+IMPL_CMPOP(GT, u16)
+IMPL_CMPOP(GT, s16)
+IMPL_CMPOP(GT, u32)
+IMPL_CMPOP(GT, s32)
+IMPL_CMPOP(GT, f32)
+
+IMPL_CMPOP(GE, u8)
+IMPL_CMPOP(GE, s8)
+IMPL_CMPOP(GE, u16)
+IMPL_CMPOP(GE, s16)
+IMPL_CMPOP(GE, u32)
+IMPL_CMPOP(GE, s32)
+IMPL_CMPOP(GE, f32)
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/common.cpp

@@ -0,0 +1,108 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <cstdlib>
+#include <iostream>
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+bool isSupportedConfiguration()
+{
+#ifdef CAROTENE_NEON
+    return true;
+#else
+    return false;
+#endif
+}
+
+namespace internal {
+
+void assertSupportedConfiguration(bool parametersSupported)
+{
+    if (!isSupportedConfiguration()) {
+        std::cerr << "internal error: attempted to use an unavailable function" << std::endl;
+        std::abort();
+    }
+
+    if (!parametersSupported) {
+        std::cerr << "internal error: attempted to use a function with unsupported parameters" << std::endl;
+        std::abort();
+    }
+}
+
+ptrdiff_t borderInterpolate(ptrdiff_t _p, size_t _len, BORDER_MODE borderType, size_t startMargin, size_t endMargin)
+{
+    ptrdiff_t p = _p + (ptrdiff_t)startMargin;
+    size_t len = _len + startMargin + endMargin;
+    if( (size_t)p < len )
+        return _p;
+    else if( borderType == BORDER_MODE_REPLICATE )
+        p = p < 0 ? 0 : (ptrdiff_t)len - 1;
+    else if( borderType == BORDER_MODE_REFLECT || borderType == BORDER_MODE_REFLECT101 )
+    {
+        s32 delta = borderType == BORDER_MODE_REFLECT101;
+        if( len == 1 )
+            return 0;
+        do
+        {
+            if( p < 0 )
+                p = -p - 1 + delta;
+            else
+                p = (ptrdiff_t)len - 1 - (p - (ptrdiff_t)len) - delta;
+        }
+        while( (size_t)p >= len );
+    }
+    else if( borderType == BORDER_MODE_WRAP )
+    {
+        if( p < 0 )
+            p -= ((p-(ptrdiff_t)len+1)/(ptrdiff_t)len)*(ptrdiff_t)len;
+        if( p >= (ptrdiff_t)len )
+            p %= (ptrdiff_t)len;
+    }
+    else if( borderType == BORDER_MODE_CONSTANT )
+        p = -1;
+    else
+        internal::assertSupportedConfiguration(false);
+    return p - (ptrdiff_t)startMargin;
+}
+
+} // namespace internal
+} // namespace CAROTENE_NS

3rdparty/carotene/src/common.hpp

@@ -0,0 +1,97 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_SRC_COMMON_HPP
+#define CAROTENE_SRC_COMMON_HPP
+
+#include <cstddef>
+#include <cstdlib>
+#include <algorithm>
+
+#if defined WITH_NEON && (defined __ARM_NEON__ || defined __ARM_NEON)
+#define CAROTENE_NEON
+#endif
+
+#ifdef CAROTENE_NEON
+#include <arm_neon.h>
+#include "intrinsics.hpp"
+#endif
+
+#include <carotene/functions.hpp>
+#include "saturate_cast.hpp"
+
+namespace CAROTENE_NS { namespace internal {
+
+inline void prefetch(const void *ptr, size_t offset = 32*10)
+{
+#if defined __GNUC__
+    __builtin_prefetch(reinterpret_cast<const char*>(ptr) + offset);
+#elif defined _MSC_VER && defined CAROTENE_NEON
+    __prefetch(reinterpret_cast<const char*>(ptr) + offset);
+#else
+    (void)ptr;
+    (void)offset;
+#endif
+}
+
+template <typename T>
+inline T *getRowPtr(T *base, ptrdiff_t stride, size_t row)
+{
+    char *baseRaw = const_cast<char *>(reinterpret_cast<const char *>(base));
+    return reinterpret_cast<T *>(baseRaw + ptrdiff_t(row) * stride);
+}
+
+void assertSupportedConfiguration(bool parametersSupported = true);
+
+ptrdiff_t borderInterpolate(ptrdiff_t _p, size_t _len, BORDER_MODE borderType, size_t startMargin = 0, size_t endMargin = 0);
+
+/*!
+ *  Aligns pointer by the certain number of bytes
+ *
+ *  This small inline function aligns the pointer by the certain number of bytes by shifting
+ *  it forward by 0 or a positive offset.
+ */
+template<typename T> inline T* alignPtr(T* ptr, size_t n=sizeof(T))
+{
+    return (T*)(((size_t)ptr + n-1) & -n);
+}
+
+}}
+
+#endif

3rdparty/carotene/src/convert_depth.cpp

@@ -0,0 +1,399 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <int shift>
+void lshiftConst(const Size2D &size,
+                 const u8 * srcBase, ptrdiff_t srcStride,
+                 s16 * dstBase, ptrdiff_t dstStride)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+            int16x8_t v_dst0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src)));
+            int16x8_t v_dst1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src)));
+
+            vst1q_s16(dst + j, vshlq_n_s16(v_dst0, shift));
+            vst1q_s16(dst + j + 8, vshlq_n_s16(v_dst1, shift));
+        }
+        for (; j < roiw8; j += 8)
+        {
+            int16x8_t v_dst = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j)));
+            vst1q_s16(dst + j, vshlq_n_s16(v_dst, shift));
+        }
+
+        for (; j < size.width; j++)
+        {
+            dst[j] = ((s16)src[j] << shift);
+        }
+    }
+}
+
+template <>
+void lshiftConst<0>(const Size2D &size,
+                    const u8 * srcBase, ptrdiff_t srcStride,
+                    s16 * dstBase, ptrdiff_t dstStride)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+            int16x8_t v_dst0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src)));
+            int16x8_t v_dst1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src)));
+
+            vst1q_s16(dst + j, v_dst0);
+            vst1q_s16(dst + j + 8, v_dst1);
+        }
+        for (; j < roiw8; j += 8)
+        {
+            int16x8_t v_dst = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src + j)));
+            vst1q_s16(dst + j, v_dst);
+        }
+
+        for (; j < size.width; j++)
+        {
+            dst[j] = (s16)src[j];
+        }
+    }
+}
+
+template <int shift>
+void rshiftConst(const Size2D &size,
+                 const s16 * srcBase, ptrdiff_t srcStride,
+                 u8 * dstBase, ptrdiff_t dstStride,
+                 CONVERT_POLICY cpolicy)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s16 * src = internal::getRowPtr(srcBase, srcStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (cpolicy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src + j);
+                int16x8_t v_src0 = vshrq_n_s16(vld1q_s16(src + j), shift),
+                          v_src1 = vshrq_n_s16(vld1q_s16(src + j + 8), shift);
+                uint8x16_t v_dst = vcombine_u8(vqmovun_s16(v_src0),
+                                               vqmovun_s16(v_src1));
+                vst1q_u8(dst + j, v_dst);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src = vshrq_n_s16(vld1q_s16(src + j), shift);
+                vst1_u8(dst + j, vqmovun_s16(v_src));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = internal::saturate_cast<u8>((src[j] >> shift));
+            }
+        }
+        else // CONVERT_POLICY_WRAP
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src + j);
+                int16x8_t v_src0 = vshrq_n_s16(vld1q_s16(src + j), shift),
+                          v_src1 = vshrq_n_s16(vld1q_s16(src + j + 8), shift);
+                int8x16_t v_dst = vcombine_s8(vmovn_s16(v_src0),
+                                              vmovn_s16(v_src1));
+                vst1q_u8(dst + j, vreinterpretq_u8_s8(v_dst));
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src = vshrq_n_s16(vld1q_s16(src + j), shift);
+                vst1_u8(dst + j, vreinterpret_u8_s8(vmovn_s16(v_src)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = (u8)((src[j] >> shift));
+            }
+        }
+    }
+}
+
+template <>
+void rshiftConst<0>(const Size2D &size,
+                    const s16 * srcBase, ptrdiff_t srcStride,
+                    u8 * dstBase, ptrdiff_t dstStride,
+                    CONVERT_POLICY cpolicy)
+{
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s16 * src = internal::getRowPtr(srcBase, srcStride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (cpolicy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src + j);
+                int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8);
+                uint8x16_t v_dst = vcombine_u8(vqmovun_s16(v_src0), vqmovun_s16(v_src1));
+                vst1q_u8(dst + j, v_dst);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src = vld1q_s16(src + j);
+                vst1_u8(dst + j, vqmovun_s16(v_src));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = internal::saturate_cast<u8>(src[j]);
+            }
+        }
+        else // CONVERT_POLICY_WRAP
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src + j);
+                int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8);
+                int8x16_t v_dst = vcombine_s8(vmovn_s16(v_src0), vmovn_s16(v_src1));
+                vst1q_u8(dst + j, vreinterpretq_u8_s8(v_dst));
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src = vld1q_s16(src + j);
+                vst1_u8(dst + j, vreinterpret_u8_s8(vmovn_s16(v_src)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = (u8)src[j];
+            }
+        }
+    }
+}
+
+typedef void (* lshiftConstFunc)(const Size2D &size,
+                                const u8 * srcBase, ptrdiff_t srcStride,
+                                s16 * dstBase, ptrdiff_t dstStride);
+
+typedef void (* rshiftConstFunc)(const Size2D &size,
+                                const s16 * srcBase, ptrdiff_t srcStride,
+                                u8 * dstBase, ptrdiff_t dstStride,
+                                CONVERT_POLICY cpolicy);
+
+} // namespace
+
+#endif
+
+void lshift(const Size2D &size,
+            const u8 * srcBase, ptrdiff_t srcStride,
+            s16 * dstBase, ptrdiff_t dstStride,
+            u32 shift)
+{
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    if (shift >= 16u)
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            std::memset(dst, 0, sizeof(s16) * size.width);
+        }
+        return;
+    }
+
+    // this ugly contruction is needed to avoid:
+    // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant
+    // return (int16x8_t)__builtin_neon_vshl_nv8hi (__a, __b, 1);
+
+    lshiftConstFunc funcs[16] =
+    {
+        lshiftConst<0>,
+        lshiftConst<1>,
+        lshiftConst<2>,
+        lshiftConst<3>,
+        lshiftConst<4>,
+        lshiftConst<5>,
+        lshiftConst<6>,
+        lshiftConst<7>,
+        lshiftConst<8>,
+        lshiftConst<9>,
+        lshiftConst<10>,
+        lshiftConst<11>,
+        lshiftConst<12>,
+        lshiftConst<13>,
+        lshiftConst<14>,
+        lshiftConst<15>
+    }, func = funcs[shift];
+
+    func(size, srcBase, srcStride, dstBase, dstStride);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)shift;
+#endif
+}
+
+void rshift(const Size2D &size,
+            const s16 * srcBase, ptrdiff_t srcStride,
+            u8 * dstBase, ptrdiff_t dstStride,
+            u32 shift, CONVERT_POLICY cpolicy)
+{
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    if (shift >= 16)
+    {
+        if (cpolicy == CONVERT_POLICY_WRAP)
+        {
+            size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+            size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+            int16x8_t v_zero = vdupq_n_s16(0);
+
+            for (size_t i = 0; i < size.height; ++i)
+            {
+                const s16 * src = internal::getRowPtr(srcBase, srcStride, i);
+                u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+                size_t j = 0;
+
+                for (; j < roiw16; j += 16)
+                {
+                    internal::prefetch(src + j);
+                    int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8);
+                    uint8x16_t v_dst = vcombine_u8(vmovn_u16(vcltq_s16(v_src0, v_zero)),
+                                                   vmovn_u16(vcltq_s16(v_src1, v_zero)));
+                    vst1q_u8(dst + j, v_dst);
+                }
+                for (; j < roiw8; j += 8)
+                {
+                    int16x8_t v_src = vld1q_s16(src + j);
+                    vst1_u8(dst + j, vmovn_u16(vcltq_s16(v_src, v_zero)));
+                }
+
+                for (; j < size.width; j++)
+                {
+                    dst[j] = src[j] >= 0 ? 0 : 255;
+                }
+            }
+        }
+        else
+        {
+            for (size_t i = 0; i < size.height; ++i)
+            {
+                u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+                std::memset(dst, 0, sizeof(u8) * size.width);
+            }
+        }
+        return;
+    }
+
+    // this ugly contruction is needed to avoid:
+    // /usr/lib/gcc/arm-linux-gnueabihf/4.8/include/arm_neon.h:3581:59: error: argument must be a constant
+    // return (int16x8_t)__builtin_neon_vshr_nv8hi (__a, __b, 1);
+
+    rshiftConstFunc funcs[16] =
+    {
+        rshiftConst<0>,
+        rshiftConst<1>,
+        rshiftConst<2>,
+        rshiftConst<3>,
+        rshiftConst<4>,
+        rshiftConst<5>,
+        rshiftConst<6>,
+        rshiftConst<7>,
+        rshiftConst<8>,
+        rshiftConst<9>,
+        rshiftConst<10>,
+        rshiftConst<11>,
+        rshiftConst<12>,
+        rshiftConst<13>,
+        rshiftConst<14>,
+        rshiftConst<15>
+    }, func = funcs[shift];
+
+    func(size, srcBase, srcStride, dstBase, dstStride, cpolicy);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)shift;
+    (void)cpolicy;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/convolution.cpp

@@ -0,0 +1,340 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "saturate_cast.hpp"
+
+namespace CAROTENE_NS {
+
+bool isConvolutionSupported(const Size2D &size, const Size2D &ksize,
+                            BORDER_MODE border)
+{
+    return isSupportedConfiguration() && size.width >= 8 &&
+        (border == BORDER_MODE_CONSTANT ||
+            border == BORDER_MODE_REPLICATE) &&
+        (ksize.width == 3) && (ksize.height == 3);
+}
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <int shift>
+int32x4_t vshrq_s32(int32x4_t value)
+{
+    return vshrq_n_s32(value, shift);
+}
+
+template <>
+int32x4_t vshrq_s32<0>(int32x4_t value)
+{
+    return value;
+}
+
+} // namespace
+
+typedef int32x4_t (* vshrq_s32_func)(int32x4_t value);
+
+#endif
+
+void convolution(const Size2D &size,
+                 const u8 * srcBase, ptrdiff_t srcStride,
+                 u8 * dstBase, ptrdiff_t dstStride,
+                 BORDER_MODE border, u8 borderValue,
+                 const Size2D & ksize, s16 * kernelBase, u32 scale)
+{
+    internal::assertSupportedConfiguration(isConvolutionSupported(size, ksize, border));
+#ifdef CAROTENE_NEON
+    const uint8x8_t v_zero_u8 = vdup_n_u8(0);
+    const uint8x8_t v_border = vdup_n_u8(borderValue);
+    const int32x4_t v_zero_s32 = vdupq_n_s32(0);
+
+    uint8x8_t tprev[3] = { v_zero_u8, v_zero_u8, v_zero_u8 },
+              tcurr[3] = { v_zero_u8, v_zero_u8, v_zero_u8 },
+              tnext[3] = { v_zero_u8, v_zero_u8, v_zero_u8 };
+    uint8x8_t t0 = v_zero_u8, t1 = v_zero_u8, t2 = v_zero_u8;
+
+    ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height;
+    static const vshrq_s32_func vshrq_s32_a[33] =
+    {
+        vshrq_s32<0>,
+        vshrq_s32<1>,
+        vshrq_s32<2>,
+        vshrq_s32<3>,
+        vshrq_s32<4>,
+        vshrq_s32<5>,
+        vshrq_s32<6>,
+        vshrq_s32<7>,
+        vshrq_s32<8>,
+        vshrq_s32<9>,
+        vshrq_s32<10>,
+        vshrq_s32<11>,
+        vshrq_s32<12>,
+        vshrq_s32<13>,
+        vshrq_s32<14>,
+        vshrq_s32<15>,
+        vshrq_s32<16>,
+        vshrq_s32<17>,
+        vshrq_s32<18>,
+        vshrq_s32<19>,
+        vshrq_s32<20>,
+        vshrq_s32<21>,
+        vshrq_s32<22>,
+        vshrq_s32<23>,
+        vshrq_s32<24>,
+        vshrq_s32<25>,
+        vshrq_s32<26>,
+        vshrq_s32<27>,
+        vshrq_s32<28>,
+        vshrq_s32<29>,
+        vshrq_s32<30>,
+        vshrq_s32<31>,
+        vshrq_s32<32>
+    };
+    vshrq_s32_func vshrq_s32_p = vshrq_s32_a[scale];
+
+    for (ptrdiff_t y = 0; y < height; ++y)
+    {
+        const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max<ptrdiff_t>(y - 1, 0));
+        const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1));
+        u8 * drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        u8 prevx[3] = { 0, 0, 0 },
+           currx[3] = { 0, 0, 0 },
+           nextx[3] = { 0, 0, 0 };
+        ptrdiff_t x = 0;
+        const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8);
+
+        // perform vertical convolution
+        for ( ; x <= bwidth; x += 8)
+        {
+            internal::prefetch(srow0 + x);
+            internal::prefetch(srow1 + x);
+            internal::prefetch(srow2 + x);
+
+            uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x);
+            uint8x8_t x1 = vld1_u8(srow1 + x);
+            uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x);
+
+            // calculate values for plain CPU part below if needed
+            if (x + 8 >= bwidth)
+            {
+                ptrdiff_t x3 = x == width ? width - 1 : x;
+                ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max<ptrdiff_t>(x3 - 1, 0);
+
+                if (border == BORDER_MODE_CONSTANT && x4 < 0)
+                    prevx[0] = prevx[1] = prevx[2] = borderValue;
+                else
+                {
+                    prevx[0] = srow0 ? srow0[x4] : borderValue;
+                    prevx[1] =         srow1[x4]              ;
+                    prevx[2] = srow2 ? srow2[x4] : borderValue;
+                }
+
+                currx[0] = srow0 ? srow0[x3] : borderValue;
+                currx[1] =         srow1[x3]              ;
+                currx[2] = srow2 ? srow2[x3] : borderValue;
+            }
+
+            // make shift
+            if (x)
+            {
+                tprev[0] = tcurr[0];
+                tcurr[0] = tnext[0];
+
+                tprev[1] = tcurr[1];
+                tcurr[1] = tnext[1];
+
+                tprev[2] = tcurr[2];
+                tcurr[2] = tnext[2];
+            }
+
+            tnext[0] = x0;
+            tnext[1] = x1;
+            tnext[2] = x2;
+
+            // make extrapolation for the first elements
+            if (!x)
+            {
+                // make border
+                if (border == BORDER_MODE_CONSTANT)
+                    tcurr[0] = tcurr[1] = tcurr[2] = v_border;
+                else if (border == BORDER_MODE_REPLICATE)
+                {
+                    tcurr[0] = vdup_n_u8(vget_lane_u8(tnext[0], 0));
+                    tcurr[1] = vdup_n_u8(vget_lane_u8(tnext[1], 0));
+                    tcurr[2] = vdup_n_u8(vget_lane_u8(tnext[2], 0));
+                }
+
+                continue;
+            }
+
+            int32x4_t v_dst0 = v_zero_s32, v_dst1 = v_zero_s32;
+
+            {
+                // combine 3 "shifted" vectors
+                t0 = vext_u8(tprev[0], tcurr[0], 7);
+                t1 = tcurr[0];
+                t2 = vext_u8(tcurr[0], tnext[0], 1);
+
+                int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0));
+                int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1));
+                int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2));
+
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[8]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[7]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[6]);
+
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[8]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[7]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[6]);
+            }
+
+            {
+                // combine 3 "shifted" vectors
+                t0 = vext_u8(tprev[1], tcurr[1], 7);
+                t1 = tcurr[1];
+                t2 = vext_u8(tcurr[1], tnext[1], 1);
+
+                int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0));
+                int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1));
+                int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2));
+
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[5]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[4]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[3]);
+
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[5]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[4]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[3]);
+            }
+
+            {
+                // combine 3 "shifted" vectors
+                t0 = vext_u8(tprev[2], tcurr[2], 7);
+                t1 = tcurr[2];
+                t2 = vext_u8(tcurr[2], tnext[2], 1);
+
+                int16x8_t t0_16s = vreinterpretq_s16_u16(vmovl_u8(t0));
+                int16x8_t t1_16s = vreinterpretq_s16_u16(vmovl_u8(t1));
+                int16x8_t t2_16s = vreinterpretq_s16_u16(vmovl_u8(t2));
+
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t0_16s), kernelBase[2]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t1_16s), kernelBase[1]);
+                v_dst0 = vmlal_n_s16(v_dst0, vget_low_s16(t2_16s), kernelBase[0]);
+
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t0_16s), kernelBase[2]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t1_16s), kernelBase[1]);
+                v_dst1 = vmlal_n_s16(v_dst1, vget_high_s16(t2_16s), kernelBase[0]);
+            }
+
+
+            // make scale
+            v_dst0 = vshrq_s32_p(v_dst0);
+            v_dst1 = vshrq_s32_p(v_dst1);
+
+            // and add them
+            vst1_u8(drow + x - 8, vqmovn_u16(vcombine_u16(vqmovun_s32(v_dst0),
+                                                          vqmovun_s32(v_dst1))));
+        }
+
+        x -= 8;
+        if (x == width)
+            --x;
+
+        for ( ; x < width; ++x)
+        {
+            // make extrapolation for the last elements
+            if (x + 1 >= width)
+            {
+                if (border == BORDER_MODE_CONSTANT)
+                {
+                    nextx[0] = borderValue;
+                    nextx[1] = borderValue;
+                    nextx[2] = borderValue;
+                }
+                else if (border == BORDER_MODE_REPLICATE)
+                {
+                    nextx[0] = srow0[x];
+                    nextx[1] = srow1[x];
+                    nextx[2] = srow2[x];
+                }
+            }
+            else
+            {
+                nextx[0] = srow0 ? srow0[x + 1] : borderValue;
+                nextx[1] =         srow1[x + 1]              ;
+                nextx[2] = srow2 ? srow2[x + 1] : borderValue;
+            }
+
+            s32 val = 0;
+            for (s32 _y = 0; _y < 3; ++_y)
+                val += prevx[_y] * kernelBase[(2 - _y) * 3 + 2] +
+                       currx[_y] * kernelBase[(2 - _y) * 3 + 1] +
+                       nextx[_y] * kernelBase[(2 - _y) * 3 + 0];
+
+            drow[x] = internal::saturate_cast<u8>(val >> scale);
+
+            // make shift
+            prevx[0] = currx[0];
+            currx[0] = nextx[0];
+
+            prevx[1] = currx[1];
+            currx[1] = nextx[1];
+
+            prevx[2] = currx[2];
+            currx[2] = nextx[2];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+    (void)ksize;
+    (void)kernelBase;
+    (void)scale;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/count_nonzero.cpp

@@ -0,0 +1,430 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <limits>
+
+namespace CAROTENE_NS {
+
+s32 countNonZero(const Size2D &_size,
+                 const u8 * srcBase, ptrdiff_t srcStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    size_t roiw16 = size.width & ~15u;
+    s32 result = 0;
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const u8* src = internal::getRowPtr( srcBase,  srcStride, k);
+        size_t i = 0;
+
+        #define COUNTNONZERO8U_BLOCK_SIZE (16*255)
+        uint8x16_t vc1 = vmovq_n_u8(1);
+        for (; i < roiw16;)
+        {
+            size_t lim = std::min(i + COUNTNONZERO8U_BLOCK_SIZE, size.width) - 16;
+            uint8x16_t vs = vmovq_n_u8(0);
+
+            for (; i <= lim; i+= 16)
+            {
+                internal::prefetch(src + i);
+                uint8x16_t vln = vld1q_u8(src + i);
+                uint8x16_t vnz = vminq_u8(vln, vc1);
+                vs = vaddq_u8(vs, vnz);
+            }
+
+            uint32x4_t vs4 = vpaddlq_u16(vpaddlq_u8(vs));
+            uint32x2_t vs2 = vadd_u32(vget_low_u32(vs4), vget_high_u32(vs4));
+
+            s32 s[2];
+            vst1_u32((u32*)s, vs2);
+
+            if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 2GB of non-zeros...
+            {
+                return 0x7fFFffFF;
+            }
+            result += (s[0] += s[1]);
+            if (s[0] < 0 || result < 0)
+            {
+                return 0x7fFFffFF;
+            }
+        }
+        for (; i < size.width; i++)
+            result += (src[i] != 0)?1:0;
+        if (result < 0)//saturate in case of overflow ~ 2GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+    }
+    return result;
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+
+    return 0;
+#endif
+}
+
+s32 countNonZero(const Size2D &_size,
+                 const u16 * srcBase, ptrdiff_t srcStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    size_t roiw8 = size.width & ~7u;
+    s32 result = 0;
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const u16* src = internal::getRowPtr( srcBase,  srcStride, k);
+        size_t i = 0;
+
+        #define COUNTNONZERO16U_BLOCK_SIZE (8*(256*256-1))
+        uint16x8_t vc1 = vmovq_n_u16(1);
+        for (; i < roiw8;)
+        {
+            size_t lim = std::min(i + COUNTNONZERO16U_BLOCK_SIZE, size.width) - 8;
+            uint16x8_t vs = vmovq_n_u16(0);
+
+            for (; i <= lim; i+= 8)
+            {
+                internal::prefetch(src + i);
+                uint16x8_t vln = vld1q_u16(src + i);
+                uint16x8_t vnz = vminq_u16(vln, vc1);
+                vs = vaddq_u16(vs, vnz);
+            }
+
+            uint32x4_t vs4 = vpaddlq_u16(vs);
+            uint32x2_t vs2 = vadd_u32(vget_low_u32(vs4), vget_high_u32(vs4));
+
+            s32 s[2];
+            vst1_u32((u32*)s, vs2);
+
+            if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 4GB of non-zeros...
+            {
+                return 0x7fFFffFF;
+            }
+            result += (s[0] += s[1]);
+            if (s[0] < 0 || result < 0)
+            {
+                return 0x7fFFffFF;
+            }
+        }
+        for (; i < size.width; i++)
+            result += (src[i] != 0)?1:0;
+        if (result < 0)//saturate in case of overflow ~ 4GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+    }
+    return result;
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+
+    return 0;
+#endif
+}
+
+s32 countNonZero(const Size2D &_size,
+                 const s32 * srcBase, ptrdiff_t srcStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    size_t roiw4 = size.width & ~3u;
+    s32 result = 0;
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const u32* src = (const u32*)internal::getRowPtr( srcBase,  srcStride, k);
+        u32 i = 0;
+
+        uint32x4_t vc1 = vmovq_n_u32(1);
+        uint32x4_t vs = vmovq_n_u32(0);
+
+        for (; i < roiw4; i += 4 )
+        {
+            internal::prefetch(src + i);
+            uint32x4_t vln = vld1q_u32(src + i);
+            uint32x4_t vnz = vminq_u32(vln, vc1);
+            vs = vqaddq_u32(vs, vnz);
+        }
+
+        uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs));
+
+        s32 s[2];
+        vst1_u32((u32*)s, vs2);
+
+        if (s[0] < 0 || s[1] < 0)//saturate in case of overflow ~ 8GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+        result += (s[0] += s[1]);
+        if (s[0] < 0 || result < 0)
+        {
+            return 0x7fFFffFF;
+        }
+
+        for (; i < size.width; i++)
+            result += (src[i] != 0)?1:0;
+        if (result < 0)//saturate in case of overflow ~ 8GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+    }
+    return result;
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+
+    return 0;
+#endif
+}
+
+s32 countNonZero(const Size2D &_size,
+                 const f32 * srcBase, ptrdiff_t srcStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    size_t roiw4 = size.width & ~3u;
+    s32 result = 0;
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const f32* src = internal::getRowPtr( srcBase,  srcStride, k);
+        size_t i = 0;
+
+        float32x4_t vc0 = vmovq_n_f32(0);
+        int32x4_t vs = vmovq_n_s32(0);
+
+        for (; i < roiw4; i += 4 )
+        {
+            internal::prefetch(src + i);
+            float32x4_t vln = vld1q_f32(src + i);
+            int32x4_t vnz = vreinterpretq_s32_u32(vmvnq_u32(vceqq_f32(vln, vc0)));
+            vs = vqaddq_s32(vs, vnz);
+        }
+
+        int32x2_t vs2 = vqneg_s32(vqadd_s32(vget_low_s32(vs), vget_high_s32(vs)));
+
+        int s[2];
+        vst1_s32(s, vs2);
+
+        result += (s[0] += s[1]);
+        if (s[0] < 0 || result < 0)//case of overflow ~ 8GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+
+        for (; i < size.width; i++)
+            result += (src[i] < std::numeric_limits<float>::min() && src[i] > -std::numeric_limits<float>::min())?0:1;
+
+        if (result < 0)
+        {
+            return 0x7fFFffFF;
+        }
+    }
+    return result;
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+
+    return 0;
+#endif
+}
+
+s32 countNonZero(const Size2D &_size,
+                 const f64 * srcBase, ptrdiff_t srcStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    size_t roiw8 = size.width & ~7u;
+    size_t roiw4 = size.width & ~3u;
+    size_t roiw2 = size.width & ~1u;
+    uint64x2_t vmask1 = vdupq_n_u64(0x7fFFffFFffFFffFFULL); //will treat denormals as non-zero
+    uint32x4_t vc0 = vmovq_n_u32(0);
+
+    s32 result = 0;
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const f64* src = internal::getRowPtr( srcBase,  srcStride, k);
+        size_t i = 0;
+
+        int32x2_t vs1 = vmov_n_s32(0);
+        int32x2_t vs2 = vmov_n_s32(0);
+        int32x2_t vs3 = vmov_n_s32(0);
+        int32x2_t vs4 = vmov_n_s32(0);
+
+        for (; i < roiw8; i += 8 )
+        {
+            internal::prefetch(src + i + 6);
+            uint64x2_t vln1 = vld1q_u64((const u64*)(src + i));
+            uint64x2_t vln2 = vld1q_u64((const u64*)(src + i + 2));
+            uint64x2_t vln3 = vld1q_u64((const u64*)(src + i + 4));
+            uint64x2_t vln4 = vld1q_u64((const u64*)(src + i + 6));
+
+            uint64x2_t vm1 = vandq_u64(vln1, vmask1);
+            uint64x2_t vm2 = vandq_u64(vln2, vmask1);
+            uint64x2_t vm3 = vandq_u64(vln3, vmask1);
+            uint64x2_t vm4 = vandq_u64(vln4, vmask1);
+
+            uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0);
+            uint32x4_t vequ2 = vceqq_u32(vreinterpretq_u32_u64(vm2), vc0);
+            uint32x4_t vequ3 = vceqq_u32(vreinterpretq_u32_u64(vm3), vc0);
+            uint32x4_t vequ4 = vceqq_u32(vreinterpretq_u32_u64(vm4), vc0);
+
+            uint32x4_t vlx1 = vmvnq_u32(vequ1);
+            uint32x4_t vlx2 = vmvnq_u32(vequ2);
+            uint32x4_t vlx3 = vmvnq_u32(vequ3);
+            uint32x4_t vlx4 = vmvnq_u32(vequ4);
+
+            int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1)));
+            int32x2_t vnz2 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx2), vget_high_u32(vlx2)));
+            int32x2_t vnz3 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx3), vget_high_u32(vlx3)));
+            int32x2_t vnz4 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx4), vget_high_u32(vlx4)));
+
+            vs1 = vqadd_s32(vs1, vnz1);
+            vs2 = vqadd_s32(vs2, vnz2);
+            vs3 = vqadd_s32(vs3, vnz3);
+            vs4 = vqadd_s32(vs4, vnz4);
+        }
+
+        if (i < roiw4)
+        {
+            internal::prefetch(src + i + 2);
+            uint64x2_t vln1 = vld1q_u64((const u64*)(src + i));
+            uint64x2_t vln2 = vld1q_u64((const u64*)(src + i + 2));
+
+            uint64x2_t vm1 = vandq_u64(vln1, vmask1);
+            uint64x2_t vm2 = vandq_u64(vln2, vmask1);
+
+            uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0);
+            uint32x4_t vequ2 = vceqq_u32(vreinterpretq_u32_u64(vm2), vc0);
+
+            uint32x4_t vlx1 = vmvnq_u32(vequ1);
+            uint32x4_t vlx2 = vmvnq_u32(vequ2);
+
+            int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1)));
+            int32x2_t vnz2 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx2), vget_high_u32(vlx2)));
+
+            vs1 = vqadd_s32(vs1, vnz1);
+            vs2 = vqadd_s32(vs2, vnz2);
+            i += 4;
+        }
+
+        if (i < roiw2)
+        {
+            internal::prefetch(src + i);
+            uint64x2_t vln1 = vld1q_u64((const u64*)(src + i));
+
+            uint64x2_t vm1 = vandq_u64(vln1, vmask1);
+
+            uint32x4_t vequ1 = vceqq_u32(vreinterpretq_u32_u64(vm1), vc0);
+
+            uint32x4_t vlx1 = vmvnq_u32(vequ1);
+
+            int32x2_t vnz1 = vreinterpret_s32_u32(vpmax_u32(vget_low_u32(vlx1), vget_high_u32(vlx1)));
+
+            vs1 = vqadd_s32(vs1, vnz1);
+            i += 2;
+        }
+
+        vs1 = vqadd_s32(vs1, vs2);
+        vs3 = vqadd_s32(vs3, vs4);
+        vs1 = vqadd_s32(vs1, vs3);
+        int32x2_t vsneg = vqneg_s32(vs1);
+
+        s32 s[2];
+        vst1_s32(s, vsneg);
+
+        result += (s[0] += s[1]);
+        if (s[0] < 0 || result < 0)//case of overflow ~ 16GB of non-zeros...
+        {
+            return 0x7fFFffFF;
+        }
+
+        for (; i < size.width; i++)
+            result += (src[i] < std::numeric_limits<double>::min() && src[i] > -std::numeric_limits<double>::min())?0:1;
+        if (result < 0)
+        {
+            return 0x7fFFffFF;
+        }
+    }
+    return result;
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+
+    return 0;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/div.cpp

@@ -0,0 +1,708 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2016, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+#include <cstring>
+#include <cfloat>
+#include <cmath>
+#include <limits>
+
+namespace CAROTENE_NS {
+
+namespace {
+
+#ifdef CAROTENE_NEON
+
+inline float32x4_t vroundq(const float32x4_t& v)
+{
+    const int32x4_t signMask = vdupq_n_s32(1 << 31), half = vreinterpretq_s32_f32(vdupq_n_f32(0.5f));
+    float32x4_t v_addition = vreinterpretq_f32_s32(vorrq_s32(half, vandq_s32(signMask, vreinterpretq_s32_f32(v))));
+    return vaddq_f32(v, v_addition);
+}
+
+template <typename T>
+inline T divSaturateQ(const T &v1, const T &v2, const float scale)
+{
+    return internal::vcombine(internal::vqmovn(divSaturateQ(internal::vmovl(internal::vget_low(v1)),
+                                                            internal::vmovl(internal::vget_low(v2)), scale)),
+                              internal::vqmovn(divSaturateQ(internal::vmovl(internal::vget_high(v1)),
+                                                            internal::vmovl(internal::vget_high(v2)), scale))
+                             );
+}
+template <>
+inline int32x4_t divSaturateQ<int32x4_t>(const int32x4_t &v1, const int32x4_t &v2, const float scale)
+{ return vcvtq_s32_f32(vroundq(vmulq_f32(vmulq_n_f32(vcvtq_f32_s32(v1), scale), internal::vrecpq_f32(vcvtq_f32_s32(v2))))); }
+template <>
+inline uint32x4_t divSaturateQ<uint32x4_t>(const uint32x4_t &v1, const uint32x4_t &v2, const float scale)
+{ return vcvtq_u32_f32(vroundq(vmulq_f32(vmulq_n_f32(vcvtq_f32_u32(v1), scale), internal::vrecpq_f32(vcvtq_f32_u32(v2))))); }
+
+inline float32x2_t vround(const float32x2_t& v)
+{
+    const int32x2_t signMask = vdup_n_s32(1 << 31), half = vreinterpret_s32_f32(vdup_n_f32(0.5f));
+    float32x2_t v_addition = vreinterpret_f32_s32(vorr_s32(half, vand_s32(signMask, vreinterpret_s32_f32(v))));
+    return vadd_f32(v, v_addition);
+}
+
+template <typename T>
+inline T divSaturate(const T &v1, const T &v2, const float scale)
+{
+    return internal::vqmovn(divSaturateQ(internal::vmovl(v1), internal::vmovl(v2), scale));
+}
+template <>
+inline int32x2_t divSaturate<int32x2_t>(const int32x2_t &v1, const int32x2_t &v2, const float scale)
+{ return vcvt_s32_f32(vround(vmul_f32(vmul_n_f32(vcvt_f32_s32(v1), scale), internal::vrecp_f32(vcvt_f32_s32(v2))))); }
+template <>
+inline uint32x2_t divSaturate<uint32x2_t>(const uint32x2_t &v1, const uint32x2_t &v2, const float scale)
+{ return vcvt_u32_f32(vround(vmul_f32(vmul_n_f32(vcvt_f32_u32(v1), scale), internal::vrecp_f32(vcvt_f32_u32(v2))))); }
+
+
+template <typename T>
+inline T divWrapQ(const T &v1, const T &v2, const float scale)
+{
+    return internal::vcombine(internal::vmovn(divWrapQ(internal::vmovl(internal::vget_low(v1)),
+                                                       internal::vmovl(internal::vget_low(v2)), scale)),
+                              internal::vmovn(divWrapQ(internal::vmovl(internal::vget_high(v1)),
+                                                       internal::vmovl(internal::vget_high(v2)), scale))
+                             );
+}
+template <>
+inline int32x4_t divWrapQ<int32x4_t>(const int32x4_t &v1, const int32x4_t &v2, const float scale)
+{ return vcvtq_s32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_s32(v1), scale), internal::vrecpq_f32(vcvtq_f32_s32(v2)))); }
+template <>
+inline uint32x4_t divWrapQ<uint32x4_t>(const uint32x4_t &v1, const uint32x4_t &v2, const float scale)
+{ return vcvtq_u32_f32(vmulq_f32(vmulq_n_f32(vcvtq_f32_u32(v1), scale), internal::vrecpq_f32(vcvtq_f32_u32(v2)))); }
+
+template <typename T>
+inline T divWrap(const T &v1, const T &v2, const float scale)
+{
+    return internal::vmovn(divWrapQ(internal::vmovl(v1), internal::vmovl(v2), scale));
+}
+template <>
+inline int32x2_t divWrap<int32x2_t>(const int32x2_t &v1, const int32x2_t &v2, const float scale)
+{ return vcvt_s32_f32(vmul_f32(vmul_n_f32(vcvt_f32_s32(v1), scale), internal::vrecp_f32(vcvt_f32_s32(v2)))); }
+template <>
+inline uint32x2_t divWrap<uint32x2_t>(const uint32x2_t &v1, const uint32x2_t &v2, const float scale)
+{ return vcvt_u32_f32(vmul_f32(vmul_n_f32(vcvt_f32_u32(v1), scale), internal::vrecp_f32(vcvt_f32_u32(v2)))); }
+
+inline  uint8x16_t vtstq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vtstq_u8 (v0, v1); }
+inline  uint16x8_t vtstq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vtstq_u16(v0, v1); }
+inline  uint32x4_t vtstq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vtstq_u32(v0, v1); }
+inline   int8x16_t vtstq(const int8x16_t   & v0, const int8x16_t   & v1) { return vreinterpretq_s8_u8  (vtstq_s8 (v0, v1)); }
+inline   int16x8_t vtstq(const int16x8_t   & v0, const int16x8_t   & v1) { return vreinterpretq_s16_u16(vtstq_s16(v0, v1)); }
+inline   int32x4_t vtstq(const int32x4_t   & v0, const int32x4_t   & v1) { return vreinterpretq_s32_u32(vtstq_s32(v0, v1)); }
+
+inline   uint8x8_t vtst(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vtst_u8 (v0, v1); }
+inline  uint16x4_t vtst(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vtst_u16(v0, v1); }
+inline  uint32x2_t vtst(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vtst_u32(v0, v1); }
+inline    int8x8_t vtst(const int8x8_t    & v0, const int8x8_t    & v1) { return vreinterpret_s8_u8  (vtst_s8 (v0, v1)); }
+inline   int16x4_t vtst(const int16x4_t   & v0, const int16x4_t   & v1) { return vreinterpret_s16_u16(vtst_s16(v0, v1)); }
+inline   int32x2_t vtst(const int32x2_t   & v0, const int32x2_t   & v1) { return vreinterpret_s32_u32(vtst_s32(v0, v1)); }
+#endif
+
+template <typename T>
+void div(const Size2D &size,
+         const T * src0Base, ptrdiff_t src0Stride,
+         const T * src1Base, ptrdiff_t src1Stride,
+         T * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    typedef typename internal::VecTraits<T>::vec128 vec128;
+    typedef typename internal::VecTraits<T>::vec64 vec64;
+
+    if (scale == 0.0f ||
+        (std::numeric_limits<T>::is_integer &&
+         (scale * std::numeric_limits<T>::max()) <  1.0f &&
+         (scale * std::numeric_limits<T>::max()) > -1.0f))
+    {
+        for (size_t y = 0; y < size.height; ++y)
+        {
+            T * dst = internal::getRowPtr(dstBase, dstStride, y);
+            std::memset(dst, 0, sizeof(T) * size.width);
+        }
+        return;
+    }
+
+    const size_t step128 = 16 / sizeof(T);
+    size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0;
+    const size_t step64 = 8 / sizeof(T);
+    size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const T * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const T * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        T * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (cpolicy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw128; j += step128)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+
+                vec128 v_src0 = internal::vld1q(src0 + j);
+                vec128 v_src1 = internal::vld1q(src1 + j);
+
+                vec128 v_mask = vtstq(v_src1,v_src1);
+                internal::vst1q(dst + j, internal::vandq(v_mask, divSaturateQ(v_src0, v_src1, scale)));
+            }
+            for (; j < roiw64; j += step64)
+            {
+                vec64 v_src0 = internal::vld1(src0 + j);
+                vec64 v_src1 = internal::vld1(src1 + j);
+
+                vec64 v_mask = vtst(v_src1,v_src1);
+                internal::vst1(dst + j, internal::vand(v_mask,divSaturate(v_src0, v_src1, scale)));
+            }
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? internal::saturate_cast<T>(scale * src0[j] / src1[j]) : 0;
+            }
+        }
+        else // CONVERT_POLICY_WRAP
+        {
+            for (; j < roiw128; j += step128)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+
+                vec128 v_src0 = internal::vld1q(src0 + j);
+                vec128 v_src1 = internal::vld1q(src1 + j);
+
+                vec128 v_mask = vtstq(v_src1,v_src1);
+                internal::vst1q(dst + j, internal::vandq(v_mask, divWrapQ(v_src0, v_src1, scale)));
+            }
+            for (; j < roiw64; j += step64)
+            {
+                vec64 v_src0 = internal::vld1(src0 + j);
+                vec64 v_src1 = internal::vld1(src1 + j);
+
+                vec64 v_mask = vtst(v_src1,v_src1);
+                internal::vst1(dst + j, internal::vand(v_mask,divWrap(v_src0, v_src1, scale)));
+            }
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? (T)((s32)trunc(scale * src0[j] / src1[j])) : 0;
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)cpolicy;
+    (void)scale;
+#endif
+}
+
+#ifdef CAROTENE_NEON
+
+template <typename T>
+inline T recipSaturateQ(const T &v2, const float scale)
+{
+    return internal::vcombine(internal::vqmovn(recipSaturateQ(internal::vmovl(internal::vget_low(v2)), scale)),
+                              internal::vqmovn(recipSaturateQ(internal::vmovl(internal::vget_high(v2)), scale))
+                             );
+}
+template <>
+inline int32x4_t recipSaturateQ<int32x4_t>(const int32x4_t &v2, const float scale)
+{ return vcvtq_s32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_s32(v2)), scale)); }
+template <>
+inline uint32x4_t recipSaturateQ<uint32x4_t>(const uint32x4_t &v2, const float scale)
+{ return vcvtq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_u32(v2)), scale)); }
+
+template <typename T>
+inline T recipSaturate(const T &v2, const float scale)
+{
+    return internal::vqmovn(recipSaturateQ(internal::vmovl(v2), scale));
+}
+template <>
+inline int32x2_t recipSaturate<int32x2_t>(const int32x2_t &v2, const float scale)
+{ return vcvt_s32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_s32(v2)), scale)); }
+template <>
+inline uint32x2_t recipSaturate<uint32x2_t>(const uint32x2_t &v2, const float scale)
+{ return vcvt_u32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_u32(v2)), scale)); }
+
+
+template <typename T>
+inline T recipWrapQ(const T &v2, const float scale)
+{
+    return internal::vcombine(internal::vmovn(recipWrapQ(internal::vmovl(internal::vget_low(v2)), scale)),
+                              internal::vmovn(recipWrapQ(internal::vmovl(internal::vget_high(v2)), scale))
+                             );
+}
+template <>
+inline int32x4_t recipWrapQ<int32x4_t>(const int32x4_t &v2, const float scale)
+{ return vcvtq_s32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_s32(v2)), scale)); }
+template <>
+inline uint32x4_t recipWrapQ<uint32x4_t>(const uint32x4_t &v2, const float scale)
+{ return vcvtq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(vcvtq_f32_u32(v2)), scale)); }
+
+template <typename T>
+inline T recipWrap(const T &v2, const float scale)
+{
+    return internal::vmovn(recipWrapQ(internal::vmovl(v2), scale));
+}
+template <>
+inline int32x2_t recipWrap<int32x2_t>(const int32x2_t &v2, const float scale)
+{ return vcvt_s32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_s32(v2)), scale)); }
+template <>
+inline uint32x2_t recipWrap<uint32x2_t>(const uint32x2_t &v2, const float scale)
+{ return vcvt_u32_f32(vmul_n_f32(internal::vrecp_f32(vcvt_f32_u32(v2)), scale)); }
+#endif
+
+template <typename T>
+void recip(const Size2D &size,
+           const T * src1Base, ptrdiff_t src1Stride,
+           T * dstBase, ptrdiff_t dstStride,
+           f32 scale,
+           CONVERT_POLICY cpolicy)
+{
+    internal::assertSupportedConfiguration();
+
+#ifdef CAROTENE_NEON
+    typedef typename internal::VecTraits<T>::vec128 vec128;
+    typedef typename internal::VecTraits<T>::vec64 vec64;
+
+    if (scale == 0.0f ||
+        (std::numeric_limits<T>::is_integer &&
+         scale <  1.0f &&
+         scale > -1.0f))
+    {
+        for (size_t y = 0; y < size.height; ++y)
+        {
+            T * dst = internal::getRowPtr(dstBase, dstStride, y);
+            std::memset(dst, 0, sizeof(T) * size.width);
+        }
+        return;
+    }
+
+    const size_t step128 = 16 / sizeof(T);
+    size_t roiw128 = size.width >= (step128 - 1) ? size.width - step128 + 1 : 0;
+    const size_t step64 = 8 / sizeof(T);
+    size_t roiw64 = size.width >= (step64 - 1) ? size.width - step64 + 1 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const T * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        T * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (cpolicy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw128; j += step128)
+            {
+                internal::prefetch(src1 + j);
+
+                vec128 v_src1 = internal::vld1q(src1 + j);
+
+                vec128 v_mask = vtstq(v_src1,v_src1);
+                internal::vst1q(dst + j, internal::vandq(v_mask, recipSaturateQ(v_src1, scale)));
+            }
+            for (; j < roiw64; j += step64)
+            {
+                vec64 v_src1 = internal::vld1(src1 + j);
+
+                vec64 v_mask = vtst(v_src1,v_src1);
+                internal::vst1(dst + j, internal::vand(v_mask, recipSaturate(v_src1, scale)));
+            }
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? internal::saturate_cast<T>(scale / src1[j]) : 0;
+            }
+        }
+        else // CONVERT_POLICY_WRAP
+        {
+            for (; j < roiw128; j += step128)
+            {
+                internal::prefetch(src1 + j);
+
+                vec128 v_src1 = internal::vld1q(src1 + j);
+
+                vec128 v_mask = vtstq(v_src1,v_src1);
+                internal::vst1q(dst + j, internal::vandq(v_mask, recipWrapQ(v_src1, scale)));
+            }
+            for (; j < roiw64; j += step64)
+            {
+                vec64 v_src1 = internal::vld1(src1 + j);
+
+                vec64 v_mask = vtst(v_src1,v_src1);
+                internal::vst1(dst + j, internal::vand(v_mask, recipWrap(v_src1, scale)));
+            }
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? (T)((s32)trunc(scale / src1[j])) : 0;
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)cpolicy;
+    (void)scale;
+#endif
+}
+
+}
+
+void div(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         u8 * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    div<u8>(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy);
+}
+
+void div(const Size2D &size,
+         const s8 * src0Base, ptrdiff_t src0Stride,
+         const s8 * src1Base, ptrdiff_t src1Stride,
+         s8 * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    div<s8>(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy);
+}
+
+void div(const Size2D &size,
+         const u16 * src0Base, ptrdiff_t src0Stride,
+         const u16 * src1Base, ptrdiff_t src1Stride,
+         u16 * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    div<u16>(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy);
+}
+
+void div(const Size2D &size,
+         const s16 * src0Base, ptrdiff_t src0Stride,
+         const s16 * src1Base, ptrdiff_t src1Stride,
+         s16 * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    div<s16>(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy);
+}
+
+void div(const Size2D &size,
+         const s32 * src0Base, ptrdiff_t src0Stride,
+         const s32 * src1Base, ptrdiff_t src1Stride,
+         s32 * dstBase, ptrdiff_t dstStride,
+         f32 scale,
+         CONVERT_POLICY cpolicy)
+{
+    div<s32>(size, src0Base, src0Stride, src1Base, src1Stride, dstBase, dstStride, scale, cpolicy);
+}
+
+void div(const Size2D &size,
+         const f32 * src0Base, ptrdiff_t src0Stride,
+         const f32 * src1Base, ptrdiff_t src1Stride,
+         f32 * dstBase, ptrdiff_t dstStride,
+         f32 scale)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (scale == 0.0f)
+    {
+        for (size_t y = 0; y < size.height; ++y)
+        {
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, y);
+            std::memset(dst, 0, sizeof(f32) * size.width);
+        }
+        return;
+    }
+
+    float32x4_t v_zero = vdupq_n_f32(0.0f);
+
+    size_t roiw128 = size.width >= 3 ? size.width - 3 : 0;
+    size_t roiw64 = size.width >= 1 ? size.width - 1 : 0;
+
+    if (std::fabs(scale - 1.0f) < FLT_EPSILON)
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+            const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            size_t j = 0;
+
+            for (; j < roiw128; j += 4)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+
+                float32x4_t v_src0 = vld1q_f32(src0 + j);
+                float32x4_t v_src1 = vld1q_f32(src1 + j);
+
+                uint32x4_t v_mask = vceqq_f32(v_src1,v_zero);
+                vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32(
+                                   vreinterpretq_u32_f32(vmulq_f32(v_src0, internal::vrecpq_f32(v_src1))), v_mask)));
+            }
+
+            for (; j < roiw64; j += 2)
+            {
+                float32x2_t v_src0 = vld1_f32(src0 + j);
+                float32x2_t v_src1 = vld1_f32(src1 + j);
+
+                uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero));
+                vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32(
+                                  vreinterpret_u32_f32(vmul_f32(v_src0, internal::vrecp_f32(v_src1))), v_mask)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? src0[j] / src1[j] : 0.0f;
+            }
+        }
+    }
+    else
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+            const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            size_t j = 0;
+
+            for (; j < roiw128; j += 4)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+
+                float32x4_t v_src0 = vld1q_f32(src0 + j);
+                float32x4_t v_src1 = vld1q_f32(src1 + j);
+
+                uint32x4_t v_mask = vceqq_f32(v_src1,v_zero);
+                vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32(
+                                   vreinterpretq_u32_f32(vmulq_f32(vmulq_n_f32(v_src0, scale),
+                                                         internal::vrecpq_f32(v_src1))), v_mask)));
+            }
+
+            for (; j < roiw64; j += 2)
+            {
+                float32x2_t v_src0 = vld1_f32(src0 + j);
+                float32x2_t v_src1 = vld1_f32(src1 + j);
+
+                uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero));
+                vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32(
+                                  vreinterpret_u32_f32(vmul_f32(vmul_n_f32(v_src0, scale),
+                                                                internal::vrecp_f32(v_src1))), v_mask)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? src0[j] * scale / src1[j] : 0.0f;
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)scale;
+#endif
+}
+
+void reciprocal(const Size2D &size,
+                const u8 * srcBase, ptrdiff_t srcStride,
+                u8 * dstBase, ptrdiff_t dstStride,
+                f32 scale,
+                CONVERT_POLICY cpolicy)
+{
+    recip<u8>(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy);
+}
+
+void reciprocal(const Size2D &size,
+                const s8 * srcBase, ptrdiff_t srcStride,
+                s8 * dstBase, ptrdiff_t dstStride,
+                f32 scale,
+                CONVERT_POLICY cpolicy)
+{
+    recip<s8>(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy);
+}
+
+void reciprocal(const Size2D &size,
+                const u16 * srcBase, ptrdiff_t srcStride,
+                u16 * dstBase, ptrdiff_t dstStride,
+                f32 scale,
+                CONVERT_POLICY cpolicy)
+{
+    recip<u16>(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy);
+}
+
+void reciprocal(const Size2D &size,
+                const s16 * srcBase, ptrdiff_t srcStride,
+                s16 * dstBase, ptrdiff_t dstStride,
+                f32 scale,
+                CONVERT_POLICY cpolicy)
+{
+    recip<s16>(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy);
+}
+
+void reciprocal(const Size2D &size,
+                const s32 * srcBase, ptrdiff_t srcStride,
+                s32 * dstBase, ptrdiff_t dstStride,
+                f32 scale,
+                CONVERT_POLICY cpolicy)
+{
+    recip<s32>(size, srcBase, srcStride, dstBase, dstStride, scale, cpolicy);
+}
+
+void reciprocal(const Size2D &size,
+                const f32 * srcBase, ptrdiff_t srcStride,
+                f32 * dstBase, ptrdiff_t dstStride,
+                f32 scale)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (scale == 0.0f)
+    {
+        for (size_t y = 0; y < size.height; ++y)
+        {
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, y);
+            std::memset(dst, 0, sizeof(f32) * size.width);
+        }
+        return;
+    }
+
+    float32x4_t v_zero = vdupq_n_f32(0.0f);
+
+    size_t roiw128 = size.width >= 3 ? size.width - 3 : 0;
+    size_t roiw64 = size.width >= 1 ? size.width - 1 : 0;
+
+    if (std::fabs(scale - 1.0f) < FLT_EPSILON)
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            const f32 * src1 = internal::getRowPtr(srcBase, srcStride, i);
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            size_t j = 0;
+
+            for (; j < roiw128; j += 4)
+            {
+                internal::prefetch(src1 + j);
+
+                float32x4_t v_src1 = vld1q_f32(src1 + j);
+
+                uint32x4_t v_mask = vceqq_f32(v_src1,v_zero);
+                vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32(
+                                   vreinterpretq_u32_f32(internal::vrecpq_f32(v_src1)), v_mask)));
+            }
+
+            for (; j < roiw64; j += 2)
+            {
+                float32x2_t v_src1 = vld1_f32(src1 + j);
+
+                uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero));
+                vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32(
+                                  vreinterpret_u32_f32(internal::vrecp_f32(v_src1)), v_mask)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? 1.0f / src1[j] : 0;
+            }
+        }
+    }
+    else
+    {
+        for (size_t i = 0; i < size.height; ++i)
+        {
+            const f32 * src1 = internal::getRowPtr(srcBase, srcStride, i);
+            f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+            size_t j = 0;
+
+            for (; j < roiw128; j += 4)
+            {
+                internal::prefetch(src1 + j);
+
+                float32x4_t v_src1 = vld1q_f32(src1 + j);
+
+                uint32x4_t v_mask = vceqq_f32(v_src1,v_zero);
+                vst1q_f32(dst + j, vreinterpretq_f32_u32(vbicq_u32(
+                                   vreinterpretq_u32_f32(vmulq_n_f32(internal::vrecpq_f32(v_src1),
+                                                                     scale)),v_mask)));
+            }
+
+            for (; j < roiw64; j += 2)
+            {
+                float32x2_t v_src1 = vld1_f32(src1 + j);
+
+                uint32x2_t v_mask = vceq_f32(v_src1,vget_low_f32(v_zero));
+                vst1_f32(dst + j, vreinterpret_f32_u32(vbic_u32(
+                                  vreinterpret_u32_f32(vmul_n_f32(internal::vrecp_f32(v_src1),
+                                                                  scale)), v_mask)));
+            }
+
+            for (; j < size.width; j++)
+            {
+                dst[j] = src1[j] ? scale / src1[j] : 0;
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)scale;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/dot_product.cpp

@@ -0,0 +1,260 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+f64 dotProduct(const Size2D &_size,
+               const u8 * src0Base, ptrdiff_t src0Stride,
+               const u8 * src1Base, ptrdiff_t src1Stride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (src0Stride == src1Stride &&
+        src0Stride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+
+// It is possible to accumulate up to 66051 uchar multiplication results in uint32 without overflow
+// We process 16 elements and accumulate two new elements per step. So we could handle 66051/2*16 elements
+#define DOT_UINT_BLOCKSIZE 66050*8
+    f64 result = 0.0;
+    for (size_t row = 0; row < size.height; ++row)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, row);
+        const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, row);
+
+        size_t i = 0;
+        uint64x2_t ws = vmovq_n_u64(0);
+
+        while(i + 16 <= size.width)
+        {
+            size_t lim = std::min(i + DOT_UINT_BLOCKSIZE, size.width) - 16;
+
+            uint32x4_t s1 = vmovq_n_u32(0);
+            uint32x4_t s2 = vmovq_n_u32(0);
+
+            for (; i <= lim; i += 16)
+            {
+                internal::prefetch(src0 + i);
+                internal::prefetch(src1 + i);
+
+                uint8x16_t vs1 = vld1q_u8(src0 + i);
+                uint8x16_t vs2 = vld1q_u8(src1 + i);
+
+                uint16x8_t vdot1 = vmull_u8(vget_low_u8(vs1), vget_low_u8(vs2));
+                uint16x8_t vdot2 = vmull_u8(vget_high_u8(vs1), vget_high_u8(vs2));
+
+                s1 = vpadalq_u16(s1, vdot1);
+                s2 = vpadalq_u16(s2, vdot2);
+            }
+
+            ws = vpadalq_u32(ws, s1);
+            ws = vpadalq_u32(ws, s2);
+        }
+
+        if(i + 8 <= size.width)
+        {
+            uint8x8_t vs1 = vld1_u8(src0 + i);
+            uint8x8_t vs2 = vld1_u8(src1 + i);
+
+            ws = vpadalq_u32(ws, vpaddlq_u16(vmull_u8(vs1, vs2)));
+            i += 8;
+        }
+
+        result += (double)vget_lane_u64(vadd_u64(vget_low_u64(ws), vget_high_u64(ws)), 0);
+
+        for (; i < size.width; ++i)
+            result += s32(src0[i]) * s32(src1[i]);
+    }
+    return result;
+#else
+    (void)_size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+
+    return 0;
+#endif
+}
+
+f64 dotProduct(const Size2D &_size,
+               const s8 * src0Base, ptrdiff_t src0Stride,
+               const s8 * src1Base, ptrdiff_t src1Stride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (src0Stride == src1Stride &&
+        src0Stride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+
+// It is possible to accumulate up to 131071 schar multiplication results in sint32 without overflow
+// We process 16 elements and accumulate two new elements per step. So we could handle 131071/2*16 elements
+#define DOT_INT_BLOCKSIZE 131070*8
+    f64 result = 0.0;
+    for (size_t row = 0; row < size.height; ++row)
+    {
+        const s8 * src0 = internal::getRowPtr(src0Base, src0Stride, row);
+        const s8 * src1 = internal::getRowPtr(src1Base, src1Stride, row);
+
+        size_t i = 0;
+        int64x2_t ws = vmovq_n_s64(0);
+
+        while(i + 16 <= size.width)
+        {
+            size_t lim = std::min(i + DOT_UINT_BLOCKSIZE, size.width) - 16;
+
+            int32x4_t s1 = vmovq_n_s32(0);
+            int32x4_t s2 = vmovq_n_s32(0);
+
+            for (; i <= lim; i += 16)
+            {
+                internal::prefetch(src0 + i);
+                internal::prefetch(src1 + i);
+
+                int8x16_t vs1 = vld1q_s8(src0 + i);
+                int8x16_t vs2 = vld1q_s8(src1 + i);
+
+                int16x8_t vdot1 = vmull_s8(vget_low_s8(vs1), vget_low_s8(vs2));
+                int16x8_t vdot2 = vmull_s8(vget_high_s8(vs1), vget_high_s8(vs2));
+
+                s1 = vpadalq_s16(s1, vdot1);
+                s2 = vpadalq_s16(s2, vdot2);
+            }
+
+            ws = vpadalq_s32(ws, s1);
+            ws = vpadalq_s32(ws, s2);
+        }
+
+        if(i + 8 <= size.width)
+        {
+            int8x8_t vs1 = vld1_s8(src0 + i);
+            int8x8_t vs2 = vld1_s8(src1 + i);
+
+            ws = vpadalq_s32(ws, vpaddlq_s16(vmull_s8(vs1, vs2)));
+            i += 8;
+        }
+
+        result += (double)vget_lane_s64(vadd_s64(vget_low_s64(ws), vget_high_s64(ws)), 0);
+
+        for (; i < size.width; ++i)
+            result += s32(src0[i]) * s32(src1[i]);
+    }
+    return result;
+#else
+    (void)_size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+
+    return 0;
+#endif
+}
+
+f64 dotProduct(const Size2D &_size,
+               const f32 * src0Base, ptrdiff_t src0Stride,
+               const f32 * src1Base, ptrdiff_t src1Stride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (src0Stride == src1Stride &&
+        src0Stride == (ptrdiff_t)(size.width * sizeof(f32)))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+
+#define DOT_FLOAT_BLOCKSIZE (1 << 13)
+    f64 result = 0.0;
+    for (size_t row = 0; row < size.height; ++row)
+    {
+        const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, row);
+        const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, row);
+
+        size_t i = 0;
+        while(i + 4 <= size.width)
+        {
+            size_t lim = std::min(i + DOT_FLOAT_BLOCKSIZE, size.width) - 4;
+            float32x4_t v_sum = vdupq_n_f32(0.0f);
+
+            for( ; i <= lim; i += 4 )
+            {
+                internal::prefetch(src0 + i);
+                internal::prefetch(src1 + i);
+                v_sum = vmlaq_f32(v_sum, vld1q_f32(src0 + i), vld1q_f32(src1 + i));
+            }
+
+            float32x2_t vres = vpadd_f32(vget_low_f32(v_sum),vget_high_f32(v_sum));
+            result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1);
+        }
+
+        if(i + 2 <= size.width)
+        {
+            float32x2_t vres = vmul_f32(vld1_f32(src0 + i), vld1_f32(src1 + i));
+            result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1);
+            i += 2;
+        }
+
+        for (; i < size.width; ++i)
+            result += src0[i] * src1[i];
+    }
+    return result;
+#else
+    (void)_size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+
+    return 0;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/fast.cpp

@@ -0,0 +1,428 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+
+/* This is FAST corner detector, contributed to OpenCV by the author, Edward Rosten.
+   Below is the original copyright and the references */
+
+/*
+Copyright (c) 2006, 2008 Edward Rosten
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ *Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+ *Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+ *Neither the name of the University of Cambridge nor the names of
+  its contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+The references are:
+ * Machine learning for high-speed corner detection,
+   E. Rosten and T. Drummond, ECCV 2006
+ * Faster and better: A machine learning approach to corner detection
+   E. Rosten, R. Porter and T. Drummond, PAMI, 2009
+*/
+
+#include "common.hpp"
+
+#include <vector>
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+namespace
+{
+
+void makeOffsets(ptrdiff_t pixel[], ptrdiff_t row_stride)
+{
+    pixel[0] = 0 + row_stride * 3;
+    pixel[1] = 1 + row_stride * 3;
+    pixel[2] = 2 + row_stride * 2;
+    pixel[3] = 3 + row_stride * 1;
+    pixel[4] = 3 + row_stride * 0;
+    pixel[5] = 3 + row_stride * -1;
+    pixel[6] = 2 + row_stride * -2;
+    pixel[7] = 1 + row_stride * -3;
+    pixel[8] = 0 + row_stride * -3;
+    pixel[9] = -1 + row_stride * -3;
+    pixel[10] = -2 + row_stride * -2;
+    pixel[11] = -3 + row_stride * -1;
+    pixel[12] = -3 + row_stride * 0;
+    pixel[13] = -3 + row_stride * 1;
+    pixel[14] = -2 + row_stride * 2;
+    pixel[15] = -1 + row_stride * 3;
+}
+
+u8 cornerScore(const u8* ptr, const ptrdiff_t pixel[])
+{
+    const s32 K = 8, N = 16 + K + 1;
+    s32 k, v = ptr[0];
+    s16 d[(N + 7) & ~7];
+    for( k = 0; k < N; k++ )
+        d[k] = (s16)(v - ptr[pixel[k]]);
+
+    int16x8_t q0 = vdupq_n_s16((s16)(-1000));
+    int16x8_t q1 = vdupq_n_s16((s16)(1000));
+
+    int16x8_t d0_7   = vld1q_s16(d +  0);
+    int16x8_t d8_15  = vld1q_s16(d +  8);
+    int16x8_t d16_23 = vld1q_s16(d + 16);
+    int16x8_t d24    = vld1q_s16(d + 24);
+
+    //k == 0
+    int16x8_t v0k0 = vextq_s16(d0_7, d8_15, 1);
+    int16x8_t v1k0 = vextq_s16(d0_7, d8_15, 2);
+    int16x8_t ak0 = vminq_s16(v0k0, v1k0);
+    int16x8_t bk0 = vmaxq_s16(v0k0, v1k0);
+
+    v0k0 = vextq_s16(d0_7, d8_15, 3);
+    ak0 = vminq_s16(ak0, v0k0);
+    bk0 = vmaxq_s16(bk0, v0k0);
+
+    v1k0 = vextq_s16(d0_7, d8_15, 4);
+    ak0 = vminq_s16(ak0, v1k0);
+    bk0 = vmaxq_s16(bk0, v1k0);
+
+    v0k0 = vextq_s16(d0_7, d8_15, 5);
+    ak0 = vminq_s16(ak0, v0k0);
+    bk0 = vmaxq_s16(bk0, v0k0);
+
+    v1k0 = vextq_s16(d0_7, d8_15, 6);
+    ak0 = vminq_s16(ak0, v1k0);
+    bk0 = vmaxq_s16(bk0, v1k0);
+
+    v0k0 = vextq_s16(d0_7, d8_15, 7);
+    ak0 = vminq_s16(ak0, v0k0);
+    bk0 = vmaxq_s16(bk0, v0k0);
+
+    ak0 = vminq_s16(ak0, d8_15);
+    bk0 = vmaxq_s16(bk0, d8_15);
+
+    q0 = vmaxq_s16(q0, vminq_s16(ak0, d0_7));
+    q1 = vminq_s16(q1, vmaxq_s16(bk0, d0_7));
+
+    v1k0 = vextq_s16(d8_15, d16_23, 1);
+    q0 = vmaxq_s16(q0, vminq_s16(ak0, v1k0));
+    q1 = vminq_s16(q1, vmaxq_s16(bk0, v1k0));
+
+    //k == 8
+    int16x8_t v0k8 = v1k0;
+    int16x8_t v1k8 = vextq_s16(d8_15, d16_23, 2);
+    int16x8_t ak8 = vminq_s16(v0k8, v1k8);
+    int16x8_t bk8 = vmaxq_s16(v0k8, v1k8);
+
+    v0k8 = vextq_s16(d8_15, d16_23, 3);
+    ak8 = vminq_s16(ak8, v0k8);
+    bk8 = vmaxq_s16(bk8, v0k8);
+
+    v1k8 = vextq_s16(d8_15, d16_23, 4);
+    ak8 = vminq_s16(ak8, v1k8);
+    bk8 = vmaxq_s16(bk8, v1k8);
+
+    v0k8 = vextq_s16(d8_15, d16_23, 5);
+    ak8 = vminq_s16(ak8, v0k8);
+    bk8 = vmaxq_s16(bk8, v0k8);
+
+    v1k8 = vextq_s16(d8_15, d16_23, 6);
+    ak8 = vminq_s16(ak8, v1k8);
+    bk8 = vmaxq_s16(bk8, v1k8);
+
+    v0k8 = vextq_s16(d8_15, d16_23, 7);
+    ak8 = vminq_s16(ak8, v0k8);
+    bk8 = vmaxq_s16(bk8, v0k8);
+
+    ak8 = vminq_s16(ak8, d16_23);
+    bk8 = vmaxq_s16(bk8, d16_23);
+
+    q0 = vmaxq_s16(q0, vminq_s16(ak8, d8_15));
+    q1 = vminq_s16(q1, vmaxq_s16(bk8, d8_15));
+
+    v1k8 = vextq_s16(d16_23, d24, 1);
+    q0 = vmaxq_s16(q0, vminq_s16(ak8, v1k8));
+    q1 = vminq_s16(q1, vmaxq_s16(bk8, v1k8));
+
+    //fin
+    int16x8_t q = vmaxq_s16(q0, vsubq_s16(vmovq_n_s16(0), q1));
+    int16x4_t q2 = vmax_s16(vget_low_s16(q), vget_high_s16(q));
+    int32x4_t q2w = vmovl_s16(q2);
+    int32x2_t q4 = vmax_s32(vget_low_s32(q2w), vget_high_s32(q2w));
+    int32x2_t q8 = vmax_s32(q4, vreinterpret_s32_s64(vshr_n_s64(vreinterpret_s64_s32(q4), 32)));
+
+    return (u8)(vget_lane_s32(q8, 0) - 1);
+}
+
+} //namespace
+#endif
+
+void FAST(const Size2D &size,
+          u8 *srcBase, ptrdiff_t srcStride,
+          KeypointStore *keypoints,
+          u8 threshold, bool nonmax_suppression)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    //keypoints.clear();
+
+    const s32 K = 8, N = 16 + K + 1;
+    ptrdiff_t i, j, k, pixel[N];
+    makeOffsets(pixel, srcStride);
+    for(k = 16; k < N; k++)
+        pixel[k] = pixel[k - 16];
+
+    uint8x16_t delta = vdupq_n_u8(128);
+    uint8x16_t t = vdupq_n_u8(threshold);
+    uint8x16_t K16 = vdupq_n_u8((u8)K);
+
+    u8 threshold_tab[512];
+    for( i = -255; i <= 255; i++ )
+        threshold_tab[i+255] = (u8)(i < -threshold ? 1 : i > threshold ? 2 : 0);
+
+    std::vector<u8> _buf((size.width+16)*3*(sizeof(ptrdiff_t) + sizeof(u8)) + 128);
+    u8* buf[3];
+    buf[0] = &_buf[0]; buf[1] = buf[0] + size.width; buf[2] = buf[1] + size.width;
+    ptrdiff_t* cpbuf[3];
+    cpbuf[0] = (ptrdiff_t*)internal::alignPtr(buf[2] + size.width, sizeof(ptrdiff_t)) + 1;
+    cpbuf[1] = cpbuf[0] + size.width + 1;
+    cpbuf[2] = cpbuf[1] + size.width + 1;
+    memset(buf[0], 0, size.width*3);
+
+    for(i = 3; i < (ptrdiff_t)size.height-2; i++)
+    {
+        const u8* ptr = internal::getRowPtr(srcBase, srcStride, i) + 3;
+        u8* curr = buf[(i - 3)%3];
+        ptrdiff_t* cornerpos = cpbuf[(i - 3)%3];
+        memset(curr, 0, size.width);
+        ptrdiff_t ncorners = 0;
+
+        if( i < (ptrdiff_t)size.height - 3 )
+        {
+            j = 3;
+
+            for(; j < (ptrdiff_t)size.width - 16 - 3; j += 16, ptr += 16)
+            {
+                internal::prefetch(ptr);
+                internal::prefetch(ptr + pixel[0]);
+                internal::prefetch(ptr + pixel[2]);
+
+                uint8x16_t v0 = vld1q_u8(ptr);
+                int8x16_t v1 = vreinterpretq_s8_u8(veorq_u8(vqsubq_u8(v0, t), delta));
+                int8x16_t v2 = vreinterpretq_s8_u8(veorq_u8(vqaddq_u8(v0, t), delta));
+
+                int8x16_t x0 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[0]), delta));
+                int8x16_t x1 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[4]), delta));
+                int8x16_t x2 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[8]), delta));
+                int8x16_t x3 = vreinterpretq_s8_u8(vsubq_u8(vld1q_u8(ptr + pixel[12]), delta));
+
+                uint8x16_t m0 =   vandq_u8(vcgtq_s8(x0, v2), vcgtq_s8(x1, v2));
+                uint8x16_t m1 =   vandq_u8(vcgtq_s8(v1, x0), vcgtq_s8(v1, x1));
+                m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x1, v2), vcgtq_s8(x2, v2)));
+                m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x1), vcgtq_s8(v1, x2)));
+                m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x2, v2), vcgtq_s8(x3, v2)));
+                m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x2), vcgtq_s8(v1, x3)));
+                m0 = vorrq_u8(m0, vandq_u8(vcgtq_s8(x3, v2), vcgtq_s8(x0, v2)));
+                m1 = vorrq_u8(m1, vandq_u8(vcgtq_s8(v1, x3), vcgtq_s8(v1, x0)));
+                m0 = vorrq_u8(m0, m1);
+
+                u64 mask[2];
+                vst1q_u64(mask, vreinterpretq_u64_u8(m0));
+
+                if( mask[0] == 0 )
+                {
+                    if (mask[1] != 0)
+                    {
+                        j -= 8;
+                        ptr -= 8;
+                    }
+                    continue;
+                }
+
+                uint8x16_t c0 = vmovq_n_u8(0);
+                uint8x16_t c1 = vmovq_n_u8(0);
+                uint8x16_t max0 = vmovq_n_u8(0);
+                uint8x16_t max1 = vmovq_n_u8(0);
+                for( k = 0; k < N; k++ )
+                {
+                    int8x16_t x = vreinterpretq_s8_u8(veorq_u8(vld1q_u8(ptr + pixel[k]), delta));
+                    m0 = vcgtq_s8(x, v2);
+                    m1 = vcgtq_s8(v1, x);
+
+                    c0 = vandq_u8(vsubq_u8(c0, m0), m0);
+                    c1 = vandq_u8(vsubq_u8(c1, m1), m1);
+
+                    max0 = vmaxq_u8(max0, c0);
+                    max1 = vmaxq_u8(max1, c1);
+                }
+
+                max0 = vmaxq_u8(max0, max1);
+                u8 m[16];
+                vst1q_u8(m, vcgtq_u8(max0, K16));
+
+                for( k = 0; k < 16; ++k )
+                    if(m[k])
+                    {
+                        cornerpos[ncorners++] = j+k;
+                        if(nonmax_suppression)
+                            curr[j+k] = cornerScore(ptr+k, pixel);
+                    }
+            }
+
+            for( ; j < (s32)size.width - 3; j++, ptr++ )
+            {
+                s32 v = ptr[0];
+                const u8* tab = &threshold_tab[0] - v + 255;
+                s32 d = tab[ptr[pixel[0]]] | tab[ptr[pixel[8]]];
+
+                if( d == 0 )
+                    continue;
+
+                d &= tab[ptr[pixel[2]]] | tab[ptr[pixel[10]]];
+                d &= tab[ptr[pixel[4]]] | tab[ptr[pixel[12]]];
+                d &= tab[ptr[pixel[6]]] | tab[ptr[pixel[14]]];
+
+                if( d == 0 )
+                    continue;
+
+                d &= tab[ptr[pixel[1]]] | tab[ptr[pixel[9]]];
+                d &= tab[ptr[pixel[3]]] | tab[ptr[pixel[11]]];
+                d &= tab[ptr[pixel[5]]] | tab[ptr[pixel[13]]];
+                d &= tab[ptr[pixel[7]]] | tab[ptr[pixel[15]]];
+
+                if( d & 1 )
+                {
+                    s32 vt = v - threshold, count = 0;
+
+                    for( k = 0; k < N; k++ )
+                    {
+                        s32 x = ptr[pixel[k]];
+                        if(x < vt)
+                        {
+                            if( ++count > K )
+                            {
+                                cornerpos[ncorners++] = j;
+                                if(nonmax_suppression)
+                                    curr[j] = cornerScore(ptr, pixel);
+                                break;
+                            }
+                        }
+                        else
+                            count = 0;
+                    }
+                }
+
+                if( d & 2 )
+                {
+                    s32 vt = v + threshold, count = 0;
+
+                    for( k = 0; k < N; k++ )
+                    {
+                        s32 x = ptr[pixel[k]];
+                        if(x > vt)
+                        {
+                            if( ++count > K )
+                            {
+                                cornerpos[ncorners++] = j;
+                                if(nonmax_suppression)
+                                    curr[j] = cornerScore(ptr, pixel);
+                                break;
+                            }
+                        }
+                        else
+                            count = 0;
+                    }
+                }
+            }
+        }
+
+        cornerpos[-1] = ncorners;
+
+        if( i == 3 )
+            continue;
+
+        const u8* prev = buf[(i - 4 + 3)%3];
+        const u8* pprev = buf[(i - 5 + 3)%3];
+        cornerpos = cpbuf[(i - 4 + 3)%3];
+        ncorners = cornerpos[-1];
+
+        for( k = 0; k < ncorners; k++ )
+        {
+            j = cornerpos[k];
+            s32 score = prev[j];
+            if( !nonmax_suppression ||
+                    (score > prev[j+1] && score > prev[j-1] &&
+                     score > pprev[j-1] && score > pprev[j] && score > pprev[j+1] &&
+                     score > curr[j-1] && score > curr[j] && score > curr[j+1]) )
+            {
+                keypoints->push((f32)j, (f32)(i-1), 7.f, -1, (f32)score);
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)keypoints;
+    (void)threshold;
+    (void)nonmax_suppression;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/fill_minmaxloc.cpp

@@ -0,0 +1,442 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T>
+void process(const T * src, size_t j0, size_t j1, size_t i,
+             T minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+             T maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    for (size_t j = j0; j < j1; ++j)
+    {
+        T val = src[j];
+
+        if (val == maxVal)
+        {
+            if (maxLocCount < maxLocCapacity)
+            {
+                maxLocPtr[maxLocCount] = j;
+                maxLocPtr[maxLocCount + 1] = i;
+            }
+            maxLocCount += 2;
+        }
+
+        if (val == minVal)
+        {
+            if (minLocCount < minLocCapacity)
+            {
+                minLocPtr[minLocCount] = j;
+                minLocPtr[minLocCount + 1] = i;
+            }
+            minLocCount += 2;
+        }
+    }
+}
+
+} // namespace
+
+#endif
+
+void fillMinMaxLocs(const Size2D & size,
+                    const u8 * srcBase, ptrdiff_t srcStride,
+                    u8 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+                    u8 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    uint8x16_t v_maxval16 = vdupq_n_u8(maxVal), v_minval16 = vdupq_n_u8(minVal);
+    uint8x8_t v_maxval8 = vdup_n_u8(maxVal), v_minval8 = vdup_n_u8(minVal);
+
+    u64 mask[2] = { 0ul };
+
+    minLocCapacity <<= 1;
+    maxLocCapacity <<= 1;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0;
+
+        for ( ; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            uint8x16_t v_src = vld1q_u8(src + j);
+
+            uint8x16_t v_maxmask = vceqq_u8(v_src, v_maxval16);
+            uint8x16_t v_minmask = vceqq_u8(v_src, v_minval16);
+            uint8x16_t v_mask = vorrq_u8(v_maxmask, v_minmask);
+
+            vst1q_u8((u8 *)&mask[0], v_mask);
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+            if (mask[1])
+                process(src, j + 8, j + 16, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+        for ( ; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src = vld1_u8(src + j);
+
+            uint8x8_t v_maxmask = vceq_u8(v_src, v_maxval8);
+            uint8x8_t v_minmask = vceq_u8(v_src, v_minval8);
+            uint8x8_t v_mask = vorr_u8(v_maxmask, v_minmask);
+
+            vst1_u8((u8 *)&mask[0], v_mask);
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+
+        process(src, j, size.width, i,
+                minVal, minLocPtr, minLocCount, minLocCapacity,
+                maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+    }
+
+    minLocCount >>= 1;
+    maxLocCount >>= 1;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)minVal;
+    (void)minLocPtr;
+    (void)minLocCount;
+    (void)minLocCapacity;
+    (void)maxVal;
+    (void)maxLocPtr;
+    (void)maxLocCount;
+    (void)maxLocCapacity;
+#endif
+}
+
+void fillMinMaxLocs(const Size2D & size,
+                    const u16 * srcBase, ptrdiff_t srcStride,
+                    u16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+                    u16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    uint16x8_t v_maxval8 = vdupq_n_u16(maxVal),
+               v_minval8 = vdupq_n_u16(minVal);
+    u64 mask[2] = { 0ul };
+
+    minLocCapacity <<= 1;
+    maxLocCapacity <<= 1;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u16 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0;
+
+        for ( ; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            uint16x8_t v_src0 = vld1q_u16(src + j), v_src1 = vld1q_u16(src + j + 8);
+
+            uint16x8_t v_mask0 = vorrq_u16(vceqq_u16(v_src0, v_maxval8), vceqq_u16(v_src0, v_minval8));
+            uint16x8_t v_mask1 = vorrq_u16(vceqq_u16(v_src1, v_maxval8), vceqq_u16(v_src1, v_minval8));
+
+            vst1q_u8((u8 *)&mask[0], vcombine_u8(vmovn_u16(v_mask0), vmovn_u16(v_mask1)));
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+            if (mask[1])
+                process(src, j + 8, j + 16, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+        for ( ; j < roiw8; j += 8)
+        {
+            internal::prefetch(src + j);
+            uint16x8_t v_src = vld1q_u16(src + j);
+
+            uint16x8_t v_maxmask = vceqq_u16(v_src, v_maxval8);
+            uint16x8_t v_minmask = vceqq_u16(v_src, v_minval8);
+            uint16x8_t v_mask = vorrq_u16(v_maxmask, v_minmask);
+
+            vst1_u8((u8 *)&mask[0], vmovn_u16(v_mask));
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+
+        process(src, j, size.width, i,
+                minVal, minLocPtr, minLocCount, minLocCapacity,
+                maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+    }
+
+    minLocCount >>= 1;
+    maxLocCount >>= 1;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)minVal;
+    (void)minLocPtr;
+    (void)minLocCount;
+    (void)minLocCapacity;
+    (void)maxVal;
+    (void)maxLocPtr;
+    (void)maxLocCount;
+    (void)maxLocCapacity;
+#endif
+}
+
+void fillMinMaxLocs(const Size2D & size,
+                    const s16 * srcBase, ptrdiff_t srcStride,
+                    s16 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+                    s16 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    int16x8_t v_maxval8 = vdupq_n_s16(maxVal),
+              v_minval8 = vdupq_n_s16(minVal);
+    u64 mask[2] = { 0ul };
+
+    minLocCapacity <<= 1;
+    maxLocCapacity <<= 1;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s16 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0;
+
+        for ( ; j < roiw16; j += 16)
+        {
+            internal::prefetch(src + j);
+            int16x8_t v_src0 = vld1q_s16(src + j), v_src1 = vld1q_s16(src + j + 8);
+
+            uint16x8_t v_mask0 = vorrq_u16(vceqq_s16(v_src0, v_maxval8), vceqq_s16(v_src0, v_minval8));
+            uint16x8_t v_mask1 = vorrq_u16(vceqq_s16(v_src1, v_maxval8), vceqq_s16(v_src1, v_minval8));
+
+            vst1q_u8((u8 *)&mask[0], vcombine_u8(vmovn_u16(v_mask0), vmovn_u16(v_mask1)));
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+            if (mask[1])
+                process(src, j + 8, j + 16, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+        for ( ; j < roiw8; j += 8)
+        {
+            internal::prefetch(src + j);
+            int16x8_t v_src = vld1q_s16(src + j);
+
+            uint16x8_t v_maxmask = vceqq_s16(v_src, v_maxval8);
+            uint16x8_t v_minmask = vceqq_s16(v_src, v_minval8);
+            uint16x8_t v_mask = vorrq_u16(v_maxmask, v_minmask);
+
+            vst1_u8((u8 *)&mask[0], vmovn_u16(v_mask));
+
+            if (mask[0])
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+
+        process(src, j, size.width, i,
+                minVal, minLocPtr, minLocCount, minLocCapacity,
+                maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+    }
+
+    minLocCount >>= 1;
+    maxLocCount >>= 1;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)minVal;
+    (void)minLocPtr;
+    (void)minLocCount;
+    (void)minLocCapacity;
+    (void)maxVal;
+    (void)maxLocPtr;
+    (void)maxLocCount;
+    (void)maxLocCapacity;
+#endif
+}
+
+void fillMinMaxLocs(const Size2D & size,
+                    const s32 * srcBase, ptrdiff_t srcStride,
+                    s32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+                    s32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    int32x4_t v_maxval4 = vdupq_n_s32(maxVal),
+              v_minval4 = vdupq_n_s32(minVal);
+    u64 mask = 0ul;
+
+    minLocCapacity <<= 1;
+    maxLocCapacity <<= 1;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s32 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0;
+
+        for ( ; j < roiw8; j += 8)
+        {
+            internal::prefetch(src + j);
+            int32x4_t v_src0 = vld1q_s32(src + j), v_src1 = vld1q_s32(src + j + 4);
+
+            uint32x4_t v_mask0 = vorrq_u32(vceqq_s32(v_src0, v_maxval4), vceqq_s32(v_src0, v_minval4));
+            uint32x4_t v_mask1 = vorrq_u32(vceqq_s32(v_src1, v_maxval4), vceqq_s32(v_src1, v_minval4));
+
+            vst1_u8((u8 *)&mask, vmovn_u16(vcombine_u16(vmovn_u32(v_mask0), vmovn_u32(v_mask1))));
+
+            if (mask)
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+
+        process(src, j, size.width, i,
+                minVal, minLocPtr, minLocCount, minLocCapacity,
+                maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+    }
+
+    minLocCount >>= 1;
+    maxLocCount >>= 1;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)minVal;
+    (void)minLocPtr;
+    (void)minLocCount;
+    (void)minLocCapacity;
+    (void)maxVal;
+    (void)maxLocPtr;
+    (void)maxLocCount;
+    (void)maxLocCapacity;
+#endif
+}
+
+void fillMinMaxLocs(const Size2D & size,
+                    const u32 * srcBase, ptrdiff_t srcStride,
+                    u32 minVal, size_t * minLocPtr, s32 & minLocCount, s32 minLocCapacity,
+                    u32 maxVal, size_t * maxLocPtr, s32 & maxLocCount, s32 maxLocCapacity)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    uint32x4_t v_maxval4 = vdupq_n_u32(maxVal),
+               v_minval4 = vdupq_n_u32(minVal);
+    u64 mask = 0ul;
+
+    minLocCapacity <<= 1;
+    maxLocCapacity <<= 1;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u32 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0;
+
+        for ( ; j < roiw8; j += 8)
+        {
+            internal::prefetch(src + j);
+            uint32x4_t v_src0 = vld1q_u32(src + j), v_src1 = vld1q_u32(src + j + 4);
+
+            uint32x4_t v_mask0 = vorrq_u32(vceqq_u32(v_src0, v_maxval4), vceqq_u32(v_src0, v_minval4));
+            uint32x4_t v_mask1 = vorrq_u32(vceqq_u32(v_src1, v_maxval4), vceqq_u32(v_src1, v_minval4));
+
+            vst1_u8((u8 *)&mask, vmovn_u16(vcombine_u16(vmovn_u32(v_mask0), vmovn_u32(v_mask1))));
+
+            if (mask)
+                process(src, j, j + 8, i,
+                        minVal, minLocPtr, minLocCount, minLocCapacity,
+                        maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+        }
+
+        process(src, j, size.width, i,
+                minVal, minLocPtr, minLocCount, minLocCapacity,
+                maxVal, maxLocPtr, maxLocCount, maxLocCapacity);
+    }
+
+    minLocCount >>= 1;
+    maxLocCount >>= 1;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)minVal;
+    (void)minLocPtr;
+    (void)minLocCount;
+    (void)minLocCapacity;
+    (void)maxVal;
+    (void)maxLocPtr;
+    (void)maxLocCount;
+    (void)maxLocCapacity;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/flip.cpp

@@ -0,0 +1,222 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+bool isFlipSupported(FLIP_MODE flipMode, u32 elemSize)
+{
+    bool supportedElemSize = (elemSize == 1) || (elemSize == 2) || (elemSize == 3) || (elemSize == 4);
+    return isSupportedConfiguration() &&
+            ((supportedElemSize && ((flipMode == FLIP_BOTH_MODE) || (flipMode == FLIP_HORIZONTAL_MODE))) ||
+             (flipMode == FLIP_VERTICAL_MODE));
+}
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T>
+void flip(const Size2D & size,
+          const void * srcBase, ptrdiff_t srcStride,
+          void * dstBase, ptrdiff_t dstStride,
+          FLIP_MODE flipMode)
+{
+    using namespace internal;
+
+    typedef typename VecTraits<T>::vec128 vec128;
+    typedef typename VecTraits<T>::vec64 vec64;
+
+    u32 step_base = 16 / sizeof(T), step_tail = 8 / sizeof(T);
+    size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0;
+    size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const T * src = getRowPtr((const T *)srcBase, srcStride, i);
+        T * dst = getRowPtr((T *)dstBase, dstStride, (flipMode & FLIP_VERTICAL_MODE) != 0 ? size.height - i - 1 : i);
+        size_t js = 0, jd = size.width;
+
+        for (; js < roiw_base; js += step_base, jd -= step_base)
+        {
+            prefetch(src + js);
+
+            vec128 v_src = vld1q(src + js);
+            vec128 v_dst = vrev64q(v_src);
+            v_dst = vcombine(vget_high(v_dst), vget_low(v_dst));
+            vst1q(dst + jd - step_base, v_dst);
+        }
+        for (; js < roiw_tail; js += step_tail, jd -= step_tail)
+        {
+            vec64 v_src = vld1(src + js);
+            vst1(dst + jd - step_tail, vrev64(v_src));
+        }
+
+        for (--jd; js < size.width; ++js, --jd)
+            dst[jd] = src[js];
+    }
+}
+
+template <typename T>
+void flip3(const Size2D & size,
+           const void * srcBase, ptrdiff_t srcStride,
+           void * dstBase, ptrdiff_t dstStride,
+           FLIP_MODE flipMode)
+{
+    using namespace internal;
+
+#ifndef ANDROID
+    typedef typename VecTraits<T, 3>::vec128 vec128;
+#endif
+    typedef typename VecTraits<T, 3>::vec64 vec64;
+
+#ifndef ANDROID
+    u32 step_base = 16 / sizeof(T), step_base3 = step_base * 3;
+    size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0;
+#endif
+    u32 step_tail = 8 / sizeof(T), step_tail3 = step_tail * 3;
+    size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const T * src = getRowPtr((const T *)srcBase, srcStride, i);
+        T * dst = getRowPtr((T *)dstBase, dstStride, (flipMode & FLIP_VERTICAL_MODE) != 0 ? size.height - i - 1 : i);
+        size_t j = 0, js = 0, jd = size.width * 3;
+
+#ifndef ANDROID
+        for (; j < roiw_base; j += step_base, js += step_base3, jd -= step_base3)
+        {
+            prefetch(src + js);
+
+            vec128 v_src = vld3q(src + js), v_dst;
+            v_src.val[0] = vrev64q(v_src.val[0]);
+            v_src.val[1] = vrev64q(v_src.val[1]);
+            v_src.val[2] = vrev64q(v_src.val[2]);
+
+            v_dst.val[0] = vcombine(vget_high(v_src.val[0]), vget_low(v_src.val[0]));
+            v_dst.val[1] = vcombine(vget_high(v_src.val[1]), vget_low(v_src.val[1]));
+            v_dst.val[2] = vcombine(vget_high(v_src.val[2]), vget_low(v_src.val[2]));
+
+            vst3q(dst + jd - step_base3, v_dst);
+        }
+#endif // ANDROID
+
+        for (; j < roiw_tail; j += step_tail, js += step_tail3, jd -= step_tail3)
+        {
+            vec64 v_src = vld3(src + js), v_dst;
+            v_dst.val[0] = vrev64(v_src.val[0]);
+            v_dst.val[1] = vrev64(v_src.val[1]);
+            v_dst.val[2] = vrev64(v_src.val[2]);
+
+            vst3(dst + jd - step_tail3, v_dst);
+        }
+
+        for (jd -= 3; j < size.width; ++j, js += 3, jd -= 3)
+        {
+            dst[jd] = src[js];
+            dst[jd + 1] = src[js + 1];
+            dst[jd + 2] = src[js + 2];
+        }
+    }
+}
+
+typedef void (* flipFunc)(const Size2D &size,
+                  const void * srcBase, ptrdiff_t srcStride,
+                  void * dstBase, ptrdiff_t dstStride,
+                  FLIP_MODE flipMode);
+
+} // namespace
+
+#endif
+
+void flip(const Size2D &size,
+          const u8 * srcBase, ptrdiff_t srcStride,
+          u8 * dstBase, ptrdiff_t dstStride,
+          FLIP_MODE flipMode, u32 elemSize)
+{
+    internal::assertSupportedConfiguration(isFlipSupported(flipMode, elemSize));
+#ifdef CAROTENE_NEON
+
+    if (flipMode == FLIP_VERTICAL_MODE)
+    {
+        for (size_t y = 0; y < size.height; ++y)
+        {
+            const u8 * src_row = internal::getRowPtr(srcBase, srcStride, y);
+            u8 * dst_row = internal::getRowPtr(dstBase, dstStride, size.height - y - 1);
+
+            std::memcpy(dst_row, src_row, elemSize * size.width);
+        }
+        return;
+    }
+
+    flipFunc func = NULL;
+
+    if (elemSize == (u32)sizeof(u8))
+        func = &flip<u8>;
+    if (elemSize == (u32)sizeof(u16))
+        func = &flip<u16>;
+    if (elemSize == (u32)sizeof(u32))
+        func = &flip<u32>;
+    if (elemSize == (u32)sizeof(u8) * 3)
+        func = &flip3<u8>;
+
+    if (func == NULL)
+        return;
+
+    func(size,
+         srcBase, srcStride,
+         dstBase, dstStride,
+         flipMode);
+
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)flipMode;
+    (void)elemSize;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/in_range.cpp

@@ -0,0 +1,195 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+inline void vnst(u8* dst, uint8x16_t v1, uint8x16_t v2) { vst1q_u8(dst, v1); vst1q_u8(dst+16, v2); }
+inline void vnst(u8* dst, uint16x8_t v1, uint16x8_t v2) { vst1q_u8(dst, vcombine_u8(vmovn_u16(v1), vmovn_u16(v2))); }
+inline void vnst(u8* dst, uint32x4_t v1, uint32x4_t v2) { vst1_u8(dst, vmovn_u16(vcombine_u16(vmovn_u32(v1), vmovn_u32(v2)))); }
+
+template <typename T, int elsize> struct vtail
+{
+    static inline void inRange(const T *, const T *, const T *,
+                               u8 *, size_t &, size_t)
+    {
+        //do nothing since there couldn't be enough data
+    }
+};
+template <typename T> struct vtail<T, 2>
+{
+    static inline void inRange(const T * src, const T * rng1, const T * rng2,
+                               u8 * dst, size_t &x, size_t width)
+    {
+        typedef typename internal::VecTraits<T>::vec128 vec128;
+        typedef typename internal::VecTraits<T>::unsign::vec128 uvec128;
+        //There no more than 15 elements in the tail, so we could handle 8 element vector only once
+        if( x + 8 < width)
+        {
+             vec128  vs = internal::vld1q( src + x);
+             vec128 vr1 = internal::vld1q(rng1 + x);
+             vec128 vr2 = internal::vld1q(rng2 + x);
+            uvec128  vd = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs));
+            internal::vst1(dst + x, internal::vmovn(vd));
+            x+=8;
+        }
+    }
+};
+template <typename T> struct vtail<T, 1>
+{
+    static inline void inRange(const T * src, const T * rng1, const T * rng2,
+                               u8 * dst, size_t &x, size_t width)
+    {
+        typedef typename internal::VecTraits<T>::vec128 vec128;
+        typedef typename internal::VecTraits<T>::unsign::vec128 uvec128;
+        typedef typename internal::VecTraits<T>::vec64 vec64;
+        typedef typename internal::VecTraits<T>::unsign::vec64 uvec64;
+        //There no more than 31 elements in the tail, so we could handle once 16+8 or 16 or 8 elements
+        if( x + 16 < width)
+        {
+             vec128  vs = internal::vld1q( src + x);
+             vec128 vr1 = internal::vld1q(rng1 + x);
+             vec128 vr2 = internal::vld1q(rng2 + x);
+            uvec128  vd = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs));
+            internal::vst1q(dst + x, vd);
+            x+=16;
+        }
+        if( x + 8 < width)
+        {
+             vec64  vs = internal::vld1( src + x);
+             vec64 vr1 = internal::vld1(rng1 + x);
+             vec64 vr2 = internal::vld1(rng2 + x);
+            uvec64  vd = internal::vand(internal::vcge(vs, vr1), internal::vcge(vr2, vs));
+            internal::vst1(dst + x, vd);
+            x+=8;
+        }
+    }
+};
+
+template <typename T>
+inline void inRangeCheck(const Size2D &_size,
+                         const T * srcBase, ptrdiff_t srcStride,
+                         const T * rng1Base, ptrdiff_t rng1Stride,
+                         const T * rng2Base, ptrdiff_t rng2Stride,
+                         u8 * dstBase, ptrdiff_t dstStride)
+{
+    typedef typename internal::VecTraits<T>::vec128 vec128;
+    typedef typename internal::VecTraits<T>::unsign::vec128 uvec128;
+
+    Size2D size(_size);
+    if (srcStride == dstStride &&
+        srcStride == rng1Stride &&
+        srcStride == rng2Stride &&
+        srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    const size_t width = size.width & ~( 32/sizeof(T) - 1 );
+
+    for(size_t j = 0; j < size.height; ++j)
+    {
+        const T *  src = internal::getRowPtr( srcBase,  srcStride, j);
+        const T * rng1 = internal::getRowPtr(rng1Base, rng1Stride, j);
+        const T * rng2 = internal::getRowPtr(rng2Base, rng2Stride, j);
+             u8 *  dst = internal::getRowPtr( dstBase,  dstStride, j);
+        size_t i = 0;
+        for( ; i < width; i += 32/sizeof(T) )
+        {
+            internal::prefetch(src + i);
+            internal::prefetch(rng1 + i);
+            internal::prefetch(rng2 + i);
+
+             vec128  vs = internal::vld1q( src + i);
+             vec128 vr1 = internal::vld1q(rng1 + i);
+             vec128 vr2 = internal::vld1q(rng2 + i);
+            uvec128 vd1 = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs));
+                     vs = internal::vld1q( src + i + 16/sizeof(T));
+                    vr1 = internal::vld1q(rng1 + i + 16/sizeof(T));
+                    vr2 = internal::vld1q(rng2 + i + 16/sizeof(T));
+            uvec128 vd2 = internal::vandq(internal::vcgeq(vs, vr1), internal::vcgeq(vr2, vs));
+            vnst(dst + i, vd1, vd2);
+        }
+        vtail<T, sizeof(T)>::inRange(src, rng1, rng2, dst, i, size.width);
+        for( ; i < size.width; i++ )
+            dst[i] = (u8)(-(rng1[i] <= src[i] && src[i] <= rng2[i]));
+    }
+}
+
+}
+
+#define INRANGEFUNC(T)                                       \
+void inRange(const Size2D &_size,                            \
+             const T * srcBase, ptrdiff_t srcStride,         \
+             const T * rng1Base, ptrdiff_t rng1Stride,       \
+             const T * rng2Base, ptrdiff_t rng2Stride,       \
+             u8 * dstBase, ptrdiff_t dstStride)              \
+{                                                            \
+    internal::assertSupportedConfiguration();                \
+    inRangeCheck(_size, srcBase, srcStride,                  \
+                 rng1Base, rng1Stride, rng2Base, rng2Stride, \
+                 dstBase, dstStride);                        \
+}
+#else
+#define INRANGEFUNC(T)                                       \
+void inRange(const Size2D &,                                 \
+             const T *, ptrdiff_t,                           \
+             const T *, ptrdiff_t,                           \
+             const T *, ptrdiff_t,                           \
+             u8 *, ptrdiff_t)                                \
+{                                                            \
+    internal::assertSupportedConfiguration();                \
+}
+#endif
+
+INRANGEFUNC(u8)
+INRANGEFUNC(s8)
+INRANGEFUNC(u16)
+INRANGEFUNC(s16)
+INRANGEFUNC(s32)
+INRANGEFUNC(f32)
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/integral.cpp

@@ -0,0 +1,238 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+void integral(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u32 * sumBase, ptrdiff_t sumStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    uint32x4_t v_zero = vmovq_n_u32(0u);
+
+    // the first iteration
+    const u8 * src = internal::getRowPtr(srcBase, srcStride, 0);
+    u32 * sum = internal::getRowPtr(sumBase, sumStride, 0);
+
+    uint32x4_t prev = v_zero;
+    size_t j = 0u;
+
+    for ( ; j + 7 < size.width; j += 8)
+    {
+        internal::prefetch(sum + j);
+        internal::prefetch(src + j);
+
+        uint8x8_t el8shr0 = vld1_u8(src + j);
+        uint8x8_t el8shr1 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 8));
+        uint8x8_t el8shr2 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 16));
+        uint8x8_t el8shr3 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 24));
+
+        uint16x8_t el8shr12 =  vaddl_u8(el8shr1, el8shr2);
+        uint16x8_t el8shr03 =  vaddl_u8(el8shr0, el8shr3);
+
+        uint16x8_t el8 = vaddq_u16(el8shr12, el8shr03);
+        uint16x4_t el4h = vadd_u16(vget_low_u16(el8), vget_high_u16(el8));
+
+        uint32x4_t vsuml = vaddw_u16(prev, vget_low_u16(el8));
+        uint32x4_t vsumh = vaddw_u16(prev, el4h);
+
+        vst1q_u32(sum + j, vsuml);
+        vst1q_u32(sum + j + 4, vsumh);
+
+        prev = vaddw_u16(prev, vdup_lane_u16(el4h, 3));
+    }
+
+    for (u32 v = vgetq_lane_u32(prev, 3); j < size.width; ++j)
+        sum[j] = (v += src[j]);
+
+    // the others
+    for (size_t i = 1; i < size.height ; ++i)
+    {
+        src = internal::getRowPtr(srcBase, srcStride, i);
+        u32 * prevSum = internal::getRowPtr(sumBase, sumStride, i - 1);
+        sum = internal::getRowPtr(sumBase, sumStride, i);
+
+        prev = v_zero;
+        j = 0u;
+
+        for ( ; j + 7 < size.width; j += 8)
+        {
+            internal::prefetch(sum + j);
+            internal::prefetch(src + j);
+
+            uint32x4_t vsuml = vld1q_u32(prevSum + j);
+            uint32x4_t vsumh = vld1q_u32(prevSum + j + 4);
+
+            uint8x8_t el8shr0 = vld1_u8(src + j);
+            uint8x8_t el8shr1 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 8));
+            uint8x8_t el8shr2 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 16));
+            uint8x8_t el8shr3 = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(el8shr0), 24));
+
+            vsuml = vaddq_u32(vsuml, prev);
+            vsumh = vaddq_u32(vsumh, prev);
+
+            uint16x8_t el8shr12 =  vaddl_u8(el8shr1, el8shr2);
+            uint16x8_t el8shr03 =  vaddl_u8(el8shr0, el8shr3);
+
+            uint16x8_t el8 = vaddq_u16(el8shr12, el8shr03);
+            uint16x4_t el4h = vadd_u16(vget_low_u16(el8), vget_high_u16(el8));
+
+            vsuml = vaddw_u16(vsuml, vget_low_u16(el8));
+            vsumh = vaddw_u16(vsumh, el4h);
+
+            vst1q_u32(sum + j, vsuml);
+            vst1q_u32(sum + j + 4, vsumh);
+
+            prev = vaddw_u16(prev, vdup_lane_u16(el4h, 3));
+        }
+
+        for (u32 v = vgetq_lane_u32(prev, 3); j < size.width; ++j)
+            sum[j] = (v += src[j]) + prevSum[j];
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)sumBase;
+    (void)sumStride;
+#endif
+}
+
+void sqrIntegral(const Size2D &size,
+                 const u8 * srcBase, ptrdiff_t srcStride,
+                 f64 * sqsumBase, ptrdiff_t sqsumStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    uint16x8_t v_zero8 = vmovq_n_u16(0u);
+
+    // the first iteration
+    const u8 * src = internal::getRowPtr(srcBase, srcStride, 0);
+    f64 * sqsum = internal::getRowPtr(sqsumBase, sqsumStride, 0);
+
+    double prev = 0.;
+    size_t j = 0u;
+
+    for ( ; j + 7 < size.width; j += 8)
+    {
+        internal::prefetch(sqsum + j);
+        internal::prefetch(src + j);
+
+        uint8x8_t vsrc = vld1_u8(src + j);
+
+        uint16x8_t el8shr0 = vmull_u8(vsrc, vsrc);
+        uint16x8_t el8shr1 = vextq_u16(v_zero8, el8shr0, 7);
+
+        uint32x4_t el8shr01l =  vaddl_u16(vget_low_u16(el8shr0), vget_low_u16(el8shr1));
+        uint32x4_t el8shr01h =  vaddl_u16(vget_high_u16(el8shr0), vget_high_u16(el8shr1));
+
+        uint32x4_t el4h = vaddq_u32(el8shr01l, el8shr01h);
+
+        uint32x2_t el2l = vadd_u32(vget_low_u32(el8shr01l), vget_high_u32(el8shr01l));
+        uint32x2_t el2hl = vadd_u32(vget_low_u32(el4h), vget_high_u32(el8shr01l));
+        uint32x2_t el2hh = vadd_u32(vget_low_u32(el4h), vget_high_u32(el4h));
+
+        u32 buf[8];
+        vst1_u32(buf, vget_low_u32(el8shr01l));
+        vst1_u32(buf+2, el2l);
+        vst1_u32(buf+4, el2hl);
+        vst1_u32(buf+6, el2hh);
+        for(u32 k=0; k < 8; k++)
+            sqsum[j+k] = prev + buf[k];
+        prev += buf[7];
+    }
+
+    for (; j < size.width; ++j)
+        sqsum[j] = (prev += src[j]*src[j]);
+
+    // the others
+    for (size_t i = 1; i < size.height ; ++i)
+    {
+        src = internal::getRowPtr(srcBase, srcStride, i);
+        f64 * prevSqSum = internal::getRowPtr(sqsumBase, sqsumStride, i - 1);
+        sqsum = internal::getRowPtr(sqsumBase, sqsumStride, i);
+
+        prev = 0.;
+        j = 0u;
+
+        for ( ; j + 7 < size.width; j += 8)
+        {
+            internal::prefetch(sqsum + j);
+            internal::prefetch(src + j);
+
+            uint8x8_t vsrc = vld1_u8(src + j);
+
+            uint16x8_t el8shr0 = vmull_u8(vsrc, vsrc);
+            uint16x8_t el8shr1 = vextq_u16(v_zero8, el8shr0, 7);
+
+            uint32x4_t el8shr01l =  vaddl_u16(vget_low_u16(el8shr0), vget_low_u16(el8shr1));
+            uint32x4_t el8shr01h =  vaddl_u16(vget_high_u16(el8shr0), vget_high_u16(el8shr1));
+
+            uint32x4_t el4h = vaddq_u32(el8shr01l, el8shr01h);
+
+            uint32x2_t el2l = vadd_u32(vget_low_u32(el8shr01l), vget_high_u32(el8shr01l));
+            uint32x2_t el2hl = vadd_u32(vget_low_u32(el4h), vget_high_u32(el8shr01l));
+            uint32x2_t el2hh = vadd_u32(vget_low_u32(el4h), vget_high_u32(el4h));
+
+            u32 buf[8];
+            vst1_u32(buf, vget_low_u32(el8shr01l));
+            vst1_u32(buf+2, el2l);
+            vst1_u32(buf+4, el2hl);
+            vst1_u32(buf+6, el2hh);
+            for(u32 k=0; k < 8; k++)
+                sqsum[j+k] = prev + prevSqSum[j+k] + buf[k];
+            prev += buf[7];
+        }
+
+        for (; j < size.width; ++j)
+            sqsum[j] = (prev += src[j]*src[j]) + prevSqSum[j];
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)sqsumBase;
+    (void)sqsumStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/intrinsics.hpp

@@ -0,0 +1,112 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_INTRINSICS_HPP
+#define CAROTENE_INTRINSICS_HPP
+
+#include <carotene/definitions.hpp>
+
+#include <arm_neon.h>
+
+namespace CAROTENE_NS { namespace internal {
+
+/////////////// Custom NEON intrinsics ///////////////////
+
+// calculate reciprocal value
+
+inline float32x4_t vrecpq_f32(float32x4_t val)
+{
+    float32x4_t reciprocal = vrecpeq_f32(val);
+    reciprocal = vmulq_f32(vrecpsq_f32(val, reciprocal), reciprocal);
+    reciprocal = vmulq_f32(vrecpsq_f32(val, reciprocal), reciprocal);
+    return reciprocal;
+}
+
+inline float32x2_t vrecp_f32(float32x2_t val)
+{
+    float32x2_t reciprocal = vrecpe_f32(val);
+    reciprocal = vmul_f32(vrecps_f32(val, reciprocal), reciprocal);
+    reciprocal = vmul_f32(vrecps_f32(val, reciprocal), reciprocal);
+    return reciprocal;
+}
+
+// caclulate sqrt value
+
+inline float32x4_t vrsqrtq_f32(float32x4_t val)
+{
+    float32x4_t e = vrsqrteq_f32(val);
+    e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(e, e), val), e);
+    e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(e, e), val), e);
+    return e;
+}
+
+inline float32x2_t vrsqrt_f32(float32x2_t val)
+{
+    float32x2_t e = vrsqrte_f32(val);
+    e = vmul_f32(vrsqrts_f32(vmul_f32(e, e), val), e);
+    e = vmul_f32(vrsqrts_f32(vmul_f32(e, e), val), e);
+    return e;
+}
+
+inline float32x4_t vsqrtq_f32(float32x4_t val)
+{
+    return vrecpq_f32(vrsqrtq_f32(val));
+}
+
+inline float32x2_t vsqrt_f32(float32x2_t val)
+{
+    return vrecp_f32(vrsqrt_f32(val));
+}
+
+// table lookup with the table in a 128-bit register
+
+inline uint8x8_t vqtbl1_u8 (uint8x16_t a, uint8x8_t b)
+{
+#ifdef __aarch64__
+    // AArch64 supports this natively
+    return ::vqtbl1_u8(a, b);
+#else
+    union { uint8x16_t v; uint8x8x2_t w; } u = { a };
+    return vtbl2_u8(u.w, b);
+#endif
+}
+
+} }
+
+#endif

3rdparty/carotene/src/laplacian.cpp

@@ -0,0 +1,713 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "saturate_cast.hpp"
+
+#include <vector>
+
+namespace CAROTENE_NS {
+
+bool isLaplacian3x3Supported(const Size2D &size, BORDER_MODE border)
+{
+    return isSupportedConfiguration() && size.width >= 8 &&
+        (border == BORDER_MODE_CONSTANT ||
+            border == BORDER_MODE_REPLICATE);
+}
+
+void Laplacian3x3(const Size2D &size,
+                  const u8 * srcBase, ptrdiff_t srcStride,
+                  u8 * dstBase, ptrdiff_t dstStride,
+                  BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isLaplacian3x3Supported(size, border));
+#ifdef CAROTENE_NEON
+    const uint16x8_t v_border_x3 = vdupq_n_u16(borderValue * 3);
+    const uint16x8_t v_zero = vdupq_n_u16(0);
+    const uint8x8_t v_border = vdup_n_u8(borderValue);
+
+    uint8x8_t vsub;
+    uint16x8_t tprev = v_zero, tcurr = v_zero, tnext = v_zero;
+    uint16x8_t t0 = v_zero, t1 = v_zero, t2 = v_zero;
+
+    ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height;
+
+    for (ptrdiff_t y = 0; y < height; ++y)
+    {
+        const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max<ptrdiff_t>(y - 1, 0));
+        const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1));
+        u8 * drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        s16 prevx = 0, currx = 0, nextx = 0;
+        ptrdiff_t x = 0;
+        const ptrdiff_t bwidth = y + 2 < height ? width : (width - 8);
+
+        // perform vertical convolution
+        for ( ; x <= bwidth; x += 8)
+        {
+            internal::prefetch(srow0 + x);
+            internal::prefetch(srow1 + x);
+            internal::prefetch(srow2 + x);
+
+            uint8x8_t x0 = !srow0 ? v_border : vld1_u8(srow0 + x);
+            uint8x8_t x1 = vld1_u8(srow1 + x);
+            uint8x8_t x2 = !srow2 ? v_border : vld1_u8(srow2 + x);
+
+            // calculate values for plain CPU part below if needed
+            if (x + 8 >= bwidth)
+            {
+                ptrdiff_t x3 = x == width ? width - 1 : x;
+                ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max<ptrdiff_t>(x3 - 1, 0);
+
+                if (border == BORDER_MODE_CONSTANT && x4 < 0)
+                    prevx = borderValue;
+                else
+                    prevx = (srow2 ? srow2[x4] : borderValue) + srow1[x4] + (srow0 ? srow0[x4] : borderValue);
+
+                currx = (srow2 ? srow2[x3] : borderValue) + srow1[x3] + (srow0 ? srow0[x3] : borderValue);
+            }
+
+            // make shift
+            if (x)
+            {
+                tprev = tcurr;
+                tcurr = tnext;
+            }
+
+            // and calculate next value
+            tnext = vaddw_u8(vaddl_u8(x0, x1), x2);
+
+            // make extrapolation for the first elements
+            if (!x)
+            {
+                // make border
+                if (border == BORDER_MODE_CONSTANT)
+                    tcurr = v_border_x3;
+                else if (border == BORDER_MODE_REPLICATE)
+                    tcurr = vdupq_n_u16(vgetq_lane_u16(tnext, 0));
+
+                vsub = x1;
+
+                continue;
+            }
+
+            // combine 3 "shifted" vectors
+            t0 = vextq_u16(tprev, tcurr, 7);
+            t1 = tcurr;
+            t2 = vextq_u16(tcurr, tnext, 1);
+
+            // and add them
+            t0 = vqaddq_u16(t0, vqaddq_u16(t1, t2));
+
+            int16x8_t tt0 = vsubq_s16(vreinterpretq_s16_u16(t0),
+                                      vreinterpretq_s16_u16(vaddw_u8(vshll_n_u8(vsub, 3), vsub)));
+            uint8x8_t it0 = vqmovun_s16(tt0);
+            vst1_u8(drow + x - 8, it0);
+
+            vsub = x1;
+        }
+
+        x -= 8;
+        if (x == width)
+            --x;
+
+        for ( ; x < width; ++x)
+        {
+            // make extrapolation for the last elements
+            if (x + 1 >= width)
+            {
+                if (border == BORDER_MODE_CONSTANT)
+                    nextx = borderValue * 3;
+                else if (border == BORDER_MODE_REPLICATE)
+                    nextx = srow2[x] + srow1[x] + srow0[x];
+            }
+            else
+            {
+                nextx = (srow2 ? srow2[x + 1] : borderValue) +
+                                 srow1[x + 1] +
+                        (srow0 ? srow0[x + 1] : borderValue);
+            }
+
+            s32 val = (prevx + currx + nextx) - 9 * srow1[x];
+            drow[x] = internal::saturate_cast<u8>((s32)val);
+
+            // make shift
+            prevx = currx;
+            currx = nextx;
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+bool isLaplacianOpenCVSupported(const Size2D &size, BORDER_MODE border)
+{
+    return isSupportedConfiguration() &&
+        size.width >= 8 && size.height >= 1 &&
+        (border == BORDER_MODE_CONSTANT   ||
+         border == BORDER_MODE_REFLECT    ||
+         border == BORDER_MODE_REFLECT101 ||
+         border == BORDER_MODE_REPLICATE);
+}
+
+void Laplacian1OpenCV(const Size2D &size,
+                      const u8 * srcBase, ptrdiff_t srcStride,
+                      s16 * dstBase, ptrdiff_t dstStride,
+                      BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border));
+#ifdef CAROTENE_NEON
+    ptrdiff_t rows = size.height, cols = size.width;
+
+    std::vector<u8> _tmp;
+    u8 *tmp = 0;
+    if (border == BORDER_MODE_CONSTANT)
+    {
+        _tmp.assign(cols + 4,borderValue);
+        tmp = &_tmp[2];
+    }
+
+    for( ptrdiff_t y = 0; y < rows; y++ )
+    {
+        const u8* v0 = 0;
+        const u8* v1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8* v2 = 0;
+        // make border
+        if (border == BORDER_MODE_REFLECT101) {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : y+1);
+            v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0);
+        } else  if (border == BORDER_MODE_CONSTANT) {
+            v0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp;
+            v2 =  y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp;
+        } else {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0);
+            v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0);
+        }
+        s16* drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        int16x8_t tcurr = vmovq_n_s16(0x0);
+        int16x8_t tnext = vmovq_n_s16(0x0);
+        int16x8_t t0, t2;
+        uint8x8_t xx0 = vmov_n_u8(0x0);
+        uint8x8_t xx1 = vmov_n_u8(0x0);
+        uint8x8_t xx2 = vmov_n_u8(0x0);
+        ptrdiff_t x = 0;
+        const ptrdiff_t bcols = y + 2 < rows ? cols : (cols - 8);
+        for( ; x <= bcols; x += 8 )
+        {
+            internal::prefetch(v0 + x);
+            internal::prefetch(v1 + x);
+            internal::prefetch(v2 + x);
+
+            uint8x8_t x0 = vld1_u8(v0 + x);
+            uint8x8_t x1 = vld1_u8(v1 + x);
+            uint8x8_t x2 = vld1_u8(v2 + x);
+
+            if(x) {
+                xx0 = xx1;
+                xx1 = xx2;
+            } else {
+                xx1 = x1;
+                // make border
+                    if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT)
+                    {
+                        xx1 = vset_lane_u8(vget_lane_u8(x1, 0),x1, 7);
+                    }
+                    else if (border == BORDER_MODE_CONSTANT)
+                    {
+                        xx1 = vset_lane_u8(borderValue, x1, 7);
+                    }
+                    else if (border == BORDER_MODE_REFLECT101)
+                    {
+                        xx1 = vset_lane_u8(vget_lane_u8(x1, 1),x1, 7);
+                    }
+            }
+            xx2 = x1;
+
+            if(x) {
+                tcurr = tnext;
+            }
+            tnext = vsubq_s16(vreinterpretq_s16_u16(vaddl_u8(x0, x2)),
+                              vreinterpretq_s16_u16(vshll_n_u8(x1, 2)));
+
+            if(!x) {
+                tcurr = tnext;
+                continue;
+            }
+            t0 = vreinterpretq_s16_u16(vmovl_u8(vext_u8(xx0, xx1, 7)));
+            t2 = vreinterpretq_s16_u16(vmovl_u8(vext_u8(xx1, xx2, 1)));
+            t0 = vaddq_s16(vqaddq_s16(t0, t2), tcurr);
+
+            vst1q_s16(drow + x - 8, t0);
+        }
+
+        x -= 8;
+        if(x == cols){
+            x--;
+        }
+
+        for( ; x < cols; x++ )
+        {
+            s16 nextx;
+            s16 prevx;
+            // make border
+            if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT)
+            {
+                prevx = x == 0 ? v1[0] : v1[x-1];
+                nextx = x == cols-1 ? v1[x] : v1[x+1];
+            }
+            else if (border == BORDER_MODE_REFLECT101)
+            {
+                prevx = x == 0 ? v1[1] : v1[x-1];
+                nextx = x == cols-1 ? v1[x-1] : v1[x+1];
+            }
+            else //if (border == BORDER_MODE_CONSTANT)
+            {
+                prevx = x == 0 ? borderValue : v1[x-1];
+                nextx = x == cols-1 ? borderValue : v1[x+1];
+            }
+            *(drow+x) = prevx + nextx - 4*v1[x] + v0[x] + v2[x];
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+void Laplacian3OpenCV(const Size2D &size,
+                      const u8 * srcBase, ptrdiff_t srcStride,
+                      s16 * dstBase, ptrdiff_t dstStride,
+                      BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border));
+#ifdef CAROTENE_NEON
+    ptrdiff_t rows = size.height, cols = size.width;
+
+    std::vector<u8> _tmp;
+    u8 *tmp = 0;
+    if (border == BORDER_MODE_CONSTANT)
+    {
+        _tmp.assign(cols + 4,borderValue);
+        tmp = &_tmp[2];
+    }
+
+    for( ptrdiff_t y = 0; y < rows; y++ )
+    {
+        const u8* v0 = 0;
+        const u8* v1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8* v2 = 0;
+        // make border
+        if (border == BORDER_MODE_REFLECT101) {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : y+1);
+            v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0);
+        } else  if (border == BORDER_MODE_CONSTANT) {
+            v0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp;
+            v2 = y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp;
+        } else {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0);
+            v2 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0);
+        }
+        s16* drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        int16x8_t tprev = vmovq_n_s16(0x0);
+        int16x8_t tcurr = vmovq_n_s16(0x0);
+        int16x8_t tnext = vmovq_n_s16(0x0);
+        int16x8_t tc = vmovq_n_s16(0x0);
+        int16x8_t t0, t2, tcnext;
+        ptrdiff_t x = 0;
+        const ptrdiff_t bcols = y + 2 < rows ? cols : (cols - 8);
+        for( ; x <= bcols; x += 8 )
+        {
+            internal::prefetch(v0 + x);
+            internal::prefetch(v1 + x);
+            internal::prefetch(v2 + x);
+
+            uint8x8_t x0 = vld1_u8(v0 + x);
+            uint8x8_t x1 = vld1_u8(v1 + x);
+            uint8x8_t x2 = vld1_u8(v2 + x);
+            tcnext = vreinterpretq_s16_u16(vshll_n_u8(x1, 2));
+
+            if(x) {
+                tprev = tcurr;
+                tcurr = tnext;
+            }
+            tnext = vreinterpretq_s16_u16(vaddl_u8(x0, x2));
+
+            if(!x) {
+                tcurr = tnext;
+                tc = tcnext;
+
+                // make border
+                    if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT)
+                    {
+                        tcurr = vsetq_lane_s16(vgetq_lane_s16(tcurr, 0),tcurr, 7);
+                    }
+                    else if (border == BORDER_MODE_CONSTANT)
+                    {
+                        tcurr = vsetq_lane_s16(borderValue, tcurr, 7);
+                    }
+                    else if (border == BORDER_MODE_REFLECT101)
+                    {
+                        tcurr = vsetq_lane_s16(vgetq_lane_s16(tcurr, 1),tcurr, 7);
+                    }
+                continue;
+            }
+
+            t0 = vextq_s16(tprev, tcurr, 7);
+            t2 = vextq_s16(tcurr, tnext, 1);
+
+            t0 = vsubq_s16(vqaddq_s16(t0, t2), tc);
+            tc = tcnext;
+
+            t0 = vshlq_n_s16(t0, 1);
+            vst1q_s16(drow + x - 8, t0);
+        }
+        x -= 8;
+        if(x == cols){
+            x--;
+        }
+
+        for( ; x < cols; x++ )
+        {
+            s16 nextx, nextx2;
+            s16 prevx, prevx2;
+            // make border
+            if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT)
+            {
+                prevx = x == 0 ? v0[0] : v0[x-1];
+                prevx2 = x == 0 ? v2[0] : v2[x-1];
+                nextx = x == cols-1 ? v0[x] : v0[x+1];
+                nextx2 = x == cols-1 ? v2[x] : v2[x+1];
+            }
+            else if (border == BORDER_MODE_REFLECT101)
+            {
+                prevx = x == 0 ? v0[1] : v0[x-1];
+                prevx2 = x == 0 ? v2[1] : v2[x-1];
+                nextx = x == cols-1 ? v0[x-1] : v0[x+1];
+                nextx2 = x == cols-1 ? v2[x-1] : v2[x+1];
+            }
+            else //if (border == BORDER_MODE_CONSTANT)
+            {
+                prevx = x == 0 ? borderValue : v0[x-1];
+                prevx2 = x == 0 ? borderValue : v2[x-1];
+                nextx = x == cols-1 ? borderValue : v0[x+1];
+                nextx2 = x == cols-1 ? borderValue : v2[x+1];
+            }
+            s16 res = prevx + nextx - 4*v1[x] + prevx2 + nextx2;
+            *(drow+x) = 2*res;
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+void Laplacian5OpenCV(const Size2D &size,
+                      const u8 * srcBase, ptrdiff_t srcStride,
+                      s16 * dstBase, ptrdiff_t dstStride,
+                      BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isLaplacianOpenCVSupported(size, border));
+#ifdef CAROTENE_NEON
+    ptrdiff_t rows = size.height, cols = size.width;
+
+    std::vector<u8> _tmp;
+    u8 *tmp = 0;
+    if (border == BORDER_MODE_CONSTANT)
+    {
+        _tmp.assign(cols + 4,borderValue);
+        tmp = &_tmp[2];
+    }
+
+    for( ptrdiff_t y = 0; y < rows; y++ )
+    {
+        const u8* v0 = 0;
+        const u8* v1 = 0;
+        const u8* v2 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8* v3 = 0;
+        const u8* v4 = 0;
+        // make border
+        if (border == BORDER_MODE_REPLICATE) {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : 0);
+            v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0);
+            v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0);
+            v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 0 ? rows-1 : 0);
+        } else if (border == BORDER_MODE_REFLECT) {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : rows > 1 ? 1-y : 0);
+            v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0);
+            v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 0 ? rows-1 : 0);
+            v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 1 ? 2*rows-(y+3) : 0);
+        } else if (border == BORDER_MODE_REFLECT101) {
+            v0 = internal::getRowPtr(srcBase, srcStride, y > 1 ? y-2 : rows > 2-y ? 2-y : 0); ///check
+            v1 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : rows > 1 ? 1 : 0);
+            v3 = internal::getRowPtr(srcBase, srcStride, y < rows-1 ? y+1 : rows > 1 ? rows-2 : 0);
+            v4 = internal::getRowPtr(srcBase, srcStride, y < rows-2 ? y+2 : rows > 2 ? 2*rows-(y+4) : 0);///bad if rows=2 y=1   rows - 4 + (2,1)
+        } else if (border == BORDER_MODE_CONSTANT) {
+            v0 = y > 1 ? internal::getRowPtr(srcBase, srcStride, y-2) : tmp;
+            v1 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp;
+            v3 = y < rows-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp;
+            v4 = y < rows-2 ? internal::getRowPtr(srcBase, srcStride, y+2) : tmp;
+        }
+        s16* drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        int16x8_t tnext, tc, t0;
+        int16x8_t tnext2, tnext3;
+        int16x8_t tnext1Old, tnext2Old, tnext3Old;
+        int16x8_t tnext4OldOldOld, tnext5OldOldOld;
+
+        int16x8_t tcurr1 = vmovq_n_s16(0x0);
+        int16x8_t tnext1 = vmovq_n_s16(0x0);
+        int16x8_t tprev1 = vmovq_n_s16(0x0);
+        int16x8_t tpprev1 = vmovq_n_s16(0x0);
+        int16x8_t tppprev1 = vmovq_n_s16(0x0);
+
+        int16x8_t tnext4Old = vmovq_n_s16(0x0);
+        int16x8_t tnext5Old = vmovq_n_s16(0x0);
+        int16x8_t tnext1OldOld = vmovq_n_s16(0x0);
+        int16x8_t tnext2OldOld = vmovq_n_s16(0x0);
+        int16x8_t tnext3OldOld = vmovq_n_s16(0x0);
+        int16x8_t tnext4OldOld = vmovq_n_s16(0x0);
+        int16x8_t tnext5OldOld = vmovq_n_s16(0x0);
+
+        // do vertical convolution
+        ptrdiff_t x = 0;
+        const ptrdiff_t bcols = y + 3 < rows ? cols : (cols - 8);
+        for( ; x <= bcols; x += 8 )
+        {
+            internal::prefetch(v0 + x);
+            internal::prefetch(v1 + x);
+            internal::prefetch(v2 + x);
+            internal::prefetch(v3 + x);
+            internal::prefetch(v4 + x);
+
+            uint8x8_t x0 = vld1_u8(v0 + x);
+            uint8x8_t x1 = vld1_u8(v1 + x);
+            uint8x8_t x2 = vld1_u8(v2 + x);
+            uint8x8_t x3 = vld1_u8(v3 + x);
+            uint8x8_t x4 = vld1_u8(v4 + x);
+            if(x) {
+                tcurr1 = tnext1;
+            }
+
+            tnext4OldOldOld = tnext4Old;
+            tnext5OldOldOld = tnext5Old;
+            tnext1Old = tnext1OldOld;
+            tnext2Old = tnext2OldOld;
+            tnext3Old = tnext3OldOld;
+            tnext4Old = tnext4OldOld;
+            tnext5Old = tnext5OldOld;
+
+            tnext3 = vreinterpretq_s16_u16(vaddq_u16(vaddl_u8(x3, x2),vaddl_u8(x2, x1)));
+            tnext3 = vshlq_n_s16(tnext3, 1);
+
+            tc = vreinterpretq_s16_u16(vsubl_u8(x4, x2));
+            tnext = vreinterpretq_s16_u16(vsubl_u8(x2, x0));
+            tnext2 = vsubq_s16(tc, tnext);
+
+            tnext1 = vaddq_s16(tnext3, tnext2);
+            // tnext1 = x0 + 2*x1 + 2*x2 + 2*x3 + x4
+
+            tnext2 = vshlq_n_s16(tnext2, 1);
+            // tnext2 = 2*x4 - 4*x2 + 2*x0
+
+            tnext3 = vsubq_s16(tnext2, vshlq_n_s16(tnext3, 1));
+            // tnext3 = 2*x0 - 4*x1 - 12*x2 - 4*x3  + 2*x4
+
+            tnext1OldOld = tnext1;
+            tnext2OldOld = tnext2;
+            tnext3OldOld = tnext3;
+            tnext4OldOld = tnext2;
+            tnext5OldOld = tnext1;
+
+            if(x) {
+                tnext1 = vextq_s16(tnext1Old, tnext1, 2);
+                tcurr1 = vextq_s16(tnext2Old, tnext2, 1);
+                tprev1 = tnext3Old;
+
+                if(x!=8) {
+                    tpprev1 = vextq_s16(tnext4OldOldOld, tnext4Old, 7);
+                    tppprev1 = vextq_s16(tnext5OldOldOld, tnext5Old, 6);
+                }
+            }
+
+            if(!x) {
+                // make border
+                if (border == BORDER_MODE_REPLICATE) {
+                    tpprev1 = vextq_s16(tnext2, tnext2, 7);
+                    tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 1),tpprev1, 0);
+
+                    tprev1 = vextq_s16(tnext1, tnext1, 6);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 0);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 1);
+                } else if (border == BORDER_MODE_REFLECT) {
+                    tpprev1 = vextq_s16(tnext2, tnext2, 7);
+                    tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 1),tpprev1, 0);
+
+                    tprev1 = vextq_s16(tnext1, tnext1, 6);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 3),tprev1, 0);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 2),tprev1, 1);
+                } else if (border == BORDER_MODE_REFLECT101) {
+                    tpprev1 = vextq_s16(tnext2, tnext2, 7);
+                    tpprev1 = vsetq_lane_s16(vgetq_lane_s16(tpprev1, 2),tpprev1, 0);
+
+                    tprev1 = vextq_s16(tnext1, tnext1, 6);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 3),tprev1, 1);
+                    tprev1 = vsetq_lane_s16(vgetq_lane_s16(tprev1, 4),tprev1, 0);
+                } else if (border == BORDER_MODE_CONSTANT) {
+                    tpprev1 = vextq_s16(tnext2, tnext2, 7);
+                    tpprev1 = vsetq_lane_s16(borderValue, tpprev1, 0);
+
+                    tprev1 = vextq_s16(tnext1, tnext1, 6);
+                    tprev1 = vsetq_lane_s16(borderValue, tprev1, 0);
+                    tprev1 = vsetq_lane_s16(borderValue, tprev1, 1);
+                }
+                tppprev1 = tprev1;
+                continue;
+            }
+
+            t0 = vaddq_s16(vaddq_s16(vqaddq_s16(tcurr1, tprev1), vqaddq_s16(tpprev1, tppprev1)), tnext1);
+            t0 = vaddq_s16(t0, t0);
+            vst1q_s16(drow + x - 8, t0);
+        }
+        x -= 8;
+        if(x >= cols - 1)
+            x = cols-2;
+
+        s16 pprevx = 0;
+        s16 prevx = 0;
+        s16 nextx = 0;
+        s16 nnextx = 0;
+
+        for( ; x < cols; x++ )
+        {
+            if (x == 0) {
+                // make border
+                if (border == BORDER_MODE_REPLICATE) {
+                    pprevx = v0[0] + 2*v1[0] + 2*v2[0] + 2*v3[0] + v4[0];
+                    prevx = 2*v0[0] - 4*v2[0] + 2*v4[0];
+                } else if (border == BORDER_MODE_REFLECT) {
+                    pprevx = v0[1] + 2*v1[1] + 2*v2[1] + 2*v3[1] + v4[1];
+                    prevx = 2*v0[0] - 4*v2[0] + 2*v4[0];
+                } else if (border == BORDER_MODE_REFLECT101) {
+                    pprevx = v0[2] + 2*v1[2] + 2*v2[2] + 2*v3[2] + v4[2];
+                    prevx = 2*v0[1] - 4*v2[1] + 2*v4[1];
+                } else if (border == BORDER_MODE_CONSTANT) {
+                    pprevx = 8 * borderValue;
+                    prevx = 0;
+                }
+            } else if (x == 1) {
+                // make border
+                if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) {
+                    pprevx = v0[0] + 2*v1[0] + 2*v2[0] + 2*v3[0] + v4[0];
+                } else if (border == BORDER_MODE_REFLECT101) {
+                    pprevx = v0[1] + 2*v1[1] + 2*v2[1] + 2*v3[1] + v4[1];
+                } else if (border == BORDER_MODE_CONSTANT) {
+                    pprevx = 8 * borderValue;
+                }
+                prevx = 2*v0[0] - 4*v2[0] + 2*v4[0];
+            } else {
+                pprevx = v0[x-2] + 2*v1[x-2] + 2*v2[x-2] + 2*v3[x-2] + v4[x-2];
+                prevx = 2*v0[x-1] - 4*v2[x-1] + 2*v4[x-1];
+            }
+            s16 currx = 2*v0[x] - 4*v1[x] - 12*v2[x] - 4*v3[x] + 2*v4[x];
+            if (x == cols-1) {
+                // make border
+                if (border == BORDER_MODE_REPLICATE) {
+                    nextx = 2*v0[x] - 4*v2[x] + 2*v4[x];
+                    nnextx = v0[x] + 2*v1[x] + 2*v2[x] + 2*v3[x] + v4[x];
+                } else if (border == BORDER_MODE_REFLECT) {
+                    nextx = 2*v0[x] - 4*v2[x] + 2*v4[x];
+                    nnextx = v0[x-1] + 2*v1[x-1] + 2*v2[x-1] + 2*v3[x-1] + v4[x-1];
+                } else if (border == BORDER_MODE_REFLECT101) {
+                    nextx = 2*v0[x-1] - 4*v2[x-1] + 2*v4[x-1];
+                    nnextx = v0[x-2] + 2*v1[x-2] + 2*v2[x-2] + 2*v3[x-2] + v4[x-2];
+                } else if (border == BORDER_MODE_CONSTANT) {
+                    nextx = 0;
+                    nnextx = 8 * borderValue;
+                }
+            } else if (x == cols-2) {
+                // make border
+                if (border == BORDER_MODE_REPLICATE || border == BORDER_MODE_REFLECT) {
+                    nnextx = v0[x+1] + 2*v1[x+1] + 2*v2[x+1] + 2*v3[x+1] + v4[x+1];
+                } else if (border == BORDER_MODE_REFLECT101) {
+                    nnextx = v0[x] + 2*v1[x] + 2*v2[x] + 2*v3[x] + v4[x];
+                } else if (border == BORDER_MODE_CONSTANT) {
+                    nnextx = 8 * borderValue;
+                }
+                nextx = 2*v0[x+1] - 4*v2[x+1] + 2*v4[x+1];
+            } else {
+                nextx = 2*v0[x+1] - 4*v2[x+1] + 2*v4[x+1];
+                nnextx = v0[x+2] + 2*v1[x+2] + 2*v2[x+2] + 2*v3[x+2] + v4[x+2];
+            }
+            s16 res = pprevx + prevx + currx + nextx + nnextx;
+            *(drow+x) = 2*res;
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/magnitude.cpp

@@ -0,0 +1,160 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+#include <cmath>
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+struct Magnitude
+{
+    typedef s16 type;
+
+    void operator() (const int16x8_t & v_src0, const int16x8_t & v_src1,
+              int16x8_t & v_dst) const
+    {
+        int16x4_t v_src0_p = vget_low_s16(v_src0), v_src1_p = vget_low_s16(v_src1);
+        float32x4_t v_sqr0 = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0_p, v_src0_p)),
+                                       vcvtq_f32_s32(vmull_s16(v_src1_p, v_src1_p)));
+        v_src0_p = vget_high_s16(v_src0);
+        v_src1_p = vget_high_s16(v_src1);
+        float32x4_t v_sqr1 = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0_p, v_src0_p)),
+                                       vcvtq_f32_s32(vmull_s16(v_src1_p, v_src1_p)));
+
+        int32x4_t v_sqrt0 = vcvtq_s32_f32(internal::vsqrtq_f32(v_sqr0));
+        int32x4_t v_sqrt1 = vcvtq_s32_f32(internal::vsqrtq_f32(v_sqr1));
+
+        v_dst = vcombine_s16(vqmovn_s32(v_sqrt0), vqmovn_s32(v_sqrt1));
+    }
+
+    void operator() (const int16x4_t & v_src0, const int16x4_t & v_src1,
+              int16x4_t & v_dst) const
+    {
+        float32x4_t v_tmp = vaddq_f32(vcvtq_f32_s32(vmull_s16(v_src0, v_src0)),
+                                      vcvtq_f32_s32(vmull_s16(v_src1, v_src1)));
+        int32x4_t v_sqrt = vcvtq_s32_f32(internal::vsqrtq_f32(v_tmp));
+        v_dst = vqmovn_s32(v_sqrt);
+    }
+
+    void operator() (const short * src0, const short * src1, short * dst) const
+    {
+        f32 src0val = (f32)src0[0], src1val = (f32)src1[0];
+        dst[0] = internal::saturate_cast<s16>((s32)sqrtf(src0val * src0val + src1val * src1val));
+    }
+};
+
+struct MagnitudeF32
+{
+    typedef f32 type;
+
+    void operator() (const float32x4_t & v_src0, const float32x4_t & v_src1,
+              float32x4_t & v_dst) const
+    {
+        v_dst = internal::vsqrtq_f32(vaddq_f32(vmulq_f32(v_src0, v_src0), vmulq_f32(v_src1, v_src1)));
+    }
+
+    void operator() (const float32x2_t & v_src0, const float32x2_t & v_src1,
+              float32x2_t & v_dst) const
+    {
+        v_dst = internal::vsqrt_f32(vadd_f32(vmul_f32(v_src0, v_src0), vmul_f32(v_src1, v_src1)));
+    }
+
+    void operator() (const f32 * src0, const f32 * src1, f32 * dst) const
+    {
+        dst[0] = sqrtf(src0[0] * src0[0] + src1[0] * src1[0]);
+    }
+};
+
+} // namespace
+
+#endif
+
+void magnitude(const Size2D &size,
+               const s16 * src0Base, ptrdiff_t src0Stride,
+               const s16 * src1Base, ptrdiff_t src1Stride,
+               s16 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride,
+                         Magnitude());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void magnitude(const Size2D &size,
+               const f32 * src0Base, ptrdiff_t src0Stride,
+               const f32 * src1Base, ptrdiff_t src1Stride,
+               f32 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride,
+                         MagnitudeF32());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/meanstddev.cpp

@@ -0,0 +1,163 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <cmath>
+
+namespace CAROTENE_NS {
+
+void meanStdDev(const Size2D &size,
+                const u8 * srcBase, ptrdiff_t srcStride,
+                f32 * pMean, f32 * pStdDev)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    f64 fsum = 0.0f, fsqsum = 0.0f;
+    sqsum(size, srcBase, srcStride, &fsum, &fsqsum, 1);
+
+    // calc mean and stddev
+    f64 itotal = 1.0 / size.total();
+    f64 mean = fsum * itotal;
+    f64 stddev = sqrt(std::max(fsqsum * itotal - mean * mean, 0.0));
+
+    if (pMean)
+        *pMean = mean;
+    if (pStdDev)
+        *pStdDev = stddev;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)pMean;
+    (void)pStdDev;
+#endif
+}
+
+void meanStdDev(const Size2D &size,
+                const u16 * srcBase, ptrdiff_t srcStride,
+                f32 * pMean, f32 * pStdDev)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t blockSize0 = 1 << 10, roiw4 = size.width & ~3;
+    f64 fsum = 0.0f, fsqsum = 0.0f;
+
+    f32 arsum[8];
+    uint32x4_t v_zero = vdupq_n_u32(0u), v_sum;
+    float32x4_t v_zero_f = vdupq_n_f32(0.0f), v_sqsum;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u16 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0u;
+
+        while (j < roiw4)
+        {
+            size_t blockSize = std::min(roiw4 - j, blockSize0) + j;
+            v_sum = v_zero;
+            v_sqsum = v_zero_f;
+
+            for ( ; j + 16 < blockSize ; j += 16)
+            {
+                internal::prefetch(src + j);
+                uint16x8_t v_src0 = vld1q_u16(src + j), v_src1 = vld1q_u16(src + j + 8);
+
+                // 0
+                uint32x4_t v_srclo = vmovl_u16(vget_low_u16(v_src0));
+                uint32x4_t v_srchi = vmovl_u16(vget_high_u16(v_src0));
+                v_sum = vaddq_u32(v_sum, vaddq_u32(v_srclo, v_srchi));
+                float32x4_t v_srclo_f = vcvtq_f32_u32(v_srclo);
+                float32x4_t v_srchi_f = vcvtq_f32_u32(v_srchi);
+                v_sqsum = vmlaq_f32(v_sqsum, v_srclo_f, v_srclo_f);
+                v_sqsum = vmlaq_f32(v_sqsum, v_srchi_f, v_srchi_f);
+
+                // 1
+                v_srclo = vmovl_u16(vget_low_u16(v_src1));
+                v_srchi = vmovl_u16(vget_high_u16(v_src1));
+                v_sum = vaddq_u32(v_sum, vaddq_u32(v_srclo, v_srchi));
+                v_srclo_f = vcvtq_f32_u32(v_srclo);
+                v_srchi_f = vcvtq_f32_u32(v_srchi);
+                v_sqsum = vmlaq_f32(v_sqsum, v_srclo_f, v_srclo_f);
+                v_sqsum = vmlaq_f32(v_sqsum, v_srchi_f, v_srchi_f);
+            }
+
+            for ( ; j < blockSize; j += 4)
+            {
+                uint32x4_t v_src = vmovl_u16(vld1_u16(src + j));
+                float32x4_t v_src_f = vcvtq_f32_u32(v_src);
+                v_sum = vaddq_u32(v_sum, v_src);
+                v_sqsum = vmlaq_f32(v_sqsum, v_src_f, v_src_f);
+            }
+
+            vst1q_f32(arsum, vcvtq_f32_u32(v_sum));
+            vst1q_f32(arsum + 4, v_sqsum);
+
+            fsum += (f64)arsum[0] + arsum[1] + arsum[2] + arsum[3];
+            fsqsum += (f64)arsum[4] + arsum[5] + arsum[6] + arsum[7];
+        }
+
+        // collect a few last elements in the current row
+        for ( ; j < size.width; ++j)
+        {
+            f32 srcval = src[j];
+            fsum += srcval;
+            fsqsum += srcval * srcval;
+        }
+    }
+
+    // calc mean and stddev
+    f64 itotal = 1.0 / size.total();
+    f64 mean = fsum * itotal;
+    f64 stddev = sqrt(std::max(fsqsum * itotal - mean * mean, 0.0));
+
+    if (pMean)
+        *pMean = mean;
+    if (pStdDev)
+        *pStdDev = stddev;
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)pMean;
+    (void)pStdDev;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/median_filter.cpp

@@ -0,0 +1,227 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+/*
+ * The code here is based on the code in
+ * <http://ndevilla.free.fr/median/median/src/optmed.c>, which is in public domain.
+ * See also <http://ndevilla.free.fr/median/median/index.html>.
+ */
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+namespace {
+
+    uint8x16_t getLeftReplicate(uint8x16_t r, u32 cn)
+    {
+        u8 buf[16+8];
+        vst1q_u8(buf+cn, r);
+        for (u32 i = 0; i < cn; ++i) buf[i] = buf[cn+i];
+        return vld1q_u8(buf);
+    }
+
+    uint8x8_t getRightReplicate(uint8x8_t r, u32 cn)
+    {
+        u8 buf[8+8];
+        vst1_u8(buf, r);
+        for (u32 i = 0; i < cn; ++i) buf[8+i] = buf[8-cn+i];
+        return vld1_u8(buf+cn);
+    }
+
+} // namespace
+
+//o------^-------^-----------------------------o 0
+//       |       |
+//o--^---v---^---|-------^---------------------o 1
+//   |       |   |       |
+//o--v-------v---|-------|-^-------^-------^---o 2
+//               |       | |       |       |
+//o------^-------v-----^-|-|-------|-------|---o 3
+//       |             | | |       |       |
+//o--^---v---^-----^---|-v-|---^---v---^---v---o 4
+//   |       |     |   |   |   |       |
+//o--v-------v---^-|---|---v---|-------|-------o 5
+//               | |   |       |       |
+//o------^-------|-|---v-------|-------v-------o 6
+//       |       | |           |
+//o--^---v---^---|-v-----------v---------------o 7
+//   |       |   |
+//o--v-------v---v-----------------------------o 8
+
+#define ELT(num, level) v ## num ## _lv ## level
+#define PIX_SORT(a, alvl, b, blvl, newlvl) \
+    PIX_MIN(a, alvl, b, blvl, newlvl); \
+    PIX_MAX(a, alvl, b, blvl, newlvl);
+
+#define SORT9 \
+    PIX_SORT(1, 00, 2, 00, 01); \
+    PIX_SORT(4, 00, 5, 00, 02); \
+    PIX_SORT(7, 00, 8, 00, 03); \
+    PIX_SORT(0, 00, 1, 01, 04); \
+    PIX_SORT(3, 00, 4, 02, 05); \
+    PIX_SORT(6, 00, 7, 03, 06); \
+    PIX_SORT(1, 04, 2, 01, 07); \
+    PIX_SORT(4, 05, 5, 02, 08); \
+    PIX_SORT(7, 06, 8, 03, 09); \
+    PIX_MAX (0, 04, 3, 05, 10); \
+    PIX_MIN (5, 08, 8, 09, 11); \
+    PIX_SORT(4, 08, 7, 09, 12); \
+    PIX_MAX (3, 10, 6, 06, 13); \
+    PIX_MAX (1, 07, 4, 12, 14); \
+    PIX_MIN (2, 07, 5, 11, 15); \
+    PIX_MIN (4, 14, 7, 12, 16); \
+    PIX_SORT(4, 16, 2, 15, 17); \
+    PIX_MAX (6, 13, 4, 17, 18); \
+    PIX_MIN (4, 18, 2, 17, 19);
+
+#endif
+
+bool isMedianFilter3x3Supported(const Size2D &size, u32 numChannels)
+{
+    return isSupportedConfiguration() && size.width >= 16 + numChannels && numChannels <= 8;
+}
+
+void medianFilter3x3(const Size2D &size, u32 numChannels,
+                     const u8 *srcBase, ptrdiff_t srcStride,
+                     const Margin &srcMargin,
+                     u8 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration(isMedianFilter3x3Supported(size, numChannels));
+#ifdef CAROTENE_NEON
+    u32 cn = numChannels;
+    size_t colsn = size.width * cn;
+
+    for (size_t i = 0; i < size.height; ++i) {
+        const u8* psrc1 = internal::getRowPtr(srcBase, srcStride, i);
+        const u8* psrc0 = i == 0 && srcMargin.top == 0 ? psrc1 : psrc1 - srcStride;
+        const u8* psrc2 = i + 1 == size.height && srcMargin.bottom == 0 ? psrc1 : psrc1 + srcStride;
+        u8* pdst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        {
+            uint8x16_t v3_lv00 = vld1q_u8(psrc0);
+            uint8x16_t v4_lv00 = vld1q_u8(psrc1);
+            uint8x16_t v5_lv00 = vld1q_u8(psrc2);
+            uint8x16_t v6_lv00 = vld1q_u8(psrc0 + cn);
+            uint8x16_t v7_lv00 = vld1q_u8(psrc1 + cn);
+            uint8x16_t v8_lv00 = vld1q_u8(psrc2 + cn);
+            uint8x16_t v0_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc0 - cn) : getLeftReplicate(v3_lv00, cn);
+            uint8x16_t v1_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc1 - cn) : getLeftReplicate(v4_lv00, cn);
+            uint8x16_t v2_lv00 = srcMargin.left > 0 ? vld1q_u8(psrc2 - cn) : getLeftReplicate(v5_lv00, cn);
+
+            goto medianBlur3x3_mainBody;
+
+            for (; j < colsn - 16; j += 16) {
+                internal::prefetch(psrc0 + j);
+                internal::prefetch(psrc1 + j);
+                internal::prefetch(psrc2 + j);
+
+                v0_lv00 = vld1q_u8(psrc0 + j - cn);
+                v1_lv00 = vld1q_u8(psrc1 + j - cn);
+                v2_lv00 = vld1q_u8(psrc2 + j - cn);
+                v3_lv00 = vld1q_u8(psrc0 + j);
+                v4_lv00 = vld1q_u8(psrc1 + j);
+                v5_lv00 = vld1q_u8(psrc2 + j);
+                v6_lv00 = vld1q_u8(psrc0 + j + cn);
+                v7_lv00 = vld1q_u8(psrc1 + j + cn);
+                v8_lv00 = vld1q_u8(psrc2 + j + cn);
+
+medianBlur3x3_mainBody:
+
+#define PIX_MIN(a, alvl, b, blvl, newlvl) uint8x16_t ELT(a, newlvl) = vminq_u8(ELT(a, alvl), ELT(b, blvl))
+#define PIX_MAX(a, alvl, b, blvl, newlvl) uint8x16_t ELT(b, newlvl) = vmaxq_u8(ELT(a, alvl), ELT(b, blvl))
+                SORT9;
+#undef PIX_MAX
+#undef PIX_MIN
+
+                vst1q_u8(pdst + j, v4_lv19);
+            }
+        }
+
+        {
+            size_t k = colsn - 8;
+            uint8x8_t v0_lv00 = vld1_u8(psrc0 + k - cn);
+            uint8x8_t v1_lv00 = vld1_u8(psrc1 + k - cn);
+            uint8x8_t v2_lv00 = vld1_u8(psrc2 + k - cn);
+            uint8x8_t v3_lv00 = vld1_u8(psrc0 + k);
+            uint8x8_t v4_lv00 = vld1_u8(psrc1 + k);
+            uint8x8_t v5_lv00 = vld1_u8(psrc2 + k);
+            uint8x8_t v6_lv00 = srcMargin.right > 0 ? vld1_u8(psrc0 + k + cn) : getRightReplicate(v3_lv00, cn);
+            uint8x8_t v7_lv00 = srcMargin.right > 0 ? vld1_u8(psrc1 + k + cn) : getRightReplicate(v4_lv00, cn);
+            uint8x8_t v8_lv00 = srcMargin.right > 0 ? vld1_u8(psrc2 + k + cn) : getRightReplicate(v5_lv00, cn);
+
+            goto medianBlur3x3_tailBody;
+
+            for (; k >= j - 8; k -= 8) {
+                v0_lv00 = vld1_u8(psrc0 + k - cn);
+                v1_lv00 = vld1_u8(psrc1 + k - cn);
+                v2_lv00 = vld1_u8(psrc2 + k - cn);
+                v3_lv00 = vld1_u8(psrc0 + k);
+                v4_lv00 = vld1_u8(psrc1 + k);
+                v5_lv00 = vld1_u8(psrc2 + k);
+                v6_lv00 = vld1_u8(psrc0 + k + cn);
+                v7_lv00 = vld1_u8(psrc1 + k + cn);
+                v8_lv00 = vld1_u8(psrc2 + k + cn);
+
+medianBlur3x3_tailBody:
+
+#define PIX_MIN(a, alvl, b, blvl, newlvl) uint8x8_t ELT(a, newlvl) = vmin_u8(ELT(a, alvl), ELT(b, blvl))
+#define PIX_MAX(a, alvl, b, blvl, newlvl) uint8x8_t ELT(b, newlvl) = vmax_u8(ELT(a, alvl), ELT(b, blvl))
+                SORT9;
+#undef PIX_MAX
+#undef PIX_MIN
+
+                vst1_u8(pdst + k, v4_lv19);
+            }
+        }
+    }
+#else
+    (void)size;
+    (void)numChannels;
+    (void)srcBase;
+    (void)srcStride;
+    (void)srcMargin;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/min_max.cpp

@@ -0,0 +1,139 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <algorithm>
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T>
+struct Min
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vminq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vmin(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = std::min(src0[0], src1[0]);
+    }
+};
+
+template <typename T>
+struct Max
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vmaxq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vmax(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = std::max(src0[0], src1[0]);
+    }
+};
+
+} // namespace
+
+#define IMPL_OP(fun, op, type)                                         \
+void fun(const Size2D &size,                                           \
+         const type * src0Base, ptrdiff_t src0Stride,                  \
+         const type * src1Base, ptrdiff_t src1Stride,                  \
+         type * dstBase, ptrdiff_t dstStride)                          \
+{                                                                      \
+    internal::assertSupportedConfiguration();                          \
+    internal::vtransform(size,                                         \
+                         src0Base, src0Stride,                         \
+                         src1Base, src1Stride,                         \
+                         dstBase, dstStride, op<type>());              \
+}
+
+#else
+
+#define IMPL_OP(fun, op, type)                    \
+void fun(const Size2D &,                          \
+         const type *, ptrdiff_t,                 \
+         const type *, ptrdiff_t,                 \
+         type *, ptrdiff_t)                       \
+{                                                 \
+    internal::assertSupportedConfiguration();     \
+}
+
+#endif
+
+#define IMPL_MINMAX(type) IMPL_OP(min, Min, type) IMPL_OP(max, Max, type)
+
+IMPL_MINMAX(u8)
+IMPL_MINMAX(s8)
+IMPL_MINMAX(u16)
+IMPL_MINMAX(s16)
+IMPL_MINMAX(u32)
+IMPL_MINMAX(s32)
+IMPL_MINMAX(f32)
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/morph.cpp

@@ -0,0 +1,728 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+bool isMorph3x3Supported(const Size2D &size, BORDER_MODE border)
+{
+    return isSupportedConfiguration() && size.width >= 16 &&
+        (border == BORDER_MODE_CONSTANT ||
+            border == BORDER_MODE_REPLICATE);
+}
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+struct ErodeVecOp
+{
+    ErodeVecOp():borderValue(0){}
+
+    ErodeVecOp(BORDER_MODE border, u8 borderValue_) :
+        borderValue(borderValue_)
+    {
+        if (border == BORDER_MODE_REPLICATE)
+            borderValue = std::numeric_limits<u8>::max();
+    }
+
+    inline uint8x16_t operator()(uint8x16_t a, uint8x16_t b) const
+    {
+        return vminq_u8(a, b);
+    }
+
+    inline uint8x8_t operator()(uint8x8_t a, uint8x8_t b) const
+    {
+        return vmin_u8(a, b);
+    }
+
+    inline u8 operator()(u8 a, u8 b) const
+    {
+        return std::min(a, b);
+    }
+
+    u8 borderValue;
+};
+
+struct DilateVecOp
+{
+    DilateVecOp():borderValue(0){}
+
+    DilateVecOp(BORDER_MODE border, u8 borderValue_) :
+        borderValue(borderValue_)
+    {
+        if (border == BORDER_MODE_REPLICATE)
+            borderValue = std::numeric_limits<u8>::min();
+    }
+
+    inline uint8x16_t operator()(uint8x16_t a, uint8x16_t b) const
+    {
+        return vmaxq_u8(a, b);
+    }
+
+    inline uint8x8_t operator()(uint8x8_t a, uint8x8_t b) const
+    {
+        return vmax_u8(a, b);
+    }
+
+    inline u8 operator()(u8 a, u8 b) const
+    {
+        return std::max(a, b);
+    }
+
+    u8 borderValue;
+};
+
+template <typename VecOp>
+void morph3x3(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u8 * dstBase, ptrdiff_t dstStride,
+              BORDER_MODE border, const VecOp & vop)
+{
+    u8 borderValue = vop.borderValue;
+    ptrdiff_t width = (ptrdiff_t)size.width, height = (ptrdiff_t)size.height;
+
+    const uint8x16_t v_zero = vdupq_n_u8(0);
+    const uint8x16_t v_border = vdupq_n_u8(borderValue);
+
+    uint8x16_t tprev = v_zero, tcurr = v_zero, tnext = v_zero;
+    uint8x16_t t0 = v_zero, t1 = v_zero, t2 = v_zero;
+
+    for (ptrdiff_t y = 0; y < height; ++y)
+    {
+        const u8 * srow0 = y == 0 && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::max<ptrdiff_t>(y - 1, 0));
+        const u8 * srow1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8 * srow2 = y + 1 == height && border == BORDER_MODE_CONSTANT ? NULL : internal::getRowPtr(srcBase, srcStride, std::min(y + 1, height - 1));
+        u8 * drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        u8 prevx = 0, currx = 0, nextx = 0;
+        ptrdiff_t x = 0;
+        const ptrdiff_t bwidth = y + 2 < height ? width : (width - 16);
+
+        // perform vertical convolution
+        for ( ; x <= bwidth; x += 16)
+        {
+            internal::prefetch(srow0 + x);
+            internal::prefetch(srow1 + x);
+            internal::prefetch(srow2 + x);
+
+            uint8x16_t x0 = !srow0 ? v_border : vld1q_u8(srow0 + x);
+            uint8x16_t x1 = vld1q_u8(srow1 + x);
+            uint8x16_t x2 = !srow2 ? v_border : vld1q_u8(srow2 + x);
+
+            // calculate values for plain CPU part below if needed
+            if (x + 16 >= bwidth)
+            {
+                ptrdiff_t x3 = x == width ? width - 1 : x;
+                ptrdiff_t x4 = border == BORDER_MODE_CONSTANT ? x3 - 1 : std::max<ptrdiff_t>(x3 - 1, 0);
+
+                if (border == BORDER_MODE_CONSTANT && x4 < 0)
+                    prevx = borderValue;
+                else
+                    prevx = vop(srow1[x4],
+                                vop(srow2 ? srow2[x4] : borderValue,
+                                    srow0 ? srow0[x4] : borderValue));
+
+                currx = vop(srow2 ? srow2[x3] : borderValue, vop(srow1[x3], srow0 ? srow0[x3] : borderValue));
+            }
+
+            // make shift
+            if (x)
+            {
+                tprev = tcurr;
+                tcurr = tnext;
+            }
+
+            // and calculate next value
+            tnext = vop(vop(x0, x1), x2);
+
+            // make extrapolation for the first elements
+            if (!x)
+            {
+                // make border
+                if (border == BORDER_MODE_CONSTANT)
+                    tcurr = v_border;
+                else if (border == BORDER_MODE_REPLICATE)
+                    tcurr = vdupq_n_u8(vgetq_lane_u8(tnext, 0));
+
+                continue;
+            }
+
+            // combine 3 "shifted" vectors
+            t0 = vextq_u8(tprev, tcurr, 15);
+            t1 = tcurr;
+            t2 = vextq_u8(tcurr, tnext, 1);
+
+            // and add them
+            t0 = vop(t0, vop(t1, t2));
+
+            vst1q_u8(drow + x - 16, t0);
+        }
+
+        x -= 16;
+        if (x == width)
+            --x;
+
+        for ( ; x < width; ++x)
+        {
+            // make extrapolation for the last elements
+            if (x + 1 >= width)
+            {
+                if (border == BORDER_MODE_CONSTANT)
+                    nextx = borderValue;
+                else if (border == BORDER_MODE_REPLICATE)
+                    nextx = vop(srow2[x], vop(srow1[x], srow0[x]));
+            }
+            else
+                nextx = vop(vop(srow2 ? srow2[x + 1] : borderValue,
+                                srow0 ? srow0[x + 1] : borderValue),
+                            srow1[x + 1]);
+
+            drow[x] = vop(prevx, vop(currx, nextx));
+
+            // make shift
+            prevx = currx;
+            currx = nextx;
+        }
+    }
+}
+
+} // namespace
+
+#endif
+
+void erode3x3(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              u8 * dstBase, ptrdiff_t dstStride,
+              BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isMorph3x3Supported(size, border));
+#ifdef CAROTENE_NEON
+    morph3x3(size,
+             srcBase, srcStride,
+             dstBase, dstStride,
+             border, ErodeVecOp(border, borderValue));
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+void dilate3x3(const Size2D &size,
+               const u8 * srcBase, ptrdiff_t srcStride,
+               u8 * dstBase, ptrdiff_t dstStride,
+               BORDER_MODE border, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isMorph3x3Supported(size, border));
+#ifdef CAROTENE_NEON
+    morph3x3(size,
+             srcBase, srcStride,
+             dstBase, dstStride,
+             border, DilateVecOp(border, borderValue));
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)border;
+    (void)borderValue;
+#endif
+}
+
+#ifdef CAROTENE_NEON
+namespace {
+
+template<class VecUpdate>
+void MorphRow(const u8* src, u8* dst, size_t width, s32 cn, size_t ksize)
+{
+    size_t i, j, k;
+    size_t width16 = (width & -16) * cn;
+    size_t width8 = (width & -8) * cn;
+    width *= cn;
+
+    if (ksize == 1)
+    {
+        for (i = 0; i < width; i++)
+            dst[i] = src[i];
+        return;
+    }
+
+    ksize = ksize*cn;
+    VecUpdate updateOp;
+    switch(cn)
+    {
+    case 1:
+        for (i = 0; i < width16; i += 16)
+        {
+            const u8* sptr = src + i;
+            uint8x16_t s = vld1q_u8(sptr);
+            internal::prefetch(sptr);
+
+            for( k = 1; k < ksize; ++k)
+                s = updateOp(s, vld1q_u8(sptr + k));
+
+            vst1q_u8(dst + i, s);
+        }
+
+        for (; i < width8; i += 8)
+        {
+            const u8* sptr = src + i;
+            uint8x8_t s = vld1_u8(sptr);
+            internal::prefetch(sptr);
+
+            for( k = 1; k < ksize; ++k)
+                s = updateOp(s, vld1_u8(sptr + k));
+
+            vst1_u8(dst + i, s);
+        }
+        break;
+    default:
+        for (i = 0; i < width16; i += 16)
+        {
+            uint8x16_t s = vld1q_u8(src + i);
+            internal::prefetch(src + i);
+
+            for (k = cn; k < ksize; k += cn)
+                s = updateOp(s, vld1q_u8(src + i + k));
+
+            vst1q_u8(dst + i, s);
+        }
+
+        for (; i < width8; i += 8)
+        {
+            uint8x8_t s = vld1_u8(src + i);
+            internal::prefetch(src + i);
+
+            for (k = cn; k < ksize; k += cn)
+                s = updateOp(s, vld1_u8(src + i + k));
+
+            vst1_u8(dst + i, s);
+        }
+        break;
+    }
+
+    ptrdiff_t i0 = i;
+    for( k = 0; k < (size_t)cn; k++, src++, dst++ )
+    {
+        for( i = i0; i <= width - cn*2; i += cn*2 )
+        {
+            const u8* s = src + i;
+            u8 m = s[cn];
+            for( j = cn*2; j < ksize; j += cn )
+                m = updateOp(m, s[j]);
+            dst[i] = updateOp(m, s[0]);
+            dst[i+cn] = updateOp(m, s[j]);
+        }
+
+        for( ; i < width; i += cn )
+        {
+            const u8* s = src + i;
+            u8 m = s[0];
+            for( j = cn; j < ksize; j += cn )
+                m = updateOp(m, s[j]);
+            dst[i] = m;
+        }
+    }
+}
+
+template<class VecUpdate>
+void MorphColumn(const u8** src, u8* dst, ptrdiff_t dststep, size_t count, size_t width, size_t ksize)
+{
+    size_t i, k;
+    size_t width32 = width & -32;
+    VecUpdate updateOp;
+
+    uint8x16_t x0,x1,s0,s1;
+    if (ksize == 3)
+    {
+        for (; count > 1; count -= 2, dst += dststep * 2, src += 2)
+        {
+            for (i = 0; i < width32; i += 32)
+            {
+                const u8* sptr = src[1] + i;
+                s0 = vld1q_u8(sptr);
+                s1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+
+                sptr = src[2] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+
+                s0 = updateOp(s0, x0);
+                s1 = updateOp(s1, x1);
+
+                sptr = src[0] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+
+                vst1q_u8(dst+i, updateOp(s0, x0));
+                vst1q_u8(dst+i+16, updateOp(s1, x1));
+
+                sptr = src[3] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+                vst1q_u8(dst + dststep + i, updateOp(s0, x0));
+                vst1q_u8(dst + dststep + i + 16, updateOp(s1, x1));
+
+            }
+            for(; i < width; i++ )
+            {
+                u8 s = src[1][i];
+
+                for( k = 2; k < ksize; k++ )
+                    s = updateOp(s, src[k][i]);
+
+                dst[i] = updateOp(s, src[0][i]);
+                dst[i+dststep] = updateOp(s, src[k][i]);
+            }
+        }
+    }
+    else if (ksize > 1)
+        for (; count > 1; count -= 2, dst += dststep*2, src += 2)
+        {
+            for (i = 0; i < width32; i += 32)
+            {
+                const u8* sptr = src[1] + i;
+                s0 = vld1q_u8(sptr);
+                s1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+                for (k = 2; k < ksize; k++)
+                {
+                    sptr = src[k] + i;
+                    x0 = vld1q_u8(sptr);
+                    x1 = vld1q_u8(sptr + 16);
+                    internal::prefetch(sptr);
+
+                    s0 = updateOp(s0, x0);
+                    s1 = updateOp(s1, x1);
+                }
+
+                sptr = src[0] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+
+                vst1q_u8(dst+i, updateOp(s0, x0));
+                vst1q_u8(dst+i+16, updateOp(s1, x1));
+
+                sptr = src[k] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+                vst1q_u8(dst + dststep + i, updateOp(s0, x0));
+                vst1q_u8(dst + dststep + i + 16, updateOp(s1, x1));
+            }
+            for(; i < width; i++ )
+            {
+                u8 s = src[1][i];
+
+                for( k = 2; k < ksize; k++ )
+                    s = updateOp(s, src[k][i]);
+
+                dst[i] = updateOp(s, src[0][i]);
+                dst[i+dststep] = updateOp(s, src[k][i]);
+            }
+        }
+
+    for (; count > 0; count--, dst += dststep, src++)
+    {
+        for (i = 0; i < width32; i += 32)
+        {
+            const u8* sptr = src[0] + i;
+            s0 = vld1q_u8(sptr);
+            s1 = vld1q_u8(sptr + 16);
+            internal::prefetch(sptr);
+
+            for (k = 1; k < ksize; k++)
+            {
+                sptr = src[k] + i;
+                x0 = vld1q_u8(sptr);
+                x1 = vld1q_u8(sptr + 16);
+                internal::prefetch(sptr);
+                s0 = updateOp(s0, x0);
+                s1 = updateOp(s1, x1);
+            }
+
+            vst1q_u8(dst + i, s0);
+            vst1q_u8(dst + i + 16, s1);
+        }
+        for(; i < width; i++ )
+        {
+            u8 s = src[0][i];
+            for( k = 1; k < ksize; k++ )
+                s = updateOp(s, src[k][i]);
+            dst[i] = s;
+        }
+    }
+}
+
+template <class Op>
+inline void morphology(const Size2D &ssize, u32 cn,
+                       const u8 * srcBase, ptrdiff_t srcStride,
+                       u8 * dstBase, ptrdiff_t dstStride,
+                       const Size2D &ksize,
+                       size_t anchorX, size_t anchorY,
+                       BORDER_MODE rowBorderType, BORDER_MODE columnBorderType,
+                       const u8 * borderValues, Margin borderMargin)
+{
+    //Temporary buffers common for all iterations
+    std::vector<u8> _srcRow(cn*(ssize.width + ksize.width - 1));
+    u8* srcRow = &_srcRow[0];
+
+    size_t bufRows = std::max<size_t>(ksize.height + 3, std::max<size_t>(anchorY, ksize.height-anchorY-1)*2+1);
+    std::vector<u8*> _rows(bufRows);
+    u8** rows = &_rows[0];
+
+    // adjust swidthcn so that the used part of buffers stays compact in memory
+    ptrdiff_t swidthcn = cn*((ssize.width + 15) & -16);// cn * (aligned ssize.width size)
+    std::vector<u8> _ringBuf(swidthcn*bufRows+16);
+    u8 * ringBuf = internal::alignPtr(&_ringBuf[0], 16);
+
+    size_t borderLength = std::max<size_t>(ksize.width - 1, 1) * cn;
+    std::vector<ptrdiff_t> _borderTab(borderLength);
+    ptrdiff_t * borderTab = &_borderTab[0];
+
+    std::vector<u8> _constBorderValue;
+    std::vector<u8> _constBorderRow;
+    u8 * constBorderValue = NULL;
+    u8 * constBorderRow = NULL;
+    if( rowBorderType == BORDER_MODE_CONSTANT || columnBorderType == BORDER_MODE_CONSTANT )
+    {
+        _constBorderValue.resize(borderLength);
+        constBorderValue = &_constBorderValue[0];
+        size_t i;
+        for(i = 0; i < cn; i++)
+            constBorderValue[i] = borderValues[i];
+        for(; i < borderLength; i++)
+            constBorderValue[i] = constBorderValue[i-cn];
+
+        if( columnBorderType == BORDER_MODE_CONSTANT )
+        {
+            _constBorderRow.resize(cn*(ssize.width + ksize.width - 1 + 16));
+            constBorderRow = internal::alignPtr(&_constBorderRow[0], 16);
+            size_t N = (ssize.width + ksize.width - 1)*cn;
+            for( i = 0; i < N; i += borderLength )
+            {
+                size_t n = std::min( borderLength, N - i );
+                for(size_t j = 0; j < n; j++)
+                    srcRow[i+j] = constBorderValue[j];
+            }
+            MorphRow<Op>(srcRow, constBorderRow, ssize.width, cn, ksize.width);
+        }
+    }
+
+    Size2D wholeSize(ssize.width + borderMargin.left + borderMargin.right,
+                     ssize.height + borderMargin.top + borderMargin.bottom);
+
+    ptrdiff_t dx1 = std::max<ptrdiff_t>(anchorX - (ptrdiff_t)borderMargin.left, 0);
+    ptrdiff_t dx2 = std::max<ptrdiff_t>((ptrdiff_t)ksize.width - anchorX - 1 - (ptrdiff_t)borderMargin.right, 0);
+    // recompute border tables
+    if( dx1 > 0 || dx2 > 0 )
+    {
+        if( rowBorderType == BORDER_MODE_CONSTANT )
+        {
+            memcpy( srcRow, &constBorderValue[0], dx1*cn );
+            memcpy( srcRow + (ssize.width + ksize.width - 1 - dx2)*cn, &constBorderValue[0], dx2*cn );
+        }
+        else
+        {
+            ptrdiff_t xofs1 = std::min<ptrdiff_t>(borderMargin.left, anchorX) - borderMargin.left;
+
+            ptrdiff_t wholeWidth = wholeSize.width;
+
+            ptrdiff_t i, j;
+            for( i = 0; i < dx1; i++ )
+            {
+                ptrdiff_t p0 = (internal::borderInterpolate(i-dx1, wholeWidth, rowBorderType) + xofs1)*cn;
+                for( j = 0; j < (ptrdiff_t)cn; j++ )
+                    borderTab[i*cn + j] = p0 + j;
+            }
+
+            for( i = 0; i < dx2; i++ )
+            {
+                ptrdiff_t p0 = (internal::borderInterpolate(wholeWidth + i, wholeWidth, rowBorderType) + xofs1)*cn;
+                for( j = 0; j < (ptrdiff_t)cn; j++ )
+                    borderTab[(i + dx1)*cn + j] = p0 + j;
+            }
+        }
+    }
+
+    ptrdiff_t startY, startY0, endY, rowCount;
+    startY = startY0 = std::max<ptrdiff_t>(borderMargin.top - anchorY, 0);
+    endY = std::min<ptrdiff_t>(borderMargin.top + ssize.height + ksize.height - anchorY - 1, wholeSize.height);
+
+    const u8* src = srcBase + (startY - borderMargin.top)*srcStride;
+    u8* dst = dstBase;
+
+    ptrdiff_t width = ssize.width, kwidth = ksize.width;
+    ptrdiff_t kheight = ksize.height, ay = anchorY;
+    ptrdiff_t width1 = ssize.width + kwidth - 1;
+    ptrdiff_t xofs1 = std::min<ptrdiff_t>(borderMargin.left, anchorX);
+    bool makeBorder = (dx1 > 0 || dx2 > 0) && rowBorderType != BORDER_MODE_CONSTANT;
+    ptrdiff_t dy = 0, i = 0;
+
+    src -= xofs1*cn;
+    ptrdiff_t count = endY - startY;
+
+    rowCount = 0;
+    for(;; dst += dstStride*i, dy += i)
+    {
+        ptrdiff_t dcount = bufRows - ay - startY - rowCount + borderMargin.top;
+        dcount = dcount > 0 ? dcount : bufRows - kheight + 1;
+        dcount = std::min(dcount, count);
+        count -= dcount;
+        for( ; dcount-- > 0; src += srcStride )
+        {
+            ptrdiff_t bi = (startY - startY0 + rowCount) % bufRows;
+            u8* brow = ringBuf + bi*swidthcn;
+
+            if( (size_t)(++rowCount) > bufRows )
+            {
+                --rowCount;
+                ++startY;
+            }
+
+            memcpy( srcRow + dx1*cn, src, (width1 - dx2 - dx1)*cn );
+
+            if( makeBorder )
+            {
+                    for( i = 0; i < (ptrdiff_t)(dx1*cn); i++ )
+                        srcRow[i] = src[borderTab[i]];
+                    for( i = 0; i < (ptrdiff_t)(dx2*cn); i++ )
+                        srcRow[i + (width1 - dx2)*cn] = src[borderTab[i+dx1*cn]];
+            }
+
+            MorphRow<Op>(srcRow, brow, width, cn, ksize.width);
+        }
+
+        ptrdiff_t max_i = std::min<ptrdiff_t>(bufRows, ssize.height - dy + (kheight - 1));
+        for( i = 0; i < max_i; i++ )
+        {
+            ptrdiff_t srcY = internal::borderInterpolate(dy + i + borderMargin.top - ay,
+                                               wholeSize.height, columnBorderType);
+            if( srcY < 0 ) // can happen only with constant border type
+                rows[i] = constBorderRow;
+            else
+            {
+                if( srcY >= startY + rowCount )
+                    break;
+                ptrdiff_t bi = (srcY - startY0) % bufRows;
+                rows[i] = ringBuf + bi*swidthcn;
+            }
+        }
+        if( i < kheight )
+            break;
+        i -= kheight - 1;
+        MorphColumn<Op>((const u8**)rows, dst, dstStride, i, ssize.width*cn, ksize.height);
+    }
+}
+
+} // namespace
+#endif // CAROTENE_NEON
+
+void erode(const Size2D &ssize, u32 cn,
+           const u8 * srcBase, ptrdiff_t srcStride,
+           u8 * dstBase, ptrdiff_t dstStride,
+           const Size2D &ksize,
+           size_t anchorX, size_t anchorY,
+           BORDER_MODE rowBorderType, BORDER_MODE columnBorderType,
+           const u8 * borderValues, Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(ssize.width > 0 && ssize.height > 0 &&
+                                           anchorX < ksize.width && anchorY < ksize.height);
+#ifdef CAROTENE_NEON
+    morphology<ErodeVecOp>(ssize, cn, srcBase, srcStride, dstBase, dstStride,
+                           ksize, anchorX, anchorY, rowBorderType, columnBorderType,
+                           borderValues, borderMargin);
+#else
+    (void)cn;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)rowBorderType;
+    (void)columnBorderType;
+    (void)borderValues;
+    (void)borderMargin;
+#endif
+}
+
+void dilate(const Size2D &ssize, u32 cn,
+            const u8 * srcBase, ptrdiff_t srcStride,
+            u8 * dstBase, ptrdiff_t dstStride,
+            const Size2D &ksize,
+            size_t anchorX, size_t anchorY,
+            BORDER_MODE rowBorderType, BORDER_MODE columnBorderType,
+            const u8 * borderValues, Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(ssize.width > 0 && ssize.height > 0 &&
+                                           anchorX < ksize.width && anchorY < ksize.height);
+#ifdef CAROTENE_NEON
+    morphology<DilateVecOp>(ssize, cn, srcBase, srcStride, dstBase, dstStride,
+                            ksize, anchorX, anchorY, rowBorderType, columnBorderType,
+                            borderValues, borderMargin);
+#else
+    (void)cn;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)rowBorderType;
+    (void)columnBorderType;
+    (void)borderValues;
+    (void)borderMargin;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/opticalflow.cpp

@@ -0,0 +1,539 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "saturate_cast.hpp"
+#include <vector>
+#include <float.h> // For FLT_EPSILON
+
+namespace CAROTENE_NS {
+
+#define CV_DESCALE(x,n)     (((x) + (1 << ((n)-1))) >> (n))
+
+/*
+ *        Pyramidal Lucas-Kanade Optical Flow level processing
+ */
+void pyrLKOptFlowLevel(const Size2D &size, s32 cn,
+                       const u8 *prevData, ptrdiff_t prevStride,
+                       const s16 *prevDerivData, ptrdiff_t prevDerivStride,
+                       const u8 *nextData, ptrdiff_t nextStride,
+                       u32 ptCount,
+                       const f32 *prevPts, f32 *nextPts,
+                       u8 *status, f32 *err,
+                       const Size2D &winSize,
+                       u32 terminationCount, f64 terminationEpsilon,
+                       u32 level, u32 maxLevel, bool useInitialFlow, bool getMinEigenVals,
+                       f32 minEigThreshold)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    f32 halfWinX = (winSize.width-1)*0.5f, halfWinY = (winSize.height-1)*0.5f;
+    s32 cn2 = cn*2;
+
+    std::vector<s16> _buf(winSize.total()*(cn + cn2));
+    s16* IWinBuf = &_buf[0];
+    s32  IWinBufStride = winSize.width*cn;
+    s16* derivIWinBuf = &_buf[winSize.total()*cn];
+    s32  derivIWinBufStride = winSize.width*cn2;
+
+    for( u32 ptidx = 0; ptidx < ptCount; ptidx++ )
+    {
+        f32 levscale = (1./(1 << level));
+        u32 ptref = ptidx << 1;
+        f32 prevPtX = prevPts[ptref+0]*levscale;
+        f32 prevPtY = prevPts[ptref+1]*levscale;
+        f32 nextPtX;
+        f32 nextPtY;
+        if( level == maxLevel )
+        {
+            if( useInitialFlow )
+            {
+                nextPtX = nextPts[ptref+0]*levscale;
+                nextPtY = nextPts[ptref+1]*levscale;
+            }
+            else
+            {
+                nextPtX = prevPtX;
+                nextPtY = prevPtY;
+            }
+        }
+        else
+        {
+            nextPtX = nextPts[ptref+0]*2.f;
+            nextPtY = nextPts[ptref+1]*2.f;
+        }
+        nextPts[ptref+0] = nextPtX;
+        nextPts[ptref+1] = nextPtY;
+
+        s32 iprevPtX, iprevPtY;
+        s32 inextPtX, inextPtY;
+        prevPtX -= halfWinX;
+        prevPtY -= halfWinY;
+        iprevPtX = floor(prevPtX);
+        iprevPtY = floor(prevPtY);
+
+        if( iprevPtX < -(s32)winSize.width || iprevPtX >= (s32)size.width ||
+            iprevPtY < -(s32)winSize.height || iprevPtY >= (s32)size.height )
+        {
+            if( level == 0 )
+            {
+                if( status )
+                    status[ptidx] = false;
+                if( err )
+                    err[ptidx] = 0;
+            }
+            continue;
+        }
+
+        f32 a = prevPtX - iprevPtX;
+        f32 b = prevPtY - iprevPtY;
+        const s32 W_BITS = 14, W_BITS1 = 14;
+        const f32 FLT_SCALE = 1.f/(1 << 20);
+        s32 iw00 = round((1.f - a)*(1.f - b)*(1 << W_BITS));
+        s32 iw01 = round(a*(1.f - b)*(1 << W_BITS));
+        s32 iw10 = round((1.f - a)*b*(1 << W_BITS));
+        s32 iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
+
+        s32 dstep = prevDerivStride/sizeof(s16);
+        f32 A11 = 0, A12 = 0, A22 = 0;
+
+        int16x4_t viw00 = vmov_n_s16((s16)iw00);
+        int16x4_t viw01 = vmov_n_s16((s16)iw01);
+        int16x4_t viw10 = vmov_n_s16((s16)iw10);
+        int16x4_t viw11 = vmov_n_s16((s16)iw11);
+
+        float32x4_t vA11 = vmovq_n_f32(0);
+        float32x4_t vA12 = vmovq_n_f32(0);
+        float32x4_t vA22 = vmovq_n_f32(0);
+
+        s32 wwcn = winSize.width*cn;
+
+        // extract the patch from the first image, compute covariation matrix of derivatives
+        s32 x = 0;
+        for(s32 y = 0; y < (s32)winSize.height; y++ )
+        {
+            const u8* src = prevData + prevStride*(y + iprevPtY) + iprevPtX*cn;
+            const s16* dsrc = prevDerivData + dstep*(y + iprevPtY) + iprevPtX*cn2;
+
+            s16* Iptr = IWinBuf + y*IWinBufStride;
+            s16* dIptr = derivIWinBuf + y*derivIWinBufStride;
+
+            internal::prefetch(src + x + prevStride * 2, 0);
+            for(x = 0; x <= wwcn - 8; x += 8)
+            {
+                uint8x8_t vsrc00 = vld1_u8(src + x);
+                uint8x8_t vsrc10 = vld1_u8(src + x + prevStride);
+                uint8x8_t vsrc01 = vld1_u8(src + x + cn);
+                uint8x8_t vsrc11 = vld1_u8(src + x + prevStride + cn);
+
+                int16x8_t vs00 = vreinterpretq_s16_u16(vmovl_u8(vsrc00));
+                int16x8_t vs10 = vreinterpretq_s16_u16(vmovl_u8(vsrc10));
+                int16x8_t vs01 = vreinterpretq_s16_u16(vmovl_u8(vsrc01));
+                int16x8_t vs11 = vreinterpretq_s16_u16(vmovl_u8(vsrc11));
+
+                int32x4_t vsuml = vmull_s16(vget_low_s16(vs00), viw00);
+                int32x4_t vsumh = vmull_s16(vget_high_s16(vs10), viw10);
+
+                vsuml = vmlal_s16(vsuml, vget_low_s16(vs01), viw01);
+                vsumh = vmlal_s16(vsumh, vget_high_s16(vs11), viw11);
+
+                vsuml = vmlal_s16(vsuml, vget_low_s16(vs10), viw10);
+                vsumh = vmlal_s16(vsumh, vget_high_s16(vs00), viw00);
+
+                vsuml = vmlal_s16(vsuml, vget_low_s16(vs11), viw11);
+                vsumh = vmlal_s16(vsumh, vget_high_s16(vs01), viw01);
+
+                int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5);
+                int16x4_t vsumnh = vrshrn_n_s32(vsumh, W_BITS1-5);
+
+                vst1q_s16(Iptr + x, vcombine_s16(vsumnl, vsumnh));
+            }
+            for(; x <= wwcn - 4; x += 4)
+            {
+                uint8x8_t vsrc00 = vld1_u8(src + x);
+                uint8x8_t vsrc10 = vld1_u8(src + x + prevStride);
+                uint8x8_t vsrc01 = vld1_u8(src + x + cn);
+                uint8x8_t vsrc11 = vld1_u8(src + x + prevStride + cn);
+
+                int16x4_t vs00 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc00)));
+                int16x4_t vs10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc10)));
+                int16x4_t vs01 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc01)));
+                int16x4_t vs11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(vsrc11)));
+
+                int32x4_t vsuml1 = vmull_s16(vs00, viw00);
+                int32x4_t vsuml2 = vmull_s16(vs01, viw01);
+                vsuml1 = vmlal_s16(vsuml1, vs10, viw10);
+                vsuml2 = vmlal_s16(vsuml2, vs11, viw11);
+                int32x4_t vsuml = vaddq_s32(vsuml1, vsuml2);
+
+                int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5);
+
+                vst1_s16(Iptr + x, vsumnl);
+            }
+
+            internal::prefetch(dsrc + dstep * 2, 0);
+            for(x = 0; x <= wwcn - 4; x += 4, dsrc += 4*2, dIptr += 4*2 )
+            {
+#if 0
+                __asm__ (
+                    "vld2.16 {d0-d1}, [%[dsrc00]]                         \n\t"
+                    "vld2.16 {d2-d3}, [%[dsrc10]]                         \n\t"
+                    "vld2.16 {d4-d5}, [%[dsrc01]]                         \n\t"
+                    "vld2.16 {d6-d7}, [%[dsrc11]]                         \n\t"
+                    "vmull.s16 q4, d3, %P[viw10]                           \n\t"
+                    "vmull.s16 q5, d0, %P[viw00]                           \n\t"
+                    "vmlal.s16 q4, d7, %P[viw11]                           \n\t"
+                    "vmlal.s16 q5, d4, %P[viw01]                           \n\t"
+                    "vmlal.s16 q4, d1, %P[viw00]                           \n\t"
+                    "vmlal.s16 q5, d2, %P[viw10]                           \n\t"
+                    "vmlal.s16 q4, d5, %P[viw01]                           \n\t"
+                    "vmlal.s16 q5, d6, %P[viw11]                            \n\t"
+                    "vrshrn.s32 d13, q4, %[W_BITS1]                       \n\t"
+                    "vrshrn.s32 d12, q5, %[W_BITS1]                       \n\t"
+                    "vmull.s16 q3, d13, d13                               \n\t"
+                    "vmull.s16 q4, d12, d12                               \n\t"
+                    "vmull.s16 q5, d13, d12                               \n\t"
+                    "vcvt.f32.s32 q3, q3                                  \n\t"
+                    "vcvt.f32.s32 q4, q4                                  \n\t"
+                    "vcvt.f32.s32 q5, q5                                  \n\t"
+                    "vadd.f32 %q[vA22], q3                                \n\t"
+                    "vadd.f32 %q[vA11], q4                                \n\t"
+                    "vadd.f32 %q[vA12], q5                                \n\t"
+                    "vst2.16 {d12-d13}, [%[out]]                          \n\t"
+                    : [vA22] "=w" (vA22),
+                      [vA11] "=w" (vA11),
+                      [vA12] "=w" (vA12)
+                    : "0" (vA22),
+                      "1" (vA11),
+                      "2" (vA12),
+                      [out] "r" (dIptr),
+                      [dsrc00] "r" (dsrc),
+                      [dsrc10] "r" (dsrc + dstep),
+                      [dsrc01] "r" (dsrc + cn2),
+                      [dsrc11] "r" (dsrc + dstep + cn2),
+                      [viw00] "w" (viw00),
+                      [viw10] "w" (viw10),
+                      [viw01] "w" (viw01),
+                      [viw11] "w" (viw11),
+                      [W_BITS1] "I" (W_BITS1)
+                    : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13"
+                );
+#else
+                int16x4x2_t vdsrc00 = vld2_s16(dsrc);
+                int16x4x2_t vdsrc10 = vld2_s16(dsrc + dstep);
+                int16x4x2_t vdsrc01 = vld2_s16(dsrc + cn2);
+                int16x4x2_t vdsrc11 = vld2_s16(dsrc + dstep + cn2);
+
+                int32x4_t vsumy = vmull_s16(vdsrc10.val[1], viw10);
+                int32x4_t vsumx = vmull_s16(vdsrc00.val[0], viw00);
+
+                vsumy = vmlal_s16(vsumy, vdsrc11.val[1], viw11);
+                vsumx = vmlal_s16(vsumx, vdsrc01.val[0], viw01);
+
+                vsumy = vmlal_s16(vsumy, vdsrc00.val[1], viw00);
+                vsumx = vmlal_s16(vsumx, vdsrc10.val[0], viw10);
+
+                vsumy = vmlal_s16(vsumy, vdsrc01.val[1], viw01);
+                vsumx = vmlal_s16(vsumx, vdsrc11.val[0], viw11);
+
+                int16x4_t vsumny = vrshrn_n_s32(vsumy, W_BITS1);
+                int16x4_t vsumnx = vrshrn_n_s32(vsumx, W_BITS1);
+
+                int32x4_t va22i = vmull_s16(vsumny, vsumny);
+                int32x4_t va11i = vmull_s16(vsumnx, vsumnx);
+                int32x4_t va12i = vmull_s16(vsumnx, vsumny);
+
+                float32x4_t va22f = vcvtq_f32_s32(va22i);
+                float32x4_t va11f = vcvtq_f32_s32(va11i);
+                float32x4_t va12f = vcvtq_f32_s32(va12i);
+
+                vA22 = vaddq_f32(vA22, va22f);
+                vA11 = vaddq_f32(vA11, va11f);
+                vA12 = vaddq_f32(vA12, va12f);
+
+                int16x4x2_t vsum;
+                vsum.val[0] = vsumnx;
+                vsum.val[1] = vsumny;
+                vst2_s16(dIptr, vsum);
+#endif
+            }
+
+            for( ; x < wwcn; x++, dsrc += 2, dIptr += 2 )
+            {
+                s32 ival = CV_DESCALE(src[x]*iw00 + src[x+cn]*iw01 +
+                                      src[x+prevStride]*iw10 + src[x+prevStride+cn]*iw11, W_BITS1-5);
+                s32 ixval = CV_DESCALE(dsrc[0]*iw00 + dsrc[cn2]*iw01 +
+                                       dsrc[dstep]*iw10 + dsrc[dstep+cn2]*iw11, W_BITS1);
+                s32 iyval = CV_DESCALE(dsrc[1]*iw00 + dsrc[cn2+1]*iw01 + dsrc[dstep+1]*iw10 +
+                                       dsrc[dstep+cn2+1]*iw11, W_BITS1);
+                Iptr[x] = (s16)ival;
+                dIptr[0] = (s16)ixval;
+                dIptr[1] = (s16)iyval;
+
+                A11 += (f32)(ixval*ixval);
+                A12 += (f32)(ixval*iyval);
+                A22 += (f32)(iyval*iyval);
+            }
+        }
+
+        f32 A11buf[2], A12buf[2], A22buf[2];
+        vst1_f32(A11buf, vadd_f32(vget_low_f32(vA11), vget_high_f32(vA11)));
+        vst1_f32(A12buf, vadd_f32(vget_low_f32(vA12), vget_high_f32(vA12)));
+        vst1_f32(A22buf, vadd_f32(vget_low_f32(vA22), vget_high_f32(vA22)));
+        A11 += A11buf[0] + A11buf[1];
+        A12 += A12buf[0] + A12buf[1];
+        A22 += A22buf[0] + A22buf[1];
+
+        A11 *= FLT_SCALE;
+        A12 *= FLT_SCALE;
+        A22 *= FLT_SCALE;
+
+        f32 D = A11*A22 - A12*A12;
+        f32 minEig = (A22 + A11 - std::sqrt((A11-A22)*(A11-A22) +
+                        4.f*A12*A12))/(2*winSize.width*winSize.height);
+
+        if( err && getMinEigenVals )
+            err[ptidx] = (f32)minEig;
+
+        if( minEig < minEigThreshold || D < FLT_EPSILON )
+        {
+            if( level == 0 && status )
+                status[ptidx] = false;
+            continue;
+        }
+
+        D = 1.f/D;
+
+        nextPtX -= halfWinX;
+        nextPtY -= halfWinY;
+        f32 prevDeltaX = 0;
+        f32 prevDeltaY = 0;
+
+        for(u32 j = 0; j < terminationCount; j++ )
+        {
+            inextPtX = floor(nextPtX);
+            inextPtY = floor(nextPtY);
+
+            if( inextPtX < -(s32)winSize.width || inextPtX >= (s32)size.width ||
+               inextPtY < -(s32)winSize.height || inextPtY >= (s32)size.height )
+            {
+                if( level == 0 && status )
+                    status[ptidx] = false;
+                break;
+            }
+
+            a = nextPtX - inextPtX;
+            b = nextPtY - inextPtY;
+            iw00 = round((1.f - a)*(1.f - b)*(1 << W_BITS));
+            iw01 = round(a*(1.f - b)*(1 << W_BITS));
+            iw10 = round((1.f - a)*b*(1 << W_BITS));
+            iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
+            f32 b1 = 0, b2 = 0;
+
+            viw00 = vmov_n_s16((s16)iw00);
+            viw01 = vmov_n_s16((s16)iw01);
+            viw10 = vmov_n_s16((s16)iw10);
+            viw11 = vmov_n_s16((s16)iw11);
+
+            float32x4_t vb1 = vmovq_n_f32(0);
+            float32x4_t vb2 = vmovq_n_f32(0);
+
+            for(s32 y = 0; y < (s32)winSize.height; y++ )
+            {
+                const u8* Jptr = nextData + nextStride*(y + inextPtY) + inextPtX*cn;
+                const s16* Iptr = IWinBuf + y*IWinBufStride;
+                const s16* dIptr = derivIWinBuf + y*derivIWinBufStride;
+
+                x = 0;
+
+                internal::prefetch(Jptr, nextStride * 2);
+                internal::prefetch(Iptr, IWinBufStride/2);
+                internal::prefetch(dIptr, derivIWinBufStride/2);
+
+                for( ; x <= wwcn - 8; x += 8, dIptr += 8*2 )
+                {
+                    uint8x8_t vj00 = vld1_u8(Jptr + x);
+                    uint8x8_t vj10 = vld1_u8(Jptr + x + nextStride);
+                    uint8x8_t vj01 = vld1_u8(Jptr + x + cn);
+                    uint8x8_t vj11 = vld1_u8(Jptr + x + nextStride + cn);
+                    int16x8_t vI = vld1q_s16(Iptr + x);
+                    int16x8x2_t vDerivI = vld2q_s16(dIptr);
+
+                    int16x8_t vs00 = vreinterpretq_s16_u16(vmovl_u8(vj00));
+                    int16x8_t vs10 = vreinterpretq_s16_u16(vmovl_u8(vj10));
+                    int16x8_t vs01 = vreinterpretq_s16_u16(vmovl_u8(vj01));
+                    int16x8_t vs11 = vreinterpretq_s16_u16(vmovl_u8(vj11));
+
+                    int32x4_t vsuml = vmull_s16(vget_low_s16(vs00), viw00);
+                    int32x4_t vsumh = vmull_s16(vget_high_s16(vs10), viw10);
+
+                    vsuml = vmlal_s16(vsuml, vget_low_s16(vs01), viw01);
+                    vsumh = vmlal_s16(vsumh, vget_high_s16(vs11), viw11);
+
+                    vsuml = vmlal_s16(vsuml, vget_low_s16(vs10), viw10);
+                    vsumh = vmlal_s16(vsumh, vget_high_s16(vs00), viw00);
+
+                    vsuml = vmlal_s16(vsuml, vget_low_s16(vs11), viw11);
+                    vsumh = vmlal_s16(vsumh, vget_high_s16(vs01), viw01);
+
+                    int16x4_t vsumnl = vrshrn_n_s32(vsuml, W_BITS1-5);
+                    int16x4_t vsumnh = vrshrn_n_s32(vsumh, W_BITS1-5);
+
+                    int16x8_t diff = vqsubq_s16(vcombine_s16(vsumnl, vsumnh), vI);
+
+                    int32x4_t vb1l = vmull_s16(vget_low_s16(diff), vget_low_s16(vDerivI.val[0]));
+                    int32x4_t vb2h = vmull_s16(vget_high_s16(diff), vget_high_s16(vDerivI.val[1]));
+                    int32x4_t vb1i = vmlal_s16(vb1l, vget_high_s16(diff), vget_high_s16(vDerivI.val[0]));
+                    int32x4_t vb2i = vmlal_s16(vb2h, vget_low_s16(diff), vget_low_s16(vDerivI.val[1]));
+
+                    float32x4_t vb1f = vcvtq_f32_s32(vb1i);
+                    float32x4_t vb2f = vcvtq_f32_s32(vb2i);
+
+                    vb1 = vaddq_f32(vb1, vb1f);
+                    vb2 = vaddq_f32(vb2, vb2f);
+                }
+
+                for( ; x < wwcn; x++, dIptr += 2 )
+                {
+                    s32 diff = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 +
+                                          Jptr[x+nextStride]*iw10 + Jptr[x+nextStride+cn]*iw11,
+                                          W_BITS1-5) - Iptr[x];
+                    b1 += (f32)(diff*dIptr[0]);
+                    b2 += (f32)(diff*dIptr[1]);
+                }
+            }
+
+            f32 bbuf[2];
+            float32x2_t vb = vpadd_f32(vadd_f32(vget_low_f32(vb1), vget_high_f32(vb1)), vadd_f32(vget_low_f32(vb2), vget_high_f32(vb2)));
+            vst1_f32(bbuf, vb);
+            b1 += bbuf[0];
+            b2 += bbuf[1];
+
+            b1 *= FLT_SCALE;
+            b2 *= FLT_SCALE;
+
+            f32 deltaX = (f32)((A12*b2 - A22*b1) * D);
+            f32 deltaY = (f32)((A12*b1 - A11*b2) * D);
+
+            nextPtX += deltaX;
+            nextPtY += deltaY;
+            nextPts[ptref+0] = nextPtX + halfWinX;
+            nextPts[ptref+1] = nextPtY + halfWinY;
+
+            if( ((double)deltaX*deltaX + (double)deltaY*deltaY) <= terminationEpsilon )
+                break;
+
+            if( j > 0 && std::abs(deltaX + prevDeltaX) < 0.01 &&
+               std::abs(deltaY + prevDeltaY) < 0.01 )
+            {
+                nextPts[ptref+0] -= deltaX*0.5f;
+                nextPts[ptref+1] -= deltaY*0.5f;
+                break;
+            }
+            prevDeltaX = deltaX;
+            prevDeltaY = deltaY;
+        }
+
+        if( status && status[ptidx] && err && level == 0 && !getMinEigenVals )
+        {
+            f32 nextPointX = nextPts[ptref+0] - halfWinX;
+            f32 nextPointY = nextPts[ptref+1] - halfWinY;
+
+            s32 inextPointX = floor(nextPointX);
+            s32 inextPointY = floor(nextPointY);
+
+            if( inextPointX < -(s32)winSize.width || inextPointX >= (s32)size.width ||
+                inextPointY < -(s32)winSize.height || inextPointY >= (s32)size.height )
+            {
+                if( status )
+                    status[ptidx] = false;
+                continue;
+            }
+
+            f32 aa = nextPointX - inextPointX;
+            f32 bb = nextPointY - inextPointY;
+            iw00 = round((1.f - aa)*(1.f - bb)*(1 << W_BITS));
+            iw01 = round(aa*(1.f - bb)*(1 << W_BITS));
+            iw10 = round((1.f - aa)*bb*(1 << W_BITS));
+            iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
+            f32 errval = 0.f;
+
+            for(s32 y = 0; y < (s32)winSize.height; y++ )
+            {
+                const u8* Jptr = nextData + nextStride*(y + inextPointY) + inextPointX*cn;
+                const s16* Iptr = IWinBuf + y*IWinBufStride;
+
+                for( x = 0; x < wwcn; x++ )
+                {
+                    s32 diff = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 +
+                                          Jptr[x+nextStride]*iw10 + Jptr[x+nextStride+cn]*iw11,
+                                          W_BITS1-5) - Iptr[x];
+                    errval += std::abs((f32)diff);
+                }
+            }
+            err[ptidx] = errval / (32*wwcn*winSize.height);
+        }
+    }
+#else
+    (void)size;
+    (void)cn;
+    (void)prevData;
+    (void)prevStride;
+    (void)prevDerivData;
+    (void)prevDerivStride;
+    (void)nextData;
+    (void)nextStride;
+    (void)prevPts;
+    (void)nextPts;
+    (void)status;
+    (void)err;
+    (void)winSize;
+    (void)terminationCount;
+    (void)terminationEpsilon;
+    (void)level;
+    (void)maxLevel;
+    (void)useInitialFlow;
+    (void)getMinEigenVals;
+    (void)minEigThreshold;
+    (void)ptCount;
+#endif
+}
+
+}//CAROTENE_NS
+

3rdparty/carotene/src/phase.cpp

@@ -0,0 +1,274 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <cfloat>
+#include <cmath>
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+#define FASTATAN2CONST(scale) \
+        f32 P1((f32)( 0.9997878412794807  * (180.0 / M_PI) * scale)), \
+        P3((f32)(-0.3258083974640975  * (180.0 / M_PI) * scale)), \
+        P5((f32)( 0.1555786518463281  * (180.0 / M_PI) * scale)), \
+        P7((f32)(-0.04432655554792128 * (180.0 / M_PI) * scale)), \
+         A_90((f32)(90.f * scale)), \
+        A_180((f32)(180.f * scale)), \
+        A_360((f32)(360.f * scale)); \
+        float32x4_t eps(vdupq_n_f32((float)DBL_EPSILON)), \
+         _90(vdupq_n_f32(A_90)), \
+        _180(vdupq_n_f32(A_180)), \
+        _360(vdupq_n_f32(A_360)), \
+           z(vdupq_n_f32(0.0f)), \
+        p1(vdupq_n_f32(P1)), \
+        p3(vdupq_n_f32(P3)), \
+        p5(vdupq_n_f32(P5)), \
+        p7(vdupq_n_f32(P7));
+
+#define FASTATAN2SCALAR(y, x, a) \
+    { \
+        f32 ax = std::abs(x), ay = std::abs(y); \
+        f32 c, c2; \
+        if (ax >= ay) \
+        { \
+            c = ay / (ax + (float)DBL_EPSILON); \
+            c2 = c * c; \
+            a = (((P7 * c2 + P5) * c2 + P3) * c2 + P1) * c; \
+        } \
+        else \
+        { \
+            c = ax / (ay + (float)DBL_EPSILON); \
+            c2 = c * c; \
+            a = A_90 - (((P7 * c2 + P5) * c2 + P3) * c2 + P1) * c; \
+        } \
+        if (x < 0) \
+            a = A_180 - a; \
+        if (y < 0) \
+            a = A_360 - a; \
+    }
+
+#define FASTATAN2VECTOR(v_y, v_x, a) \
+    { \
+        float32x4_t ax = vabsq_f32(v_x), ay = vabsq_f32(v_y); \
+        float32x4_t tmin = vminq_f32(ax, ay), tmax = vmaxq_f32(ax, ay); \
+        float32x4_t c = vmulq_f32(tmin, internal::vrecpq_f32(vaddq_f32(tmax, eps))); \
+        float32x4_t c2 = vmulq_f32(c, c); \
+        a = vmulq_f32(c2, p7); \
+ \
+        a = vmulq_f32(vaddq_f32(a, p5), c2); \
+        a = vmulq_f32(vaddq_f32(a, p3), c2); \
+        a = vmulq_f32(vaddq_f32(a, p1), c); \
+ \
+        a = vbslq_f32(vcgeq_f32(ax, ay), a, vsubq_f32(_90, a)); \
+        a = vbslq_f32(vcltq_f32(v_x, z), vsubq_f32(_180, a), a); \
+        a = vbslq_f32(vcltq_f32(v_y, z), vsubq_f32(_360, a), a); \
+ \
+    }
+
+} // namespace
+
+#endif
+
+void phase(const Size2D &size,
+           const s16 * src0Base, ptrdiff_t src0Stride,
+           const s16 * src1Base, ptrdiff_t src1Stride,
+           u8 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    FASTATAN2CONST(256.0f / 360.0f)
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    float32x4_t v_05 = vdupq_n_f32(0.5f);
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s16 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        u8 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw16; j += 16)
+        {
+            internal::prefetch(src0 + j);
+            internal::prefetch(src1 + j);
+
+            int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8);
+            int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8);
+
+            // 0
+            float32x4_t v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00)));
+            float32x4_t v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10)));
+            float32x4_t v_dst32f0;
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0)
+
+            v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00)));
+            v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10)));
+            float32x4_t v_dst32f1;
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1)
+
+            uint16x8_t v_dst16s0 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))),
+                                                vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05))));
+
+            // 1
+            v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src01)));
+            v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src11)));
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0)
+
+            v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src01)));
+            v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src11)));
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1)
+
+            uint16x8_t v_dst16s1 = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))),
+                                                vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05))));
+
+            vst1q_u8(dst + j, vcombine_u8(vmovn_u16(v_dst16s0),
+                                          vmovn_u16(v_dst16s1)));
+        }
+        for (; j < roiw8; j += 8)
+        {
+            int16x8_t v_src0 = vld1q_s16(src0 + j);
+            int16x8_t v_src1 = vld1q_s16(src1 + j);
+
+            float32x4_t v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src0)));
+            float32x4_t v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src1)));
+            float32x4_t v_dst32f0;
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f0)
+
+            v_src0_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src0)));
+            v_src1_p = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src1)));
+            float32x4_t v_dst32f1;
+            FASTATAN2VECTOR(v_src1_p, v_src0_p, v_dst32f1)
+
+            uint16x8_t v_dst = vcombine_u16(vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f0, v_05))),
+                                            vmovn_u32(vcvtq_u32_f32(vaddq_f32(v_dst32f1, v_05))));
+
+            vst1_u8(dst + j, vmovn_u16(v_dst));
+        }
+
+        for (; j < size.width; j++)
+        {
+            f32 x = src0[j], y = src1[j];
+            f32 a;
+            FASTATAN2SCALAR(y, x, a)
+            dst[j] = (u8)(s32)floor(a + 0.5f);
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void phase(const Size2D &size,
+           const f32 * src0Base, ptrdiff_t src0Stride,
+           const f32 * src1Base, ptrdiff_t src1Stride,
+           f32 * dstBase, ptrdiff_t dstStride,
+           f32 scale)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    FASTATAN2CONST(scale)
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw8; j += 8)
+        {
+            internal::prefetch(src0 + j);
+            internal::prefetch(src1 + j);
+
+            float32x4_t v_src00 = vld1q_f32(src0 + j), v_src01 = vld1q_f32(src0 + j + 4);
+            float32x4_t v_src10 = vld1q_f32(src1 + j), v_src11 = vld1q_f32(src1 + j + 4);
+
+            float32x4_t v_dst32f;
+            // 0
+            FASTATAN2VECTOR(v_src10, v_src00, v_dst32f)
+            vst1q_f32(dst + j,     v_dst32f);
+            // 1
+            FASTATAN2VECTOR(v_src11, v_src01, v_dst32f)
+            vst1q_f32(dst + j + 4, v_dst32f);
+        }
+        if(j + 4 <= size.width)
+        {
+            float32x4_t v_src0 = vld1q_f32(src0 + j);
+            float32x4_t v_src1 = vld1q_f32(src1 + j);
+
+            float32x4_t v_dst32f;
+            FASTATAN2VECTOR(v_src1, v_src0, v_dst32f)
+            vst1q_f32(dst + j, v_dst32f);
+            j += 4;
+        }
+
+        for (; j < size.width; j++)
+        {
+            f32 a;
+            FASTATAN2SCALAR(src1[j], src0[j], a)
+            dst[j] = a;
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)scale;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/reduce.cpp

@@ -0,0 +1,460 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+void reduceColSum(const Size2D &size,
+                  const u8 * srcBase, ptrdiff_t srcStride,
+                  s32 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memset(dstBase, 0, size.width*sizeof(s32));
+    size_t i = 0;
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const u8* src_address = srcBase + i;
+
+        int32x4_t sll = vmovq_n_s32(0);
+        int32x4_t slh = vmovq_n_s32(0);
+        int32x4_t shl = vmovq_n_s32(0);
+        int32x4_t shh = vmovq_n_s32(0);
+
+        for (size_t h = 0; h < size.height; h += 256)
+        {
+            size_t lim = std::min(h + 256, size.height);
+
+            uint16x8_t sl = vmovq_n_u16(0);
+            uint16x8_t sh = vmovq_n_u16(0);
+
+            for (size_t k = h; k < lim; ++k, src_address += srcStride)
+            {
+                internal::prefetch(src_address + srcStride, 0);
+
+                uint8x16_t v = vld1q_u8(src_address);
+
+                sl = vaddw_u8(sl, vget_low_u8(v));
+                sh = vaddw_u8(sh, vget_high_u8(v));
+            }
+
+            int32x4_t vsll = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(sl)));
+            int32x4_t vslh = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(sl)));
+            int32x4_t vshl = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(sh)));
+            int32x4_t vshh = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(sh)));
+
+            sll = vqaddq_s32(sll, vsll);
+            slh = vqaddq_s32(slh, vslh);
+            shl = vqaddq_s32(shl, vshl);
+            shh = vqaddq_s32(shh, vshh);
+        }
+
+        vst1q_s32(dstBase + i + 0, sll);
+        vst1q_s32(dstBase + i + 4, slh);
+        vst1q_s32(dstBase + i + 8, shl);
+        vst1q_s32(dstBase + i + 12, shh);
+    }
+
+    for(size_t h = 0; h < size.height; ++h)
+    {
+        for(size_t j = i ; j < size.width; j++ )
+        {
+            if (((u32)(dstBase[j] += srcBase[j + srcStride * h])) > 0x7fFFffFFu)
+                dstBase[j] = 0x7fFFffFF;
+        }
+    }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+void reduceColMax(const Size2D &size,
+                  const u8 * srcBase, ptrdiff_t srcStride,
+                  u8 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memcpy(dstBase, srcBase, size.width);
+    size_t i = 0;
+    for (; i + 16*4 <= size.width; i += 16*4)
+    {
+        const u8* src_address = srcBase + i;
+
+        uint8x16_t s1 = vld1q_u8(src_address + 0);
+        uint8x16_t s2 = vld1q_u8(src_address + 16);
+        uint8x16_t s3 = vld1q_u8(src_address + 32);
+        uint8x16_t s4 = vld1q_u8(src_address + 48);
+
+        src_address += srcStride;
+
+        for(size_t h = 1; h < size.height; ++h, src_address += srcStride)
+        {
+            internal::prefetch(src_address + srcStride, 0);
+            internal::prefetch(src_address + srcStride, 32);
+
+            uint8x16_t v1 = vld1q_u8(src_address + 0);
+            uint8x16_t v2 = vld1q_u8(src_address + 16);
+            uint8x16_t v3 = vld1q_u8(src_address + 32);
+            uint8x16_t v4 = vld1q_u8(src_address + 48);
+
+            s1 = vmaxq_u8(s1, v1);
+            s2 = vmaxq_u8(s2, v2);
+            s3 = vmaxq_u8(s3, v3);
+            s4 = vmaxq_u8(s4, v4);
+        }
+
+        vst1q_u8(dstBase + i + 0, s1);
+        vst1q_u8(dstBase + i + 16, s2);
+        vst1q_u8(dstBase + i + 32, s3);
+        vst1q_u8(dstBase + i + 48, s4);
+    }
+
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const u8* src_address = srcBase + i;
+        uint8x16_t s1 = vld1q_u8(src_address);
+        src_address += srcStride;
+        for(size_t h = 1; h < size.height; ++h, src_address += srcStride)
+        {
+            internal::prefetch(src_address + srcStride, 0);
+
+            uint8x16_t v1 = vld1q_u8(src_address);
+            s1 = vmaxq_u8(s1, v1);
+        }
+        vst1q_u8(dstBase + i, s1);
+    }
+
+    if (i < size.width)
+        for(size_t h = 1; h < size.height; ++h)
+            for(size_t j = i ; j < size.width; j++ )
+                dstBase[j] = std::max(dstBase[j], srcBase[j + srcStride * h]);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+void reduceColMin(const Size2D &size,
+                  const u8 * srcBase, ptrdiff_t srcStride,
+                  u8 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memcpy(dstBase, srcBase, size.width);
+    size_t i = 0;
+    for (; i + 16*4 <= size.width; i += 16*4)
+    {
+        const u8* src_address = srcBase + i;
+
+        uint8x16_t s1 = vld1q_u8(src_address + 0);
+        uint8x16_t s2 = vld1q_u8(src_address + 16);
+        uint8x16_t s3 = vld1q_u8(src_address + 32);
+        uint8x16_t s4 = vld1q_u8(src_address + 48);
+
+        src_address += srcStride;
+
+        for(size_t h = 1; h < size.height; ++h, src_address += srcStride)
+        {
+            internal::prefetch(src_address + srcStride, 0);
+            internal::prefetch(src_address + srcStride, 32);
+
+            uint8x16_t v1 = vld1q_u8(src_address + 0);
+            uint8x16_t v2 = vld1q_u8(src_address + 16);
+            uint8x16_t v3 = vld1q_u8(src_address + 32);
+            uint8x16_t v4 = vld1q_u8(src_address + 48);
+
+            s1 = vminq_u8(s1, v1);
+            s2 = vminq_u8(s2, v2);
+            s3 = vminq_u8(s3, v3);
+            s4 = vminq_u8(s4, v4);
+        }
+
+        vst1q_u8(dstBase + i + 0, s1);
+        vst1q_u8(dstBase + i + 16, s2);
+        vst1q_u8(dstBase + i + 32, s3);
+        vst1q_u8(dstBase + i + 48, s4);
+    }
+
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const u8* src_address = srcBase + i;
+        uint8x16_t s1 = vld1q_u8(src_address);
+        src_address += srcStride;
+        for(size_t h = 1; h < size.height; ++h, src_address += srcStride)
+        {
+            internal::prefetch(src_address + srcStride, 0);
+
+            uint8x16_t v1 = vld1q_u8(src_address);
+            s1 = vminq_u8(s1, v1);
+        }
+        vst1q_u8(dstBase + i, s1);
+    }
+
+    if (i < size.width)
+        for(size_t h = 1; h < size.height; ++h)
+            for(size_t j = i ; j < size.width; j++ )
+                dstBase[j] = std::min(dstBase[j], srcBase[j + srcStride * h]);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+void reduceColSum(const Size2D &size,
+                  const f32 * srcBase, ptrdiff_t srcStride,
+                  f32 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memcpy(dstBase, srcBase, size.width*sizeof(f32));
+    size_t srcstep = srcStride/sizeof(f32);
+    size_t i = 0;
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const f32* src_address = srcBase + i;
+
+        float32x4_t s1 = vld1q_f32(src_address + 0);
+        float32x4_t s2 = vld1q_f32(src_address + 4);
+        float32x4_t s3 = vld1q_f32(src_address + 8);
+        float32x4_t s4 = vld1q_f32(src_address + 12);
+
+        src_address += srcstep;
+
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+            internal::prefetch(src_address + srcstep, 32);
+
+            float32x4_t v1 = vld1q_f32(src_address + 0);
+            float32x4_t v2 = vld1q_f32(src_address + 4);
+            float32x4_t v3 = vld1q_f32(src_address + 8);
+            float32x4_t v4 = vld1q_f32(src_address + 12);
+
+            s1 = vaddq_f32(s1, v1);
+            s2 = vaddq_f32(s2, v2);
+            s3 = vaddq_f32(s3, v3);
+            s4 = vaddq_f32(s4, v4);
+        }
+
+        vst1q_f32(dstBase + i + 0, s1);
+        vst1q_f32(dstBase + i + 4, s2);
+        vst1q_f32(dstBase + i + 8, s3);
+        vst1q_f32(dstBase + i + 12, s4);
+    }
+
+    for (; i + 4 <= size.width; i += 4)
+    {
+        const f32* src_address = srcBase + i;
+        float32x4_t s1 = vld1q_f32(src_address);
+        src_address += srcstep;
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+
+            float32x4_t v1 = vld1q_f32(src_address);
+            s1 = vaddq_f32(s1, v1);
+        }
+        vst1q_f32(dstBase + i, s1);
+    }
+
+    if (i < size.width)
+        for(size_t h = 1; h < size.height; ++h)
+        {
+            for(size_t j = i ; j < size.width; j++ )
+            {
+                dstBase[j] += srcBase[j + srcstep * h];
+            }
+        }
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+void reduceColMax(const Size2D &size,
+                  const f32 * srcBase, ptrdiff_t srcStride,
+                  f32 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memcpy(dstBase, srcBase, size.width*sizeof(f32));
+    size_t srcstep = srcStride/sizeof(f32);
+    size_t i = 0;
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const f32* src_address = srcBase + i;
+
+        float32x4_t s1 = vld1q_f32(src_address + 0);
+        float32x4_t s2 = vld1q_f32(src_address + 4);
+        float32x4_t s3 = vld1q_f32(src_address + 8);
+        float32x4_t s4 = vld1q_f32(src_address + 12);
+
+        src_address += srcstep;
+
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+            internal::prefetch(src_address + srcstep, 32);
+
+            float32x4_t v1 = vld1q_f32(src_address + 0);
+            float32x4_t v2 = vld1q_f32(src_address + 4);
+            float32x4_t v3 = vld1q_f32(src_address + 8);
+            float32x4_t v4 = vld1q_f32(src_address + 12);
+
+            s1 = vmaxq_f32(s1, v1);
+            s2 = vmaxq_f32(s2, v2);
+            s3 = vmaxq_f32(s3, v3);
+            s4 = vmaxq_f32(s4, v4);
+        }
+
+        vst1q_f32(dstBase + i + 0, s1);
+        vst1q_f32(dstBase + i + 4, s2);
+        vst1q_f32(dstBase + i + 8, s3);
+        vst1q_f32(dstBase + i + 12, s4);
+    }
+
+    for (; i + 4 <= size.width; i += 4)
+    {
+        const f32* src_address = srcBase + i;
+        float32x4_t s1 = vld1q_f32(src_address);
+        src_address += srcstep;
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+
+            float32x4_t v1 = vld1q_f32(src_address);
+            s1 = vmaxq_f32(s1, v1);
+        }
+        vst1q_f32(dstBase + i, s1);
+    }
+
+    if (i < size.width)
+        for(size_t h = 1; h < size.height; ++h)
+            for(size_t j = i ; j < size.width; j++ )
+                dstBase[j] = std::max(dstBase[j], srcBase[j + srcstep * h]);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+void reduceColMin(const Size2D &size,
+                  const f32 * srcBase, ptrdiff_t srcStride,
+                  f32 * dstBase)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    memcpy(dstBase, srcBase, size.width*sizeof(f32));
+    size_t srcstep = srcStride/sizeof(f32);
+    size_t i = 0;
+    for (; i + 16 <= size.width; i += 16)
+    {
+        const f32* src_address = srcBase + i;
+
+        float32x4_t s1 = vld1q_f32(src_address + 0);
+        float32x4_t s2 = vld1q_f32(src_address + 4);
+        float32x4_t s3 = vld1q_f32(src_address + 8);
+        float32x4_t s4 = vld1q_f32(src_address + 12);
+
+        src_address += srcstep;
+
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+            internal::prefetch(src_address + srcstep, 32);
+
+            float32x4_t v1 = vld1q_f32(src_address + 0);
+            float32x4_t v2 = vld1q_f32(src_address + 4);
+            float32x4_t v3 = vld1q_f32(src_address + 8);
+            float32x4_t v4 = vld1q_f32(src_address + 12);
+
+            s1 = vminq_f32(s1, v1);
+            s2 = vminq_f32(s2, v2);
+            s3 = vminq_f32(s3, v3);
+            s4 = vminq_f32(s4, v4);
+        }
+
+        vst1q_f32(dstBase + i + 0, s1);
+        vst1q_f32(dstBase + i + 4, s2);
+        vst1q_f32(dstBase + i + 8, s3);
+        vst1q_f32(dstBase + i + 12, s4);
+    }
+
+    for (; i + 4 <= size.width; i += 4)
+    {
+        const f32* src_address = srcBase + i;
+        float32x4_t s1 = vld1q_f32(src_address);
+        src_address += srcstep;
+        for(size_t h = 1; h < size.height; ++h, src_address += srcstep)
+        {
+            internal::prefetch(src_address + srcstep, 0);
+
+            float32x4_t v1 = vld1q_f32(src_address);
+            s1 = vminq_f32(s1, v1);
+        }
+        vst1q_f32(dstBase + i, s1);
+    }
+
+    if (i < size.width)
+        for(size_t h = 1; h < size.height; ++h)
+            for(size_t j = i ; j < size.width; j++ )
+                dstBase[j] = std::min(dstBase[j], srcBase[j + srcstep * h]);
+#else
+    (void)size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/remap.cpp

@@ -0,0 +1,694 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "remap.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace internal {
+
+void remapNearestNeighborReplicate(const Size2D size,
+                                   const u8 * srcBase,
+                                   const s32 * map,
+                                   u8 * dstBase, ptrdiff_t dstStride)
+{
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32), y);
+        u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y);
+
+        for (size_t x = 0; x < size.width; ++x)
+        {
+            dst_row[x] = srcBase[map_row[x]];
+        }
+    }
+}
+
+void remapNearestNeighborConst(const Size2D size,
+                               const u8 * srcBase,
+                               const s32 * map,
+                               u8 * dstBase, ptrdiff_t dstStride,
+                               u8 borderValue)
+{
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32), y);
+        u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y);
+
+        for (size_t x = 0; x < size.width; ++x)
+        {
+            s32 src_idx = map_row[x];
+            dst_row[x] = src_idx >= 0 ? srcBase[map_row[x]] : borderValue;
+        }
+    }
+}
+
+void remapLinearReplicate(const Size2D size,
+                          const u8 * srcBase,
+                          const s32 * map,
+                          const f32 * coeffs,
+                          u8 * dstBase, ptrdiff_t dstStride)
+{
+    int16x8_t v_zero16 = vdupq_n_s16(0);
+
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32) * 4, y);
+        const f32 * coeff_row = internal::getRowPtr(coeffs, size.width * sizeof(f32) * 2, y);
+
+        u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y);
+
+        size_t x = 0;
+        for ( ; x + 8 < size.width; x += 8)
+        {
+            int16x8_t v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2)]], v_zero16, 0);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 4]], v_src00, 1);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 8]], v_src00, 2);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 12]], v_src00, 3);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 16]], v_src00, 4);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 20]], v_src00, 5);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 24]], v_src00, 6);
+            v_src00 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 28]], v_src00, 7);
+
+            int16x8_t v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 1]], v_zero16, 0);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 5]], v_src01, 1);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 9]], v_src01, 2);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 13]], v_src01, 3);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 17]], v_src01, 4);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 21]], v_src01, 5);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 25]], v_src01, 6);
+            v_src01 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 29]], v_src01, 7);
+
+            int16x8_t v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 2]], v_zero16, 0);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 6]], v_src10, 1);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 10]], v_src10, 2);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 14]], v_src10, 3);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 18]], v_src10, 4);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 22]], v_src10, 5);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 26]], v_src10, 6);
+            v_src10 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 30]], v_src10, 7);
+
+            int16x8_t v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 3]], v_zero16, 0);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 7]], v_src11, 1);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 11]], v_src11, 2);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 15]], v_src11, 3);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 19]], v_src11, 4);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 23]], v_src11, 5);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 27]], v_src11, 6);
+            v_src11 = vsetq_lane_s16(srcBase[map_row[(x << 2) + 31]], v_src11, 7);
+
+            // first part
+            float32x4_t v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00)));
+            float32x4_t v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10)));
+
+            float32x4x2_t v_coeff = vld2q_f32(coeff_row + (x << 1));
+            float32x4_t v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src01),
+                                                                               vget_low_s16(v_src00))), v_coeff.val[0]);
+            float32x4_t v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src11),
+                                                                               vget_low_s16(v_src10))), v_coeff.val[0]);
+
+            float32x4_t v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]);
+            uint16x4_t v_dst0 = vmovn_u32(vcvtq_u32_f32(v_dst));
+
+            // second part
+            v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00)));
+            v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10)));
+
+            v_coeff = vld2q_f32(coeff_row + (x << 1) + 8);
+            v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src01),
+                                                                   vget_high_s16(v_src00))), v_coeff.val[0]);
+            v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src11),
+                                                                   vget_high_s16(v_src10))), v_coeff.val[0]);
+
+            v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]);
+            uint16x4_t v_dst1 = vmovn_u32(vcvtq_u32_f32(v_dst));
+
+            // store
+            vst1_u8(dst_row + x, vmovn_u16(vcombine_u16(v_dst0, v_dst1)));
+        }
+
+        for ( ; x < size.width; ++x)
+        {
+            s32 src00_index = map_row[(x << 2)];
+            s32 src10_index = map_row[(x << 2) + 2];
+            f32 dst_val_0 = (srcBase[map_row[(x << 2) + 1]] - srcBase[src00_index]) * coeff_row[x << 1] +
+                             srcBase[src00_index];
+            f32 dst_val_1 = (srcBase[map_row[(x << 2) + 3]] - srcBase[src10_index]) * coeff_row[x << 1] +
+                             srcBase[src10_index];
+            dst_row[x] = floorf((dst_val_1 - dst_val_0) * coeff_row[(x << 1) + 1] + dst_val_0);
+        }
+    }
+}
+
+void remapLinearConst(const Size2D size,
+                      const u8 * srcBase,
+                      const s32 * map,
+                      const f32 * coeffs,
+                      u8 * dstBase, ptrdiff_t dstStride,
+                      u8 borderValue)
+{
+    int16x8_t v_zero16 = vdupq_n_s16(0);
+
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const s32 * map_row = internal::getRowPtr(map, size.width * sizeof(s32) * 4, y);
+        const f32 * coeff_row = internal::getRowPtr(coeffs, size.width * sizeof(f32) * 2, y);
+
+        u8 * dst_row = internal::getRowPtr(dstBase, dstStride, y);
+
+        size_t x = 0;
+        for ( ; x + 8 < size.width; x += 8)
+        {
+            int16x8_t v_src00 = vsetq_lane_s16(map_row[(x << 2)] >= 0 ? srcBase[map_row[(x << 2)]] : borderValue, v_zero16, 0);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) +  4] >= 0 ? srcBase[map_row[(x << 2) +  4]] : borderValue, v_src00, 1);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) +  8] >= 0 ? srcBase[map_row[(x << 2) +  8]] : borderValue, v_src00, 2);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) + 12] >= 0 ? srcBase[map_row[(x << 2) + 12]] : borderValue, v_src00, 3);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) + 16] >= 0 ? srcBase[map_row[(x << 2) + 16]] : borderValue, v_src00, 4);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) + 20] >= 0 ? srcBase[map_row[(x << 2) + 20]] : borderValue, v_src00, 5);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) + 24] >= 0 ? srcBase[map_row[(x << 2) + 24]] : borderValue, v_src00, 6);
+            v_src00 = vsetq_lane_s16(map_row[(x << 2) + 28] >= 0 ? srcBase[map_row[(x << 2) + 28]] : borderValue, v_src00, 7);
+
+            int16x8_t v_src01 = vsetq_lane_s16(map_row[(x << 2) + 1] >= 0 ? srcBase[map_row[(x << 2) + 1]] : borderValue, v_zero16, 0);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) +  5] >= 0 ? srcBase[map_row[(x << 2) +  5]] : borderValue, v_src01, 1);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) +  9] >= 0 ? srcBase[map_row[(x << 2) +  9]] : borderValue, v_src01, 2);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) + 13] >= 0 ? srcBase[map_row[(x << 2) + 13]] : borderValue, v_src01, 3);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) + 17] >= 0 ? srcBase[map_row[(x << 2) + 17]] : borderValue, v_src01, 4);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) + 21] >= 0 ? srcBase[map_row[(x << 2) + 21]] : borderValue, v_src01, 5);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) + 25] >= 0 ? srcBase[map_row[(x << 2) + 25]] : borderValue, v_src01, 6);
+            v_src01 = vsetq_lane_s16(map_row[(x << 2) + 29] >= 0 ? srcBase[map_row[(x << 2) + 29]] : borderValue, v_src01, 7);
+
+            int16x8_t v_src10 = vsetq_lane_s16(map_row[(x << 2) + 2] >= 0 ? srcBase[map_row[(x << 2) + 2]] : borderValue, v_zero16, 0);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) +  6] >= 0 ? srcBase[map_row[(x << 2) +  6]] : borderValue, v_src10, 1);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 10] >= 0 ? srcBase[map_row[(x << 2) + 10]] : borderValue, v_src10, 2);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 14] >= 0 ? srcBase[map_row[(x << 2) + 14]] : borderValue, v_src10, 3);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 18] >= 0 ? srcBase[map_row[(x << 2) + 18]] : borderValue, v_src10, 4);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 22] >= 0 ? srcBase[map_row[(x << 2) + 22]] : borderValue, v_src10, 5);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 26] >= 0 ? srcBase[map_row[(x << 2) + 26]] : borderValue, v_src10, 6);
+            v_src10 = vsetq_lane_s16(map_row[(x << 2) + 30] >= 0 ? srcBase[map_row[(x << 2) + 30]] : borderValue, v_src10, 7);
+
+            int16x8_t v_src11 = vsetq_lane_s16(map_row[(x << 2) + 3] >= 0 ? srcBase[map_row[(x << 2) + 3]] : borderValue, v_zero16, 0);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) +  7] >= 0 ? srcBase[map_row[(x << 2) +  7]] : borderValue, v_src11, 1);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 11] >= 0 ? srcBase[map_row[(x << 2) + 11]] : borderValue, v_src11, 2);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 15] >= 0 ? srcBase[map_row[(x << 2) + 15]] : borderValue, v_src11, 3);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 19] >= 0 ? srcBase[map_row[(x << 2) + 19]] : borderValue, v_src11, 4);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 23] >= 0 ? srcBase[map_row[(x << 2) + 23]] : borderValue, v_src11, 5);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 27] >= 0 ? srcBase[map_row[(x << 2) + 27]] : borderValue, v_src11, 6);
+            v_src11 = vsetq_lane_s16(map_row[(x << 2) + 31] >= 0 ? srcBase[map_row[(x << 2) + 31]] : borderValue, v_src11, 7);
+
+            // first part
+            float32x4_t v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src00)));
+            float32x4_t v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_src10)));
+
+            float32x4x2_t v_coeff = vld2q_f32(coeff_row + (x << 1));
+            float32x4_t v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src01),
+                                                                               vget_low_s16(v_src00))), v_coeff.val[0]);
+            float32x4_t v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_low_s16(v_src11),
+                                                                               vget_low_s16(v_src10))), v_coeff.val[0]);
+
+            float32x4_t v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]);
+            uint16x4_t v_dst0 = vmovn_u32(vcvtq_u32_f32(v_dst));
+
+            // second part
+            v_src00_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src00)));
+            v_src10_f = vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_src10)));
+
+            v_coeff = vld2q_f32(coeff_row + (x << 1) + 8);
+            v_dst_0 = vmlaq_f32(v_src00_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src01),
+                                                                   vget_high_s16(v_src00))), v_coeff.val[0]);
+            v_dst_1 = vmlaq_f32(v_src10_f, vcvtq_f32_s32(vsubl_s16(vget_high_s16(v_src11),
+                                                                   vget_high_s16(v_src10))), v_coeff.val[0]);
+
+            v_dst = vmlaq_f32(v_dst_0, vsubq_f32(v_dst_1, v_dst_0), v_coeff.val[1]);
+            uint16x4_t v_dst1 = vmovn_u32(vcvtq_u32_f32(v_dst));
+
+            // store
+            vst1_u8(dst_row + x, vmovn_u16(vcombine_u16(v_dst0, v_dst1)));
+        }
+
+        for ( ; x < size.width; ++x)
+        {
+            s16 src00 = map_row[(x << 2) + 0] >= 0 ? srcBase[map_row[(x << 2) + 0]] : borderValue;
+            s16 src01 = map_row[(x << 2) + 1] >= 0 ? srcBase[map_row[(x << 2) + 1]] : borderValue;
+            s16 src10 = map_row[(x << 2) + 2] >= 0 ? srcBase[map_row[(x << 2) + 2]] : borderValue;
+            s16 src11 = map_row[(x << 2) + 3] >= 0 ? srcBase[map_row[(x << 2) + 3]] : borderValue;
+
+            f32 dst_val_0 = (src01 - src00) * coeff_row[(x << 1)] + src00;
+            f32 dst_val_1 = (src11 - src10) * coeff_row[(x << 1)] + src10;
+            dst_row[x] = floorf((dst_val_1 - dst_val_0) * coeff_row[(x << 1) + 1] + dst_val_0);
+        }
+    }
+}
+
+} // namespace internal
+
+#endif // CAROTENE_NEON
+
+bool isRemapNearestNeighborSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+bool isRemapLinearSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+void remapNearestNeighbor(const Size2D &ssize, const Size2D &dsize,
+                          const u8 * srcBase, ptrdiff_t srcStride,
+                          const f32 * tableBase, ptrdiff_t tableStride,
+                          u8 * dstBase, ptrdiff_t dstStride,
+                          BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isRemapNearestNeighborSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16];
+    s32 * map = alignPtr(_map, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x2_t v_width2 = vdup_n_s32(ssize.width - 1), v_height2 = vdup_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride);
+    int32x2_t v_step2 = vdup_n_s32(srcStride);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+        int32x2_t v_zero2 = vdup_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1);
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0;
+                    for ( ; x + 8 <= blockWidth; x += 8)
+                    {
+                        float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)),
+                                      v_table1 = vld2q_f32(table_row + (x << 1) + 8);
+
+                        int32x4_t v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table0.val[0])));
+                        int32x4_t v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table0.val[1])));
+                        int32x4_t v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4);
+                        vst1q_s32(map_row + x, v_dst_index);
+
+                        v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table1.val[0])));
+                        v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table1.val[1])));
+                        v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4);
+                        vst1q_s32(map_row + x + 4, v_dst_index);
+                    }
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1));
+
+                        int32x4_t v_dst_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_table0.val[0])));
+                        int32x4_t v_dst_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_table0.val[1])));
+                        int32x4_t v_dst_index = vmlaq_s32(v_dst_x, v_dst_y, v_step4);
+                        vst1q_s32(map_row + x, v_dst_index);
+                    }
+
+                    for ( ; x + 2 <= blockWidth; x += 2)
+                    {
+                        float32x2x2_t v_table0 = vld2_f32(table_row + (x << 1));
+
+                        int32x2_t v_dst_x = vmax_s32(v_zero2, vmin_s32(v_width2, vcvt_s32_f32(v_table0.val[0])));
+                        int32x2_t v_dst_y = vmax_s32(v_zero2, vmin_s32(v_height2, vcvt_s32_f32(v_table0.val[1])));
+                        int32x2_t v_dst_index = vmla_s32(v_dst_x, v_dst_y, v_step2);
+                        vst1_s32(map_row + x, v_dst_index);
+                    }
+
+                    for ( ; x < blockWidth; ++x)
+                    {
+                        s32 src_x = std::max(0, std::min<s32>(ssize.width - 1, (s32)floorf(table_row[(x << 1) + 0])));
+                        s32 src_y = std::max(0, std::min<s32>(ssize.height - 1, (s32)floorf(table_row[(x << 1) + 1])));
+                        map_row[x] = src_y * srcStride + src_x;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                              getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+        int32x2_t v_m1_2 = vdup_n_s32(-1);
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+        float32x2_t v_zero2 = vdup_n_f32(0.0f);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1);
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0;
+                    for ( ; x + 8 <= blockWidth; x += 8)
+                    {
+                        float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1)),
+                                      v_table1 = vld2q_f32(table_row + (x << 1) + 8);
+
+                        int32x4_t v_dst_x = vcvtq_s32_f32(v_table0.val[0]);
+                        int32x4_t v_dst_y = vcvtq_s32_f32(v_table0.val[1]);
+                        uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table0.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)),
+                                                      vandq_u32(vcgeq_f32(v_table0.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4)));
+                        int32x4_t v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4);
+                        vst1q_s32(map_row + x, v_dst_index);
+
+                        v_dst_x = vcvtq_s32_f32(v_table1.val[0]);
+                        v_dst_y = vcvtq_s32_f32(v_table1.val[1]);
+                        v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table1.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)),
+                                           vandq_u32(vcgeq_f32(v_table1.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4)));
+                        v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4);
+                        vst1q_s32(map_row + x + 4, v_dst_index);
+                    }
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4x2_t v_table0 = vld2q_f32(table_row + (x << 1));
+
+                        int32x4_t v_dst_x = vcvtq_s32_f32(v_table0.val[0]);
+                        int32x4_t v_dst_y = vcvtq_s32_f32(v_table0.val[1]);
+                        uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_table0.val[0], v_zero4), vcleq_s32(v_dst_x, v_width4)),
+                                                      vandq_u32(vcgeq_f32(v_table0.val[1], v_zero4), vcleq_s32(v_dst_y, v_height4)));
+                        int32x4_t v_dst_index = vbslq_s32(v_mask, vmlaq_s32(v_dst_x, v_dst_y, v_step4), v_m1_4);
+                        vst1q_s32(map_row + x, v_dst_index);
+                    }
+
+                    for ( ; x + 2 <= blockWidth; x += 2)
+                    {
+                        float32x2x2_t v_table0 = vld2_f32(table_row + (x << 1));
+
+                        int32x2_t v_dst_x = vcvt_s32_f32(v_table0.val[0]);
+                        int32x2_t v_dst_y = vcvt_s32_f32(v_table0.val[1]);
+                        uint32x2_t v_mask = vand_u32(vand_u32(vcge_f32(v_table0.val[0], v_zero2), vcle_s32(v_dst_x, v_width2)),
+                                                     vand_u32(vcge_f32(v_table0.val[1], v_zero2), vcle_s32(v_dst_y, v_height2)));
+                        int32x2_t v_dst_index = vbsl_s32(v_mask, vmla_s32(v_dst_x, v_dst_y, v_step2), v_m1_2);
+                        vst1_s32(map_row + x, v_dst_index);
+                    }
+
+                    for ( ; x < blockWidth; ++x)
+                    {
+                        s32 src_x = (s32)floorf(table_row[(x << 1) + 0]);
+                        s32 src_y = (s32)floorf(table_row[(x << 1) + 1]);
+                        map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) &&
+                                     (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                          getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)tableBase;
+    (void)tableStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+void remapLinear(const Size2D &ssize, const Size2D &dsize,
+                 const u8 * srcBase, ptrdiff_t srcStride,
+                 const f32 * tableBase, ptrdiff_t tableStride,
+                 u8 * dstBase, ptrdiff_t dstStride,
+                 BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isRemapLinearSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16];
+    f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16];
+
+    s32 * map = alignPtr(_map, 16);
+    f32 * coeffs = alignPtr(_coeffs, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1);
+    float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1);
+
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0;
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4x2_t v_table = vld2q_f32(table_row + (x << 1));
+
+                        int32x4_t v_src_x = vcvtq_s32_f32(v_table.val[0]);
+                        int32x4_t v_src_y = vcvtq_s32_f32(v_table.val[1]);
+
+                        float32x4x2_t  v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_table.val[0], vcvtq_f32_s32(v_src_x));
+                        v_coeff.val[1] = vsubq_f32(v_table.val[1], vcvtq_f32_s32(v_src_y));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x);
+                        v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y);
+
+                        int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x));
+                        int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y));
+                        int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x)));
+                        int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y)));
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+                    }
+
+                    for ( ; x < blockWidth; ++x)
+                    {
+                        f32 src_x_f = table_row[(x << 1) + 0];
+                        f32 src_y_f = table_row[(x << 1) + 1];
+
+                        s32 src0_x = (s32)floorf(src_x_f);
+                        s32 src0_y = (s32)floorf(src_y_f);
+
+                        coeff_row[x << 1] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        s32 src1_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y + 1));
+                        src0_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y));
+                        s32 src1_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x + 1));
+                        src0_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x));
+
+                        map_row[(x << 2) + 0] = src0_y * srcStride + src0_x;
+                        map_row[(x << 2) + 1] = src0_y * srcStride + src1_x;
+                        map_row[(x << 2) + 2] = src1_y * srcStride + src0_x;
+                        map_row[(x << 2) + 3] = src1_y * srcStride + src1_x;
+                    }
+                }
+
+                remapLinearReplicate(Size2D(blockWidth, blockHeight),
+                                     srcBase, &map[0], &coeffs[0],
+                                     getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    const f32 * table_row = getRowPtr(tableBase, tableStride, i + y) + (j << 1);
+
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0;
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4x2_t v_table = vld2q_f32(table_row + (x << 1));
+
+                        int32x4_t v_src_x0 = vcvtq_s32_f32(v_table.val[0]);
+                        int32x4_t v_src_y0 = vcvtq_s32_f32(v_table.val[1]);
+
+                        float32x4x2_t v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_table.val[0], vcvtq_f32_s32(v_src_x0));
+                        v_coeff.val[1] = vsubq_f32(v_table.val[1], vcvtq_f32_s32(v_src_y0));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0);
+                        v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0);
+
+                        int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1);
+                        int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1);
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4);
+
+                        uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_table.val[0], v_zero4), vcleq_s32(v_src_x0, v_width4));
+                        uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_table.val[0], v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4));
+                        uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_table.val[1], v_zero4), vcleq_s32(v_src_y0, v_height4));
+                        uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_table.val[1], v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4));
+
+                        v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4);
+                        v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4);
+                        v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4);
+                        v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+                    }
+
+                    for ( ; x < blockWidth; ++x)
+                    {
+                        f32 src_x_f = table_row[(x << 1) + 0];
+                        f32 src_y_f = table_row[(x << 1) + 1];
+
+                        s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1;
+                        s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1;
+
+                        coeff_row[(x << 1)] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1;
+                        map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1;
+                    }
+                }
+
+                remapLinearConst(Size2D(blockWidth, blockHeight),
+                                 srcBase, &map[0], &coeffs[0],
+                                 getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)tableBase;
+    (void)tableStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/remap.hpp

@@ -0,0 +1,85 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_SRC_REMAP_HPP
+#define CAROTENE_SRC_REMAP_HPP
+
+#include "common.hpp"
+
+#include <cmath>
+
+#ifdef CAROTENE_NEON
+
+namespace CAROTENE_NS { namespace internal {
+
+enum
+{
+    BLOCK_SIZE = 32
+};
+
+
+void remapNearestNeighborReplicate(const Size2D size,
+                                   const u8 * srcBase,
+                                   const s32 * map,
+                                   u8 * dstBase, ptrdiff_t dstStride);
+
+void remapNearestNeighborConst(const Size2D size,
+                               const u8 * srcBase,
+                               const s32 * map,
+                               u8 * dstBase, ptrdiff_t dstStride,
+                               u8 borderValue);
+
+void remapLinearReplicate(const Size2D size,
+                          const u8 * srcBase,
+                          const s32 * map,
+                          const f32 * coeffs,
+                          u8 * dstBase, ptrdiff_t dstStride);
+
+void remapLinearConst(const Size2D size,
+                      const u8 * srcBase,
+                      const s32 * map,
+                      const f32 * coeffs,
+                      u8 * dstBase, ptrdiff_t dstStride,
+                      u8 borderValue);
+
+} }
+
+#endif // CAROTENE_NEON
+
+#endif // CAROTENE_SRC_REMAP_HPP

3rdparty/carotene/src/saturate_cast.hpp

@@ -0,0 +1,199 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_SATURATE_CAST_HPP
+#define CAROTENE_SATURATE_CAST_HPP
+
+#include <algorithm>
+#include <climits>
+#include <cmath>
+
+#if defined _MSC_VER && defined _M_ARM
+# include <intrin.h>
+#endif
+
+#include <carotene/definitions.hpp>
+#include <carotene/types.hpp>
+
+namespace CAROTENE_NS { namespace internal {
+
+#if defined _MSC_VER && defined _M_ARM
+
+__declspec(naked) static void vcvtr_s32_f64_imp(f64 d)
+{
+    (void)d;
+    __emit(0xEEBD);  // vcvtr.s32.f64 s0, d0
+    __emit(0x0B40);
+    __emit(0xEE10);  // vmov r0, s0
+    __emit(0x0A10);
+    __emit(0x4770);  // bx lr
+}
+
+# define CAROTENE_ROUND_FLT(x) return ((s32 (*)(f64))vcvtr_s32_f64_imp)((f64)x);
+# define CAROTENE_ROUND_DBL(x) return ((s32 (*)(f64))vcvtr_s32_f64_imp)(x);
+
+#elif defined CV_ICC || defined __GNUC__
+
+# if defined(__VFP_FP__) && !defined(__SOFTFP__) && !(defined _DEBUG || defined DEBUG) && !defined(__CUDACC__)
+#  define CAROTENE_ROUND_FLT(value) {                              \
+    register union { f32 f; s32 i; } result;                    \
+    asm ("ftosis  %0, %1 \n" : "=w" (result.f) : "w" (value) ); \
+    return result.i; }
+#  define CAROTENE_ROUND_DBL(value) {                      \
+    register union {f32 f; s32 i;} __tegra_result;      \
+    asm (                                               \
+        "ftosid  %0, %P1\n"                             \
+        : "=w" (__tegra_result.f)                       \
+        : "w" (value)                                   \
+    );                                                  \
+    return __tegra_result.i;                            \
+    }
+# else
+#  define CAROTENE_ROUND_FLT(x) return (s32)lrintf(value);
+#  define CAROTENE_ROUND_DBL(value) return (s32)lrint(value);
+# endif
+
+#endif
+
+inline s32 round(f32 value)
+{
+#ifdef CAROTENE_ROUND_FLT
+    CAROTENE_ROUND_FLT(value)
+#else
+    s32 intpart = (s32)(value);
+    f32 fractpart = value - intpart;
+    if ((fractpart != 0.5 && fractpart != -0.5) || ((intpart % 2) != 0))
+        return (s32)(value + (value >= 0 ? 0.5 : -0.5));
+    else
+        return intpart;
+#endif
+}
+
+inline s32 round(f64 value)
+{
+#ifdef CAROTENE_ROUND_DBL
+    CAROTENE_ROUND_DBL(value)
+#else
+    s32 intpart = (s32)(value);
+    f64 fractpart = value - intpart;
+    if ((fractpart != 0.5 && fractpart != -0.5) || ((intpart % 2) != 0))
+        return (s32)(value + (value >= 0 ? 0.5 : -0.5));
+    else
+        return intpart;
+#endif
+}
+/////////////// saturate_cast (used in image & signal processing) ///////////////////
+
+template<typename _Tp> inline _Tp saturate_cast(u8 v)    { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(s8 v)    { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(u16 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(s16 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(u32 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(s32 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(s64 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(u64 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(f32 v)   { return _Tp(v); }
+template<typename _Tp> inline _Tp saturate_cast(f64 v)   { return _Tp(v); }
+
+template<> inline u8 saturate_cast<u8>(s8 v)      { return (u8)std::max((s32)v, 0); }
+template<> inline u8 saturate_cast<u8>(u16 v)     { return (u8)std::min((u32)v, (u32)UCHAR_MAX); }
+template<> inline u8 saturate_cast<u8>(s32 v)     { return (u8)((u32)v <= UCHAR_MAX ? v : v > 0 ? UCHAR_MAX : 0); }
+template<> inline u8 saturate_cast<u8>(s16 v)     { return saturate_cast<u8>((s32)v); }
+template<> inline u8 saturate_cast<u8>(u32 v)     { return (u8)std::min(v, (u32)UCHAR_MAX); }
+template<> inline u8 saturate_cast<u8>(s64 v)     { return (u8)((u64)v <= UCHAR_MAX ? v : v > 0 ? UCHAR_MAX : 0); }
+template<> inline u8 saturate_cast<u8>(u64 v)     { return (u8)std::min(v, (u64)UCHAR_MAX); }
+template<> inline u8 saturate_cast<u8>(f32 v)     { return saturate_cast<u8>(round(v)); }
+template<> inline u8 saturate_cast<u8>(f64 v)     { return saturate_cast<u8>(round(v)); }
+
+template<> inline s8 saturate_cast<s8>(u8 v)      { return (s8)std::min((s32)v, SCHAR_MAX); }
+template<> inline s8 saturate_cast<s8>(u16 v)     { return (s8)std::min((u32)v, (u32)SCHAR_MAX); }
+template<> inline s8 saturate_cast<s8>(s32 v)     { return (s8)((u32)(v-SCHAR_MIN) <= (u32)UCHAR_MAX ? v : v > 0 ? SCHAR_MAX : SCHAR_MIN); }
+template<> inline s8 saturate_cast<s8>(s16 v)     { return saturate_cast<s8>((s32)v); }
+template<> inline s8 saturate_cast<s8>(u32 v)     { return (s8)std::min(v, (u32)SCHAR_MAX); }
+template<> inline s8 saturate_cast<s8>(s64 v)     { return (s8)((u64)(v-SCHAR_MIN) <= (u64)UCHAR_MAX ? v : v > 0 ? SCHAR_MAX : SCHAR_MIN); }
+template<> inline s8 saturate_cast<s8>(u64 v)     { return (s8)std::min(v, (u64)SCHAR_MAX); }
+template<> inline s8 saturate_cast<s8>(f32 v)     { return saturate_cast<s8>(round(v)); }
+template<> inline s8 saturate_cast<s8>(f64 v)     { return saturate_cast<s8>(round(v)); }
+
+template<> inline u16 saturate_cast<u16>(s8 v)    { return (u16)std::max((s32)v, 0); }
+template<> inline u16 saturate_cast<u16>(s16 v)   { return (u16)std::max((s32)v, 0); }
+template<> inline u16 saturate_cast<u16>(s32 v)   { return (u16)((u32)v <= (u32)USHRT_MAX ? v : v > 0 ? USHRT_MAX : 0); }
+template<> inline u16 saturate_cast<u16>(u32 v)   { return (u16)std::min(v, (u32)USHRT_MAX); }
+template<> inline u16 saturate_cast<u16>(s64 v)   { return (u16)((u64)v <= (u64)USHRT_MAX ? v : v > 0 ? USHRT_MAX : 0); }
+template<> inline u16 saturate_cast<u16>(u64 v)   { return (u16)std::min(v, (u64)USHRT_MAX); }
+template<> inline u16 saturate_cast<u16>(f32 v)   { return saturate_cast<u16>(round(v)); }
+template<> inline u16 saturate_cast<u16>(f64 v)   { return saturate_cast<u16>(round(v)); }
+
+template<> inline s16 saturate_cast<s16>(u16 v)   { return (s16)std::min((s32)v, SHRT_MAX); }
+template<> inline s16 saturate_cast<s16>(s32 v)   { return (s16)((u32)(v - SHRT_MIN) <= (u32)USHRT_MAX ? v : v > 0 ? SHRT_MAX : SHRT_MIN); }
+template<> inline s16 saturate_cast<s16>(u32 v)   { return (s16)std::min(v, (u32)SHRT_MAX); }
+template<> inline s16 saturate_cast<s16>(s64 v)   { return (s16)((u64)(v - SHRT_MIN) <= (u64)USHRT_MAX ? v : v > 0 ? SHRT_MAX : SHRT_MIN); }
+template<> inline s16 saturate_cast<s16>(u64 v)   { return (s16)std::min(v, (u64)SHRT_MAX); }
+template<> inline s16 saturate_cast<s16>(f32 v)   { return saturate_cast<s16>(round(v)); }
+template<> inline s16 saturate_cast<s16>(f64 v)   { return saturate_cast<s16>(round(v)); }
+
+template<> inline u32 saturate_cast<u32>(s8 v)    { return (u32)std::max(v, (s8)0); }
+template<> inline u32 saturate_cast<u32>(s16 v)   { return (u32)std::max(v, (s16)0); }
+template<> inline u32 saturate_cast<u32>(s32 v)   { return (u32)std::max(v, (s32)0); }
+template<> inline u32 saturate_cast<u32>(s64 v)   { return (u32)((u64)v <= (u64)UINT_MAX ? v : v > 0 ? UINT_MAX : 0); }
+template<> inline u32 saturate_cast<u32>(u64 v)   { return (u32)std::min(v, (u64)UINT_MAX); }
+//OpenCV like f32/f64 -> u32 conversion
+//we intentionally do not clip negative numbers, to make -1 become 0xffffffff etc.
+template<> inline u32 saturate_cast<u32>(f32 v)   { return round(v); }
+template<> inline u32 saturate_cast<u32>(f64 v)   { return round(v); }
+//Negative clipping implementation
+//template<> inline u32 saturate_cast<u32>(f32 v)   { return saturate_cast<u32>(round(v)); }
+//template<> inline u32 saturate_cast<u32>(f64 v)   { return saturate_cast<u32>(round(v)); }
+
+template<> inline s32 saturate_cast<s32>(u32 v)   { return (s32)std::min(v, (u32)INT_MAX); }
+template<> inline s32 saturate_cast<s32>(s64 v)   { return (s32)((u64)(v - INT_MIN) <= (u64)UINT_MAX ? v : v > 0 ? INT_MAX : INT_MIN); }
+template<> inline s32 saturate_cast<s32>(u64 v)   { return (s32)std::min(v, (u64)INT_MAX); }
+template<> inline s32 saturate_cast<s32>(f32 v)   { return round(v); }
+template<> inline s32 saturate_cast<s32>(f64 v)   { return round(v); }
+
+template<> inline u64 saturate_cast<u64>(s8 v)    { return (u64)std::max(v, (s8)0); }
+template<> inline u64 saturate_cast<u64>(s16 v)   { return (u64)std::max(v, (s16)0); }
+template<> inline u64 saturate_cast<u64>(s32 v)   { return (u64)std::max(v, (s32)0); }
+template<> inline u64 saturate_cast<u64>(s64 v)   { return (u64)std::max(v, (s64)0); }
+
+template<> inline s64 saturate_cast<s64>(u64 v)   { return (s64)std::min(v, (u64)LLONG_MAX); }
+
+} }
+
+#endif

3rdparty/carotene/src/scharr.cpp

@@ -0,0 +1,219 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <vector>
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+bool isScharr3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin)
+{
+    return (dx == 0 && dy == 1 &&
+                   isSeparableFilter3x3Supported(size, border, 3, 1, borderMargin)) ||
+           (dx == 1 && dy == 0 &&
+                   isSeparableFilter3x3Supported(size, border, 1, 3, borderMargin));
+}
+
+void Scharr3x3(const Size2D &size,
+               const u8 * srcBase, ptrdiff_t srcStride,
+               s16 * dstBase, ptrdiff_t dstStride,
+               s32 dx, s32 dy,
+               BORDER_MODE border, u8 borderValue, Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(isScharr3x3Supported(size, border, dx, dy, borderMargin));
+#ifdef CAROTENE_NEON
+    static s16 dw[] = {3, 10, 3};
+
+    if (dy == 1)
+        SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride,
+                           3, 1, dw, 0,
+                           border, borderValue, borderMargin);
+    else
+        SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride,
+                           1, 3, 0, dw,
+                           border, borderValue, borderMargin);
+#else
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderValue;
+#endif
+}
+
+void ScharrDeriv(const Size2D &size, s32 cn,
+                 const u8 * srcBase, ptrdiff_t srcStride,
+                 s16 * dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t colsn = size.width*cn;
+    size_t roiw8 = colsn > 7 ? colsn - 7 : 0;
+
+    ptrdiff_t delta = (ptrdiff_t)(((size.width + 2)*cn + 15) & -16);//align size
+    std::vector<s16> _tempBuf((delta << 1) + 64);
+    s16 *trow0 = internal::alignPtr(&_tempBuf[cn], 16), *trow1 = internal::alignPtr(trow0 + delta, 16);
+
+    int16x8_t vc3 = vmovq_n_s16(3);
+    int16x8_t vc10 = vmovq_n_s16(10);
+    uint8x8_t v8c10 = vmov_n_u8(10);
+
+    for(size_t y = 0; y < size.height; y++ )
+    {
+        const u8* srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : size.height > 1 ? 1 : 0);
+        const u8* srow1 = internal::getRowPtr(srcBase, srcStride, y);
+        const u8* srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height > 1 ? size.height-2 : 0);
+        s16* drow = internal::getRowPtr(dstBase, dstStride, y);
+
+        // do vertical convolution
+        size_t x = 0;
+        for( ; x < roiw8; x += 8 )
+        {
+            internal::prefetch(srow0 + x);
+            internal::prefetch(srow1 + x);
+            internal::prefetch(srow2 + x);
+#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7
+            __asm__ (
+                "vld1.8 {d0}, [%[src0]]                                \n\t"
+                "vld1.8 {d2}, [%[src2]]                                \n\t"
+                "vld1.8 {d1}, [%[src1]]                                \n\t"
+                "vaddl.u8 q2, d2, d0                                   \n\t"
+                "vmull.u8 q3, d1, %[vc10]                              \n\t"
+                "vsubl.u8 q4, d2, d0                                   \n\t"
+                "vmla.s16 q3, q2, %q[vc3]                              \n\t"
+                "vst1.16 {d8-d9}, [%[out1],:128]                       \n\t"
+                "vst1.16 {d6-d7}, [%[out0],:128]                       \n\t"
+                :
+                : [out0] "r" (trow0 + x),
+                  [out1] "r" (trow1 + x),
+                  [src0] "r" (srow0 + x),
+                  [src1] "r" (srow1 + x),
+                  [src2] "r" (srow2 + x),
+                  [vc10] "w" (v8c10), [vc3] "w" (vc3)
+                : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15"
+            );
+#else
+            uint8x8_t s0 = vld1_u8(srow0 + x);
+            uint8x8_t s1 = vld1_u8(srow1 + x);
+            uint8x8_t s2 = vld1_u8(srow2 + x);
+
+            int16x8_t s1x10 = vreinterpretq_s16_u16(vmull_u8(s1, v8c10));
+            int16x8_t s02 = vreinterpretq_s16_u16(vaddl_u8(s2, s0));
+            int16x8_t t1 = vreinterpretq_s16_u16(vsubl_u8(s2, s0));
+            int16x8_t t0 = vmlaq_s16(s1x10, s02, vc3);
+
+            vst1q_s16(trow1 + x, t1);
+            vst1q_s16(trow0 + x, t0);
+#endif
+        }
+        for( ; x < colsn; x++ )
+        {
+            trow0[x] = (s16)((srow0[x] + srow2[x])*3 + srow1[x]*10);
+            trow1[x] = (s16)(srow2[x] - srow0[x]);
+        }
+
+        // make border
+        size_t x0 = (size.width > 1 ? cn : 0), x1 = (size.width > 1 ? (size.width-2)*cn : 0);
+        for( s32 k = 0; k < cn; k++ )
+        {
+            trow0[-cn + k] = trow0[x0 + k]; trow0[colsn + k] = trow0[x1 + k];
+            trow1[-cn + k] = trow1[x0 + k]; trow1[colsn + k] = trow1[x1 + k];
+        }
+
+        // do horizontal convolution, interleave the results and store them to dst
+        x = 0;
+        for( ; x < roiw8; x += 8 )
+        {
+#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 6
+            __asm__ (
+                "vld1.16 {d4-d5}, [%[s2ptr]]                           \n\t"
+                "vld1.16 {d8-d9}, [%[s4ptr]]                           \n\t"
+                "vld1.16 {d6-d7}, [%[s3ptr],:128]                      \n\t"
+                "vld1.16 {d0-d1}, [%[s0ptr]]                           \n\t"
+                "vld1.16 {d2-d3}, [%[s1ptr]]                           \n\t"
+                "vadd.i16 q7, q2, q4                                   \n\t"
+                "vmul.s16 q6, q3, %q[vc10]                             \n\t"
+                "vsub.s16 q5, q1, q0                                   \n\t"
+                "vmla.s16 q6, q7, %q[vc3]                              \n\t"
+                "vst2.16 {d10-d13}, [%[out]]                           \n\t"
+                :
+                : [out] "r" (drow + x * 2),
+                  [s0ptr] "r" (trow0 + x - cn),
+                  [s1ptr] "r" (trow0 + x + cn),
+                  [s2ptr] "r" (trow1 + x - cn),
+                  [s3ptr] "r" (trow1 + x),
+                  [s4ptr] "r" (trow1 + x + cn),
+                  [vc10] "w" (vc10), [vc3] "w" (vc3)
+                : "d0","d1","d2","d3","d4","d5","d6","d7","d8","d9","d10","d11","d12","d13","d14","d15"
+            );
+#else
+            int16x8_t s0 = vld1q_s16(trow0 + x - cn);
+            int16x8_t s1 = vld1q_s16(trow0 + x + cn);
+            int16x8_t s2 = vld1q_s16(trow1 + x - cn);
+            int16x8_t s3 = vld1q_s16(trow1 + x);
+            int16x8_t s4 = vld1q_s16(trow1 + x + cn);
+
+            int16x8_t s3x10 = vmulq_s16(s3, vc10);
+            int16x8_t s24 = vaddq_s16(s2, s4);
+
+            int16x8x2_t vr;
+            vr.val[0] = vsubq_s16(s1, s0);
+            vr.val[1] = vmlaq_s16(s3x10, s24, vc3);
+
+            vst2q_s16(drow + x*2, vr);
+#endif
+        }
+        for( ; x < colsn; x++ )
+        {
+            drow[x*2] = (s16)(trow0[x+cn] - trow0[x-cn]);
+            drow[x*2+1] = (s16)((trow1[x+cn] + trow1[x-cn])*3 + trow1[x]*10);
+        }
+    }
+#else
+    (void)size;
+    (void)cn;
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/separable_filter.cpp

@@ -0,0 +1,109 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include "separable_filter.hpp"
+
+namespace CAROTENE_NS {
+
+bool isSeparableFilter3x3Supported(const Size2D &size, BORDER_MODE border, s32 dx, s32 dy, Margin borderMargin)
+{
+    return isSupportedConfiguration() &&
+        size.width >= 9 && size.height >= 1 &&
+        (size.height + borderMargin.top + borderMargin.bottom) >= 2  &&
+        (dx >= 0) && (dx < 4) && (dy >= 0) && (dy < 4) &&
+        (border == BORDER_MODE_CONSTANT   ||
+         border == BORDER_MODE_REFLECT    ||
+         border == BORDER_MODE_REFLECT101 ||
+         border == BORDER_MODE_REPLICATE   );
+}
+
+void SeparableFilter3x3(const Size2D &size,
+                        const u8 * srcBase, ptrdiff_t srcStride,
+                        s16 * dstBase, ptrdiff_t dstStride,
+                        const u8 rowFilter, const u8 colFilter, const s16 *xw, const s16 *yw,
+                        BORDER_MODE border, u8 borderValue, Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(isSeparableFilter3x3Supported(size, border, rowFilter, colFilter, borderMargin));
+#ifdef CAROTENE_NEON
+    if(!((xw || rowFilter < 3) && (yw || colFilter < 3)))
+        std::abort();//Couldn't call generic filter without provided weights
+
+    typedef void (*sepFilter3x3_8u16s_func)(const Size2D&, const u8*, ptrdiff_t, s16*, ptrdiff_t,
+                                            const s16*, const s16*, BORDER_MODE, u8, Margin);
+
+    static sepFilter3x3_8u16s_func quickFilters[4][4]=
+    {
+    /*d0y*/{ /*d0x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_121,    internal::ColFilter3x3S16_121>::process,
+             /*dx*/  internal::sepFilter3x3<internal::RowFilter3x3S16_m101,   internal::ColFilter3x3S16_121>::process,
+             /*d2x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_1m21,   internal::ColFilter3x3S16_121>::process,
+             /*dNx*/ internal::sepFilter3x3<internal::RowFilter3x3S16Generic, internal::ColFilter3x3S16_121>::process},
+
+    /*dy */{ /*d0x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_121,    internal::ColFilter3x3S16_m101>::process,
+             /*dx*/  internal::sepFilter3x3<internal::RowFilter3x3S16_m101,   internal::ColFilter3x3S16_m101>::process,
+             /*d2x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_1m21,   internal::ColFilter3x3S16_m101>::process,
+             /*dNx*/ internal::sepFilter3x3<internal::RowFilter3x3S16Generic, internal::ColFilter3x3S16_m101>::process},
+
+    /*d2y*/{ /*d0x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_121,    internal::ColFilter3x3S16_1m21>::process,
+             /*dx*/  internal::sepFilter3x3<internal::RowFilter3x3S16_m101,   internal::ColFilter3x3S16_1m21>::process,
+             /*d2x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_1m21,   internal::ColFilter3x3S16_1m21>::process,
+             /*dNx*/ internal::sepFilter3x3<internal::RowFilter3x3S16Generic, internal::ColFilter3x3S16_1m21>::process},
+
+    /*dNy*/{ /*d0x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_121,    internal::ColFilter3x3S16Generic>::process,
+             /*dx*/  internal::sepFilter3x3<internal::RowFilter3x3S16_m101,   internal::ColFilter3x3S16Generic>::process,
+             /*d2x*/ internal::sepFilter3x3<internal::RowFilter3x3S16_1m21,   internal::ColFilter3x3S16Generic>::process,
+             /*dNx*/ internal::sepFilter3x3<internal::RowFilter3x3S16Generic, internal::ColFilter3x3S16Generic>::process}
+    };
+
+    quickFilters[colFilter][rowFilter](size, srcBase, srcStride, dstBase, dstStride,
+                                       xw, yw, border, borderValue, borderMargin);
+#else
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)xw;
+    (void)yw;
+    (void)borderValue;
+#endif
+}
+
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/sobel.cpp

@@ -0,0 +1,317 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include <vector>
+
+#include "common.hpp"
+
+namespace CAROTENE_NS {
+
+bool isSobel3x3Supported(const Size2D &size, BORDER_MODE border,
+                         s32 dx, s32 dy, Margin borderMargin)
+{
+    return dx < 3 && dx >= 0 &&
+           dy < 3 && dy >= 0 &&
+           (dx + dy) > 0 &&
+           isSeparableFilter3x3Supported(size, border, dx, dy, borderMargin);
+}
+
+void Sobel3x3(const Size2D &size,
+              const u8 * srcBase, ptrdiff_t srcStride,
+              s16 * dstBase, ptrdiff_t dstStride,
+              s32 dx, s32 dy,
+              BORDER_MODE borderType, u8 borderValue, Margin borderMargin)
+{
+    internal::assertSupportedConfiguration(isSobel3x3Supported(size, borderType, dx, dy, borderMargin));
+#ifdef CAROTENE_NEON
+    SeparableFilter3x3(size, srcBase, srcStride, dstBase, dstStride,
+                       dx, dy, 0, 0,
+                       borderType, borderValue, borderMargin);
+#else
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderValue;
+#endif
+}
+
+bool isSobel3x3f32Supported(const Size2D &size, BORDER_MODE border,
+                            s32 dx, s32 dy)
+{
+    return isSupportedConfiguration() &&
+           dx < 3 && dx >= 0 &&
+           dy < 3 && dy >= 0 &&
+           (dx + dy) > 0 &&
+           size.width >= 4 && size.height >= 2 &&
+           (border == BORDER_MODE_CONSTANT   ||
+            border == BORDER_MODE_REFLECT    ||
+            border == BORDER_MODE_REFLECT101 ||
+            border == BORDER_MODE_REPLICATE   );
+}
+
+void Sobel3x3(const Size2D &size,
+              const f32 * srcBase, ptrdiff_t srcStride,
+              f32 * dstBase, ptrdiff_t dstStride,
+              s32 dx, s32 dy,
+              BORDER_MODE borderType, f32 borderValue)
+{
+    internal::assertSupportedConfiguration(isSobel3x3f32Supported(size, borderType, dx, dy));
+#ifdef CAROTENE_NEON
+    std::vector<f32> _tmp;
+    f32 *tmp = 0;
+    if (borderType == BORDER_MODE_CONSTANT)
+    {
+        _tmp.assign(size.width + 2, borderValue);
+        tmp = &_tmp[1];
+    }
+
+    ptrdiff_t delta = (ptrdiff_t)((size.width + 2 + 31) & -32);//align size
+    std::vector<f32> _tempBuf((delta << 1) + 64);
+    f32 *trow0 = internal::alignPtr(&_tempBuf[1], 32), *trow1 = internal::alignPtr(trow0 + delta, 32);
+
+    for( size_t y = 0; y < size.height; y++ )
+    {
+        const f32* srow0;
+        const f32* srow1 = internal::getRowPtr(srcBase, srcStride, y);
+        const f32* srow2;
+        f32* drow = internal::getRowPtr(dstBase, dstStride, y > 0 ? y-1 : 0);
+        f32* drow1 = internal::getRowPtr(dstBase, dstStride, y);
+        if (borderType == BORDER_MODE_REFLECT101) {
+            srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 1);
+            srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-2);
+        } else  if (borderType == BORDER_MODE_CONSTANT) {
+            srow0 = y > 0 ? internal::getRowPtr(srcBase, srcStride, y-1) : tmp;
+            srow2 =  y < size.height-1 ? internal::getRowPtr(srcBase, srcStride, y+1) : tmp;
+        } else { // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE
+            srow0 = internal::getRowPtr(srcBase, srcStride, y > 0 ? y-1 : 0);
+            srow2 = internal::getRowPtr(srcBase, srcStride, y < size.height-1 ? y+1 : size.height-1);
+        }
+
+        float32x4_t tprev = vmovq_n_f32(0.f);
+        float32x4_t tcurr = vmovq_n_f32(0.f);
+        float32x4_t tnext = vmovq_n_f32(0.f);
+        float32x4_t t0, t1, t2;
+        // do vertical convolution
+        size_t x = 0, bcolsn = y + 2 < size.height ? size.width : (size.width - 4);
+        for( ; x <= bcolsn; x += 4 )
+        {
+            internal::prefetch(srow0 + x);
+            internal::prefetch(srow1 + x);
+            internal::prefetch(srow2 + x);
+
+            float32x4_t x0 = vld1q_f32(srow0 + x);
+            float32x4_t x1 = vld1q_f32(srow1 + x);
+            float32x4_t x2 = vld1q_f32(srow2 + x);
+
+            tprev = tcurr;
+            tcurr = tnext;
+            if(!dy)
+            {
+                tnext = vaddq_f32(vaddq_f32(vaddq_f32(x1, x1), x2), x0);
+            }
+            else if(dy == 2)
+            {
+                tnext = vsubq_f32(vsubq_f32(x2, x1), vsubq_f32(x1, x0));
+            }
+            else
+            {
+                tnext = vsubq_f32(x2, x0);
+            }
+
+            if(!x) {
+                tcurr = tnext;
+                // make border
+                if (borderType == BORDER_MODE_CONSTANT)
+                {
+                    tcurr = vsetq_lane_f32(borderValue,tcurr, 3);
+                }
+                else if (borderType == BORDER_MODE_REFLECT101)
+                {
+                    tcurr = vsetq_lane_f32(vgetq_lane_f32(tcurr, 1),tcurr, 3);
+                }
+                else // BORDER_MODE_REFLECT || BORDER_MODE_REPLICATE
+                {
+                    tcurr = vsetq_lane_f32(vgetq_lane_f32(tcurr, 0),tcurr, 3);
+                }
+                continue;
+            }
+
+            internal::prefetch(trow0 + x);
+            internal::prefetch(trow1 + x);
+
+            t0 = vextq_f32(tprev, tcurr, 3);
+            t1 = tcurr;
+            t2 = vextq_f32(tcurr, tnext, 1);
+            if(!dx)
+            {
+                t0 = vaddq_f32(t0, vaddq_f32(vaddq_f32(t1, t1), t2));
+            }
+            else if(dx == 2)
+            {
+                t0 = vsubq_f32(vsubq_f32(t2, t1), vsubq_f32(t1, t0));
+            }
+            else
+            {
+                t0 = vsubq_f32(t2, t0);
+            }
+
+            if(!(y%2))
+            {
+                vst1q_f32(trow0 + x - 4, t0);
+            }
+            else
+            {
+                vst1q_f32(trow1 + x - 4, t0);
+            }
+        }
+        x -= 4;
+        if(x == size.width){
+            x--;
+        }
+        f32 prevx = 0, rowx = 0, nextx = 0;
+        if(!dy)
+        {
+            prevx = x > 0 ? srow2[x-1] + 2*srow1[x-1] + srow0[x-1] :
+                    (borderType == BORDER_MODE_REFLECT101 ? srow2[1] + 2*srow1[1] + srow0[1] :
+                    (borderType == BORDER_MODE_CONSTANT   ? 4*borderValue :
+                                                            srow2[0] + 2*srow1[0] + srow0[0]) );
+            rowx  = srow2[x] + 2*srow1[x] + srow0[x];
+        }
+        else if(dy == 2)
+        {
+            prevx = x > 0 ? srow2[x-1] - 2*srow1[x-1] + srow0[x-1] :
+                    (borderType == BORDER_MODE_REFLECT101 ? srow2[1] - 2*srow1[1] + srow0[1] :
+                    (borderType == BORDER_MODE_CONSTANT   ? 0.f :
+                                                            srow2[0] - 2*srow1[0] + srow0[0]) );
+            rowx  = srow2[x] - 2*srow1[x] + srow0[x];
+        }
+        else
+        {
+            prevx = x > 0 ? srow2[x-1] - srow0[x-1] :
+                    (borderType == BORDER_MODE_REFLECT101 ? srow2[1] - srow0[1] :
+                    (borderType == BORDER_MODE_CONSTANT   ? 0.f :
+                                                            srow2[0] - srow0[0]) );
+            rowx  = srow2[x] - srow0[x];
+        }
+
+        for( ; x < size.width; x++ )
+        {
+            if(x+1 == size.width) {
+                // make border
+                if (borderType == BORDER_MODE_CONSTANT)
+                {
+                    if(!dy) {
+                        nextx = 4*borderValue;
+                    } else {
+                        nextx = 0.f;
+                    }
+                } else if (borderType == BORDER_MODE_REFLECT101)
+                {
+                    if(!dy) {
+                        nextx = srow2[x-1] + 2*srow1[x-1] + srow0[x-1];
+                    } else if(dy == 2) {
+                        nextx = srow2[x-1] - 2*srow1[x-1] + srow0[x-1];
+                    } else {
+                        nextx = srow2[x-1] - srow0[x-1];
+                    }
+                } else {
+                    if(!dy) {
+                        nextx = srow2[x] + 2*srow1[x] + srow0[x];
+                    } else if(dy == 2) {
+                        nextx = srow2[x] - 2*srow1[x] + srow0[x];
+                    } else {
+                        nextx = srow2[x] - srow0[x];
+                    }
+                }
+            } else {
+                if(!dy) {
+                    nextx = srow2[x+1] + 2*srow1[x+1] + srow0[x+1];
+                } else if(dy == 2) {
+                    nextx = srow2[x+1] - 2*srow1[x+1] + srow0[x+1];
+                } else {
+                    nextx = srow2[x+1] - srow0[x+1];
+                }
+            }
+            f32 res;
+            if(dx==1) {
+                res = nextx - prevx;
+            } else if(!dx) {
+                res = prevx + 2*rowx + nextx;
+            } else {
+                res = prevx - 2*rowx + nextx;
+            }
+            if(!(y%2)) {
+                *(trow0+x) = res;
+            } else {
+                *(trow1+x) = res;
+            }
+            prevx = rowx;
+            rowx = nextx;
+        }
+
+        if(y>0) {
+            for(size_t x1 = 0; x1 < size.width; x1++ )
+            {
+                if(y%2)
+                    *(drow + x1) = trow0[x1];
+                else
+                    *(drow + x1) = trow1[x1];
+            }
+        }
+        if(y == size.height-1) {
+            for(size_t x1 = 0; x1 < size.width; x1++ )
+            {
+                if(!(y%2))
+                    *(drow1 + x1) = trow0[x1];
+                else
+                    *(drow1 + x1) = trow1[x1];
+            }
+        }
+    }
+#else
+    (void)srcBase;
+    (void)srcStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderValue;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/sub.cpp

@@ -0,0 +1,621 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+#ifdef CAROTENE_NEON
+
+namespace {
+
+template <typename T, typename WT>
+struct SubWrap
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vsubq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vsub(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = (T)((WT)src0[0] - (WT)src1[0]);
+    }
+};
+
+template <typename T, typename WT>
+struct SubSaturate
+{
+    typedef T type;
+
+    void operator() (const typename internal::VecTraits<T>::vec128 & v_src0,
+                     const typename internal::VecTraits<T>::vec128 & v_src1,
+                     typename internal::VecTraits<T>::vec128 & v_dst) const
+    {
+        v_dst = internal::vqsubq(v_src0, v_src1);
+    }
+
+    void operator() (const typename internal::VecTraits<T>::vec64 & v_src0,
+                     const typename internal::VecTraits<T>::vec64 & v_src1,
+                     typename internal::VecTraits<T>::vec64 & v_dst) const
+    {
+        v_dst = internal::vqsub(v_src0, v_src1);
+    }
+
+    void operator() (const T * src0, const T * src1, T * dst) const
+    {
+        dst[0] = internal::saturate_cast<T>((WT)src0[0] - (WT)src1[0]);
+    }
+};
+
+} // namespace
+
+#endif
+
+void sub(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         u8 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<u8, s16>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<u8, s16>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        u16 * dstu16 = internal::getRowPtr((u16 *)dstBase, dstStride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw32; j += 32)
+        {
+            internal::prefetch(src0 + j);
+            internal::prefetch(src1 + j);
+            uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16);
+            uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16);
+            vst1q_u16(dstu16 + j, vsubl_u8(vget_low_u8(v_src00), vget_low_u8(v_src10)));
+            vst1q_u16(dstu16 + j + 8, vsubl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10)));
+            vst1q_u16(dstu16 + j + 16, vsubl_u8(vget_low_u8(v_src01), vget_low_u8(v_src11)));
+            vst1q_u16(dstu16 + j + 24, vsubl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11)));
+        }
+        for (; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src0 = vld1_u8(src0 + j);
+            uint8x8_t v_src1 = vld1_u8(src1 + j);
+            vst1q_u16(dstu16 + j, vsubl_u8(v_src0, v_src1));
+        }
+
+        for (; j < size.width; j++)
+            dst[j] = (s16)src0[j] - (s16)src1[j];
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void sub(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         f32 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw32 = size.width >= 31 ? size.width - 31 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        f32 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        for (; j < roiw32; j += 32)
+        {
+            internal::prefetch(src0 + j);
+            internal::prefetch(src1 + j);
+            uint8x16_t v_src00 = vld1q_u8(src0 + j), v_src01 = vld1q_u8(src0 + j + 16);
+            uint8x16_t v_src10 = vld1q_u8(src1 + j), v_src11 = vld1q_u8(src1 + j + 16);
+            int16x8_t vsl = vreinterpretq_s16_u16(vsubl_u8( vget_low_u8(v_src00),  vget_low_u8(v_src10)));
+            int16x8_t vsh = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(v_src00), vget_high_u8(v_src10)));
+
+            vst1q_f32(dst + j +  0, vcvtq_f32_s32(vmovl_s16(  vget_low_s16(vsl) )));
+            vst1q_f32(dst + j +  4, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsl) )));
+            vst1q_f32(dst + j +  8, vcvtq_f32_s32(vmovl_s16(  vget_low_s16(vsh) )));
+            vst1q_f32(dst + j + 12, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsh) )));
+
+            vsl = vreinterpretq_s16_u16(vsubl_u8( vget_low_u8(v_src01),  vget_low_u8(v_src11)));
+            vsh = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(v_src01), vget_high_u8(v_src11)));
+
+            vst1q_f32(dst + j + 16, vcvtq_f32_s32(vmovl_s16(  vget_low_s16(vsl) )));
+            vst1q_f32(dst + j + 20, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsl) )));
+            vst1q_f32(dst + j + 24, vcvtq_f32_s32(vmovl_s16(  vget_low_s16(vsh) )));
+            vst1q_f32(dst + j + 28, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vsh) )));
+        }
+        for (; j < roiw8; j += 8)
+        {
+            uint8x8_t v_src0 = vld1_u8(src0 + j);
+            uint8x8_t v_src1 = vld1_u8(src1 + j);
+
+            int16x8_t vs = vreinterpretq_s16_u16(vsubl_u8(v_src0, v_src1));
+            vst1q_f32(dst + j + 0, vcvtq_f32_s32(vmovl_s16(  vget_low_s16(vs) )));
+            vst1q_f32(dst + j + 4, vcvtq_f32_s32(vmovl_s16( vget_high_s16(vs) )));
+        }
+        for(; j < size.width; j++)
+            dst[j] = (f32)src0[j] - (f32)src1[j];
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+void sub(const Size2D &size,
+         const u8 * src0Base, ptrdiff_t src0Stride,
+         const s16 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const s16 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (policy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                uint8x16_t v_src0 = vld1q_u8(src0 + j);
+                int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0)));
+                int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0)));
+                int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8);
+                int16x8_t v_dst0 = vqsubq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vqsubq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j)));
+                int16x8_t v_src1 = vld1q_s16(src1 + j);
+                int16x8_t v_dst = vqsubq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = internal::saturate_cast<s16>((s32)src0[j] - (s32)src1[j]);
+        }
+        else
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                uint8x16_t v_src0 = vld1q_u8(src0 + j);
+                int16x8_t v_src00 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src0)));
+                int16x8_t v_src01 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src0)));
+                int16x8_t v_src10 = vld1q_s16(src1 + j), v_src11 = vld1q_s16(src1 + j + 8);
+                int16x8_t v_dst0 = vsubq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vsubq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src0 + j)));
+                int16x8_t v_src1 = vld1q_s16(src1 + j);
+                int16x8_t v_dst = vsubq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = (s16)((s32)src0[j] - (s32)src1[j]);
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const s16 * src0Base, ptrdiff_t src0Stride,
+         const u8 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    size_t roiw16 = size.width >= 15 ? size.width - 15 : 0;
+    size_t roiw8 = size.width >= 7 ? size.width - 7 : 0;
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const s16 * src0 = internal::getRowPtr(src0Base, src0Stride, i);
+        const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, i);
+        s16 * dst = internal::getRowPtr(dstBase, dstStride, i);
+        size_t j = 0;
+
+        if (policy == CONVERT_POLICY_SATURATE)
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8);
+                uint8x16_t v_src1 = vld1q_u8(src1 + j);
+                int16x8_t v_src10 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src1)));
+                int16x8_t v_src11 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src1)));
+                int16x8_t v_dst0 = vqsubq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vqsubq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vld1q_s16(src0 + j);
+                int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src1 + j)));
+                int16x8_t v_dst = vqsubq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = internal::saturate_cast<s16>((s32)src0[j] - (s32)src1[j]);
+        }
+        else
+        {
+            for (; j < roiw16; j += 16)
+            {
+                internal::prefetch(src0 + j);
+                internal::prefetch(src1 + j);
+                int16x8_t v_src00 = vld1q_s16(src0 + j), v_src01 = vld1q_s16(src0 + j + 8);
+                uint8x16_t v_src1 = vld1q_u8(src1 + j);
+                int16x8_t v_src10 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_src1)));
+                int16x8_t v_src11 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_src1)));
+                int16x8_t v_dst0 = vsubq_s16(v_src00, v_src10);
+                int16x8_t v_dst1 = vsubq_s16(v_src01, v_src11);
+                vst1q_s16(dst + j, v_dst0);
+                vst1q_s16(dst + j + 8, v_dst1);
+            }
+            for (; j < roiw8; j += 8)
+            {
+                int16x8_t v_src0 = vld1q_s16(src0 + j);
+                int16x8_t v_src1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(src1 + j)));
+                int16x8_t v_dst = vsubq_s16(v_src0, v_src1);
+                vst1q_s16(dst + j, v_dst);
+            }
+
+            for (; j < size.width; j++)
+                dst[j] = (s16)((s32)src0[j] - (s32)src1[j]);
+        }
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const s8 * src0Base, ptrdiff_t src0Stride,
+         const s8 * src1Base, ptrdiff_t src1Stride,
+         s8 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<s8, s16>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<s8, s16>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const s16 * src0Base, ptrdiff_t src0Stride,
+         const s16 * src1Base, ptrdiff_t src1Stride,
+         s16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<s16, s32>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<s16, s32>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const u16 * src0Base, ptrdiff_t src0Stride,
+         const u16 * src1Base, ptrdiff_t src1Stride,
+         u16 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<u16, s32>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<u16, s32>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const s32 * src0Base, ptrdiff_t src0Stride,
+         const s32 * src1Base, ptrdiff_t src1Stride,
+         s32 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<s32, s64>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<s32, s64>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const u32 * src0Base, ptrdiff_t src0Stride,
+         const u32 * src1Base, ptrdiff_t src1Stride,
+         u32 *dstBase, ptrdiff_t dstStride,
+         CONVERT_POLICY policy)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    if (policy == CONVERT_POLICY_SATURATE)
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubSaturate<u32, s64>());
+    }
+    else
+    {
+        internal::vtransform(size,
+                             src0Base, src0Stride,
+                             src1Base, src1Stride,
+                             dstBase, dstStride,
+                             SubWrap<u32, s64>());
+    }
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)policy;
+#endif
+}
+
+void sub(const Size2D &size,
+         const f32 * src0Base, ptrdiff_t src0Stride,
+         const f32 * src1Base, ptrdiff_t src1Stride,
+         f32 *dstBase, ptrdiff_t dstStride)
+{
+    internal::assertSupportedConfiguration();
+#ifdef CAROTENE_NEON
+    internal::vtransform(size,
+                         src0Base, src0Stride,
+                         src1Base, src1Stride,
+                         dstBase, dstStride,
+                         SubWrap<f32, f32>());
+#else
+    (void)size;
+    (void)src0Base;
+    (void)src0Stride;
+    (void)src1Base;
+    (void)src1Stride;
+    (void)dstBase;
+    (void)dstStride;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/sum.cpp

@@ -0,0 +1,385 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include "vtransform.hpp"
+
+namespace CAROTENE_NS {
+
+bool isSumSupported(u32 channels)
+{
+    return (channels && channels < 5);
+}
+
+void sum(const Size2D &_size,
+         const u8 * srcBase, ptrdiff_t srcStride,
+         u32 * sumdst, u32 channels)
+{
+    internal::assertSupportedConfiguration(isSumSupported(channels));
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    const ptrdiff_t width = size.width * channels;
+
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const u8* src = internal::getRowPtr( srcBase,  srcStride, k);
+        ptrdiff_t i = 0;
+
+        if (channels == 3)
+        {
+            uint32x4_t vs1231 = vdupq_n_u32(0);
+            uint32x4_t vs3123 = vdupq_n_u32(0);
+            uint32x4_t vs2312 = vdupq_n_u32(0);
+            for (; i <= width - 257*8*3; i += 257*8*3, src += 257*8*3)
+            {
+                uint16x8_t s1 = vmovl_u8(vld1_u8(src +  0));
+                uint16x8_t s2 = vmovl_u8(vld1_u8(src +  8));
+                uint16x8_t s3 = vmovl_u8(vld1_u8(src + 16));
+
+                for (ptrdiff_t j = 8*3; j < 257*8*3; j+= 8*3)
+                {
+                    internal::prefetch(src + j + 24);
+                    s1 = vaddw_u8(s1, vld1_u8(src + j +  0));
+                    s2 = vaddw_u8(s2, vld1_u8(src + j +  8));
+                    s3 = vaddw_u8(s3, vld1_u8(src + j + 16));
+                }
+
+                vs1231 = vqaddq_u32(vs1231, vaddl_u16(vget_low_u16(s1), vget_high_u16(s2)));
+                vs3123 = vqaddq_u32(vs3123, vaddl_u16(vget_low_u16(s2), vget_high_u16(s3)));
+                vs2312 = vqaddq_u32(vs2312, vaddl_u16(vget_low_u16(s3), vget_high_u16(s1)));
+            }
+            if (i <= width - 8*3)
+            {
+                uint16x8_t s1 = vmovl_u8(vld1_u8(src +  0));
+                uint16x8_t s2 = vmovl_u8(vld1_u8(src +  8));
+                uint16x8_t s3 = vmovl_u8(vld1_u8(src + 16));
+
+                for (i += 8*3, src += 8*3; i <= width - 8*3; i += 8*3, src += 8*3)
+                {
+                    internal::prefetch(src + 24);
+                    s1 = vaddw_u8(s1, vld1_u8(src +  0));
+                    s2 = vaddw_u8(s2, vld1_u8(src +  8));
+                    s3 = vaddw_u8(s3, vld1_u8(src + 16));
+                }
+
+                vs1231 = vqaddq_u32(vs1231, vaddl_u16(vget_low_u16(s1), vget_high_u16(s2)));
+                vs3123 = vqaddq_u32(vs3123, vaddl_u16(vget_low_u16(s2), vget_high_u16(s3)));
+                vs2312 = vqaddq_u32(vs2312, vaddl_u16(vget_low_u16(s3), vget_high_u16(s1)));
+            }
+
+            u32 sum[12];
+            vst1q_u32(sum+0, vs1231);
+            vst1q_u32(sum+4, vs2312);
+            vst1q_u32(sum+8, vs3123);
+
+            for (; i < width; i += 3, src += 3)
+            {
+                sumdst[0] += src[0];
+                sumdst[1] += src[1];
+                sumdst[2] += src[2];
+            }
+
+            sumdst[0] += sum[0] + sum[3] + sum[6] + sum[9];
+            sumdst[1] += sum[1] + sum[4] + sum[7] + sum[10];
+            sumdst[2] += sum[2] + sum[5] + sum[8] + sum[11];
+        }
+        else
+        {
+            uint32x4_t vs = vdupq_n_u32(0);
+            for (; i <= width - 257*8; i += 257*8, src += 257 * 8)
+            {
+                uint16x8_t s1 = vmovl_u8(vld1_u8(src));
+
+                for (int j = 8; j < 257 * 8; j += 8)
+                {
+                    internal::prefetch(src + j);
+                    s1 = vaddw_u8(s1, vld1_u8(src + j));
+                }
+
+                vs = vqaddq_u32(vs, vaddl_u16(vget_low_u16(s1), vget_high_u16(s1)));
+            }
+            if (i < width - 7)
+            {
+                uint16x8_t s1 = vmovl_u8(vld1_u8(src));
+
+                for(i+=8,src+=8; i < width-7; i+=8,src+=8)
+                {
+                    internal::prefetch(src);
+                    s1 = vaddw_u8(s1, vld1_u8(src));
+                }
+                vs = vqaddq_u32(vs, vaddl_u16(vget_low_u16(s1), vget_high_u16(s1)));
+            }
+
+            if (channels == 1)
+            {
+                uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs));
+                uint32x2_t vs1 = vreinterpret_u32_u64(vpaddl_u32(vs2));
+
+                u32 s0 = vget_lane_u32(vs1, 0);
+                for(; i < width; ++i,++src)
+                    s0 += src[0];
+                sumdst[0] += s0;
+            }
+            else if (channels == 4)
+            {
+                vst1q_u32(sumdst, vqaddq_u32(vs, vld1q_u32(sumdst)));
+
+                for(; i < width; i+=4,src+=4)
+                {
+                    sumdst[0] += src[0];
+                    sumdst[1] += src[1];
+                    sumdst[2] += src[2];
+                    sumdst[3] += src[3];
+                }
+            }
+            else//if (channels == 2)
+            {
+                uint32x2_t vs2 = vqadd_u32(vget_low_u32(vs), vget_high_u32(vs));
+                vst1_u32(sumdst, vqadd_u32(vs2, vld1_u32(sumdst)));
+
+                for(; i < width; i+=2,src+=2)
+                {
+                    sumdst[0] += src[0];
+                    sumdst[1] += src[1];
+                }
+            }
+        }//channels != 3
+    }
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)sumdst;
+    (void)channels;
+#endif
+}
+
+void sum(const Size2D &_size,
+         const f32 * srcBase, ptrdiff_t srcStride,
+         f64 * sumdst, u32 channels)
+{
+    internal::assertSupportedConfiguration(isSumSupported(channels));
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    const ptrdiff_t width = size.width * channels;
+
+    for(size_t k = 0; k < size.height; ++k)
+    {
+        const f32* src = internal::getRowPtr( srcBase,  srcStride, k);
+        ptrdiff_t i = 0;
+
+        if (channels == 3)
+        {
+            float32x4_t vs1231 = vdupq_n_f32(0);
+            float32x4_t vs2312 = vdupq_n_f32(0);
+            float32x4_t vs3123 = vdupq_n_f32(0);
+            for(; i <= width-12; i += 12)
+            {
+                internal::prefetch(src + i + 12);
+                vs1231 = vaddq_f32(vs1231, vld1q_f32(src + i + 0));
+                vs2312 = vaddq_f32(vs2312, vld1q_f32(src + i + 4));
+                vs3123 = vaddq_f32(vs3123, vld1q_f32(src + i + 8));
+            }
+
+            f32 s[12];
+            vst1q_f32(s + 0, vs1231);
+            vst1q_f32(s + 4, vs2312);
+            vst1q_f32(s + 8, vs3123);
+
+            sumdst[0] += s[0] + s[3] + s[6] + s[9];
+            sumdst[1] += s[1] + s[4] + s[7] + s[10];
+            sumdst[2] += s[2] + s[5] + s[8] + s[11];
+            for( ; i < width; i+=3)
+            {
+                sumdst[0] += src[i];
+                sumdst[1] += src[i+1];
+                sumdst[2] += src[i+2];
+            }
+        }
+        else
+        {
+            float32x4_t vs = vdupq_n_f32(0);
+            for(; i <= width-4; i += 4)
+            {
+                internal::prefetch(src + i);
+                vs = vaddq_f32(vs, vld1q_f32(src+i));
+            }
+
+            if (channels == 1)
+            {
+                float32x2_t vs2 = vpadd_f32(vget_low_f32(vs), vget_high_f32(vs));
+                f32 s[2];
+                vst1_f32(s, vs2);
+
+                sumdst[0] += s[0] + s[1];
+                for( ; i < width; i++)
+                    sumdst[0] += src[i];
+            }
+            else if (channels == 4)
+            {
+                f32 s[4];
+                vst1q_f32(s, vs);
+
+                sumdst[0] += s[0];
+                sumdst[1] += s[1];
+                sumdst[2] += s[2];
+                sumdst[3] += s[3];
+            }
+            else//if (channels == 2)
+            {
+                float32x2_t vs2 = vadd_f32(vget_low_f32(vs), vget_high_f32(vs));
+                f32 s[2];
+                vst1_f32(s, vs2);
+
+                sumdst[0] += s[0];
+                sumdst[1] += s[1];
+
+                if(i < width)
+                {
+                    sumdst[0] += src[i];
+                    sumdst[1] += src[i+1];
+                }
+            }
+        }//channels != 3
+    }
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)sumdst;
+    (void)channels;
+#endif
+}
+
+bool isSqsumSupported(u32 channels)
+{
+    return (channels && ((4/channels)*channels == 4));
+}
+
+void sqsum(const Size2D &_size,
+           const u8 * srcBase, ptrdiff_t srcStride,
+           f64 * sumdst, f64 * sqsumdst, u32 channels)
+{
+    internal::assertSupportedConfiguration(isSqsumSupported(channels));
+#ifdef CAROTENE_NEON
+    Size2D size(_size);
+    if (srcStride == (ptrdiff_t)(size.width*channels))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+    const size_t width = size.width * channels;
+
+    size_t blockSize0 = 1 << 23;
+    size_t roiw8 = width & ~7;
+
+    uint32x4_t v_zero = vdupq_n_u32(0u);
+
+    for (size_t i = 0; i < size.height; ++i)
+    {
+        const u8 * src = internal::getRowPtr(srcBase, srcStride, i);
+        size_t j = 0u;
+
+        while (j < roiw8)
+        {
+            size_t blockSize = std::min(roiw8 - j, blockSize0) + j;
+            uint32x4_t v_sum = v_zero;
+            uint32x4_t v_sqsum = v_zero;
+
+            for ( ; j < blockSize ; j += 8, src += 8)
+            {
+                internal::prefetch(src);
+                uint8x8_t v_src0 = vld1_u8(src);
+
+                uint16x8_t v_src = vmovl_u8(v_src0);
+                uint16x4_t v_srclo = vget_low_u16(v_src), v_srchi = vget_high_u16(v_src);
+                v_sum = vaddq_u32(v_sum, vaddl_u16(v_srclo, v_srchi));
+                v_sqsum = vmlal_u16(v_sqsum, v_srclo, v_srclo);
+                v_sqsum = vmlal_u16(v_sqsum, v_srchi, v_srchi);
+            }
+
+            u32 arsum[8];
+            vst1q_u32(arsum, v_sum);
+            vst1q_u32(arsum + 4, v_sqsum);
+
+            sumdst[0] += (f64)arsum[0];
+            sumdst[1 % channels] += (f64)arsum[1];
+            sumdst[2 % channels] += (f64)arsum[2];
+            sumdst[3 % channels] += (f64)arsum[3];
+            sqsumdst[0] += (f64)arsum[4];
+            sqsumdst[1 % channels] += (f64)arsum[5];
+            sqsumdst[2 % channels] += (f64)arsum[6];
+            sqsumdst[3 % channels] += (f64)arsum[7];
+        }
+        // collect a few last elements in the current row
+        // it's ok to process channels elements per step
+        // since we could handle 1,2 or 4 channels
+        // we always have channels-fold amount of elements remaining
+        for ( ; j < width; j+=channels, src+=channels)
+        {
+            for (u32 kk = 0; kk < channels; kk++)
+            {
+                u32 srcval = src[kk];
+                sumdst[kk] += srcval;
+                sqsumdst[kk] += srcval * srcval;
+            }
+        }
+    }
+#else
+    (void)_size;
+    (void)srcBase;
+    (void)srcStride;
+    (void)sumdst;
+    (void)sqsumdst;
+    (void)channels;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/template_matching.cpp

@@ -0,0 +1,241 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2013-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "common.hpp"
+
+#include <vector>
+#include <cstring>
+
+namespace CAROTENE_NS {
+
+#define ENABLE4LINESMATCHING false  //Disabled since overall time for simultaneous 4 lines matching is greater than
+                                    //time for simultaneous 2 lines matching for the same amount of data
+
+bool isMatchTemplateSupported(const Size2D &tmplSize)
+{
+    return isSupportedConfiguration() &&
+           tmplSize.width >= 8 && // Actually the function could process even shorter templates
+                                  // but there will be no NEON optimization in this case
+           (tmplSize.width * tmplSize.height) <= 256;
+}
+
+void matchTemplate(const Size2D &srcSize,
+                   const u8 * srcBase, ptrdiff_t srcStride,
+                   const Size2D &tmplSize,
+                   const u8 * tmplBase, ptrdiff_t tmplStride,
+                   f32 * dstBase, ptrdiff_t dstStride,
+                   bool normalize)
+{
+    internal::assertSupportedConfiguration(isMatchTemplateSupported(tmplSize));
+#ifdef CAROTENE_NEON
+    const size_t tmplW = tmplSize.width;
+    const size_t tmplH = tmplSize.height;
+    const size_t dstW  = srcSize.width  - tmplSize.width  + 1;
+    const size_t dstH  = srcSize.height - tmplSize.height + 1;
+
+    //template correlation part
+    {
+#if ENABLE4LINESMATCHING
+        const size_t dstroiw4 = dstW & ~3u;
+#endif
+        const size_t dstroiw2 = dstW & ~1u;
+        const size_t tmplroiw = tmplW & ~7u;
+        const size_t dstride = dstStride >> 2;
+
+        f32 *corr = dstBase;
+        const u8  *imgrrow = srcBase;
+        for(size_t r = 0; r < dstH; ++r, corr+=dstride, imgrrow+=srcStride)
+        {
+            size_t c = 0;
+#if ENABLE4LINESMATCHING
+            for(; c < dstroiw4; c+=4)
+            {
+                u32 dot[4] = {0, 0, 0, 0};
+                uint32x4_t vdot0 = vmovq_n_u32(0);
+                uint32x4_t vdot1 = vmovq_n_u32(0);
+                uint32x4_t vdot2 = vmovq_n_u32(0);
+                uint32x4_t vdot3 = vmovq_n_u32(0);
+
+                const u8  *img = imgrrow;
+                const u8 *tmpl = tmplBase;
+                for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride)
+                {
+                    size_t j = 0;
+                    for(; j < tmplroiw; j+=8)
+                    {
+                        uint8x8_t vtmpl = vld1_u8(tmpl + j);
+
+                        uint8x8_t vimg0 = vld1_u8(img + j + c + 0);
+                        uint8x8_t vimg1 = vld1_u8(img + j + c + 1);
+                        uint8x8_t vimg2 = vld1_u8(img + j + c + 2);
+                        uint8x8_t vimg3 = vld1_u8(img + j + c + 3);
+
+                        uint16x8_t vd0 = vmull_u8(vtmpl, vimg0);
+                        uint16x8_t vd1 = vmull_u8(vtmpl, vimg1);
+                        uint16x8_t vd2 = vmull_u8(vtmpl, vimg2);
+                        uint16x8_t vd3 = vmull_u8(vtmpl, vimg3);
+
+                        vdot0 = vpadalq_u16(vdot0, vd0);
+                        vdot1 = vpadalq_u16(vdot1, vd1);
+                        vdot2 = vpadalq_u16(vdot2, vd2);
+                        vdot3 = vpadalq_u16(vdot3, vd3);
+                    }
+                    for(; j < tmplW; ++j)
+                    {
+                        dot[0] += tmpl[j] * img[j + c + 0];
+                        dot[1] += tmpl[j] * img[j + c + 1];
+                        dot[2] += tmpl[j] * img[j + c + 2];
+                        dot[3] += tmpl[j] * img[j + c + 3];
+                    }
+                }
+                uint32x4_t vdotx   = vld1q_u32(dot);
+                uint32x2_t vdot_0  = vpadd_u32(vget_low_u32(vdot0), vget_high_u32(vdot0));
+                uint32x2_t vdot_1  = vpadd_u32(vget_low_u32(vdot1), vget_high_u32(vdot1));
+                uint32x2_t vdot_2  = vpadd_u32(vget_low_u32(vdot2), vget_high_u32(vdot2));
+                uint32x2_t vdot_3  = vpadd_u32(vget_low_u32(vdot3), vget_high_u32(vdot3));
+                uint32x2_t vdot_01 = vpadd_u32(vdot_0, vdot_1);
+                uint32x2_t vdot_23 = vpadd_u32(vdot_2, vdot_3);
+
+                vst1q_f32(corr + c, vcvtq_f32_u32(vaddq_u32(vdotx, vcombine_u32(vdot_01, vdot_23))));
+            }
+#endif
+
+            for(; c < dstroiw2; c+=2)
+            {
+                u32 dot[2] = {0, 0};
+                uint32x4_t vdot0 = vmovq_n_u32(0);
+                uint32x4_t vdot1 = vmovq_n_u32(0);
+                const u8  *img = imgrrow;
+                const u8 *tmpl = tmplBase;
+                for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride)
+                {
+                    size_t j = 0;
+                    for(; j < tmplroiw; j+=8)
+                    {
+                        uint8x8_t vtmpl = vld1_u8(tmpl + j);
+
+                        uint8x8_t vimg0 = vld1_u8(img + j + c + 0);
+                        uint8x8_t vimg1 = vld1_u8(img + j + c + 1);
+
+                        uint16x8_t vd0 = vmull_u8(vtmpl, vimg0);
+                        uint16x8_t vd1 = vmull_u8(vtmpl, vimg1);
+
+                        vdot0 = vpadalq_u16(vdot0, vd0);
+                        vdot1 = vpadalq_u16(vdot1, vd1);
+                    }
+                    for(; j < tmplW; ++j)
+                    {
+                        dot[0] += tmpl[j] * img[j + c + 0];
+                        dot[1] += tmpl[j] * img[j + c + 1];
+                    }
+                }
+                uint32x2_t vdotx  = vld1_u32(dot);
+                uint32x2_t vdot_0 = vpadd_u32(vget_low_u32(vdot0), vget_high_u32(vdot0));
+                uint32x2_t vdot_1 = vpadd_u32(vget_low_u32(vdot1), vget_high_u32(vdot1));
+                uint32x2_t vdot_  = vpadd_u32(vdot_0, vdot_1);
+                vst1_f32(corr + c, vcvt_f32_u32(vadd_u32(vdotx, vdot_)));
+            }
+
+            for(; c < dstW; ++c)
+            {
+                u32 dot = 0;
+                uint32x4_t vdot = vmovq_n_u32(0);
+                const u8  *img = imgrrow;
+                const u8 *tmpl = tmplBase;
+                for(size_t i = 0; i < tmplH; ++i, tmpl+=tmplStride, img+=srcStride)
+                {
+                    size_t j = 0;
+                    for(; j < tmplroiw; j+=8)
+                    {
+                        uint8x8_t vtmpl = vld1_u8(tmpl + j);
+                        uint8x8_t vimg  = vld1_u8(img + j + c);
+                        uint16x8_t vd   = vmull_u8(vtmpl, vimg);
+                        vdot = vpadalq_u16(vdot, vd);
+                    }
+                    for(; j < tmplW; ++j)
+                        dot += tmpl[j] * img[j + c];
+                }
+                u32 wdot[2];
+                vst1_u32(wdot, vpadd_u32(vget_low_u32(vdot), vget_high_u32(vdot)));
+                dot += wdot[0] + wdot[1];
+                corr[c] = (f32)dot;
+            }
+        }
+    }
+
+    if(normalize)
+    {
+        f32 tn = std::sqrt((f32)normL2(tmplSize, tmplBase, tmplStride));
+
+        size_t iw = srcSize.width+1;
+        size_t ih = srcSize.height+1;
+        std::vector<f64> _sqsum(iw*ih);
+        f64 *sqsum = &_sqsum[0];
+        memset(sqsum, 0, iw*sizeof(f64));
+        for(size_t i = 1; i < ih; ++i)
+            sqsum[iw*i] = 0.;
+        sqrIntegral(srcSize, srcBase, srcStride, sqsum + iw + 1, iw*sizeof(f64));
+
+        for(size_t i = 0; i < dstH; ++i)
+        {
+            f32 *result = internal::getRowPtr(dstBase, dstStride, i);
+            for(size_t j = 0; j < dstW; ++j)
+            {
+                double s2 = sqsum[iw*i + j] +
+                            sqsum[iw*(i + tmplSize.height) + j + tmplSize.width] -
+                            sqsum[iw*(i + tmplSize.height) + j] -
+                            sqsum[iw*i + j + tmplSize.width];
+
+                result[j] /= tn * std::sqrt(s2);
+            }
+        }
+    }
+#else
+    (void)srcSize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)tmplBase;
+    (void)tmplStride;
+    (void)dstBase;
+    (void)dstStride;
+    (void)normalize;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/vtransform.hpp

@@ -0,0 +1,689 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef CAROTENE_SRC_VTRANSFORM_HPP
+#define CAROTENE_SRC_VTRANSFORM_HPP
+
+#include "common.hpp"
+
+#include <carotene/types.hpp>
+
+#ifdef CAROTENE_NEON
+
+namespace CAROTENE_NS { namespace internal {
+
+////////////////////////////// Type Traits ///////////////////////
+
+template <typename T, int cn = 1>
+struct VecTraits;
+
+template <> struct VecTraits< u8, 1> { typedef  uint8x16_t vec128; typedef   uint8x8_t vec64; typedef VecTraits<  u8, 1> unsign; };
+template <> struct VecTraits< s8, 1> { typedef   int8x16_t vec128; typedef    int8x8_t vec64; typedef VecTraits<  u8, 1> unsign; };
+template <> struct VecTraits<u16, 1> { typedef  uint16x8_t vec128; typedef  uint16x4_t vec64; typedef VecTraits< u16, 1> unsign; };
+template <> struct VecTraits<s16, 1> { typedef   int16x8_t vec128; typedef   int16x4_t vec64; typedef VecTraits< u16, 1> unsign; };
+template <> struct VecTraits<s32, 1> { typedef   int32x4_t vec128; typedef   int32x2_t vec64; typedef VecTraits< u32, 1> unsign; };
+template <> struct VecTraits<u32, 1> { typedef  uint32x4_t vec128; typedef  uint32x2_t vec64; typedef VecTraits< u32, 1> unsign; };
+template <> struct VecTraits<s64, 1> { typedef   int64x2_t vec128; typedef   int64x1_t vec64; typedef VecTraits< u64, 1> unsign; };
+template <> struct VecTraits<u64, 1> { typedef  uint64x2_t vec128; typedef  uint64x1_t vec64; typedef VecTraits< u64, 1> unsign; };
+template <> struct VecTraits<f32, 1> { typedef float32x4_t vec128; typedef float32x2_t vec64; typedef VecTraits< u32, 1> unsign; };
+
+template <> struct VecTraits< u8, 2> { typedef  uint8x16x2_t vec128; typedef   uint8x8x2_t vec64; typedef VecTraits<  u8, 2> unsign; };
+template <> struct VecTraits< s8, 2> { typedef   int8x16x2_t vec128; typedef    int8x8x2_t vec64; typedef VecTraits<  u8, 2> unsign; };
+template <> struct VecTraits<u16, 2> { typedef  uint16x8x2_t vec128; typedef  uint16x4x2_t vec64; typedef VecTraits< u16, 2> unsign; };
+template <> struct VecTraits<s16, 2> { typedef   int16x8x2_t vec128; typedef   int16x4x2_t vec64; typedef VecTraits< u16, 2> unsign; };
+template <> struct VecTraits<s32, 2> { typedef   int32x4x2_t vec128; typedef   int32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; };
+template <> struct VecTraits<u32, 2> { typedef  uint32x4x2_t vec128; typedef  uint32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; };
+template <> struct VecTraits<s64, 2> { typedef   int64x2x2_t vec128; typedef   int64x1x2_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<u64, 2> { typedef  uint64x2x2_t vec128; typedef  uint64x1x2_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<f32, 2> { typedef float32x4x2_t vec128; typedef float32x2x2_t vec64; typedef VecTraits< u32, 2> unsign; };
+
+template <> struct VecTraits< u8, 3> { typedef  uint8x16x3_t vec128; typedef   uint8x8x3_t vec64; typedef VecTraits<  u8, 3> unsign; };
+template <> struct VecTraits< s8, 3> { typedef   int8x16x3_t vec128; typedef    int8x8x3_t vec64; typedef VecTraits<  u8, 3> unsign; };
+template <> struct VecTraits<u16, 3> { typedef  uint16x8x3_t vec128; typedef  uint16x4x3_t vec64; typedef VecTraits< u16, 3> unsign; };
+template <> struct VecTraits<s16, 3> { typedef   int16x8x3_t vec128; typedef   int16x4x3_t vec64; typedef VecTraits< u16, 3> unsign; };
+template <> struct VecTraits<s32, 3> { typedef   int32x4x3_t vec128; typedef   int32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; };
+template <> struct VecTraits<u32, 3> { typedef  uint32x4x3_t vec128; typedef  uint32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; };
+template <> struct VecTraits<s64, 3> { typedef   int64x2x3_t vec128; typedef   int64x1x3_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<u64, 3> { typedef  uint64x2x3_t vec128; typedef  uint64x1x3_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<f32, 3> { typedef float32x4x3_t vec128; typedef float32x2x3_t vec64; typedef VecTraits< u32, 3> unsign; };
+
+template <> struct VecTraits< u8, 4> { typedef  uint8x16x4_t vec128; typedef   uint8x8x4_t vec64; typedef VecTraits<  u8, 3> unsign; };
+template <> struct VecTraits< s8, 4> { typedef   int8x16x4_t vec128; typedef    int8x8x4_t vec64; typedef VecTraits<  u8, 3> unsign; };
+template <> struct VecTraits<u16, 4> { typedef  uint16x8x4_t vec128; typedef  uint16x4x4_t vec64; typedef VecTraits< u16, 3> unsign; };
+template <> struct VecTraits<s16, 4> { typedef   int16x8x4_t vec128; typedef   int16x4x4_t vec64; typedef VecTraits< u16, 3> unsign; };
+template <> struct VecTraits<s32, 4> { typedef   int32x4x4_t vec128; typedef   int32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; };
+template <> struct VecTraits<u32, 4> { typedef  uint32x4x4_t vec128; typedef  uint32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; };
+template <> struct VecTraits<s64, 4> { typedef   int64x2x4_t vec128; typedef   int64x1x4_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<u64, 4> { typedef  uint64x2x4_t vec128; typedef  uint64x1x4_t vec64; typedef VecTraits< u64, 2> unsign; };
+template <> struct VecTraits<f32, 4> { typedef float32x4x4_t vec128; typedef float32x2x4_t vec64; typedef VecTraits< u32, 3> unsign; };
+
+////////////////////////////// vld1q ///////////////////////
+
+inline  uint8x16_t vld1q(const u8  * ptr) { return  vld1q_u8(ptr); }
+inline   int8x16_t vld1q(const s8  * ptr) { return  vld1q_s8(ptr); }
+inline  uint16x8_t vld1q(const u16 * ptr) { return vld1q_u16(ptr); }
+inline   int16x8_t vld1q(const s16 * ptr) { return vld1q_s16(ptr); }
+inline  uint32x4_t vld1q(const u32 * ptr) { return vld1q_u32(ptr); }
+inline   int32x4_t vld1q(const s32 * ptr) { return vld1q_s32(ptr); }
+inline float32x4_t vld1q(const f32 * ptr) { return vld1q_f32(ptr); }
+
+////////////////////////////// vld1 ///////////////////////
+
+inline   uint8x8_t vld1(const u8  * ptr) { return  vld1_u8(ptr); }
+inline    int8x8_t vld1(const s8  * ptr) { return  vld1_s8(ptr); }
+inline  uint16x4_t vld1(const u16 * ptr) { return vld1_u16(ptr); }
+inline   int16x4_t vld1(const s16 * ptr) { return vld1_s16(ptr); }
+inline  uint32x2_t vld1(const u32 * ptr) { return vld1_u32(ptr); }
+inline   int32x2_t vld1(const s32 * ptr) { return vld1_s32(ptr); }
+inline float32x2_t vld1(const f32 * ptr) { return vld1_f32(ptr); }
+
+////////////////////////////// vld2q ///////////////////////
+
+inline  uint8x16x2_t vld2q(const u8  * ptr) { return  vld2q_u8(ptr); }
+inline   int8x16x2_t vld2q(const s8  * ptr) { return  vld2q_s8(ptr); }
+inline  uint16x8x2_t vld2q(const u16 * ptr) { return vld2q_u16(ptr); }
+inline   int16x8x2_t vld2q(const s16 * ptr) { return vld2q_s16(ptr); }
+inline  uint32x4x2_t vld2q(const u32 * ptr) { return vld2q_u32(ptr); }
+inline   int32x4x2_t vld2q(const s32 * ptr) { return vld2q_s32(ptr); }
+inline float32x4x2_t vld2q(const f32 * ptr) { return vld2q_f32(ptr); }
+
+////////////////////////////// vld2 ///////////////////////
+
+inline   uint8x8x2_t vld2(const u8  * ptr) { return  vld2_u8(ptr); }
+inline    int8x8x2_t vld2(const s8  * ptr) { return  vld2_s8(ptr); }
+inline  uint16x4x2_t vld2(const u16 * ptr) { return vld2_u16(ptr); }
+inline   int16x4x2_t vld2(const s16 * ptr) { return vld2_s16(ptr); }
+inline  uint32x2x2_t vld2(const u32 * ptr) { return vld2_u32(ptr); }
+inline   int32x2x2_t vld2(const s32 * ptr) { return vld2_s32(ptr); }
+inline float32x2x2_t vld2(const f32 * ptr) { return vld2_f32(ptr); }
+
+////////////////////////////// vld3q ///////////////////////
+
+inline  uint8x16x3_t vld3q(const u8  * ptr) { return  vld3q_u8(ptr); }
+inline   int8x16x3_t vld3q(const s8  * ptr) { return  vld3q_s8(ptr); }
+inline  uint16x8x3_t vld3q(const u16 * ptr) { return vld3q_u16(ptr); }
+inline   int16x8x3_t vld3q(const s16 * ptr) { return vld3q_s16(ptr); }
+inline  uint32x4x3_t vld3q(const u32 * ptr) { return vld3q_u32(ptr); }
+inline   int32x4x3_t vld3q(const s32 * ptr) { return vld3q_s32(ptr); }
+inline float32x4x3_t vld3q(const f32 * ptr) { return vld3q_f32(ptr); }
+
+////////////////////////////// vld3 ///////////////////////
+
+inline   uint8x8x3_t vld3(const u8  * ptr) { return  vld3_u8(ptr); }
+inline    int8x8x3_t vld3(const s8  * ptr) { return  vld3_s8(ptr); }
+inline  uint16x4x3_t vld3(const u16 * ptr) { return vld3_u16(ptr); }
+inline   int16x4x3_t vld3(const s16 * ptr) { return vld3_s16(ptr); }
+inline  uint32x2x3_t vld3(const u32 * ptr) { return vld3_u32(ptr); }
+inline   int32x2x3_t vld3(const s32 * ptr) { return vld3_s32(ptr); }
+inline float32x2x3_t vld3(const f32 * ptr) { return vld3_f32(ptr); }
+
+////////////////////////////// vld4q ///////////////////////
+
+inline  uint8x16x4_t vld4q(const u8  * ptr) { return  vld4q_u8(ptr); }
+inline   int8x16x4_t vld4q(const s8  * ptr) { return  vld4q_s8(ptr); }
+inline  uint16x8x4_t vld4q(const u16 * ptr) { return vld4q_u16(ptr); }
+inline   int16x8x4_t vld4q(const s16 * ptr) { return vld4q_s16(ptr); }
+inline  uint32x4x4_t vld4q(const u32 * ptr) { return vld4q_u32(ptr); }
+inline   int32x4x4_t vld4q(const s32 * ptr) { return vld4q_s32(ptr); }
+inline float32x4x4_t vld4q(const f32 * ptr) { return vld4q_f32(ptr); }
+
+////////////////////////////// vld4 ///////////////////////
+
+inline   uint8x8x4_t vld4(const u8  * ptr) { return  vld4_u8(ptr); }
+inline    int8x8x4_t vld4(const s8  * ptr) { return  vld4_s8(ptr); }
+inline  uint16x4x4_t vld4(const u16 * ptr) { return vld4_u16(ptr); }
+inline   int16x4x4_t vld4(const s16 * ptr) { return vld4_s16(ptr); }
+inline  uint32x2x4_t vld4(const u32 * ptr) { return vld4_u32(ptr); }
+inline   int32x2x4_t vld4(const s32 * ptr) { return vld4_s32(ptr); }
+inline float32x2x4_t vld4(const f32 * ptr) { return vld4_f32(ptr); }
+
+////////////////////////////// vst1q ///////////////////////
+
+inline void vst1q(u8  * ptr, const uint8x16_t  & v) { return vst1q_u8(ptr,  v); }
+inline void vst1q(s8  * ptr, const int8x16_t   & v) { return vst1q_s8(ptr,  v); }
+inline void vst1q(u16 * ptr, const uint16x8_t  & v) { return vst1q_u16(ptr, v); }
+inline void vst1q(s16 * ptr, const int16x8_t   & v) { return vst1q_s16(ptr, v); }
+inline void vst1q(u32 * ptr, const uint32x4_t  & v) { return vst1q_u32(ptr, v); }
+inline void vst1q(s32 * ptr, const int32x4_t   & v) { return vst1q_s32(ptr, v); }
+inline void vst1q(f32 * ptr, const float32x4_t & v) { return vst1q_f32(ptr, v); }
+
+////////////////////////////// vst1 ///////////////////////
+
+inline void vst1(u8  * ptr, const uint8x8_t   & v) { return vst1_u8(ptr,  v); }
+inline void vst1(s8  * ptr, const int8x8_t    & v) { return vst1_s8(ptr,  v); }
+inline void vst1(u16 * ptr, const uint16x4_t  & v) { return vst1_u16(ptr, v); }
+inline void vst1(s16 * ptr, const int16x4_t   & v) { return vst1_s16(ptr, v); }
+inline void vst1(u32 * ptr, const uint32x2_t  & v) { return vst1_u32(ptr, v); }
+inline void vst1(s32 * ptr, const int32x2_t   & v) { return vst1_s32(ptr, v); }
+inline void vst1(f32 * ptr, const float32x2_t & v) { return vst1_f32(ptr, v); }
+
+////////////////////////////// vst2q ///////////////////////
+
+inline void vst2q(u8  * ptr, const uint8x16x2_t  & v) { return vst2q_u8(ptr,  v); }
+inline void vst2q(s8  * ptr, const int8x16x2_t   & v) { return vst2q_s8(ptr,  v); }
+inline void vst2q(u16 * ptr, const uint16x8x2_t  & v) { return vst2q_u16(ptr, v); }
+inline void vst2q(s16 * ptr, const int16x8x2_t   & v) { return vst2q_s16(ptr, v); }
+inline void vst2q(u32 * ptr, const uint32x4x2_t  & v) { return vst2q_u32(ptr, v); }
+inline void vst2q(s32 * ptr, const int32x4x2_t   & v) { return vst2q_s32(ptr, v); }
+inline void vst2q(f32 * ptr, const float32x4x2_t & v) { return vst2q_f32(ptr, v); }
+
+////////////////////////////// vst2 ///////////////////////
+
+inline void vst2(u8  * ptr, const uint8x8x2_t   & v) { return vst2_u8(ptr,  v); }
+inline void vst2(s8  * ptr, const int8x8x2_t    & v) { return vst2_s8(ptr,  v); }
+inline void vst2(u16 * ptr, const uint16x4x2_t  & v) { return vst2_u16(ptr, v); }
+inline void vst2(s16 * ptr, const int16x4x2_t   & v) { return vst2_s16(ptr, v); }
+inline void vst2(u32 * ptr, const uint32x2x2_t  & v) { return vst2_u32(ptr, v); }
+inline void vst2(s32 * ptr, const int32x2x2_t   & v) { return vst2_s32(ptr, v); }
+inline void vst2(f32 * ptr, const float32x2x2_t & v) { return vst2_f32(ptr, v); }
+
+////////////////////////////// vst3q ///////////////////////
+
+inline void vst3q(u8  * ptr, const uint8x16x3_t  & v) { return vst3q_u8(ptr,  v); }
+inline void vst3q(s8  * ptr, const int8x16x3_t   & v) { return vst3q_s8(ptr,  v); }
+inline void vst3q(u16 * ptr, const uint16x8x3_t  & v) { return vst3q_u16(ptr, v); }
+inline void vst3q(s16 * ptr, const int16x8x3_t   & v) { return vst3q_s16(ptr, v); }
+inline void vst3q(u32 * ptr, const uint32x4x3_t  & v) { return vst3q_u32(ptr, v); }
+inline void vst3q(s32 * ptr, const int32x4x3_t   & v) { return vst3q_s32(ptr, v); }
+inline void vst3q(f32 * ptr, const float32x4x3_t & v) { return vst3q_f32(ptr, v); }
+
+////////////////////////////// vst3 ///////////////////////
+
+inline void vst3(u8  * ptr, const uint8x8x3_t   & v) { return vst3_u8(ptr,  v); }
+inline void vst3(s8  * ptr, const int8x8x3_t    & v) { return vst3_s8(ptr,  v); }
+inline void vst3(u16 * ptr, const uint16x4x3_t  & v) { return vst3_u16(ptr, v); }
+inline void vst3(s16 * ptr, const int16x4x3_t   & v) { return vst3_s16(ptr, v); }
+inline void vst3(u32 * ptr, const uint32x2x3_t  & v) { return vst3_u32(ptr, v); }
+inline void vst3(s32 * ptr, const int32x2x3_t   & v) { return vst3_s32(ptr, v); }
+inline void vst3(f32 * ptr, const float32x2x3_t & v) { return vst3_f32(ptr, v); }
+
+////////////////////////////// vst4q ///////////////////////
+
+inline void vst4q(u8  * ptr, const uint8x16x4_t  & v) { return vst4q_u8(ptr,  v); }
+inline void vst4q(s8  * ptr, const int8x16x4_t   & v) { return vst4q_s8(ptr,  v); }
+inline void vst4q(u16 * ptr, const uint16x8x4_t  & v) { return vst4q_u16(ptr, v); }
+inline void vst4q(s16 * ptr, const int16x8x4_t   & v) { return vst4q_s16(ptr, v); }
+inline void vst4q(u32 * ptr, const uint32x4x4_t  & v) { return vst4q_u32(ptr, v); }
+inline void vst4q(s32 * ptr, const int32x4x4_t   & v) { return vst4q_s32(ptr, v); }
+inline void vst4q(f32 * ptr, const float32x4x4_t & v) { return vst4q_f32(ptr, v); }
+
+////////////////////////////// vst4 ///////////////////////
+
+inline void vst4(u8  * ptr, const uint8x8x4_t   & v) { return vst4_u8(ptr,  v); }
+inline void vst4(s8  * ptr, const int8x8x4_t    & v) { return vst4_s8(ptr,  v); }
+inline void vst4(u16 * ptr, const uint16x4x4_t  & v) { return vst4_u16(ptr, v); }
+inline void vst4(s16 * ptr, const int16x4x4_t   & v) { return vst4_s16(ptr, v); }
+inline void vst4(u32 * ptr, const uint32x2x4_t  & v) { return vst4_u32(ptr, v); }
+inline void vst4(s32 * ptr, const int32x2x4_t   & v) { return vst4_s32(ptr, v); }
+inline void vst4(f32 * ptr, const float32x2x4_t & v) { return vst4_f32(ptr, v); }
+
+////////////////////////////// vabdq ///////////////////////
+
+inline  uint8x16_t vabdq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vabdq_u8 (v0, v1); }
+inline   int8x16_t vabdq(const int8x16_t   & v0, const int8x16_t   & v1) { return vabdq_s8 (v0, v1); }
+inline  uint16x8_t vabdq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vabdq_u16(v0, v1); }
+inline   int16x8_t vabdq(const int16x8_t   & v0, const int16x8_t   & v1) { return vabdq_s16(v0, v1); }
+inline  uint32x4_t vabdq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vabdq_u32(v0, v1); }
+inline   int32x4_t vabdq(const int32x4_t   & v0, const int32x4_t   & v1) { return vabdq_s32(v0, v1); }
+inline float32x4_t vabdq(const float32x4_t & v0, const float32x4_t & v1) { return vabdq_f32(v0, v1); }
+
+////////////////////////////// vabd ///////////////////////
+
+inline   uint8x8_t vabd(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vabd_u8 (v0, v1); }
+inline    int8x8_t vabd(const int8x8_t    & v0, const int8x8_t    & v1) { return vabd_s8 (v0, v1); }
+inline  uint16x4_t vabd(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vabd_u16(v0, v1); }
+inline   int16x4_t vabd(const int16x4_t   & v0, const int16x4_t   & v1) { return vabd_s16(v0, v1); }
+inline  uint32x2_t vabd(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vabd_u32(v0, v1); }
+inline   int32x2_t vabd(const int32x2_t   & v0, const int32x2_t   & v1) { return vabd_s32(v0, v1); }
+inline float32x2_t vabd(const float32x2_t & v0, const float32x2_t & v1) { return vabd_f32(v0, v1); }
+
+////////////////////////////// vminq ///////////////////////
+
+inline  uint8x16_t vminq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vminq_u8 (v0, v1); }
+inline   int8x16_t vminq(const int8x16_t   & v0, const int8x16_t   & v1) { return vminq_s8 (v0, v1); }
+inline  uint16x8_t vminq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vminq_u16(v0, v1); }
+inline   int16x8_t vminq(const int16x8_t   & v0, const int16x8_t   & v1) { return vminq_s16(v0, v1); }
+inline  uint32x4_t vminq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vminq_u32(v0, v1); }
+inline   int32x4_t vminq(const int32x4_t   & v0, const int32x4_t   & v1) { return vminq_s32(v0, v1); }
+inline float32x4_t vminq(const float32x4_t & v0, const float32x4_t & v1) { return vminq_f32(v0, v1); }
+
+////////////////////////////// vmin ///////////////////////
+
+inline   uint8x8_t vmin(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vmin_u8 (v0, v1); }
+inline    int8x8_t vmin(const int8x8_t    & v0, const int8x8_t    & v1) { return vmin_s8 (v0, v1); }
+inline  uint16x4_t vmin(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vmin_u16(v0, v1); }
+inline   int16x4_t vmin(const int16x4_t   & v0, const int16x4_t   & v1) { return vmin_s16(v0, v1); }
+inline  uint32x2_t vmin(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vmin_u32(v0, v1); }
+inline   int32x2_t vmin(const int32x2_t   & v0, const int32x2_t   & v1) { return vmin_s32(v0, v1); }
+inline float32x2_t vmin(const float32x2_t & v0, const float32x2_t & v1) { return vmin_f32(v0, v1); }
+
+////////////////////////////// vmaxq ///////////////////////
+
+inline  uint8x16_t vmaxq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vmaxq_u8 (v0, v1); }
+inline   int8x16_t vmaxq(const int8x16_t   & v0, const int8x16_t   & v1) { return vmaxq_s8 (v0, v1); }
+inline  uint16x8_t vmaxq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vmaxq_u16(v0, v1); }
+inline   int16x8_t vmaxq(const int16x8_t   & v0, const int16x8_t   & v1) { return vmaxq_s16(v0, v1); }
+inline  uint32x4_t vmaxq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vmaxq_u32(v0, v1); }
+inline   int32x4_t vmaxq(const int32x4_t   & v0, const int32x4_t   & v1) { return vmaxq_s32(v0, v1); }
+inline float32x4_t vmaxq(const float32x4_t & v0, const float32x4_t & v1) { return vmaxq_f32(v0, v1); }
+
+////////////////////////////// vmax ///////////////////////
+
+inline   uint8x8_t vmax(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vmax_u8 (v0, v1); }
+inline    int8x8_t vmax(const int8x8_t    & v0, const int8x8_t    & v1) { return vmax_s8 (v0, v1); }
+inline  uint16x4_t vmax(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vmax_u16(v0, v1); }
+inline   int16x4_t vmax(const int16x4_t   & v0, const int16x4_t   & v1) { return vmax_s16(v0, v1); }
+inline  uint32x2_t vmax(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vmax_u32(v0, v1); }
+inline   int32x2_t vmax(const int32x2_t   & v0, const int32x2_t   & v1) { return vmax_s32(v0, v1); }
+inline float32x2_t vmax(const float32x2_t & v0, const float32x2_t & v1) { return vmax_f32(v0, v1); }
+
+////////////////////////////// vdupq_n ///////////////////////
+
+inline  uint8x16_t vdupq_n(const u8  & val) { return  vdupq_n_u8(val); }
+inline   int8x16_t vdupq_n(const s8  & val) { return  vdupq_n_s8(val); }
+inline  uint16x8_t vdupq_n(const u16 & val) { return vdupq_n_u16(val); }
+inline   int16x8_t vdupq_n(const s16 & val) { return vdupq_n_s16(val); }
+inline  uint32x4_t vdupq_n(const u32 & val) { return vdupq_n_u32(val); }
+inline   int32x4_t vdupq_n(const s32 & val) { return vdupq_n_s32(val); }
+inline  uint64x2_t vdupq_n(const u64 & val) { return vdupq_n_u64(val); }
+inline   int64x2_t vdupq_n(const s64 & val) { return vdupq_n_s64(val); }
+inline float32x4_t vdupq_n(const f32 & val) { return vdupq_n_f32(val); }
+
+////////////////////////////// vdup_n ///////////////////////
+
+inline   uint8x8_t vdup_n(const u8  & val) { return  vdup_n_u8(val); }
+inline    int8x8_t vdup_n(const s8  & val) { return  vdup_n_s8(val); }
+inline  uint16x4_t vdup_n(const u16 & val) { return vdup_n_u16(val); }
+inline   int16x4_t vdup_n(const s16 & val) { return vdup_n_s16(val); }
+inline  uint32x2_t vdup_n(const u32 & val) { return vdup_n_u32(val); }
+inline   int32x2_t vdup_n(const s32 & val) { return vdup_n_s32(val); }
+inline  uint64x1_t vdup_n(const u64 & val) { return vdup_n_u64(val); }
+inline   int64x1_t vdup_n(const s64 & val) { return vdup_n_s64(val); }
+inline float32x2_t vdup_n(const f32 & val) { return vdup_n_f32(val); }
+
+////////////////////////////// vget_low ///////////////////////
+
+inline uint8x8_t   vget_low(const uint8x16_t  & v) { return vget_low_u8 (v); }
+inline int8x8_t    vget_low(const int8x16_t   & v) { return vget_low_s8 (v); }
+inline uint16x4_t  vget_low(const uint16x8_t  & v) { return vget_low_u16(v); }
+inline int16x4_t   vget_low(const int16x8_t   & v) { return vget_low_s16(v); }
+inline uint32x2_t  vget_low(const uint32x4_t  & v) { return vget_low_u32(v); }
+inline int32x2_t   vget_low(const int32x4_t   & v) { return vget_low_s32(v); }
+inline float32x2_t vget_low(const float32x4_t & v) { return vget_low_f32(v); }
+
+////////////////////////////// vget_high ///////////////////////
+
+inline uint8x8_t   vget_high(const uint8x16_t  & v) { return vget_high_u8 (v); }
+inline int8x8_t    vget_high(const int8x16_t   & v) { return vget_high_s8 (v); }
+inline uint16x4_t  vget_high(const uint16x8_t  & v) { return vget_high_u16(v); }
+inline int16x4_t   vget_high(const int16x8_t   & v) { return vget_high_s16(v); }
+inline uint32x2_t  vget_high(const uint32x4_t  & v) { return vget_high_u32(v); }
+inline int32x2_t   vget_high(const int32x4_t   & v) { return vget_high_s32(v); }
+inline float32x2_t vget_high(const float32x4_t & v) { return vget_high_f32(v); }
+
+////////////////////////////// vcombine ///////////////////////
+
+inline   uint8x16_t vcombine(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vcombine_u8 (v0, v1); }
+inline    int8x16_t vcombine(const int8x8_t    & v0, const int8x8_t    & v1) { return vcombine_s8 (v0, v1); }
+inline  uint16x8_t  vcombine(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vcombine_u16(v0, v1); }
+inline   int16x8_t  vcombine(const int16x4_t   & v0, const int16x4_t   & v1) { return vcombine_s16(v0, v1); }
+inline  uint32x4_t  vcombine(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vcombine_u32(v0, v1); }
+inline   int32x4_t  vcombine(const int32x2_t   & v0, const int32x2_t   & v1) { return vcombine_s32(v0, v1); }
+inline float32x4_t  vcombine(const float32x2_t & v0, const float32x2_t & v1) { return vcombine_f32(v0, v1); }
+
+////////////////////////////// vaddq ///////////////////////
+
+inline  uint8x16_t vaddq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vaddq_u8 (v0, v1); }
+inline   int8x16_t vaddq(const int8x16_t   & v0, const int8x16_t   & v1) { return vaddq_s8 (v0, v1); }
+inline  uint16x8_t vaddq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vaddq_u16(v0, v1); }
+inline   int16x8_t vaddq(const int16x8_t   & v0, const int16x8_t   & v1) { return vaddq_s16(v0, v1); }
+inline  uint32x4_t vaddq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vaddq_u32(v0, v1); }
+inline   int32x4_t vaddq(const int32x4_t   & v0, const int32x4_t   & v1) { return vaddq_s32(v0, v1); }
+inline float32x4_t vaddq(const float32x4_t & v0, const float32x4_t & v1) { return vaddq_f32(v0, v1); }
+
+////////////////////////////// vadd ///////////////////////
+
+inline   uint8x8_t vadd(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vadd_u8 (v0, v1); }
+inline    int8x8_t vadd(const int8x8_t    & v0, const int8x8_t    & v1) { return vadd_s8 (v0, v1); }
+inline  uint16x4_t vadd(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vadd_u16(v0, v1); }
+inline   int16x4_t vadd(const int16x4_t   & v0, const int16x4_t   & v1) { return vadd_s16(v0, v1); }
+inline  uint32x2_t vadd(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vadd_u32(v0, v1); }
+inline   int32x2_t vadd(const int32x2_t   & v0, const int32x2_t   & v1) { return vadd_s32(v0, v1); }
+inline float32x2_t vadd(const float32x2_t & v0, const float32x2_t & v1) { return vadd_f32(v0, v1); }
+
+////////////////////////////// vqaddq ///////////////////////
+
+inline  uint8x16_t vqaddq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vqaddq_u8 (v0, v1); }
+inline   int8x16_t vqaddq(const int8x16_t   & v0, const int8x16_t   & v1) { return vqaddq_s8 (v0, v1); }
+inline  uint16x8_t vqaddq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vqaddq_u16(v0, v1); }
+inline   int16x8_t vqaddq(const int16x8_t   & v0, const int16x8_t   & v1) { return vqaddq_s16(v0, v1); }
+inline  uint32x4_t vqaddq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vqaddq_u32(v0, v1); }
+inline   int32x4_t vqaddq(const int32x4_t   & v0, const int32x4_t   & v1) { return vqaddq_s32(v0, v1); }
+
+////////////////////////////// vqadd ///////////////////////
+
+inline   uint8x8_t vqadd(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vqadd_u8 (v0, v1); }
+inline    int8x8_t vqadd(const int8x8_t    & v0, const int8x8_t    & v1) { return vqadd_s8 (v0, v1); }
+inline  uint16x4_t vqadd(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vqadd_u16(v0, v1); }
+inline   int16x4_t vqadd(const int16x4_t   & v0, const int16x4_t   & v1) { return vqadd_s16(v0, v1); }
+inline  uint32x2_t vqadd(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vqadd_u32(v0, v1); }
+inline   int32x2_t vqadd(const int32x2_t   & v0, const int32x2_t   & v1) { return vqadd_s32(v0, v1); }
+
+////////////////////////////// vsubq ///////////////////////
+
+inline  uint8x16_t vsubq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vsubq_u8 (v0, v1); }
+inline   int8x16_t vsubq(const int8x16_t   & v0, const int8x16_t   & v1) { return vsubq_s8 (v0, v1); }
+inline  uint16x8_t vsubq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vsubq_u16(v0, v1); }
+inline   int16x8_t vsubq(const int16x8_t   & v0, const int16x8_t   & v1) { return vsubq_s16(v0, v1); }
+inline  uint32x4_t vsubq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vsubq_u32(v0, v1); }
+inline   int32x4_t vsubq(const int32x4_t   & v0, const int32x4_t   & v1) { return vsubq_s32(v0, v1); }
+inline float32x4_t vsubq(const float32x4_t & v0, const float32x4_t & v1) { return vsubq_f32(v0, v1); }
+
+////////////////////////////// vsub ///////////////////////
+
+inline   uint8x8_t vsub(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vsub_u8 (v0, v1); }
+inline    int8x8_t vsub(const int8x8_t    & v0, const int8x8_t    & v1) { return vsub_s8 (v0, v1); }
+inline  uint16x4_t vsub(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vsub_u16(v0, v1); }
+inline   int16x4_t vsub(const int16x4_t   & v0, const int16x4_t   & v1) { return vsub_s16(v0, v1); }
+inline  uint32x2_t vsub(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vsub_u32(v0, v1); }
+inline   int32x2_t vsub(const int32x2_t   & v0, const int32x2_t   & v1) { return vsub_s32(v0, v1); }
+inline float32x2_t vsub(const float32x2_t & v0, const float32x2_t & v1) { return vsub_f32(v0, v1); }
+
+////////////////////////////// vqsubq ///////////////////////
+
+inline  uint8x16_t vqsubq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vqsubq_u8 (v0, v1); }
+inline   int8x16_t vqsubq(const int8x16_t   & v0, const int8x16_t   & v1) { return vqsubq_s8 (v0, v1); }
+inline  uint16x8_t vqsubq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vqsubq_u16(v0, v1); }
+inline   int16x8_t vqsubq(const int16x8_t   & v0, const int16x8_t   & v1) { return vqsubq_s16(v0, v1); }
+inline  uint32x4_t vqsubq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vqsubq_u32(v0, v1); }
+inline   int32x4_t vqsubq(const int32x4_t   & v0, const int32x4_t   & v1) { return vqsubq_s32(v0, v1); }
+inline  uint64x2_t vqsubq(const uint64x2_t  & v0, const uint64x2_t  & v1) { return vqsubq_u64(v0, v1); }
+inline   int64x2_t vqsubq(const int64x2_t   & v0, const int64x2_t   & v1) { return vqsubq_s64(v0, v1); }
+
+////////////////////////////// vqsub ///////////////////////
+
+inline   uint8x8_t vqsub(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vqsub_u8 (v0, v1); }
+inline    int8x8_t vqsub(const int8x8_t    & v0, const int8x8_t    & v1) { return vqsub_s8 (v0, v1); }
+inline  uint16x4_t vqsub(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vqsub_u16(v0, v1); }
+inline   int16x4_t vqsub(const int16x4_t   & v0, const int16x4_t   & v1) { return vqsub_s16(v0, v1); }
+inline  uint32x2_t vqsub(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vqsub_u32(v0, v1); }
+inline   int32x2_t vqsub(const int32x2_t   & v0, const int32x2_t   & v1) { return vqsub_s32(v0, v1); }
+inline  uint64x1_t vqsub(const uint64x1_t  & v0, const uint64x1_t  & v1) { return vqsub_u64(v0, v1); }
+inline   int64x1_t vqsub(const int64x1_t   & v0, const int64x1_t   & v1) { return vqsub_s64(v0, v1); }
+
+////////////////////////////// vmull ///////////////////////
+
+inline  uint16x8_t vmull(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vmull_u8 (v0, v1); }
+inline   int16x8_t vmull(const int8x8_t    & v0, const int8x8_t    & v1) { return vmull_s8 (v0, v1); }
+inline  uint32x4_t vmull(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vmull_u16(v0, v1); }
+inline   int32x4_t vmull(const int16x4_t   & v0, const int16x4_t   & v1) { return vmull_s16(v0, v1); }
+inline  uint64x2_t vmull(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vmull_u32(v0, v1); }
+inline   int64x2_t vmull(const int32x2_t   & v0, const int32x2_t   & v1) { return vmull_s32(v0, v1); }
+
+////////////////////////////// vrev64q ///////////////////////
+
+inline uint8x16_t  vrev64q(const uint8x16_t  & v) { return vrev64q_u8 (v); }
+inline int8x16_t   vrev64q(const int8x16_t   & v) { return vrev64q_s8 (v); }
+inline uint16x8_t  vrev64q(const uint16x8_t  & v) { return vrev64q_u16(v); }
+inline int16x8_t   vrev64q(const int16x8_t   & v) { return vrev64q_s16(v); }
+inline uint32x4_t  vrev64q(const uint32x4_t  & v) { return vrev64q_u32(v); }
+inline int32x4_t   vrev64q(const int32x4_t   & v) { return vrev64q_s32(v); }
+inline float32x4_t vrev64q(const float32x4_t & v) { return vrev64q_f32(v); }
+
+////////////////////////////// vrev64 ///////////////////////
+
+inline uint8x8_t   vrev64(const uint8x8_t   & v) { return vrev64_u8 (v); }
+inline int8x8_t    vrev64(const int8x8_t    & v) { return vrev64_s8 (v); }
+inline uint16x4_t  vrev64(const uint16x4_t  & v) { return vrev64_u16(v); }
+inline int16x4_t   vrev64(const int16x4_t   & v) { return vrev64_s16(v); }
+inline uint32x2_t  vrev64(const uint32x2_t  & v) { return vrev64_u32(v); }
+inline int32x2_t   vrev64(const int32x2_t   & v) { return vrev64_s32(v); }
+inline float32x2_t vrev64(const float32x2_t & v) { return vrev64_f32(v); }
+
+////////////////////////////// vceqq ///////////////////////
+
+inline  uint8x16_t vceqq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vceqq_u8 (v0, v1); }
+inline  uint8x16_t vceqq(const int8x16_t   & v0, const int8x16_t   & v1) { return vceqq_s8 (v0, v1); }
+inline  uint16x8_t vceqq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vceqq_u16(v0, v1); }
+inline  uint16x8_t vceqq(const int16x8_t   & v0, const int16x8_t   & v1) { return vceqq_s16(v0, v1); }
+inline  uint32x4_t vceqq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vceqq_u32(v0, v1); }
+inline  uint32x4_t vceqq(const int32x4_t   & v0, const int32x4_t   & v1) { return vceqq_s32(v0, v1); }
+inline  uint32x4_t vceqq(const float32x4_t & v0, const float32x4_t & v1) { return vceqq_f32(v0, v1); }
+
+////////////////////////////// vceq ///////////////////////
+
+inline   uint8x8_t vceq(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vceq_u8 (v0, v1); }
+inline   uint8x8_t vceq(const int8x8_t    & v0, const int8x8_t    & v1) { return vceq_s8 (v0, v1); }
+inline  uint16x4_t vceq(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vceq_u16(v0, v1); }
+inline  uint16x4_t vceq(const int16x4_t   & v0, const int16x4_t   & v1) { return vceq_s16(v0, v1); }
+inline  uint32x2_t vceq(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vceq_u32(v0, v1); }
+inline  uint32x2_t vceq(const int32x2_t   & v0, const int32x2_t   & v1) { return vceq_s32(v0, v1); }
+inline  uint32x2_t vceq(const float32x2_t & v0, const float32x2_t & v1) { return vceq_f32(v0, v1); }
+
+////////////////////////////// vcgtq ///////////////////////
+
+inline  uint8x16_t vcgtq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vcgtq_u8 (v0, v1); }
+inline  uint8x16_t vcgtq(const int8x16_t   & v0, const int8x16_t   & v1) { return vcgtq_s8 (v0, v1); }
+inline  uint16x8_t vcgtq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vcgtq_u16(v0, v1); }
+inline  uint16x8_t vcgtq(const int16x8_t   & v0, const int16x8_t   & v1) { return vcgtq_s16(v0, v1); }
+inline  uint32x4_t vcgtq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vcgtq_u32(v0, v1); }
+inline  uint32x4_t vcgtq(const int32x4_t   & v0, const int32x4_t   & v1) { return vcgtq_s32(v0, v1); }
+inline  uint32x4_t vcgtq(const float32x4_t & v0, const float32x4_t & v1) { return vcgtq_f32(v0, v1); }
+
+////////////////////////////// vcgt ///////////////////////
+
+inline   uint8x8_t vcgt(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vcgt_u8 (v0, v1); }
+inline   uint8x8_t vcgt(const int8x8_t    & v0, const int8x8_t    & v1) { return vcgt_s8 (v0, v1); }
+inline  uint16x4_t vcgt(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vcgt_u16(v0, v1); }
+inline  uint16x4_t vcgt(const int16x4_t   & v0, const int16x4_t   & v1) { return vcgt_s16(v0, v1); }
+inline  uint32x2_t vcgt(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vcgt_u32(v0, v1); }
+inline  uint32x2_t vcgt(const int32x2_t   & v0, const int32x2_t   & v1) { return vcgt_s32(v0, v1); }
+inline  uint32x2_t vcgt(const float32x2_t & v0, const float32x2_t & v1) { return vcgt_f32(v0, v1); }
+
+////////////////////////////// vcgeq ///////////////////////
+
+inline  uint8x16_t vcgeq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vcgeq_u8 (v0, v1); }
+inline  uint8x16_t vcgeq(const int8x16_t   & v0, const int8x16_t   & v1) { return vcgeq_s8 (v0, v1); }
+inline  uint16x8_t vcgeq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vcgeq_u16(v0, v1); }
+inline  uint16x8_t vcgeq(const int16x8_t   & v0, const int16x8_t   & v1) { return vcgeq_s16(v0, v1); }
+inline  uint32x4_t vcgeq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vcgeq_u32(v0, v1); }
+inline  uint32x4_t vcgeq(const int32x4_t   & v0, const int32x4_t   & v1) { return vcgeq_s32(v0, v1); }
+inline  uint32x4_t vcgeq(const float32x4_t & v0, const float32x4_t & v1) { return vcgeq_f32(v0, v1); }
+
+////////////////////////////// vcge ///////////////////////
+
+inline   uint8x8_t vcge(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vcge_u8 (v0, v1); }
+inline   uint8x8_t vcge(const int8x8_t    & v0, const int8x8_t    & v1) { return vcge_s8 (v0, v1); }
+inline  uint16x4_t vcge(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vcge_u16(v0, v1); }
+inline  uint16x4_t vcge(const int16x4_t   & v0, const int16x4_t   & v1) { return vcge_s16(v0, v1); }
+inline  uint32x2_t vcge(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vcge_u32(v0, v1); }
+inline  uint32x2_t vcge(const int32x2_t   & v0, const int32x2_t   & v1) { return vcge_s32(v0, v1); }
+inline  uint32x2_t vcge(const float32x2_t & v0, const float32x2_t & v1) { return vcge_f32(v0, v1); }
+
+////////////////////////////// vandq ///////////////////////
+
+inline  uint8x16_t vandq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vandq_u8 (v0, v1); }
+inline   int8x16_t vandq(const int8x16_t   & v0, const int8x16_t   & v1) { return vandq_s8 (v0, v1); }
+inline  uint16x8_t vandq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vandq_u16(v0, v1); }
+inline   int16x8_t vandq(const int16x8_t   & v0, const int16x8_t   & v1) { return vandq_s16(v0, v1); }
+inline  uint32x4_t vandq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vandq_u32(v0, v1); }
+inline   int32x4_t vandq(const int32x4_t   & v0, const int32x4_t   & v1) { return vandq_s32(v0, v1); }
+
+////////////////////////////// vand ///////////////////////
+
+inline   uint8x8_t vand(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vand_u8 (v0, v1); }
+inline    int8x8_t vand(const int8x8_t    & v0, const int8x8_t    & v1) { return vand_s8 (v0, v1); }
+inline  uint16x4_t vand(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vand_u16(v0, v1); }
+inline   int16x4_t vand(const int16x4_t   & v0, const int16x4_t   & v1) { return vand_s16(v0, v1); }
+inline  uint32x2_t vand(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vand_u32(v0, v1); }
+inline   int32x2_t vand(const int32x2_t   & v0, const int32x2_t   & v1) { return vand_s32(v0, v1); }
+
+////////////////////////////// vmovn ///////////////////////
+
+inline uint8x8_t   vmovn(const uint16x8_t  & v) { return vmovn_u16(v); }
+inline int8x8_t    vmovn(const int16x8_t   & v) { return vmovn_s16(v); }
+inline uint16x4_t  vmovn(const uint32x4_t  & v) { return vmovn_u32(v); }
+inline int16x4_t   vmovn(const int32x4_t   & v) { return vmovn_s32(v); }
+inline uint32x2_t  vmovn(const uint64x2_t  & v) { return vmovn_u64(v); }
+inline int32x2_t   vmovn(const int64x2_t   & v) { return vmovn_s64(v); }
+
+////////////////////////////// vqmovn ///////////////////////
+
+inline uint8x8_t   vqmovn(const uint16x8_t  & v) { return vqmovn_u16(v); }
+inline int8x8_t    vqmovn(const int16x8_t   & v) { return vqmovn_s16(v); }
+inline uint16x4_t  vqmovn(const uint32x4_t  & v) { return vqmovn_u32(v); }
+inline int16x4_t   vqmovn(const int32x4_t   & v) { return vqmovn_s32(v); }
+inline uint32x2_t  vqmovn(const uint64x2_t  & v) { return vqmovn_u64(v); }
+inline int32x2_t   vqmovn(const int64x2_t   & v) { return vqmovn_s64(v); }
+
+////////////////////////////// vmovl ///////////////////////
+
+inline uint16x8_t  vmovl(const uint8x8_t   & v) { return  vmovl_u8(v); }
+inline int16x8_t   vmovl(const int8x8_t    & v) { return  vmovl_s8(v); }
+inline uint32x4_t  vmovl(const uint16x4_t  & v) { return vmovl_u16(v); }
+inline int32x4_t   vmovl(const int16x4_t   & v) { return vmovl_s16(v); }
+
+////////////////////////////// vmvnq ///////////////////////
+
+inline uint8x16_t  vmvnq(const uint8x16_t  & v) { return vmvnq_u8 (v); }
+inline int8x16_t   vmvnq(const int8x16_t   & v) { return vmvnq_s8 (v); }
+inline uint16x8_t  vmvnq(const uint16x8_t  & v) { return vmvnq_u16(v); }
+inline int16x8_t   vmvnq(const int16x8_t   & v) { return vmvnq_s16(v); }
+inline uint32x4_t  vmvnq(const uint32x4_t  & v) { return vmvnq_u32(v); }
+inline int32x4_t   vmvnq(const int32x4_t   & v) { return vmvnq_s32(v); }
+
+////////////////////////////// vmvn ///////////////////////
+
+inline uint8x8_t   vmvn(const uint8x8_t   & v) { return vmvn_u8 (v); }
+inline int8x8_t    vmvn(const int8x8_t    & v) { return vmvn_s8 (v); }
+inline uint16x4_t  vmvn(const uint16x4_t  & v) { return vmvn_u16(v); }
+inline int16x4_t   vmvn(const int16x4_t   & v) { return vmvn_s16(v); }
+inline uint32x2_t  vmvn(const uint32x2_t  & v) { return vmvn_u32(v); }
+inline int32x2_t   vmvn(const int32x2_t   & v) { return vmvn_s32(v); }
+
+////////////////////////////// vbicq ///////////////////////
+
+inline  uint8x16_t vbicq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vbicq_u8 (v0, v1); }
+inline   int8x16_t vbicq(const int8x16_t   & v0, const int8x16_t   & v1) { return vbicq_s8 (v0, v1); }
+inline  uint16x8_t vbicq(const uint16x8_t  & v0, const uint16x8_t  & v1) { return vbicq_u16(v0, v1); }
+inline   int16x8_t vbicq(const int16x8_t   & v0, const int16x8_t   & v1) { return vbicq_s16(v0, v1); }
+inline  uint32x4_t vbicq(const uint32x4_t  & v0, const uint32x4_t  & v1) { return vbicq_u32(v0, v1); }
+inline   int32x4_t vbicq(const int32x4_t   & v0, const int32x4_t   & v1) { return vbicq_s32(v0, v1); }
+inline  uint64x2_t vbicq(const uint64x2_t  & v0, const uint64x2_t  & v1) { return vbicq_u64(v0, v1); }
+inline   int64x2_t vbicq(const int64x2_t   & v0, const int64x2_t   & v1) { return vbicq_s64(v0, v1); }
+
+////////////////////////////// vbic ///////////////////////
+
+inline   uint8x8_t vbic(const uint8x8_t   & v0, const uint8x8_t   & v1) { return vbic_u8 (v0, v1); }
+inline    int8x8_t vbic(const int8x8_t    & v0, const int8x8_t    & v1) { return vbic_s8 (v0, v1); }
+inline  uint16x4_t vbic(const uint16x4_t  & v0, const uint16x4_t  & v1) { return vbic_u16(v0, v1); }
+inline   int16x4_t vbic(const int16x4_t   & v0, const int16x4_t   & v1) { return vbic_s16(v0, v1); }
+inline  uint32x2_t vbic(const uint32x2_t  & v0, const uint32x2_t  & v1) { return vbic_u32(v0, v1); }
+inline   int32x2_t vbic(const int32x2_t   & v0, const int32x2_t   & v1) { return vbic_s32(v0, v1); }
+inline  uint64x1_t vbic(const uint64x1_t  & v0, const uint64x1_t  & v1) { return vbic_u64(v0, v1); }
+inline   int64x1_t vbic(const int64x1_t   & v0, const int64x1_t   & v1) { return vbic_s64(v0, v1); }
+
+////////////////////////////// vtransform ///////////////////////
+
+template <typename Op>
+void vtransform(Size2D size,
+                const typename Op::type * src0Base, ptrdiff_t src0Stride,
+                const typename Op::type * src1Base, ptrdiff_t src1Stride,
+                typename Op::type * dstBase, ptrdiff_t dstStride, const Op & op)
+{
+    typedef typename Op::type type;
+    typedef typename VecTraits<type>::vec128 vec128;
+    typedef typename VecTraits<type>::vec64 vec64;
+
+    if (src0Stride == src1Stride && src0Stride == dstStride &&
+        src0Stride == (ptrdiff_t)(size.width * sizeof(type)))
+    {
+        size.width *= size.height;
+        size.height = 1;
+    }
+
+    const size_t step_base = 32 / sizeof(type);
+    size_t roiw_base = size.width >= (step_base - 1) ? size.width - step_base + 1 : 0;
+    const size_t step_tail = 8 / sizeof(type);
+    size_t roiw_tail = size.width >= (step_tail - 1) ? size.width - step_tail + 1 : 0;
+
+    for (size_t y = 0; y < size.height; ++y)
+    {
+        const type * src0 = internal::getRowPtr(src0Base, src0Stride, y);
+        const type * src1 = internal::getRowPtr(src1Base, src1Stride, y);
+        typename Op::type * dst = internal::getRowPtr(dstBase, dstStride, y);
+        size_t x = 0;
+
+        for( ; x < roiw_base; x += step_base )
+        {
+            internal::prefetch(src0 + x);
+            internal::prefetch(src1 + x);
+
+            vec128 v_src00 = vld1q(src0 + x), v_src01 = vld1q(src0 + x + 16 / sizeof(type));
+            vec128 v_src10 = vld1q(src1 + x), v_src11 = vld1q(src1 + x + 16 / sizeof(type));
+            vec128 v_dst;
+
+            op(v_src00, v_src10, v_dst);
+            vst1q(dst + x, v_dst);
+
+            op(v_src01, v_src11, v_dst);
+            vst1q(dst + x + 16 / sizeof(type), v_dst);
+        }
+        for( ; x < roiw_tail; x += step_tail )
+        {
+            vec64 v_src0 = vld1(src0 + x);
+            vec64 v_src1 = vld1(src1 + x);
+            vec64 v_dst;
+
+            op(v_src0, v_src1, v_dst);
+            vst1(dst + x, v_dst);
+        }
+
+        for (; x < size.width; ++x)
+        {
+            op(src0 + x, src1 + x, dst + x);
+        }
+    }
+}
+
+} }
+
+#endif // CAROTENE_NEON
+
+#endif

3rdparty/carotene/src/warp_affine.cpp

@@ -0,0 +1,434 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "remap.hpp"
+
+namespace CAROTENE_NS {
+
+bool isWarpAffineNearestNeighborSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+bool isWarpAffineLinearSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+void warpAffineNearestNeighbor(const Size2D &ssize, const Size2D &dsize,
+                               const u8 * srcBase, ptrdiff_t srcStride,
+                               const f32 * m,
+                               u8 * dstBase, ptrdiff_t dstStride,
+                               BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isWarpAffineNearestNeighborSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16];
+    s32 * map = alignPtr(_map, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride);
+    float32x4_t v_4 = vdupq_n_f32(4.0f);
+
+    float32x4_t v_m0 = vdupq_n_f32(m[0]);
+    float32x4_t v_m1 = vdupq_n_f32(m[1]);
+    float32x4_t v_m2 = vdupq_n_f32(m[2]);
+    float32x4_t v_m3 = vdupq_n_f32(m[3]);
+    float32x4_t v_m4 = vdupq_n_f32(m[4]);
+    float32x4_t v_m5 = vdupq_n_f32(m[5]);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+
+                        int32x4_t v_src_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_src_xf)));
+                        int32x4_t v_src_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_src_yf)));
+                        int32x4_t v_src_index = vmlaq_s32(v_src_x, v_src_y, v_step4);
+                        vst1q_s32(map_row + x, v_src_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 src_x_f = m[0] * x_ + yx;
+                        f32 src_y_f = m[1] * x_ + yy;
+                        s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f);
+
+                        src_x = std::max(0, std::min<s32>(ssize.width - 1, src_x));
+                        src_y = std::max(0, std::min<s32>(ssize.height - 1, src_y));
+                        map_row[x] = src_y * srcStride + src_x;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                                        getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+
+                        int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf);
+                        uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x, v_width4)),
+                                                      vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y, v_height4)));
+                        int32x4_t v_src_index = vbslq_s32(v_mask, vmlaq_s32(v_src_x, v_src_y, v_step4), v_m1_4);
+                        vst1q_s32(map_row + x, v_src_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 src_x_f = m[0] * x_ + yx;
+                        f32 src_y_f = m[1] * x_ + yy;
+                        s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f);
+
+                        map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) &&
+                                     (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                                    getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)m;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+void warpAffineLinear(const Size2D &ssize, const Size2D &dsize,
+                      const u8 * srcBase, ptrdiff_t srcStride,
+                      const f32 * m,
+                      u8 * dstBase, ptrdiff_t dstStride,
+                      BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isWarpAffineLinearSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16];
+    f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16];
+    s32 * map = alignPtr(_map, 16);
+    f32 * coeffs = alignPtr(_coeffs, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1);
+    float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f);
+
+    float32x4_t v_m0 = vdupq_n_f32(m[0]);
+    float32x4_t v_m1 = vdupq_n_f32(m[1]);
+    float32x4_t v_m2 = vdupq_n_f32(m[2]);
+    float32x4_t v_m3 = vdupq_n_f32(m[3]);
+    float32x4_t v_m4 = vdupq_n_f32(m[4]);
+    float32x4_t v_m5 = vdupq_n_f32(m[5]);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_), v_4 = vdupq_n_f32(4.0f);
+                    float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+
+                        int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf);
+
+                        float32x4x2_t v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x));
+                        v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x);
+                        v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y);
+
+                        int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x));
+                        int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y));
+                        int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x)));
+                        int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y)));
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 src_x_f = m[0] * x_ + yx;
+                        f32 src_y_f = m[1] * x_ + yy;
+
+                        s32 src0_x = (s32)floorf(src_x_f);
+                        s32 src0_y = (s32)floorf(src_y_f);
+
+                        coeff_row[(x << 1) + 0] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        s32 src1_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y + 1));
+                        src0_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y));
+                        s32 src1_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x + 1));
+                        src0_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x));
+
+                        map_row[(x << 2) + 0] = src0_y * srcStride + src0_x;
+                        map_row[(x << 2) + 1] = src0_y * srcStride + src1_x;
+                        map_row[(x << 2) + 2] = src1_y * srcStride + src0_x;
+                        map_row[(x << 2) + 3] = src1_y * srcStride + src1_x;
+                    }
+                }
+
+                remapLinearReplicate(Size2D(blockWidth, blockHeight),
+                                     srcBase, &map[0], &coeffs[0],
+                                     getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_), v_4 = vdupq_n_f32(4.0f);
+                    float32x4_t v_yx = vmlaq_f32(v_m4, v_m2, v_y), v_yy = vmlaq_f32(v_m5, v_m3, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+
+                        int32x4_t v_src_x0 = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y0 = vcvtq_s32_f32(v_src_yf);
+
+                        float32x4x2_t v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x0));
+                        v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y0));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0);
+                        v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0);
+
+                        int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1);
+                        int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1);
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4);
+
+                        uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x0, v_width4));
+                        uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_xf, v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4));
+                        uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y0, v_height4));
+                        uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_yf, v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4));
+
+                        v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4);
+                        v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4);
+                        v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4);
+                        v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[2] * y_ + m[4], yy = m[3] * y_ + m[5];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 src_x_f = m[0] * x_ + yx;
+                        f32 src_y_f = m[1] * x_ + yy;
+
+                        s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1;
+                        s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1;
+
+                        coeff_row[(x << 1) + 0] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1;
+                        map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1;
+                    }
+                }
+
+                remapLinearConst(Size2D(blockWidth, blockHeight),
+                                 srcBase, &map[0], &coeffs[0],
+                                 getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)m;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/carotene/src/warp_perspective.cpp

@@ -0,0 +1,464 @@
+/*
+ * By downloading, copying, installing or using the software you agree to this license.
+ * If you do not agree to this license, do not download, install,
+ * copy or use the software.
+ *
+ *
+ *                           License Agreement
+ *                For Open Source Computer Vision Library
+ *                        (3-clause BSD License)
+ *
+ * Copyright (C) 2015, NVIDIA Corporation, all rights reserved.
+ * Third party copyrights are property of their respective owners.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *   * Neither the names of the copyright holders nor the names of the contributors
+ *     may be used to endorse or promote products derived from this software
+ *     without specific prior written permission.
+ *
+ * This software is provided by the copyright holders and contributors "as is" and
+ * any express or implied warranties, including, but not limited to, the implied
+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
+ * In no event shall copyright holders or contributors be liable for any direct,
+ * indirect, incidental, special, exemplary, or consequential damages
+ * (including, but not limited to, procurement of substitute goods or services;
+ * loss of use, data, or profits; or business interruption) however caused
+ * and on any theory of liability, whether in contract, strict liability,
+ * or tort (including negligence or otherwise) arising in any way out of
+ * the use of this software, even if advised of the possibility of such damage.
+ */
+
+
+
+#include "remap.hpp"
+
+namespace CAROTENE_NS {
+
+bool isWarpPerspectiveNearestNeighborSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+bool isWarpPerspectiveLinearSupported(const Size2D &ssize)
+{
+#if SIZE_MAX > UINT32_MAX
+    return !(ssize.width > 0xffffFFFF || ssize.height > 0xffffFFFF) && // Restrict image size since internal index evaluation
+                                                                       // is performed with u32
+           isSupportedConfiguration();
+#else
+    (void)ssize;
+    return isSupportedConfiguration();
+#endif
+}
+
+void warpPerspectiveNearestNeighbor(const Size2D &ssize, const Size2D &dsize,
+                                    const u8 * srcBase, ptrdiff_t srcStride,
+                                    const f32 * m,
+                                    u8 * dstBase, ptrdiff_t dstStride,
+                                    BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isWarpPerspectiveNearestNeighborSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[BLOCK_SIZE * BLOCK_SIZE + 16];
+    s32 * map = alignPtr(_map, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride);
+    float32x4_t v_4 = vdupq_n_f32(4.0f);
+
+    float32x4_t v_m0 = vdupq_n_f32(m[0]);
+    float32x4_t v_m1 = vdupq_n_f32(m[1]);
+    float32x4_t v_m2 = vdupq_n_f32(m[2]);
+    float32x4_t v_m3 = vdupq_n_f32(m[3]);
+    float32x4_t v_m4 = vdupq_n_f32(m[4]);
+    float32x4_t v_m5 = vdupq_n_f32(m[5]);
+    float32x4_t v_m6 = vdupq_n_f32(m[6]);
+    float32x4_t v_m7 = vdupq_n_f32(m[7]);
+    float32x4_t v_m8 = vdupq_n_f32(m[8]);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y),
+                        v_yw = vmlaq_f32(v_m8, v_m5, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+                        float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x));
+                        v_src_xf = vmulq_f32(v_wf, v_src_xf);
+                        v_src_yf = vmulq_f32(v_wf, v_src_yf);
+
+                        int32x4_t v_src_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vcvtq_s32_f32(v_src_xf)));
+                        int32x4_t v_src_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vcvtq_s32_f32(v_src_yf)));
+                        int32x4_t v_src_index = vmlaq_s32(v_src_x, v_src_y, v_step4);
+                        vst1q_s32(map_row + x, v_src_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 w_f = 1.0f / (m[2] * x_ + yw);
+                        f32 src_x_f = (m[0] * x_ + yx) * w_f;
+                        f32 src_y_f = (m[1] * x_ + yy) * w_f;
+                        s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f);
+
+                        src_x = std::max(0, std::min<s32>(ssize.width - 1, src_x));
+                        src_y = std::max(0, std::min<s32>(ssize.height - 1, src_y));
+                        map_row[x] = src_y * srcStride + src_x;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborReplicate(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                              getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(&map[0], blockWidth * sizeof(s32), y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y),
+                        v_yw = vmlaq_f32(v_m8, v_m5, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+                        float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x));
+                        v_src_xf = vmulq_f32(v_wf, v_src_xf);
+                        v_src_yf = vmulq_f32(v_wf, v_src_yf);
+
+                        int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf);
+                        uint32x4_t v_mask = vandq_u32(vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x, v_width4)),
+                                                      vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y, v_height4)));
+                        int32x4_t v_src_index = vbslq_s32(v_mask, vmlaq_s32(v_src_x, v_src_y, v_step4), v_m1_4);
+                        vst1q_s32(map_row + x, v_src_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 w_f = 1.0f / (m[2] * x_ + yw);
+                        f32 src_x_f = (m[0] * x_ + yx) * w_f;
+                        f32 src_y_f = (m[1] * x_ + yy) * w_f;
+                        s32 src_x = floorf(src_x_f), src_y = floorf(src_y_f);
+
+                        map_row[x] = (src_x >= 0) && (src_x < (s32)ssize.width) &&
+                                     (src_y >= 0) && (src_y < (s32)ssize.height) ? src_y * srcStride + src_x : -1;
+                    }
+                }
+
+                // make remap
+                remapNearestNeighborConst(Size2D(blockWidth, blockHeight), srcBase, &map[0],
+                                          getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)m;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+void warpPerspectiveLinear(const Size2D &ssize, const Size2D &dsize,
+                           const u8 * srcBase, ptrdiff_t srcStride,
+                           const f32 * m,
+                           u8 * dstBase, ptrdiff_t dstStride,
+                           BORDER_MODE borderMode, u8 borderValue)
+{
+    internal::assertSupportedConfiguration(isWarpPerspectiveLinearSupported(ssize));
+#ifdef CAROTENE_NEON
+    using namespace internal;
+
+    s32 _map[((BLOCK_SIZE * BLOCK_SIZE) << 2) + 16];
+    f32 _coeffs[((BLOCK_SIZE * BLOCK_SIZE) << 1) + 16];
+    s32 * map = alignPtr(_map, 16);
+    f32 * coeffs = alignPtr(_coeffs, 16);
+
+    int32x4_t v_width4 = vdupq_n_s32(ssize.width - 1), v_height4 = vdupq_n_s32(ssize.height - 1);
+    int32x4_t v_step4 = vdupq_n_s32(srcStride), v_1 = vdupq_n_s32(1);
+    float32x4_t v_zero4f = vdupq_n_f32(0.0f), v_one4f = vdupq_n_f32(1.0f);
+
+    float32x4_t v_4 = vdupq_n_f32(4.0f);
+
+    float32x4_t v_m0 = vdupq_n_f32(m[0]);
+    float32x4_t v_m1 = vdupq_n_f32(m[1]);
+    float32x4_t v_m2 = vdupq_n_f32(m[2]);
+    float32x4_t v_m3 = vdupq_n_f32(m[3]);
+    float32x4_t v_m4 = vdupq_n_f32(m[4]);
+    float32x4_t v_m5 = vdupq_n_f32(m[5]);
+    float32x4_t v_m6 = vdupq_n_f32(m[6]);
+    float32x4_t v_m7 = vdupq_n_f32(m[7]);
+    float32x4_t v_m8 = vdupq_n_f32(m[8]);
+
+    if (borderMode == BORDER_MODE_REPLICATE)
+    {
+        int32x4_t v_zero4 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y),
+                        v_yw = vmlaq_f32(v_m8, v_m5, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+                        float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x));
+                        v_src_xf = vmulq_f32(v_wf, v_src_xf);
+                        v_src_yf = vmulq_f32(v_wf, v_src_yf);
+
+                        int32x4_t v_src_x = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y = vcvtq_s32_f32(v_src_yf);
+
+                        float32x4x2_t v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x));
+                        v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x = vbslq_s32(v_maskx, vsubq_s32(v_src_x, v_1), v_src_x);
+                        v_src_y = vbslq_s32(v_masky, vsubq_s32(v_src_y, v_1), v_src_y);
+
+                        int32x4_t v_dst0_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, v_src_x));
+                        int32x4_t v_dst0_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, v_src_y));
+                        int32x4_t v_dst1_x = vmaxq_s32(v_zero4, vminq_s32(v_width4, vaddq_s32(v_1, v_src_x)));
+                        int32x4_t v_dst1_y = vmaxq_s32(v_zero4, vminq_s32(v_height4, vaddq_s32(v_1, v_src_y)));
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_dst0_x, v_dst0_y, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_dst1_x, v_dst0_y, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_dst0_x, v_dst1_y, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_dst1_x, v_dst1_y, v_step4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 w_f = 1.0f / (m[2] * x_ + yw);
+                        f32 src_x_f = (m[0] * x_ + yx) * w_f;
+                        f32 src_y_f = (m[1] * x_ + yy) * w_f;
+
+                        s32 src0_x = (s32)floorf(src_x_f);
+                        s32 src0_y = (s32)floorf(src_y_f);
+
+                        coeff_row[(x << 1) + 0] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        s32 src1_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y + 1));
+                        src0_y = std::max(0, std::min<s32>(ssize.height - 1, src0_y));
+                        s32 src1_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x + 1));
+                        src0_x = std::max(0, std::min<s32>(ssize.width - 1, src0_x));
+
+                        map_row[(x << 2) + 0] = src0_y * srcStride + src0_x;
+                        map_row[(x << 2) + 1] = src0_y * srcStride + src1_x;
+                        map_row[(x << 2) + 2] = src1_y * srcStride + src0_x;
+                        map_row[(x << 2) + 3] = src1_y * srcStride + src1_x;
+                    }
+                }
+
+                remapLinearReplicate(Size2D(blockWidth, blockHeight),
+                                     srcBase, &map[0], &coeffs[0],
+                                     getRowPtr(dstBase, dstStride, i) + j, dstStride);
+            }
+        }
+    }
+    else if (borderMode == BORDER_MODE_CONSTANT)
+    {
+        float32x4_t v_zero4 = vdupq_n_f32(0.0f);
+        int32x4_t v_m1_4 = vdupq_n_s32(-1);
+
+        for (size_t i = 0; i < dsize.height; i += BLOCK_SIZE)
+        {
+            size_t blockHeight = std::min<size_t>(BLOCK_SIZE, dsize.height - i);
+            for (size_t j = 0; j < dsize.width; j += BLOCK_SIZE)
+            {
+                size_t blockWidth = std::min<size_t>(BLOCK_SIZE, dsize.width - j);
+
+                // compute table
+                for (size_t y = 0; y < blockHeight; ++y)
+                {
+                    s32 * map_row = getRowPtr(map, blockWidth * sizeof(s32) * 4, y);
+                    f32 * coeff_row = getRowPtr(coeffs, blockWidth * sizeof(f32) * 2, y);
+
+                    size_t x = 0, y_ = y + i;
+                    f32 indeces[4] = { j + 0.0f, j + 1.0f, j + 2.0f, j + 3.0f };
+                    float32x4_t v_x = vld1q_f32(indeces), v_y = vdupq_n_f32(y_);
+                    float32x4_t v_yx = vmlaq_f32(v_m6, v_m3, v_y), v_yy = vmlaq_f32(v_m7, v_m4, v_y),
+                        v_yw = vmlaq_f32(v_m8, v_m5, v_y);
+
+                    for ( ; x + 4 <= blockWidth; x += 4)
+                    {
+                        float32x4_t v_src_xf = vmlaq_f32(v_yx, v_m0, v_x);
+                        float32x4_t v_src_yf = vmlaq_f32(v_yy, v_m1, v_x);
+                        float32x4_t v_wf = vrecpq_f32(vmlaq_f32(v_yw, v_m2, v_x));
+                        v_src_xf = vmulq_f32(v_wf, v_src_xf);
+                        v_src_yf = vmulq_f32(v_wf, v_src_yf);
+
+                        int32x4_t v_src_x0 = vcvtq_s32_f32(v_src_xf);
+                        int32x4_t v_src_y0 = vcvtq_s32_f32(v_src_yf);
+
+                        float32x4x2_t v_coeff;
+                        v_coeff.val[0] = vsubq_f32(v_src_xf, vcvtq_f32_s32(v_src_x0));
+                        v_coeff.val[1] = vsubq_f32(v_src_yf, vcvtq_f32_s32(v_src_y0));
+                        uint32x4_t v_maskx = vcltq_f32(v_coeff.val[0], v_zero4f);
+                        uint32x4_t v_masky = vcltq_f32(v_coeff.val[1], v_zero4f);
+                        v_coeff.val[0] = vbslq_f32(v_maskx, vaddq_f32(v_one4f, v_coeff.val[0]), v_coeff.val[0]);
+                        v_coeff.val[1] = vbslq_f32(v_masky, vaddq_f32(v_one4f, v_coeff.val[1]), v_coeff.val[1]);
+                        v_src_x0 = vbslq_s32(v_maskx, vsubq_s32(v_src_x0, v_1), v_src_x0);
+                        v_src_y0 = vbslq_s32(v_masky, vsubq_s32(v_src_y0, v_1), v_src_y0);
+
+                        int32x4_t v_src_x1 = vaddq_s32(v_src_x0, v_1);
+                        int32x4_t v_src_y1 = vaddq_s32(v_src_y0, v_1);
+
+                        int32x4x4_t v_dst_index;
+                        v_dst_index.val[0] = vmlaq_s32(v_src_x0, v_src_y0, v_step4);
+                        v_dst_index.val[1] = vmlaq_s32(v_src_x1, v_src_y0, v_step4);
+                        v_dst_index.val[2] = vmlaq_s32(v_src_x0, v_src_y1, v_step4);
+                        v_dst_index.val[3] = vmlaq_s32(v_src_x1, v_src_y1, v_step4);
+
+                        uint32x4_t v_mask_x0 = vandq_u32(vcgeq_f32(v_src_xf, v_zero4), vcleq_s32(v_src_x0, v_width4));
+                        uint32x4_t v_mask_x1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_xf, v_one4f), v_zero4), vcleq_s32(v_src_x1, v_width4));
+                        uint32x4_t v_mask_y0 = vandq_u32(vcgeq_f32(v_src_yf, v_zero4), vcleq_s32(v_src_y0, v_height4));
+                        uint32x4_t v_mask_y1 = vandq_u32(vcgeq_f32(vaddq_f32(v_src_yf, v_one4f), v_zero4), vcleq_s32(v_src_y1, v_height4));
+
+                        v_dst_index.val[0] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y0), v_dst_index.val[0], v_m1_4);
+                        v_dst_index.val[1] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y0), v_dst_index.val[1], v_m1_4);
+                        v_dst_index.val[2] = vbslq_s32(vandq_u32(v_mask_x0, v_mask_y1), v_dst_index.val[2], v_m1_4);
+                        v_dst_index.val[3] = vbslq_s32(vandq_u32(v_mask_x1, v_mask_y1), v_dst_index.val[3], v_m1_4);
+
+                        vst2q_f32(coeff_row + (x << 1), v_coeff);
+                        vst4q_s32(map_row + (x << 2), v_dst_index);
+
+                        v_x = vaddq_f32(v_x, v_4);
+                    }
+
+                    f32 yx = m[3] * y_ + m[6], yy = m[4] * y_ + m[7], yw = m[5] * y_ + m[8];
+                    for (ptrdiff_t x_ = x + j; x < blockWidth; ++x, ++x_)
+                    {
+                        f32 w_f = 1.0f / (m[2] * x_ + yw);
+                        f32 src_x_f = (m[0] * x_ + yx) * w_f;
+                        f32 src_y_f = (m[1] * x_ + yy) * w_f;
+
+                        s32 src0_x = (s32)floorf(src_x_f), src1_x = src0_x + 1;
+                        s32 src0_y = (s32)floorf(src_y_f), src1_y = src0_y + 1;
+
+                        coeff_row[(x << 1) + 0] = src_x_f - src0_x;
+                        coeff_row[(x << 1) + 1] = src_y_f - src0_y;
+
+                        map_row[(x << 2) + 0] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 1] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src0_y >= 0) && (src0_y < (s32)ssize.height) ? src0_y * srcStride + src1_x : -1;
+                        map_row[(x << 2) + 2] = (src0_x >= 0) && (src0_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src0_x : -1;
+                        map_row[(x << 2) + 3] = (src1_x >= 0) && (src1_x < (s32)ssize.width) &&
+                                                (src1_y >= 0) && (src1_y < (s32)ssize.height) ? src1_y * srcStride + src1_x : -1;
+                    }
+                }
+
+                remapLinearConst(Size2D(blockWidth, blockHeight),
+                                 srcBase, &map[0], &coeffs[0],
+                                 getRowPtr(dstBase, dstStride, i) + j, dstStride, borderValue);
+            }
+        }
+    }
+#else
+    (void)ssize;
+    (void)dsize;
+    (void)srcBase;
+    (void)srcStride;
+    (void)m;
+    (void)dstBase;
+    (void)dstStride;
+    (void)borderMode;
+    (void)borderValue;
+#endif
+}
+
+} // namespace CAROTENE_NS

3rdparty/cpufeatures/CMakeLists.txt

@@ -0,0 +1,31 @@
+if(NOT ANDROID)
+  message("cpufeatures is ANDROID project")
+endif()
+
+set(CPUFEATURES_ROOT "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH "Android cpufeatures project sources (for example, <android-ndk>/sources/android/cpufeatures)")
+
+set(CPUFEATURES_INCLUDE_DIRS ${CPUFEATURES_ROOT} CACHE INTERNAL "")
+set(CPUFEATURES_LIBRARIES cpufeatures CACHE INTERNAL "")
+
+if(NOT DEFINED CPUFEATURES_SOURCES)
+  set(CPUFEATURES_SOURCES ${CPUFEATURES_ROOT}/cpu-features.c ${CPUFEATURES_ROOT}/cpu-features.h)
+endif()
+
+include_directories(${CPUFEATURES_INCLUDE_DIRS})
+add_library(cpufeatures STATIC ${CPUFEATURES_SOURCES})
+
+set_target_properties(cpufeatures
+  PROPERTIES OUTPUT_NAME cpufeatures
+  DEBUG_POSTFIX "${OPENCV_DEBUG_POSTFIX}"
+  COMPILE_PDB_NAME cpufeatures
+  COMPILE_PDB_NAME_DEBUG "cpufeatures${OPENCV_DEBUG_POSTFIX}"
+  ARCHIVE_OUTPUT_DIRECTORY ${3P_LIBRARY_OUTPUT_PATH}
+)
+
+if(ENABLE_SOLUTION_FOLDERS)
+  set_target_properties(cpufeatures PROPERTIES FOLDER "3rdparty")
+endif()
+
+if(NOT BUILD_SHARED_LIBS)
+  ocv_install_target(cpufeatures EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+endif()

3rdparty/cpufeatures/LICENSE

@@ -0,0 +1,13 @@
+Copyright (C) 2016 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

3rdparty/cpufeatures/README.md

@@ -0,0 +1,4 @@
+The Android NDK provides a small library named cpufeatures that your app can use at runtime to detect the target device's CPU family and the optional features it supports.
+It is designed to work as-is on all official Android platform versions.
+
+https://developer.android.com/ndk/guides/cpu-features.html

3rdparty/cpufeatures/cpu-features.c

@@ -27,6 +27,10 @@
  */
 
 /* ChangeLog for this library:
+ *
+ * NDK r10e?: Add MIPS MSA feature.
+ *
+ * NDK r10: Support for 64-bit CPUs (Intel, ARM & MIPS).
  *
  * NDK r8d: Add android_setCpu().
  *
@@ -56,16 +60,17 @@
  *
  * NDK r4: Initial release
  */
-#include <sys/system_properties.h>
-#ifdef __arm__
-#include <machine/cpu-features.h>
-#endif
-#include <pthread.h>
+
 #include "cpu-features.h"
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <fcntl.h>
-#include <errno.h>
+#include <sys/system_properties.h>
+#include <unistd.h>
 
 static  pthread_once_t     g_once;
 static  int                g_inited;
@@ -73,16 +78,12 @@ static  AndroidCpuFamily   g_cpuFamily;
 static  uint64_t           g_cpuFeatures;
 static  int                g_cpuCount;
 
-static const int  android_cpufeatures_debug = 0;
-
 #ifdef __arm__
-#  define DEFAULT_CPU_FAMILY  ANDROID_CPU_FAMILY_ARM
-#elif defined __i386__
-#  define DEFAULT_CPU_FAMILY  ANDROID_CPU_FAMILY_X86
-#else
-#  define DEFAULT_CPU_FAMILY  ANDROID_CPU_FAMILY_UNKNOWN
+static  uint32_t           g_cpuIdArm;
 #endif
 
+static const int android_cpufeatures_debug = 0;
+
 #define  D(...) \
     do { \
         if (android_cpufeatures_debug) { \
@@ -109,6 +110,25 @@ static __inline__ void x86_cpuid(int func, int values[4])
     values[2] = c;
     values[3] = d;
 }
+#elif defined(__x86_64__)
+static __inline__ void x86_cpuid(int func, int values[4])
+{
+    int64_t a, b, c, d;
+    /* We need to preserve ebx since we're compiling PIC code */
+    /* this means we can't use "=b" for the second output register */
+    __asm__ __volatile__ ( \
+      "push %%rbx\n"
+      "cpuid\n" \
+      "mov %%rbx, %1\n"
+      "pop %%rbx\n"
+      : "=a" (a), "=r" (b), "=c" (c), "=d" (d) \
+      : "a" (func) \
+    );
+    values[0] = a;
+    values[1] = b;
+    values[2] = c;
+    values[3] = d;
+}
 #endif
 
 /* Get the size of a file by reading it until the end. This is needed
@@ -118,7 +138,8 @@ static __inline__ void x86_cpuid(int func, int values[4])
 static int
 get_file_size(const char* pathname)
 {
-    int fd, ret, result = 0;
+
+   int fd, result = 0;
     char buffer[256];
 
     fd = open(pathname, O_RDONLY);
@@ -178,6 +199,7 @@ read_file(const char*  pathname, char*  buffer, size_t  buffsize)
     return count;
 }
 
+#ifdef __arm__
 /* Extract the content of a the first occurence of a given field in
  * the content of /proc/cpuinfo and return it as a heap-allocated
  * string that must be freed by the caller.
@@ -190,12 +212,11 @@ extract_cpuinfo_field(const char* buffer, int buflen, const char* field)
     int  fieldlen = strlen(field);
     const char* bufend = buffer + buflen;
     char* result = NULL;
-    int len, ignore;
+    int len;
     const char *p, *q;
 
     /* Look for first field occurence, and ensures it starts the line. */
     p = buffer;
-    bufend = buffer + buflen;
     for (;;) {
         p = memmem(p, bufend-p, field, fieldlen);
         if (p == NULL)
@@ -232,10 +253,6 @@ EXIT:
     return result;
 }
 
-/* Like strlen(), but for constant string literals */
-#define STRLEN_CONST(x)  ((sizeof(x)-1)
-
-
 /* Checks that a space-separated list of items contains one given 'item'.
  * Returns 1 if found, 0 otherwise.
  */
@@ -268,8 +285,9 @@ has_list_item(const char* list, const char* item)
     }
     return 0;
 }
+#endif /* __arm__ */
 
-/* Parse an decimal integer starting from 'input', but not going further
+/* Parse a number starting from 'input', but not going further
  * than 'limit'. Return the value into '*result'.
  *
  * NOTE: Does not skip over leading spaces, or deal with sign characters.
@@ -280,15 +298,23 @@ has_list_item(const char* list, const char* item)
  * be <= 'limit').
  */
 static const char*
-parse_decimal(const char* input, const char* limit, int* result)
+parse_number(const char* input, const char* limit, int base, int* result)
 {
     const char* p = input;
     int val = 0;
     while (p < limit) {
         int d = (*p - '0');
-        if ((unsigned)d >= 10U)
-            break;
-        val = val*10 + d;
+        if ((unsigned)d >= 10U) {
+            d = (*p - 'a');
+            if ((unsigned)d >= 6U)
+              d = (*p - 'A');
+            if ((unsigned)d >= 6U)
+              break;
+            d += 10;
+        }
+        if (d >= base)
+          break;
+        val = val*base + d;
         p++;
     }
     if (p == input)
@@ -298,6 +324,20 @@ parse_decimal(const char* input, const char* limit, int* result)
     return p;
 }
 
+static const char*
+parse_decimal(const char* input, const char* limit, int* result)
+{
+    return parse_number(input, limit, 10, result);
+}
+
+#ifdef __arm__
+static const char*
+parse_hexadecimal(const char* input, const char* limit, int* result)
+{
+    return parse_number(input, limit, 16, result);
+}
+#endif /* __arm__ */
+
 /* This small data type is used to represent a CPU list / mask, as read
  * from sysfs on Linux. See http://www.kernel.org/doc/Documentation/cputopology.txt
  *
@@ -408,6 +448,18 @@ cpulist_read_from(CpuList* list, const char* filename)
 
     cpulist_parse(list, file, filelen);
 }
+#if defined(__aarch64__)
+// see <uapi/asm/hwcap.h> kernel header
+#define HWCAP_FP                (1 << 0)
+#define HWCAP_ASIMD             (1 << 1)
+#define HWCAP_AES               (1 << 3)
+#define HWCAP_PMULL             (1 << 4)
+#define HWCAP_SHA1              (1 << 5)
+#define HWCAP_SHA2              (1 << 6)
+#define HWCAP_CRC32             (1 << 7)
+#endif
+
+#if defined(__arm__)
 
 // See <asm/hwcap.h> kernel header.
 #define HWCAP_VFP       (1 << 6)
@@ -419,27 +471,104 @@ cpulist_read_from(CpuList* list, const char* filename)
 #define HWCAP_IDIVA     (1 << 17)
 #define HWCAP_IDIVT     (1 << 18)
 
+// see <uapi/asm/hwcap.h> kernel header
+#define HWCAP2_AES     (1 << 0)
+#define HWCAP2_PMULL   (1 << 1)
+#define HWCAP2_SHA1    (1 << 2)
+#define HWCAP2_SHA2    (1 << 3)
+#define HWCAP2_CRC32   (1 << 4)
+
+// This is the list of 32-bit ARMv7 optional features that are _always_
+// supported by ARMv8 CPUs, as mandated by the ARM Architecture Reference
+// Manual.
+#define HWCAP_SET_FOR_ARMV8  \
+  ( HWCAP_VFP | \
+    HWCAP_NEON | \
+    HWCAP_VFPv3 | \
+    HWCAP_VFPv4 | \
+    HWCAP_IDIVA | \
+    HWCAP_IDIVT )
+#endif
+
+#if defined(__mips__)
+// see <uapi/asm/hwcap.h> kernel header
+#define HWCAP_MIPS_R6           (1 << 0)
+#define HWCAP_MIPS_MSA          (1 << 1)
+#endif
+
+#if defined(__arm__) || defined(__aarch64__) || defined(__mips__)
+
 #define AT_HWCAP 16
+#define AT_HWCAP2 26
+
+// Probe the system's C library for a 'getauxval' function and call it if
+// it exits, or return 0 for failure. This function is available since API
+// level 20.
+//
+// This code does *NOT* check for '__ANDROID_API__ >= 20' to support the
+// edge case where some NDK developers use headers for a platform that is
+// newer than the one really targetted by their application.
+// This is typically done to use newer native APIs only when running on more
+// recent Android versions, and requires careful symbol management.
+//
+// Note that getauxval() can't really be re-implemented here, because
+// its implementation does not parse /proc/self/auxv. Instead it depends
+// on values  that are passed by the kernel at process-init time to the
+// C runtime initialization layer.
+#if 1
+// OpenCV calls CPU features check during library initialization stage
+// (under other dlopen() call).
+// Unfortunatelly, calling dlopen() recursively is not supported on some old
+// Android versions. Android fix is here:
+// - https://android-review.googlesource.com/#/c/32951/
+// - GitHub mirror: https://github.com/android/platform_bionic/commit/e19d702b8e330cef87e0983733c427b5f7842144
+__attribute__((weak)) unsigned long getauxval(unsigned long); // Lets linker to handle this symbol
+static uint32_t
+get_elf_hwcap_from_getauxval(int hwcap_type) {
+    uint32_t ret = 0;
+    if(getauxval != 0) {
+        ret = (uint32_t)getauxval(hwcap_type);
+    } else {
+        D("getauxval() is not available\n");
+    }
+    return ret;
+}
+#else
+static uint32_t
+get_elf_hwcap_from_getauxval(int hwcap_type) {
+    typedef unsigned long getauxval_func_t(unsigned long);
+
+    dlerror();
+    void* libc_handle = dlopen("libc.so", RTLD_NOW);
+    if (!libc_handle) {
+        D("Could not dlopen() C library: %s\n", dlerror());
+        return 0;
+    }
+
+    uint32_t ret = 0;
+    getauxval_func_t* func = (getauxval_func_t*)
+            dlsym(libc_handle, "getauxval");
+    if (!func) {
+        D("Could not find getauxval() in C library\n");
+    } else {
+        // Note: getauxval() returns 0 on failure. Doesn't touch errno.
+        ret = (uint32_t)(*func)(hwcap_type);
+    }
+    dlclose(libc_handle);
+    return ret;
+}
+#endif
+#endif
 
 #if defined(__arm__)
-/* Compute the ELF HWCAP flags.
- */
+// Parse /proc/self/auxv to extract the ELF HW capabilities bitmap for the
+// current CPU. Note that this file is not accessible from regular
+// application processes on some Android platform releases.
+// On success, return new ELF hwcaps, or 0 on failure.
 static uint32_t
-get_elf_hwcap(const char* cpuinfo, int cpuinfo_len)
-{
-  /* IMPORTANT:
-   *   Accessing /proc/self/auxv doesn't work anymore on all
-   *   platform versions. More specifically, when running inside
-   *   a regular application process, most of /proc/self/ will be
-   *   non-readable, including /proc/self/auxv. This doesn't
-   *   happen however if the application is debuggable, or when
-   *   running under the "shell" UID, which is why this was not
-   *   detected appropriately.
-   */
-#if 0
-    uint32_t result = 0;
+get_elf_hwcap_from_proc_self_auxv(void) {
     const char filepath[] = "/proc/self/auxv";
-    int fd = open(filepath, O_RDONLY);
+    int fd = TEMP_FAILURE_RETRY(open(filepath, O_RDONLY));
     if (fd < 0) {
         D("Could not open %s: %s\n", filepath, strerror(errno));
         return 0;
@@ -447,11 +576,10 @@ get_elf_hwcap(const char* cpuinfo, int cpuinfo_len)
 
     struct { uint32_t tag; uint32_t value; } entry;
 
+    uint32_t result = 0;
     for (;;) {
-        int ret = read(fd, (char*)&entry, sizeof entry);
+        int ret = TEMP_FAILURE_RETRY(read(fd, (char*)&entry, sizeof entry));
         if (ret < 0) {
-            if (errno == EINTR)
-                continue;
             D("Error while reading %s: %s\n", filepath, strerror(errno));
             break;
         }
@@ -465,12 +593,33 @@ get_elf_hwcap(const char* cpuinfo, int cpuinfo_len)
     }
     close(fd);
     return result;
-#else
-    // Recreate ELF hwcaps by parsing /proc/cpuinfo Features tag.
+}
+
+/* Compute the ELF HWCAP flags from the content of /proc/cpuinfo.
+ * This works by parsing the 'Features' line, which lists which optional
+ * features the device's CPU supports, on top of its reference
+ * architecture.
+ */
+static uint32_t
+get_elf_hwcap_from_proc_cpuinfo(const char* cpuinfo, int cpuinfo_len) {
     uint32_t hwcaps = 0;
+    long architecture = 0;
+    char* cpuArch = extract_cpuinfo_field(cpuinfo, cpuinfo_len, "CPU architecture");
+    if (cpuArch) {
+        architecture = strtol(cpuArch, NULL, 10);
+        free(cpuArch);
+
+        if (architecture >= 8L) {
+            // This is a 32-bit ARM binary running on a 64-bit ARM64 kernel.
+            // The 'Features' line only lists the optional features that the
+            // device's CPU supports, compared to its reference architecture
+            // which are of no use for this process.
+            D("Faking 32-bit ARM HWCaps on ARMv%ld CPU\n", architecture);
+            return HWCAP_SET_FOR_ARMV8;
+        }
+    }
 
     char* cpuFeatures = extract_cpuinfo_field(cpuinfo, cpuinfo_len, "Features");
-
     if (cpuFeatures != NULL) {
         D("Found cpuFeatures = '%s'\n", cpuFeatures);
 
@@ -496,7 +645,6 @@ get_elf_hwcap(const char* cpuinfo, int cpuinfo_len)
         free(cpuFeatures);
     }
     return hwcaps;
-#endif
 }
 #endif  /* __arm__ */
 
@@ -526,12 +674,19 @@ get_cpu_count(void)
 static void
 android_cpuInitFamily(void)
 {
-#if defined(__ARM_ARCH__)
+#if defined(__arm__)
     g_cpuFamily = ANDROID_CPU_FAMILY_ARM;
 #elif defined(__i386__)
     g_cpuFamily = ANDROID_CPU_FAMILY_X86;
-#elif defined(_MIPS_ARCH)
+#elif defined(__mips64)
+/* Needs to be before __mips__ since the compiler defines both */
+    g_cpuFamily = ANDROID_CPU_FAMILY_MIPS64;
+#elif defined(__mips__)
     g_cpuFamily = ANDROID_CPU_FAMILY_MIPS;
+#elif defined(__aarch64__)
+    g_cpuFamily = ANDROID_CPU_FAMILY_ARM64;
+#elif defined(__x86_64__)
+    g_cpuFamily = ANDROID_CPU_FAMILY_X86_64;
 #else
     g_cpuFamily = ANDROID_CPU_FAMILY_UNKNOWN;
 #endif
@@ -576,11 +731,8 @@ android_cpuInit(void)
 
     D("found cpuCount = %d\n", g_cpuCount);
 
-#ifdef __ARM_ARCH__
+#ifdef __arm__
     {
-        char*  features = NULL;
-        char*  architecture = NULL;
-
         /* Extract architecture from the "CPU Architecture" field.
          * The list is well-known, unlike the the output of
          * the 'Processor' field which can vary greatly.
@@ -601,10 +753,7 @@ android_cpuInit(void)
             /* read the initial decimal number, ignore the rest */
             archNumber = strtol(cpuArch, &end, 10);
 
-            /* Here we assume that ARMv8 will be upwards compatible with v7
-             * in the future. Unfortunately, there is no 'Features' field to
-             * indicate that Thumb-2 is supported.
-             */
+            /* Note that ARMv8 is upwards compatible with ARMv7. */
             if (end > cpuArch && archNumber >= 7) {
                 hasARMv7 = 1;
             }
@@ -645,7 +794,19 @@ android_cpuInit(void)
         }
 
         /* Extract the list of CPU features from ELF hwcaps */
-        uint32_t hwcaps = get_elf_hwcap(cpuinfo, cpuinfo_len);
+        uint32_t hwcaps = 0;
+        hwcaps = get_elf_hwcap_from_getauxval(AT_HWCAP);
+        if (!hwcaps) {
+            D("Parsing /proc/self/auxv to extract ELF hwcaps!\n");
+            hwcaps = get_elf_hwcap_from_proc_self_auxv();
+        }
+        if (!hwcaps) {
+            // Parsing /proc/self/auxv will fail from regular application
+            // processes on some Android platform versions, when this happens
+            // parse proc/cpuinfo instead.
+            D("Parsing /proc/cpuinfo to extract ELF hwcaps!\n");
+            hwcaps = get_elf_hwcap_from_proc_cpuinfo(cpuinfo, cpuinfo_len);
+        }
 
         if (hwcaps != 0) {
             int has_vfp = (hwcaps & HWCAP_VFP);
@@ -697,22 +858,163 @@ android_cpuInit(void)
                 g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_VFPv2 |
                                  ANDROID_CPU_ARM_FEATURE_ARMv7;
 
-            // Note that some buggy kernels do not report these even when
-            // the CPU actually support the division instructions. However,
-            // assume that if 'vfpv4' is detected, then the CPU supports
-            // sdiv/udiv properly.
-            if (has_idiva || has_vfpv4)
+            if (has_idiva)
                 g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_IDIV_ARM;
-            if (has_idivt || has_vfpv4)
+            if (has_idivt)
                 g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_IDIV_THUMB2;
 
             if (has_iwmmxt)
                 g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_iWMMXt;
         }
-    }
-#endif /* __ARM_ARCH__ */
 
-#ifdef __i386__
+        /* Extract the list of CPU features from ELF hwcaps2 */
+        uint32_t hwcaps2 = 0;
+        hwcaps2 = get_elf_hwcap_from_getauxval(AT_HWCAP2);
+        if (hwcaps2 != 0) {
+            int has_aes     = (hwcaps2 & HWCAP2_AES);
+            int has_pmull   = (hwcaps2 & HWCAP2_PMULL);
+            int has_sha1    = (hwcaps2 & HWCAP2_SHA1);
+            int has_sha2    = (hwcaps2 & HWCAP2_SHA2);
+            int has_crc32   = (hwcaps2 & HWCAP2_CRC32);
+
+            if (has_aes)
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_AES;
+            if (has_pmull)
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_PMULL;
+            if (has_sha1)
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_SHA1;
+            if (has_sha2)
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_SHA2;
+            if (has_crc32)
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_CRC32;
+        }
+        /* Extract the cpuid value from various fields */
+        // The CPUID value is broken up in several entries in /proc/cpuinfo.
+        // This table is used to rebuild it from the entries.
+        static const struct CpuIdEntry {
+            const char* field;
+            char        format;
+            char        bit_lshift;
+            char        bit_length;
+        } cpu_id_entries[] = {
+            { "CPU implementer", 'x', 24, 8 },
+            { "CPU variant", 'x', 20, 4 },
+            { "CPU part", 'x', 4, 12 },
+            { "CPU revision", 'd', 0, 4 },
+        };
+        size_t i;
+        D("Parsing /proc/cpuinfo to recover CPUID\n");
+        for (i = 0;
+             i < sizeof(cpu_id_entries)/sizeof(cpu_id_entries[0]);
+             ++i) {
+            const struct CpuIdEntry* entry = &cpu_id_entries[i];
+            char* value = extract_cpuinfo_field(cpuinfo,
+                                                cpuinfo_len,
+                                                entry->field);
+            if (value == NULL)
+                continue;
+
+            D("field=%s value='%s'\n", entry->field, value);
+            char* value_end = value + strlen(value);
+            int val = 0;
+            const char* start = value;
+            const char* p;
+            if (value[0] == '0' && (value[1] == 'x' || value[1] == 'X')) {
+              start += 2;
+              p = parse_hexadecimal(start, value_end, &val);
+            } else if (entry->format == 'x')
+              p = parse_hexadecimal(value, value_end, &val);
+            else
+              p = parse_decimal(value, value_end, &val);
+
+            if (p > (const char*)start) {
+              val &= ((1 << entry->bit_length)-1);
+              val <<= entry->bit_lshift;
+              g_cpuIdArm |= (uint32_t) val;
+            }
+
+            free(value);
+        }
+
+        // Handle kernel configuration bugs that prevent the correct
+        // reporting of CPU features.
+        static const struct CpuFix {
+            uint32_t  cpuid;
+            uint64_t  or_flags;
+        } cpu_fixes[] = {
+            /* The Nexus 4 (Qualcomm Krait) kernel configuration
+             * forgets to report IDIV support. */
+            { 0x510006f2, ANDROID_CPU_ARM_FEATURE_IDIV_ARM |
+                          ANDROID_CPU_ARM_FEATURE_IDIV_THUMB2 },
+            { 0x510006f3, ANDROID_CPU_ARM_FEATURE_IDIV_ARM |
+                          ANDROID_CPU_ARM_FEATURE_IDIV_THUMB2 },
+        };
+        size_t n;
+        for (n = 0; n < sizeof(cpu_fixes)/sizeof(cpu_fixes[0]); ++n) {
+            const struct CpuFix* entry = &cpu_fixes[n];
+
+            if (g_cpuIdArm == entry->cpuid)
+                g_cpuFeatures |= entry->or_flags;
+        }
+
+        // Special case: The emulator-specific Android 4.2 kernel fails
+        // to report support for the 32-bit ARM IDIV instruction.
+        // Technically, this is a feature of the virtual CPU implemented
+        // by the emulator. Note that it could also support Thumb IDIV
+        // in the future, and this will have to be slightly updated.
+        char* hardware = extract_cpuinfo_field(cpuinfo,
+                                               cpuinfo_len,
+                                               "Hardware");
+        if (hardware) {
+            if (!strcmp(hardware, "Goldfish") &&
+                g_cpuIdArm == 0x4100c080 &&
+                (g_cpuFamily & ANDROID_CPU_ARM_FEATURE_ARMv7) != 0) {
+                g_cpuFeatures |= ANDROID_CPU_ARM_FEATURE_IDIV_ARM;
+            }
+            free(hardware);
+        }
+    }
+#endif /* __arm__ */
+#ifdef __aarch64__
+    {
+        /* Extract the list of CPU features from ELF hwcaps */
+        uint32_t hwcaps = 0;
+        hwcaps = get_elf_hwcap_from_getauxval(AT_HWCAP);
+        if (hwcaps != 0) {
+            int has_fp      = (hwcaps & HWCAP_FP);
+            int has_asimd   = (hwcaps & HWCAP_ASIMD);
+            int has_aes     = (hwcaps & HWCAP_AES);
+            int has_pmull   = (hwcaps & HWCAP_PMULL);
+            int has_sha1    = (hwcaps & HWCAP_SHA1);
+            int has_sha2    = (hwcaps & HWCAP_SHA2);
+            int has_crc32   = (hwcaps & HWCAP_CRC32);
+
+            if(has_fp == 0) {
+                D("ERROR: Floating-point unit missing, but is required by Android on AArch64 CPUs\n");
+            }
+            if(has_asimd == 0) {
+                D("ERROR: ASIMD unit missing, but is required by Android on AArch64 CPUs\n");
+            }
+
+            if (has_fp)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_FP;
+            if (has_asimd)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_ASIMD;
+            if (has_aes)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_AES;
+            if (has_pmull)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_PMULL;
+            if (has_sha1)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_SHA1;
+            if (has_sha2)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_SHA2;
+            if (has_crc32)
+                g_cpuFeatures |= ANDROID_CPU_ARM64_FEATURE_CRC32;
+        }
+    }
+#endif /* __aarch64__ */
+
+#if defined(__i386__) || defined(__x86_64__)
     int regs[4];
 
 /* According to http://en.wikipedia.org/wiki/CPUID */
@@ -732,10 +1034,50 @@ android_cpuInit(void)
     if ((regs[2] & (1 << 23)) != 0) {
         g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_POPCNT;
     }
+    if ((regs[2] & (1 << 19)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_SSE4_1;
+    }
+    if ((regs[2] & (1 << 20)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_SSE4_2;
+    }
     if (vendorIsIntel && (regs[2] & (1 << 22)) != 0) {
         g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_MOVBE;
     }
+    if ((regs[2] & (1 << 25)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_AES_NI;
+    }
+    if ((regs[2] & (1 << 28)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_AVX;
+    }
+    if ((regs[2] & (1 << 30)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_RDRAND;
+    }
+
+    x86_cpuid(7, regs);
+    if ((regs[1] & (1 << 5)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_AVX2;
+    }
+    if ((regs[1] & (1 << 29)) != 0) {
+        g_cpuFeatures |= ANDROID_CPU_X86_FEATURE_SHA_NI;
+    }
+
+
 #endif
+#if defined( __mips__)
+    {   /* MIPS and MIPS64 */
+        /* Extract the list of CPU features from ELF hwcaps */
+        uint32_t hwcaps = 0;
+        hwcaps = get_elf_hwcap_from_getauxval(AT_HWCAP);
+        if (hwcaps != 0) {
+            int has_r6      = (hwcaps & HWCAP_MIPS_R6);
+            int has_msa     = (hwcaps & HWCAP_MIPS_MSA);
+            if (has_r6)
+                g_cpuFeatures |= ANDROID_CPU_MIPS_FEATURE_R6;
+            if (has_msa)
+                g_cpuFeatures |= ANDROID_CPU_MIPS_FEATURE_MSA;
+        }
+    }
+#endif /* __mips__ */
 
     free(cpuinfo);
 }
@@ -785,6 +1127,25 @@ android_setCpu(int cpu_count, uint64_t cpu_features)
     return 1;
 }
 
+#ifdef __arm__
+uint32_t
+android_getCpuIdArm(void)
+{
+    pthread_once(&g_once, android_cpuInit);
+    return g_cpuIdArm;
+}
+
+int
+android_setCpuArm(int cpu_count, uint64_t cpu_features, uint32_t cpu_id)
+{
+    if (!android_setCpu(cpu_count, cpu_features))
+        return 0;
+
+    g_cpuIdArm = cpu_id;
+    return 1;
+}
+#endif  /* __arm__ */
+
 /*
  * Technical note: Making sense of ARM's FPU architecture versions.
  *

3rdparty/cpufeatures/cpu-features.h

@@ -33,21 +33,40 @@
 
 __BEGIN_DECLS
 
+/* A list of valid values returned by android_getCpuFamily().
+ * They describe the CPU Architecture of the current process.
+ */
 typedef enum {
     ANDROID_CPU_FAMILY_UNKNOWN = 0,
     ANDROID_CPU_FAMILY_ARM,
     ANDROID_CPU_FAMILY_X86,
     ANDROID_CPU_FAMILY_MIPS,
+    ANDROID_CPU_FAMILY_ARM64,
+    ANDROID_CPU_FAMILY_X86_64,
+    ANDROID_CPU_FAMILY_MIPS64,
 
     ANDROID_CPU_FAMILY_MAX  /* do not remove */
 
 } AndroidCpuFamily;
 
-/* Return family of the device's CPU */
-extern AndroidCpuFamily   android_getCpuFamily(void);
+/* Return the CPU family of the current process.
+ *
+ * Note that this matches the bitness of the current process. I.e. when
+ * running a 32-bit binary on a 64-bit capable CPU, this will return the
+ * 32-bit CPU family value.
+ */
+extern AndroidCpuFamily android_getCpuFamily(void);
 
-/* The list of feature flags for ARM CPUs that can be recognized by the
- * library. Value details are:
+/* Return a bitmap describing a set of optional CPU features that are
+ * supported by the current device's CPU. The exact bit-flags returned
+ * depend on the value returned by android_getCpuFamily(). See the
+ * documentation for the ANDROID_CPU_*_FEATURE_* flags below for details.
+ */
+extern uint64_t android_getCpuFeatures(void);
+
+/* The list of feature flags for ANDROID_CPU_FAMILY_ARM that can be
+ * recognized by the library (see note below for 64-bit ARM). Value details
+ * are:
  *
  *   VFPv2:
  *     CPU supports the VFPv2 instruction set. Many, but not all, ARMv6 CPUs
@@ -103,6 +122,27 @@ extern AndroidCpuFamily   android_getCpuFamily(void);
  *     ARM CPU. This is only available on a few XScale-based CPU designs
  *     sold by Marvell. Pretty rare in practice.
  *
+ *   AES:
+ *     CPU supports AES instructions. These instructions are only
+ *     available for 32-bit applications running on ARMv8 CPU.
+ *
+ *   CRC32:
+ *     CPU supports CRC32 instructions. These instructions are only
+ *     available for 32-bit applications running on ARMv8 CPU.
+ *
+ *   SHA2:
+ *     CPU supports SHA2 instructions. These instructions are only
+ *     available for 32-bit applications running on ARMv8 CPU.
+ *
+ *   SHA1:
+ *     CPU supports SHA1 instructions. These instructions are only
+ *     available for 32-bit applications running on ARMv8 CPU.
+ *
+ *   PMULL:
+ *     CPU supports 64-bit PMULL and PMULL2 instructions. These
+ *     instructions are only available for 32-bit applications
+ *     running on ARMv8 CPU.
+ *
  * If you want to tell the compiler to generate code that targets one of
  * the feature set above, you should probably use one of the following
  * flags (for more details, see technical note at the end of this file):
@@ -150,6 +190,13 @@ extern AndroidCpuFamily   android_getCpuFamily(void);
  *
  *   -mcpu=iwmmxt
  *     Allows the use of iWMMXt instrinsics with GCC.
+ *
+ * IMPORTANT NOTE: These flags should only be tested when
+ * android_getCpuFamily() returns ANDROID_CPU_FAMILY_ARM, i.e. this is a
+ * 32-bit process.
+ *
+ * When running a 64-bit ARM process on an ARMv8 CPU,
+ * android_getCpuFeatures() will return a different set of bitflags
  */
 enum {
     ANDROID_CPU_ARM_FEATURE_ARMv7       = (1 << 0),
@@ -164,18 +211,84 @@ enum {
     ANDROID_CPU_ARM_FEATURE_IDIV_ARM    = (1 << 9),
     ANDROID_CPU_ARM_FEATURE_IDIV_THUMB2 = (1 << 10),
     ANDROID_CPU_ARM_FEATURE_iWMMXt      = (1 << 11),
+    ANDROID_CPU_ARM_FEATURE_AES         = (1 << 12),
+    ANDROID_CPU_ARM_FEATURE_PMULL       = (1 << 13),
+    ANDROID_CPU_ARM_FEATURE_SHA1        = (1 << 14),
+    ANDROID_CPU_ARM_FEATURE_SHA2        = (1 << 15),
+    ANDROID_CPU_ARM_FEATURE_CRC32       = (1 << 16),
 };
 
+/* The bit flags corresponding to the output of android_getCpuFeatures()
+ * when android_getCpuFamily() returns ANDROID_CPU_FAMILY_ARM64. Value details
+ * are:
+ *
+ *   FP:
+ *     CPU has Floating-point unit.
+ *
+ *   ASIMD:
+ *     CPU has Advanced SIMD unit.
+ *
+ *   AES:
+ *     CPU supports AES instructions.
+ *
+ *   CRC32:
+ *     CPU supports CRC32 instructions.
+ *
+ *   SHA2:
+ *     CPU supports SHA2 instructions.
+ *
+ *   SHA1:
+ *     CPU supports SHA1 instructions.
+ *
+ *   PMULL:
+ *     CPU supports 64-bit PMULL and PMULL2 instructions.
+ */
+enum {
+    ANDROID_CPU_ARM64_FEATURE_FP      = (1 << 0),
+    ANDROID_CPU_ARM64_FEATURE_ASIMD   = (1 << 1),
+    ANDROID_CPU_ARM64_FEATURE_AES     = (1 << 2),
+    ANDROID_CPU_ARM64_FEATURE_PMULL   = (1 << 3),
+    ANDROID_CPU_ARM64_FEATURE_SHA1    = (1 << 4),
+    ANDROID_CPU_ARM64_FEATURE_SHA2    = (1 << 5),
+    ANDROID_CPU_ARM64_FEATURE_CRC32   = (1 << 6),
+};
+
+/* The bit flags corresponding to the output of android_getCpuFeatures()
+ * when android_getCpuFamily() returns ANDROID_CPU_FAMILY_X86 or
+ * ANDROID_CPU_FAMILY_X86_64.
+ */
 enum {
     ANDROID_CPU_X86_FEATURE_SSSE3  = (1 << 0),
     ANDROID_CPU_X86_FEATURE_POPCNT = (1 << 1),
     ANDROID_CPU_X86_FEATURE_MOVBE  = (1 << 2),
+    ANDROID_CPU_X86_FEATURE_SSE4_1 = (1 << 3),
+    ANDROID_CPU_X86_FEATURE_SSE4_2 = (1 << 4),
+    ANDROID_CPU_X86_FEATURE_AES_NI = (1 << 5),
+    ANDROID_CPU_X86_FEATURE_AVX =    (1 << 6),
+    ANDROID_CPU_X86_FEATURE_RDRAND = (1 << 7),
+    ANDROID_CPU_X86_FEATURE_AVX2 =   (1 << 8),
+    ANDROID_CPU_X86_FEATURE_SHA_NI = (1 << 9),
+};
+
+/* The bit flags corresponding to the output of android_getCpuFeatures()
+ * when android_getCpuFamily() returns ANDROID_CPU_FAMILY_MIPS
+ * or ANDROID_CPU_FAMILY_MIPS64.  Values are:
+ *
+ *   R6:
+ *     CPU executes MIPS Release 6 instructions natively, and
+ *     supports obsoleted R1..R5 instructions only via kernel traps.
+ *
+ *   MSA:
+ *     CPU supports Mips SIMD Architecture instructions.
+ */
+enum {
+    ANDROID_CPU_MIPS_FEATURE_R6    = (1 << 0),
+    ANDROID_CPU_MIPS_FEATURE_MSA   = (1 << 1),
 };
 
-extern uint64_t    android_getCpuFeatures(void);
 
 /* Return the number of CPU cores detected on this device. */
-extern int         android_getCpuCount(void);
+extern int android_getCpuCount(void);
 
 /* The following is used to force the CPU count and features
  * mask in sandboxed processes. Under 4.1 and higher, these processes
@@ -190,6 +303,21 @@ extern int         android_getCpuCount(void);
 extern int android_setCpu(int      cpu_count,
                           uint64_t cpu_features);
 
+#ifdef __arm__
+/* Retrieve the ARM 32-bit CPUID value from the kernel.
+ * Note that this cannot work on sandboxed processes under 4.1 and
+ * higher, unless you called android_setCpuArm() before.
+ */
+extern uint32_t android_getCpuIdArm(void);
+
+/* An ARM-specific variant of android_setCpu() that also allows you
+ * to set the ARM CPUID field.
+ */
+extern int android_setCpuArm(int      cpu_count,
+                             uint64_t cpu_features,
+                             uint32_t cpu_id);
+#endif
+
 __END_DECLS
 
 #endif /* CPU_FEATURES_H */

3rdparty/ffmpeg/ffmpeg.cmake

@@ -1,25 +1,37 @@
-# Binary branch name: ffmpeg/master_20150703
-# Binaries were created for OpenCV: e379ea6ed60b0caad4d4e3eea096e9d850cb8c86
-set(FFMPEG_BINARIES_COMMIT "8aeefc4efe3215de89d8c7e114ae6f7a6091b8eb")
-set(FFMPEG_FILE_HASH_BIN32 "89c783eee1c47bfc733f08334ec2e31c")
-set(FFMPEG_FILE_HASH_BIN64 "35fe6ccdda6d7a04e9056b0d73b98e76")
-set(FFMPEG_FILE_HASH_CMAKE "8606f947a780071f8fcce8cbf39ceef5")
+# Binary branch name: ffmpeg/master_20170418
+# Binaries were created for OpenCV: b993b9b7c7f6f5f37d10acacb2962812228410ba
+set(FFMPEG_BINARIES_COMMIT "86c4a841055f2612774e85b4292bb20e5fe8a783")
+set(FFMPEG_FILE_HASH_BIN32 "3dea5f7f009b44601fe95728328e0f9e")
+set(FFMPEG_FILE_HASH_BIN64 "9debe701975ef074bd6661981f3f0716")
+set(FFMPEG_FILE_HASH_CMAKE "208c00f03d2f6f39fa6262649e0bfc8d")
 
-set(FFMPEG_DOWNLOAD_URL ${OPENCV_FFMPEG_URL};$ENV{OPENCV_FFMPEG_URL};https://raw.githubusercontent.com/Itseez/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/)
+function(download_win_ffmpeg script_var)
+  set(${script_var} "" PARENT_SCOPE)
 
-ocv_download(PACKAGE opencv_ffmpeg.dll
-             HASH ${FFMPEG_FILE_HASH_BIN32}
-             URL ${FFMPEG_DOWNLOAD_URL}
-             DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR})
+  set(ids BIN32 BIN64 CMAKE)
+  set(name_BIN32 "opencv_ffmpeg.dll")
+  set(name_BIN64 "opencv_ffmpeg_64.dll")
+  set(name_CMAKE "ffmpeg_version.cmake")
 
-ocv_download(PACKAGE opencv_ffmpeg_64.dll
-             HASH ${FFMPEG_FILE_HASH_BIN64}
-             URL ${FFMPEG_DOWNLOAD_URL}
-             DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR})
+  set(FFMPEG_DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/3rdparty/ffmpeg")
 
-ocv_download(PACKAGE ffmpeg_version.cmake
-             HASH ${FFMPEG_FILE_HASH_CMAKE}
-             URL ${FFMPEG_DOWNLOAD_URL}
-             DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR})
-
-include(${CMAKE_CURRENT_LIST_DIR}/ffmpeg_version.cmake)
+  set(status TRUE)
+  foreach(id ${ids})
+    ocv_download(FILENAME ${name_${id}}
+               HASH ${FFMPEG_FILE_HASH_${id}}
+               URL
+                 "$ENV{OPENCV_FFMPEG_URL}"
+                 "${OPENCV_FFMPEG_URL}"
+                 "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/"
+               DESTINATION_DIR ${FFMPEG_DOWNLOAD_DIR}
+               ID FFMPEG
+               RELATIVE_URL
+               STATUS res)
+    if(NOT res)
+      set(status FALSE)
+    endif()
+  endforeach()
+  if(status)
+    set(${script_var} "${FFMPEG_DOWNLOAD_DIR}/ffmpeg_version.cmake" PARENT_SCOPE)
+  endif()
+endfunction()

3rdparty/ippicv/.gitignore

@@ -1,2 +0,0 @@
-downloads/
-unpack/

3rdparty/ippicv/CMakeLists.txt

@@ -0,0 +1,42 @@
+# ----------------------------------------------------------------------------
+#  CMake file for IPP IW. See root CMakeLists.txt
+#
+# ----------------------------------------------------------------------------
+project(${IPP_IW_LIBRARY})
+
+ocv_include_directories(${IPP_INCLUDE_DIRS} ${IPP_IW_PATH}/include)
+add_definitions(-DIW_BUILD)
+if(HAVE_IPP_ICV_ONLY)
+  add_definitions(-DICV_BASE)
+endif()
+
+file(GLOB lib_srcs ${IPP_IW_PATH}/src/*.c)
+file(GLOB lib_hdrs ${IPP_IW_PATH}/include/*.h ${IPP_IW_PATH}/include/iw/*.h ${IPP_IW_PATH}/include/iw++/*.hpp)
+
+# ----------------------------------------------------------------------------------
+#         Define the library target:
+# ----------------------------------------------------------------------------------
+
+add_library(${IPP_IW_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs})
+
+if(UNIX)
+  if(CMAKE_COMPILER_IS_GNUCXX OR CV_ICC)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -Wno-unused-function")
+  endif()
+endif()
+
+set_target_properties(${IPP_IW_LIBRARY}
+  PROPERTIES OUTPUT_NAME ${IPP_IW_LIBRARY}
+  DEBUG_POSTFIX "${OPENCV_DEBUG_POSTFIX}"
+  COMPILE_PDB_NAME ${IPP_IW_LIBRARY}
+  COMPILE_PDB_NAME_DEBUG "${IPP_IW_LIBRARY}${OPENCV_DEBUG_POSTFIX}"
+  ARCHIVE_OUTPUT_DIRECTORY ${3P_LIBRARY_OUTPUT_PATH}
+  )
+
+if(ENABLE_SOLUTION_FOLDERS)
+  set_target_properties(${IPP_IW_LIBRARY} PROPERTIES FOLDER "3rdparty")
+endif()
+
+if(NOT BUILD_SHARED_LIBS)
+  ocv_install_target(${IPP_IW_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+endif()

3rdparty/ippicv/downloader.cmake

@@ -1,110 +0,0 @@
-#
-# The script downloads ICV package
-#
-# On return this will define:
-# OPENCV_ICV_PATH - path to unpacked downloaded package
-#
-
-function(_icv_downloader)
-  # Commit SHA in the opencv_3rdparty repo
-  set(IPPICV_BINARIES_COMMIT "81a676001ca8075ada498583e4166079e5744668")
-  # Define actual ICV versions
-  if(APPLE)
-    set(OPENCV_ICV_PACKAGE_NAME "ippicv_macosx_20151201.tgz")
-    set(OPENCV_ICV_PACKAGE_HASH "4ff1fde9a7cfdfe7250bfcd8334e0f2f")
-    set(OPENCV_ICV_PLATFORM "macosx")
-    set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_osx")
-  elseif(UNIX)
-    if(ANDROID AND NOT (ANDROID_ABI STREQUAL x86 OR ANDROID_ABI STREQUAL x86_64))
-      return()
-    endif()
-    set(OPENCV_ICV_PACKAGE_NAME "ippicv_linux_20151201.tgz")
-    set(OPENCV_ICV_PACKAGE_HASH "808b791a6eac9ed78d32a7666804320e")
-    set(OPENCV_ICV_PLATFORM "linux")
-    set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_lnx")
-  elseif(WIN32 AND NOT ARM)
-    set(OPENCV_ICV_PACKAGE_NAME "ippicv_windows_20151201.zip")
-    set(OPENCV_ICV_PACKAGE_HASH "04e81ce5d0e329c3fbc606ae32cad44d")
-    set(OPENCV_ICV_PLATFORM "windows")
-    set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_win")
-  else()
-    return() # Not supported
-  endif()
-
-  set(OPENCV_ICV_UNPACK_PATH "${CMAKE_CURRENT_LIST_DIR}/unpack")
-  set(OPENCV_ICV_PATH "${OPENCV_ICV_UNPACK_PATH}${OPENCV_ICV_PACKAGE_SUBDIR}")
-
-  if(DEFINED OPENCV_ICV_PACKAGE_DOWNLOADED
-       AND OPENCV_ICV_PACKAGE_DOWNLOADED STREQUAL OPENCV_ICV_PACKAGE_HASH
-       AND EXISTS ${OPENCV_ICV_PATH})
-    # Package has been downloaded and checked by the previous build
-    set(OPENCV_ICV_PATH "${OPENCV_ICV_PATH}" PARENT_SCOPE)
-    return()
-  else()
-    if(EXISTS ${OPENCV_ICV_UNPACK_PATH})
-      message(STATUS "ICV: Removing previous unpacked package: ${OPENCV_ICV_UNPACK_PATH}")
-      file(REMOVE_RECURSE ${OPENCV_ICV_UNPACK_PATH})
-    endif()
-  endif()
-  unset(OPENCV_ICV_PACKAGE_DOWNLOADED CACHE)
-
-  set(OPENCV_ICV_PACKAGE_ARCHIVE "${CMAKE_CURRENT_LIST_DIR}/downloads/${OPENCV_ICV_PLATFORM}-${OPENCV_ICV_PACKAGE_HASH}/${OPENCV_ICV_PACKAGE_NAME}")
-  get_filename_component(OPENCV_ICV_PACKAGE_ARCHIVE_DIR "${OPENCV_ICV_PACKAGE_ARCHIVE}" PATH)
-  if(EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}")
-    file(MD5 "${OPENCV_ICV_PACKAGE_ARCHIVE}" archive_md5)
-    if(NOT archive_md5 STREQUAL OPENCV_ICV_PACKAGE_HASH)
-      message(WARNING "ICV: Local copy of ICV package has invalid MD5 hash: ${archive_md5} (expected: ${OPENCV_ICV_PACKAGE_HASH})")
-      file(REMOVE "${OPENCV_ICV_PACKAGE_ARCHIVE}")
-      file(REMOVE_RECURSE "${OPENCV_ICV_PACKAGE_ARCHIVE_DIR}")
-    endif()
-  endif()
-
-  if(NOT EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}")
-    if(NOT DEFINED OPENCV_ICV_URL)
-      if(DEFINED ENV{OPENCV_ICV_URL})
-        set(OPENCV_ICV_URL $ENV{OPENCV_ICV_URL})
-      else()
-        set(OPENCV_ICV_URL "https://raw.githubusercontent.com/Itseez/opencv_3rdparty/${IPPICV_BINARIES_COMMIT}/ippicv")
-      endif()
-    endif()
-
-    file(MAKE_DIRECTORY ${OPENCV_ICV_PACKAGE_ARCHIVE_DIR})
-    message(STATUS "ICV: Downloading ${OPENCV_ICV_PACKAGE_NAME}...")
-    file(DOWNLOAD "${OPENCV_ICV_URL}/${OPENCV_ICV_PACKAGE_NAME}" "${OPENCV_ICV_PACKAGE_ARCHIVE}"
-         TIMEOUT 600 STATUS __status
-         EXPECTED_MD5 ${OPENCV_ICV_PACKAGE_HASH})
-    if(NOT __status EQUAL 0)
-      message(FATAL_ERROR "ICV: Failed to download ICV package: ${OPENCV_ICV_PACKAGE_NAME}. Status=${__status}")
-    else()
-      # Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step
-      # on wrong hash
-      file(MD5 "${OPENCV_ICV_PACKAGE_ARCHIVE}" archive_md5)
-      if(NOT archive_md5 STREQUAL OPENCV_ICV_PACKAGE_HASH)
-        message(FATAL_ERROR "ICV: Downloaded copy of ICV package has invalid MD5 hash: ${archive_md5} (expected: ${OPENCV_ICV_PACKAGE_HASH})")
-      endif()
-    endif()
-  endif()
-
-  ocv_assert(EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}")
-  ocv_assert(NOT EXISTS "${OPENCV_ICV_UNPACK_PATH}")
-  file(MAKE_DIRECTORY ${OPENCV_ICV_UNPACK_PATH})
-  ocv_assert(EXISTS "${OPENCV_ICV_UNPACK_PATH}")
-
-  message(STATUS "ICV: Unpacking ${OPENCV_ICV_PACKAGE_NAME} to ${OPENCV_ICV_UNPACK_PATH}...")
-  execute_process(COMMAND ${CMAKE_COMMAND} -E tar xz "${OPENCV_ICV_PACKAGE_ARCHIVE}"
-                  WORKING_DIRECTORY "${OPENCV_ICV_UNPACK_PATH}"
-                  RESULT_VARIABLE __result)
-
-  if(NOT __result EQUAL 0)
-    message(FATAL_ERROR "ICV: Failed to unpack ICV package from ${OPENCV_ICV_PACKAGE_ARCHIVE} to ${OPENCV_ICV_UNPACK_PATH} with error ${__result}")
-  endif()
-
-  ocv_assert(EXISTS "${OPENCV_ICV_PATH}")
-
-  set(OPENCV_ICV_PACKAGE_DOWNLOADED "${OPENCV_ICV_PACKAGE_HASH}" CACHE INTERNAL "ICV package hash")
-
-  message(STATUS "ICV: Package successfully downloaded")
-  set(OPENCV_ICV_PATH "${OPENCV_ICV_PATH}" PARENT_SCOPE)
-endfunction()
-
-_icv_downloader()

3rdparty/ippicv/ippicv.cmake

@@ -0,0 +1,56 @@
+function(download_ippicv root_var)
+  set(${root_var} "" PARENT_SCOPE)
+
+  # Commit SHA in the opencv_3rdparty repo
+  set(IPPICV_COMMIT "a62e20676a60ee0ad6581e217fe7e4bada3b95db")
+  # Define actual ICV versions
+  if(APPLE)
+    set(OPENCV_ICV_PLATFORM "macosx")
+    set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_mac")
+    if(X86_64)
+      set(OPENCV_ICV_NAME "ippicv_2017u2_mac_intel64_20170418.tgz")
+      set(OPENCV_ICV_HASH "0c25953c99dbb499ff502485a9356d8d")
+    else()
+      set(OPENCV_ICV_NAME "ippicv_2017u2_mac_ia32_20170418.tgz")
+      set(OPENCV_ICV_HASH "5f225948f3f64067c681293c098d50d8")
+    endif()
+  elseif((UNIX AND NOT ANDROID) OR (UNIX AND ANDROID_ABI MATCHES "x86"))
+    set(OPENCV_ICV_PLATFORM "linux")
+    set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_lnx")
+    if(X86_64)
+      set(OPENCV_ICV_NAME "ippicv_2017u2_lnx_intel64_20170418.tgz")
+      set(OPENCV_ICV_HASH "87cbdeb627415d8e4bc811156289fa3a")
+    else()
+      set(OPENCV_ICV_NAME "ippicv_2017u2_lnx_ia32_20170418.tgz")
+      set(OPENCV_ICV_HASH "f2cece00d802d4dea86df52ed095257e")
+    endif()
+  elseif(WIN32 AND NOT ARM)
+    set(OPENCV_ICV_PLATFORM "windows")
+    set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_win")
+    if(X86_64)
+      set(OPENCV_ICV_NAME "ippicv_2017u2_win_intel64_20170418.zip")
+      set(OPENCV_ICV_HASH "75060a0c662c0800f48995b7e9b085f6")
+    else()
+      set(OPENCV_ICV_NAME "ippicv_2017u2_win_ia32_20170418.zip")
+      set(OPENCV_ICV_HASH "60fcf3ccd9a2ebc9e432ffb5cb91638b")
+    endif()
+  else()
+    return()
+  endif()
+
+  set(THE_ROOT "${OpenCV_BINARY_DIR}/3rdparty/ippicv")
+  ocv_download(FILENAME ${OPENCV_ICV_NAME}
+               HASH ${OPENCV_ICV_HASH}
+               URL
+                 "${OPENCV_IPPICV_URL}"
+                 "$ENV{OPENCV_IPPICV_URL}"
+                 "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_COMMIT}/ippicv/"
+               DESTINATION_DIR "${THE_ROOT}"
+               ID IPPICV
+               STATUS res
+               UNPACK RELATIVE_URL)
+
+  if(res)
+    set(${root_var} "${THE_ROOT}/${OPENCV_ICV_PACKAGE_SUBDIR}" PARENT_SCOPE)
+  endif()
+endfunction()

3rdparty/ittnotify/CMakeLists.txt

@@ -0,0 +1,68 @@
+# ----------------------------------------------------------------------------
+#  CMake file for Intel ITT API. See root CMakeLists.txt
+#
+# ----------------------------------------------------------------------------
+
+if(NOT ITT_LIBRARY)
+  set(ITT_LIBRARY "ittnotify")
+endif()
+project(${ITT_LIBRARY} C)
+
+if(NOT WIN32)
+  include(CheckLibraryExists)
+  if(COMMAND CHECK_LIBRARY_EXISTS)
+    CHECK_LIBRARY_EXISTS(dl dlerror "" HAVE_DL_LIBRARY)
+  endif()
+endif()
+
+ocv_include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include")
+set(ITT_INCLUDE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+set(ITT_PUBLIC_HDRS
+    include/ittnotify.h
+    include/jitprofiling.h
+    include/libittnotify.h
+    include/llvm_jit_event_listener.hpp
+)
+set(ITT_PRIVATE_HDRS
+    src/ittnotify/disable_warnings.h
+    src/ittnotify/ittnotify_config.h
+    src/ittnotify/ittnotify_static.h
+    src/ittnotify/ittnotify_types.h
+)
+set(ITT_SRCS
+    src/ittnotify/ittnotify_static.c
+    src/ittnotify/jitprofiling.c
+)
+
+add_library(${ITT_LIBRARY} STATIC ${ITT_SRCS} ${ITT_PUBLIC_HDRS} ${ITT_PRIVATE_HDRS})
+
+if(NOT WIN32)
+  if(HAVE_DL_LIBRARY)
+    target_link_libraries(${ITT_LIBRARY} dl)
+  endif()
+endif()
+
+if(UNIX)
+  if(CMAKE_COMPILER_IS_GNUCXX OR CV_ICC)
+     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+  endif()
+endif()
+
+set_target_properties(${ITT_LIBRARY} PROPERTIES
+        OUTPUT_NAME ${ITT_LIBRARY}
+        DEBUG_POSTFIX "${OPENCV_DEBUG_POSTFIX}"
+        COMPILE_PDB_NAME ${ITT_LIBRARY}
+        COMPILE_PDB_NAME_DEBUG "${ITT_LIBRARY}${OPENCV_DEBUG_POSTFIX}"
+        ARCHIVE_OUTPUT_DIRECTORY ${3P_LIBRARY_OUTPUT_PATH}
+    )
+
+ocv_warnings_disable(CMAKE_C_FLAGS -Wundef -Wsign-compare)
+
+if(ENABLE_SOLUTION_FOLDERS)
+  set_target_properties(${ITT_LIBRARY} PROPERTIES FOLDER "3rdparty")
+endif()
+
+if(NOT BUILD_SHARED_LIBS)
+  ocv_install_target(${ITT_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+endif()

3rdparty/ittnotify/include/jitprofiling.h

@@ -0,0 +1,694 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#ifndef __JITPROFILING_H__
+#define __JITPROFILING_H__
+
+/**
+ * @brief JIT Profiling APIs
+ *
+ * The JIT Profiling API is used to report information about just-in-time
+ * generated code that can be used by performance tools. The user inserts
+ * calls in the code generator to report information before JIT-compiled
+ * code goes to execution. This information is collected at runtime and used
+ * by tools like Intel(R) VTune(TM) Amplifier to display performance metrics
+ * associated with JIT-compiled code.
+ *
+ * These APIs can be used to\n
+ * - **Profile trace-based and method-based JIT-compiled
+ * code**. Some examples of environments that you can profile with these APIs:
+ * dynamic JIT compilation of JavaScript code traces, JIT execution in OpenCL(TM)
+ * software technology, Java/.NET managed execution environments, and custom
+ * ISV JIT engines.
+ * @code
+ * #include <jitprofiling.h>
+ *
+ * if (iJIT_IsProfilingActive != iJIT_SAMPLING_ON) {
+ *     return;
+ * }
+ *
+ * iJIT_Method_Load jmethod = {0};
+ * jmethod.method_id = iJIT_GetNewMethodID();
+ * jmethod.method_name = "method_name";
+ * jmethod.class_file_name = "class_name";
+ * jmethod.source_file_name = "source_file_name";
+ * jmethod.method_load_address = code_addr;
+ * jmethod.method_size = code_size;
+ *
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod);
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_SHUTDOWN, NULL);
+ * @endcode
+ *
+ *  * Expected behavior:
+ *    * If any iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an
+ *      already reported method, then such a method becomes invalid and its
+ *      memory region is treated as unloaded. VTune Amplifier displays the metrics
+ *      collected by the method until it is overwritten.
+ *    * If supplied line number information contains multiple source lines for
+ *      the same assembly instruction (code location), then VTune Amplifier picks up
+ *      the first line number.
+ *    * Dynamically generated code can be associated with a module name.
+ *      Use the iJIT_Method_Load_V2 structure.\n
+ *      Clarification of some cases:
+ *        * If you register a function with the same method ID multiple times,
+ *          specifying different module names, then the VTune Amplifier picks up
+ *          the module name registered first. If you want to distinguish the same
+ *          function between different JIT engines, supply different method IDs for
+ *          each function. Other symbolic information (for example, source file)
+ *          can be identical.
+ *
+ * - **Analyze split functions** (multiple joint or disjoint code regions
+ * belonging to the same function) **including re-JIT**
+ * with potential overlapping of code regions in time, which is common in
+ * resource-limited environments.
+ * @code
+ * #include <jitprofiling.h>
+ *
+ * unsigned int method_id = iJIT_GetNewMethodID();
+ *
+ * iJIT_Method_Load a = {0};
+ * a.method_id = method_id;
+ * a.method_load_address = 0x100;
+ * a.method_size = 0x20;
+ *
+ * iJIT_Method_Load b = {0};
+ * b.method_id = method_id;
+ * b.method_load_address = 0x200;
+ * b.method_size = 0x30;
+ *
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a);
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&b);
+ * @endcode
+ *
+ *  * Expected behaviour:
+ *      * If a iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an
+ *        already reported method, then such a method becomes invalid and
+ *        its memory region is treated as unloaded.
+ *      * All code regions reported with the same method ID are considered as
+ *        belonging to the same method. Symbolic information (method name,
+ *        source file name) will be taken from the first notification, and all
+ *        subsequent notifications with the same method ID will be processed
+ *        only for line number table information. So, the VTune Amplifier will map
+ *        samples to a source line using the line number table from the current
+ *        notification while taking the source file name from the very first one.\n
+ *        Clarification of some cases:\n
+ *          * If you register a second code region with a different source file
+ *          name and the same method ID, then this information will be saved and
+ *          will not be considered as an extension of the first code region, but
+ *          VTune Amplifier will use the source file of the first code region and map
+ *          performance metrics incorrectly.
+ *          * If you register a second code region with the same source file as
+ *          for the first region and the same method ID, then the source file will be
+ *          discarded but VTune Amplifier will map metrics to the source file correctly.
+ *          * If you register a second code region with a null source file and
+ *          the same method ID, then provided line number info will be associated
+ *          with the source file of the first code region.
+ *
+ * - **Explore inline functions** including multi-level hierarchy of
+ * nested inline methods which shows how performance metrics are distributed through them.
+ * @code
+ * #include <jitprofiling.h>
+ *
+ *  //                                    method_id   parent_id
+ *  //   [-- c --]                          3000        2000
+ *  //                  [---- d -----]      2001        1000
+ *  //  [---- b ----]                       2000        1000
+ *  // [------------ a ----------------]    1000         n/a
+ *
+ * iJIT_Method_Load a = {0};
+ * a.method_id = 1000;
+ *
+ * iJIT_Method_Inline_Load b = {0};
+ * b.method_id = 2000;
+ * b.parent_method_id = 1000;
+ *
+ * iJIT_Method_Inline_Load c = {0};
+ * c.method_id = 3000;
+ * c.parent_method_id = 2000;
+ *
+ * iJIT_Method_Inline_Load d = {0};
+ * d.method_id = 2001;
+ * d.parent_method_id = 1000;
+ *
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a);
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&b);
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&c);
+ * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&d);
+ * @endcode
+ *
+ *  * Requirements:
+ *      * Each inline (iJIT_Method_Inline_Load) method should be associated
+ *        with two method IDs: one for itself; one for its immediate parent.
+ *      * Address regions of inline methods of the same parent method cannot
+ *        overlap each other.
+ *      * Execution of the parent method must not be started until it and all
+ *        its inline methods are reported.
+ *  * Expected behaviour:
+ *      * In case of nested inline methods an order of
+ *        iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED events is not important.
+ *      * If any event overwrites either inline method or top parent method,
+ *        then the parent, including inline methods, becomes invalid and its memory
+ *        region is treated as unloaded.
+ *
+ * **Life time of allocated data**\n
+ * The client sends an event notification to the agent with event-specific
+ * data, which is a structure. The pointers in the structure refer to memory
+ * allocated by the client, which responsible for releasing it. The pointers are
+ * used by the iJIT_NotifyEvent method to copy client's data in a trace file,
+ * and they are not used after the iJIT_NotifyEvent method returns.
+ */
+
+/**
+ * @defgroup jitapi JIT Profiling
+ * @ingroup internal
+ * @{
+ */
+
+/**
+ * @brief Enumerator for the types of notifications
+ */
+typedef enum iJIT_jvm_event
+{
+    iJVM_EVENT_TYPE_SHUTDOWN = 2,               /**<\brief Send this to shutdown the agent.
+                                                 * Use NULL for event data. */
+
+    iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED = 13,  /**<\brief Send when dynamic code is
+                                                 * JIT compiled and loaded into
+                                                 * memory by the JIT engine, but
+                                                 * before the code is executed.
+                                                 * Use iJIT_Method_Load as event
+                                                 * data. */
+/** @cond exclude_from_documentation */
+    iJVM_EVENT_TYPE_METHOD_UNLOAD_START,    /**<\brief Send when compiled dynamic
+                                             * code is being unloaded from memory.
+                                             * Use iJIT_Method_Load as event data.*/
+/** @endcond */
+
+    iJVM_EVENT_TYPE_METHOD_UPDATE,   /**<\brief Send to provide new content for
+                                      * a previously reported dynamic code.
+                                      * The previous content will be invalidated
+                                      * starting from the time of the notification.
+                                      * Use iJIT_Method_Load as event data but
+                                      * required fields are following:
+                                      * - method_id    identify the code to update.
+                                      * - method_load_address    specify start address
+                                      *                          within identified code range
+                                      *                          where update should be started.
+                                      * - method_size            specify length of updated code
+                                      *                          range. */
+
+
+    iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, /**<\brief Send when an inline dynamic
+                                                  * code is JIT compiled and loaded
+                                                  * into memory by the JIT engine,
+                                                  * but before the parent code region
+                                                  * starts executing.
+                                                  * Use iJIT_Method_Inline_Load as event data.*/
+
+/** @cond exclude_from_documentation */
+    iJVM_EVENT_TYPE_METHOD_UPDATE_V2,
+/** @endcond */
+
+    iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 = 21, /**<\brief Send when a dynamic code is
+                                                   * JIT compiled and loaded into
+                                                   * memory by the JIT engine, but
+                                                   * before the code is executed.
+                                                   * Use iJIT_Method_Load_V2 as event data. */
+
+    iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3       /**<\brief Send when a dynamic code is
+                                                   * JIT compiled and loaded into
+                                                   * memory by the JIT engine, but
+                                                   * before the code is executed.
+                                                   * Use iJIT_Method_Load_V3 as event data. */
+} iJIT_JVM_EVENT;
+
+/**
+ * @brief Enumerator for the agent's mode
+ */
+typedef enum _iJIT_IsProfilingActiveFlags
+{
+    iJIT_NOTHING_RUNNING           = 0x0000,    /**<\brief The agent is not running;
+                                                 * iJIT_NotifyEvent calls will
+                                                 * not be processed. */
+    iJIT_SAMPLING_ON               = 0x0001,    /**<\brief The agent is running and
+                                                 * ready to process notifications. */
+} iJIT_IsProfilingActiveFlags;
+
+/**
+ * @brief Description of a single entry in the line number information of a code region.
+ * @details A table of line number entries gives information about how the reported code region
+ * is mapped to source file.
+ * Intel(R) VTune(TM) Amplifier uses line number information to attribute
+ * the samples (virtual address) to a line number. \n
+ * It is acceptable to report different code addresses for the same source line:
+ * @code
+ *   Offset LineNumber
+ *      1       2
+ *      12      4
+ *      15      2
+ *      18      1
+ *      21      30
+ *
+ *  VTune Amplifier constructs the following table using the client data
+ *
+ *   Code subrange  Line number
+ *      0-1             2
+ *      1-12            4
+ *      12-15           2
+ *      15-18           1
+ *      18-21           30
+ * @endcode
+ */
+typedef struct _LineNumberInfo
+{
+    unsigned int Offset;     /**<\brief Offset from the begining of the code region. */
+    unsigned int LineNumber; /**<\brief Matching source line number offset (from beginning of source file). */
+
+} *pLineNumberInfo, LineNumberInfo;
+
+/**
+ * @brief Enumerator for the code architecture.
+ */
+typedef enum _iJIT_CodeArchitecture
+{
+    iJIT_CA_NATIVE = 0, /**<\brief Native to the process architecture that is calling it. */
+
+    iJIT_CA_32,         /**<\brief 32-bit machine code. */
+
+    iJIT_CA_64          /**<\brief 64-bit machine code. */
+
+} iJIT_CodeArchitecture;
+
+#pragma pack(push, 8)
+
+/**
+ * @brief Description of a JIT-compiled method
+ * @details When you use the iJIT_Method_Load structure to describe
+ *  the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED
+ *  as an event type to report it.
+ */
+typedef struct _iJIT_Method_Load
+{
+    unsigned int method_id; /**<\brief Unique method ID. Cannot be 0.
+                             *  You must either use the API function
+                             *  iJIT_GetNewMethodID to get a valid and unique
+                             *  method ID, or else manage ID uniqueness
+                             *  and correct range by yourself.\n
+                             *  You must use the same method ID for all code
+                             *  regions of the same method, otherwise different
+                             *  method IDs specify different methods. */
+
+    char* method_name; /**<\brief The name of the method. It can be optionally
+                        *  prefixed with its class name and appended with
+                        *  its complete signature. Can't be NULL. */
+
+    void* method_load_address; /**<\brief The start virtual address of the method code
+                                *  region. If NULL, data provided with
+                                *  event are not accepted. */
+
+    unsigned int method_size; /**<\brief The code size of the method in memory.
+                               *  If 0, then data provided with the event are not
+                               *  accepted. */
+
+    unsigned int line_number_size; /**<\brief The number of entries in the line number
+                                    *  table.0 if none. */
+
+    pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info
+                                        *  array. Can be NULL if
+                                        *  line_number_size is 0. See
+                                        *  LineNumberInfo Structure for a
+                                        *  description of a single entry in
+                                        *  the line number info array */
+
+    unsigned int class_id; /**<\brief This field is obsolete. */
+
+    char* class_file_name; /**<\brief Class name. Can be NULL.*/
+
+    char* source_file_name; /**<\brief Source file name. Can be NULL.*/
+
+} *piJIT_Method_Load, iJIT_Method_Load;
+
+/**
+ * @brief Description of a JIT-compiled method
+ * @details When you use the iJIT_Method_Load_V2 structure to describe
+ *  the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2
+ *  as an event type to report it.
+ */
+typedef struct _iJIT_Method_Load_V2
+{
+    unsigned int method_id; /**<\brief Unique method ID. Cannot be 0.
+                             *  You must either use the API function
+                             *  iJIT_GetNewMethodID to get a valid and unique
+                             *  method ID, or else manage ID uniqueness
+                             *  and correct range by yourself.\n
+                             *  You must use the same method ID for all code
+                             *  regions of the same method, otherwise different
+                             *  method IDs specify different methods. */
+
+    char* method_name; /**<\brief The name of the method. It can be optionally
+                        *  prefixed with its class name and appended with
+                        *  its complete signature. Can't be  NULL. */
+
+    void* method_load_address; /**<\brief The start virtual address of the method code
+                                *  region. If NULL, then data provided with the
+                                *  event are not accepted. */
+
+    unsigned int method_size; /**<\brief The code size of the method in memory.
+                               *  If 0, then data provided with the event are not
+                               *  accepted. */
+
+    unsigned int line_number_size; /**<\brief The number of entries in the line number
+                                    *  table. 0 if none. */
+
+    pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info
+                                        *  array. Can be NULL if
+                                        *  line_number_size is 0. See
+                                        *  LineNumberInfo Structure for a
+                                        *  description of a single entry in
+                                        *  the line number info array. */
+
+    char* class_file_name; /**<\brief Class name. Can be NULL. */
+
+    char* source_file_name; /**<\brief Source file name. Can be NULL. */
+
+    char* module_name; /**<\brief Module name. Can be NULL.
+                           The module name can be useful for distinguishing among
+                           different JIT engines. VTune Amplifier will display
+                           reported methods grouped by specific module. */
+
+} *piJIT_Method_Load_V2, iJIT_Method_Load_V2;
+
+/**
+ * @brief Description of a JIT-compiled method
+ * @details The iJIT_Method_Load_V3 structure is the same as iJIT_Method_Load_V2
+ *  with a newly introduced 'arch' field that specifies architecture of the code region.
+ *  When you use the iJIT_Method_Load_V3 structure to describe
+ *  the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3
+ *  as an event type to report it.
+ */
+typedef struct _iJIT_Method_Load_V3
+{
+    unsigned int method_id; /**<\brief Unique method ID. Cannot be 0.
+                             *  You must either use the API function
+                             *  iJIT_GetNewMethodID to get a valid and unique
+                             *  method ID, or manage ID uniqueness
+                             *  and correct range by yourself.\n
+                             *  You must use the same method ID for all code
+                             *  regions of the same method, otherwise they are
+                             *  treated as regions of different methods. */
+
+    char* method_name; /**<\brief The name of the method. It can be optionally
+                        *  prefixed with its class name and appended with
+                        *  its complete signature. Cannot be  NULL. */
+
+    void* method_load_address; /**<\brief The start virtual address of the method code
+                                *  region. If NULL, then data provided with the
+                                *  event are not accepted. */
+
+    unsigned int method_size; /**<\brief The code size of the method in memory.
+                               *  If 0, then data provided with the event are not
+                               *  accepted. */
+
+    unsigned int line_number_size; /**<\brief The number of entries in the line number
+                                    *  table. 0 if none. */
+
+    pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info
+                                        *  array. Can be NULL if
+                                        *  line_number_size is 0. See
+                                        *  LineNumberInfo Structure for a
+                                        *  description of a single entry in
+                                        *  the line number info array. */
+
+    char* class_file_name; /**<\brief Class name. Can be NULL. */
+
+    char* source_file_name; /**<\brief Source file name. Can be NULL. */
+
+    char* module_name; /**<\brief Module name. Can be NULL.
+                        *  The module name can be useful for distinguishing among
+                        *  different JIT engines. VTune Amplifier will display
+                        *  reported methods grouped by specific module. */
+
+    iJIT_CodeArchitecture module_arch; /**<\brief Architecture of the method's code region.
+                                        *  By default, it is the same as the process
+                                        *  architecture that is calling it.
+                                        *  For example, you can use it if your 32-bit JIT
+                                        *  engine generates 64-bit code.
+                                        *
+                                        *  If JIT engine reports both 32-bit and 64-bit types
+                                        *  of methods then VTune Amplifier splits the methods
+                                        *  with the same module name but with different
+                                        *  architectures in two different modules. VTune Amplifier
+                                        *  modifies the original name provided with a 64-bit method
+                                        *  version by ending it with '(64)' */
+
+} *piJIT_Method_Load_V3, iJIT_Method_Load_V3;
+
+/**
+ * @brief Description of an inline JIT-compiled method
+ * @details When you use the_iJIT_Method_Inline_Load structure to describe
+ *  the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED
+ *  as an event type to report it.
+ */
+typedef struct _iJIT_Method_Inline_Load
+{
+    unsigned int method_id; /**<\brief Unique method ID. Cannot be 0.
+                             *  You must either use the API function
+                             *  iJIT_GetNewMethodID to get a valid and unique
+                             *  method ID, or else manage ID uniqueness
+                             *  and correct range by yourself. */
+
+    unsigned int parent_method_id; /**<\brief Unique immediate parent's method ID.
+                                    *  Cannot be 0.
+                                    *  You must either use the API function
+                                    *  iJIT_GetNewMethodID to get a valid and unique
+                                    *  method ID, or else manage ID uniqueness
+                                    *  and correct range by yourself. */
+
+    char* method_name; /**<\brief The name of the method. It can be optionally
+                        *  prefixed with its class name and appended with
+                        *  its complete signature. Can't be NULL. */
+
+    void* method_load_address;  /** <\brief The virtual address on which the method
+                                 *  is inlined. If NULL, then data provided with
+                                 *  the event are not accepted. */
+
+    unsigned int method_size; /**<\brief The code size of the method in memory.
+                               *  If 0, then data provided with the event are not
+                               *  accepted. */
+
+    unsigned int line_number_size; /**<\brief The number of entries in the line number
+                                    *  table. 0 if none. */
+
+    pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info
+                                        *  array. Can be NULL if
+                                        *  line_number_size is 0. See
+                                        *  LineNumberInfo Structure for a
+                                        *  description of a single entry in
+                                        *  the line number info array */
+
+    char* class_file_name; /**<\brief Class name. Can be NULL.*/
+
+    char* source_file_name; /**<\brief Source file name. Can be NULL.*/
+
+} *piJIT_Method_Inline_Load, iJIT_Method_Inline_Load;
+
+/** @cond exclude_from_documentation */
+/**
+ * @brief Description of a segment type
+ * @details Use the segment type to specify a type of data supplied
+ * with the iJVM_EVENT_TYPE_METHOD_UPDATE_V2 event to be applied to
+ * a certain code trace.
+ */
+typedef enum _iJIT_SegmentType
+{
+    iJIT_CT_UNKNOWN = 0,
+
+    iJIT_CT_CODE,           /**<\brief Executable code. */
+
+    iJIT_CT_DATA,           /**<\brief Data (not executable code).
+                             * VTune Amplifier uses the format string
+                             * (see iJIT_Method_Update) to represent
+                             * this data in the VTune Amplifier GUI */
+
+    iJIT_CT_KEEP,           /**<\brief Use the previous markup for the trace.
+                             * Can be used for the following
+                             * iJVM_EVENT_TYPE_METHOD_UPDATE_V2 events,
+                             * if the type of the previously reported segment
+                             * type is the same. */
+    iJIT_CT_EOF
+} iJIT_SegmentType;
+
+/**
+ * @brief Description of a dynamic update of the content within JIT-compiled method
+ * @details The JIT engine may generate the methods that are updated at runtime
+ * partially by mixed (data + executable code) content. When you use the iJIT_Method_Update
+ * structure to describe the update of the content within a JIT-compiled method,
+ * use iJVM_EVENT_TYPE_METHOD_UPDATE_V2 as an event type to report it.
+ *
+ * On the first Update event, VTune Amplifier copies the original code range reported by
+ * the iJVM_EVENT_TYPE_METHOD_LOAD event, then modifies it with the supplied bytes and
+ * adds the modified range to the original method. For next update events, VTune Amplifier
+ * does the same but it uses the latest modified version of a code region for update.
+ * Eventually, VTune Amplifier GUI displays multiple code ranges for the method reported by
+ * the iJVM_EVENT_TYPE_METHOD_LOAD event.
+ * Notes:
+ * - Multiple update events with different types for the same trace are allowed
+ *   but they must be reported for the same code ranges.
+ *   Example,
+ * @code
+ *                      [-- data---]        Allowed
+ *          [-- code --]                    Allowed
+ *        [code]                            Ignored
+ *                      [-- data---]        Allowed
+ *          [-- code --]                    Allowed
+ *      [------------ trace ---------]
+ * @endcode
+ * - The types of previously reported events can be changed but they must be reported
+ *   for the same code ranges.
+ *   Example,
+ * @code
+ *          [-- data---]                    Allowed
+ *          [-- code --]                    Allowed
+ *          [-- data---]                    Allowed
+ *          [-- code --]                    Allowed
+ *      [------------ trace ---------]
+ * @endcode
+ */
+
+typedef struct _iJIT_Method_Update
+{
+    void* load_address;         /**<\brief Start address of the update within a method */
+
+    unsigned int size;          /**<\brief The update size */
+
+    iJIT_SegmentType type;      /**<\brief Type of the update */
+
+    const char* data_format;    /**<\brief C string that contains a format string
+                                 * that follows the same specifications as format in printf.
+                                 * The format string is used for iJIT_CT_CODE only
+                                 * and cannot be NULL.
+                                 * Format can be changed on the fly. */
+} *piJIT_Method_Update, iJIT_Method_Update;
+
+/** @endcond */
+
+#pragma pack(pop)
+
+/** @cond exclude_from_documentation */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+#ifndef JITAPI_CDECL
+#  if defined WIN32 || defined _WIN32
+#    define JITAPI_CDECL __cdecl
+#  else /* defined WIN32 || defined _WIN32 */
+#    if defined _M_IX86 || defined __i386__
+#      define JITAPI_CDECL __attribute__ ((cdecl))
+#    else  /* _M_IX86 || __i386__ */
+#      define JITAPI_CDECL /* actual only on x86_64 platform */
+#    endif /* _M_IX86 || __i386__ */
+#  endif /* defined WIN32 || defined _WIN32 */
+#endif /* JITAPI_CDECL */
+
+#define JITAPI JITAPI_CDECL
+/** @endcond */
+
+/**
+ * @brief Generates a new unique method ID.
+ *
+ * You must use this API to obtain unique and valid method IDs for methods or
+ * traces reported to the agent if you don't have your own mechanism to generate
+ * unique method IDs.
+ *
+ * @return a new unique method ID. When out of unique method IDs, this API
+ * returns 0, which is not an accepted value.
+ */
+unsigned int JITAPI iJIT_GetNewMethodID(void);
+
+/**
+ * @brief Returns the current mode of the agent.
+ *
+ * @return iJIT_SAMPLING_ON, indicating that agent is running, or
+ * iJIT_NOTHING_RUNNING if no agent is running.
+ */
+iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void);
+
+/**
+ * @brief Reports infomation about JIT-compiled code to the agent.
+ *
+ * The reported information is used to attribute samples obtained from any
+ * Intel(R) VTune(TM) Amplifier collector. This API needs to be called
+ * after JIT compilation and before the first entry into the JIT-compiled
+ * code.
+ *
+ * @param[in] event_type - type of the data sent to the agent
+ * @param[in] EventSpecificData - pointer to event-specific data
+ * 
+ * @returns 1 on success, otherwise 0.
+ */
+int JITAPI iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+/** @endcond */
+
+/** @} jitapi group */
+
+#endif /* __JITPROFILING_H__ */

3rdparty/ittnotify/include/libittnotify.h

@@ -0,0 +1,71 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#ifndef _LIBITTNOTIFY_H_
+#define _LIBITTNOTIFY_H_
+
+#ifndef __ITT_INTERNAL_INCLUDE
+#  if defined WIN32 || defined _WIN32
+#    pragma message("WARNING!!! Include file libittnotify.h is deprecated and should not be included anymore")
+#  else /* WIN32 */
+#    warning "Include file libittnotify.h is deprecated and should not be included anymore"
+#  endif /* WIN32 */
+#endif /* __ITT_INTERNAL_INCLUDE */
+#include "legacy/ittnotify.h"
+
+#endif /* _LIBITTNOTIFY_H_ */

3rdparty/ittnotify/include/llvm_jit_event_listener.hpp

@@ -0,0 +1,241 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+/*
+ * This file implements an interface bridge from Low-Level Virtual Machine
+ * llvm::JITEventListener to Intel JIT Profiling API.  It passes the function
+ * and line information to the appropriate functions in the JIT profiling
+ * interface so that any LLVM-based JIT engine can emit the JIT code
+ * notifications that the profiler will receive.
+ *
+ * Usage model:
+ *
+ * 1. Register the listener implementation instance with the execution engine:
+ *
+ *    #include <llvm_jit_event_listener.hpp>
+ *    ...
+ *    ExecutionEngine *TheExecutionEngine;
+ *    ...
+ *    TheExecutionEngine = EngineBuilder(TheModule).create();
+ *    ...
+ *    __itt_llvm_jit_event_listener jitListener;
+ *    TheExecutionEngine->RegisterJITEventListener(&jitListener);
+ *    ...
+ *
+ * 2. When compiling make sure to add the ITT API include directory to the
+ *    compiler include directories, ITT API library directory to the linker
+ *    library directories and link with jitprofling static library.
+ */
+
+#ifndef __ITT_LLVM_JIT_EVENT_LISTENER_HPP__
+#define __ITT_LLVM_JIT_EVENT_LISTENER_HPP__
+
+#include "jitprofiling.h"
+
+#include <llvm/Function.h>
+#include <llvm/ExecutionEngine/JITEventListener.h>
+#include <llvm/ADT/StringRef.h>
+#include <llvm/Analysis/DebugInfo.h>
+
+#include <map>
+#include <cassert>
+
+// Uncomment the line below to turn on logging to stderr
+#define JITPROFILING_DEBUG_ENABLE
+
+// Some elementary logging support
+#ifdef JITPROFILING_DEBUG_ENABLE
+#include <cstdio>
+#include <cstdarg>
+static void _jit_debug(const char* format, ...)
+{
+    va_list args;
+    va_start(args, format);
+    vfprintf(stderr, format, args);
+    va_end(args);
+}
+// Use the macro as JITDEBUG(("foo: %d", foo_val));
+#define JITDEBUG(x) \
+    do { \
+        _jit_debug("jit-listener: "); \
+        _jit_debug x; \
+    } \
+    while (0)
+#else
+#define JITDEBUG(x)
+#endif
+
+// LLVM JIT event listener, translates the notifications to the JIT profiling
+// API information.
+class __itt_llvm_jit_event_listener : public llvm::JITEventListener
+{
+public:
+    __itt_llvm_jit_event_listener() {}
+
+public:
+    virtual void NotifyFunctionEmitted(const llvm::Function &F,
+        void *Code, size_t Size, const EmittedFunctionDetails &Details)
+    {
+        std::string name = F.getName().str();
+        JITDEBUG(("function jitted:\n"));
+        JITDEBUG(("  addr=0x%08x\n", (int)Code));
+        JITDEBUG(("  name=`%s'\n", name.c_str()));
+        JITDEBUG(("  code-size=%d\n", (int)Size));
+        JITDEBUG(("  line-infos-count=%d\n", Details.LineStarts.size()));
+
+        // The method must not be in the map - the entry must have been cleared
+        // from the map in NotifyFreeingMachineCode in case of rejitting.
+        assert(m_addr2MethodId.find(Code) == m_addr2MethodId.end());
+
+        int mid = iJIT_GetNewMethodID();
+        m_addr2MethodId[Code] = mid;
+
+        iJIT_Method_Load mload;
+        memset(&mload, 0, sizeof mload);
+        mload.method_id = mid;
+
+        // Populate the method size and name information
+        // TODO: The JIT profiling API should have members as const char pointers.
+        mload.method_name = (char*)name.c_str();
+        mload.method_load_address = Code;
+        mload.method_size = (unsigned int)Size;
+
+        // Populate line information now.
+        // From the JIT API documentation it is not quite clear whether the
+        // line information can be given in ranges, so we'll populate it for
+        // every byte of the function, hmm.
+        std::string srcFilePath;
+        std::vector<LineNumberInfo> lineInfos;
+        char *addr = (char*)Code;
+        char *lineAddr = addr;          // Exclusive end point at which current
+                                        // line info changes.
+        const llvm::DebugLoc* loc = 0;  // Current line info
+        int lineIndex = -1;             // Current index into the line info table
+        for (int i = 0; i < Size; ++i, ++addr) {
+            while (addr >= lineAddr) {
+                if (lineIndex >= 0 && lineIndex < Details.LineStarts.size()) {
+                    loc = &Details.LineStarts[lineIndex].Loc;
+                    std::string p = getSrcFilePath(F.getContext(), *loc);
+                    assert(srcFilePath.empty() || p == srcFilePath);
+                    srcFilePath = p;
+                } else {
+                    loc = NULL;
+                }
+                lineIndex++;
+                if (lineIndex >= 0 && lineIndex < Details.LineStarts.size()) {
+                    lineAddr = (char*)Details.LineStarts[lineIndex].Address;
+                } else {
+                    lineAddr = addr + Size;
+                }
+            }
+            if (loc) {
+                int line = loc->getLine();
+                LineNumberInfo info = { i, line };
+                lineInfos.push_back(info);
+                JITDEBUG(("  addr 0x%08x -> line %d\n", addr, line));
+            }
+        }
+        if (!lineInfos.empty()) {
+            mload.line_number_size = lineInfos.size();
+            JITDEBUG(("  translated to %d line infos to JIT", (int)lineInfos.size()));
+            mload.line_number_table = &lineInfos[0];
+            mload.source_file_name = (char*)srcFilePath.c_str();
+        }
+
+        iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, &mload);
+    }
+
+    virtual void NotifyFreeingMachineCode(void *OldPtr)
+    {
+        JITDEBUG(("function unjitted\n"));
+        JITDEBUG(("  addr=0x%08x\n", (int)OldPtr));
+        Addr2MethodId::iterator it = m_addr2MethodId.find(OldPtr);
+        assert(it != m_addr2MethodId.end());
+        iJIT_Method_Id mid = { it->second };
+        iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, &mid);
+        m_addr2MethodId.erase(it);
+    }
+
+private:
+    std::string getSrcFilePath(const llvm::LLVMContext& ctx, const llvm::DebugLoc& loc)
+    {
+        llvm::MDNode* node = loc.getAsMDNode(ctx);
+        llvm::DILocation srcLoc(node);
+        return srcLoc.getDirectory().str() + "/" + srcLoc.getFilename().str();
+    }
+
+private:
+    /// Don't copy
+    __itt_llvm_jit_event_listener(const __itt_llvm_jit_event_listener&);
+    __itt_llvm_jit_event_listener& operator=(const __itt_llvm_jit_event_listener&);
+
+private:
+    typedef std::vector<LineNumberInfo> LineInfoList;
+
+    // The method unload notification in VTune JIT profiling API takes the
+    // method ID, not method address so have to maintain the mapping.  Is
+    // there a more efficient and simple way to do this like attaching the
+    // method ID information somehow to the LLVM function instance?
+    //
+    // TODO: It would be more convenient for the JIT API to take the method
+    // address, not method ID.
+    typedef std::map<const void*, int> Addr2MethodId;
+    Addr2MethodId m_addr2MethodId;
+};
+
+#endif // Header guard

3rdparty/ittnotify/src/ittnotify/LICENSE.BSD

@@ -0,0 +1,7 @@
+Copyright (c) 2011, Intel Corporation
+All rights reserved.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+•	Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+•	Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+•	Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3rdparty/ittnotify/src/ittnotify/LICENSE.GPL

@@ -0,0 +1,65 @@
+The GNU General Public License (GPL)
+Version 2, June 1991
+Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+Everyone is permitted to copy and distribute verbatim copies
+of this license document, but changing it is not allowed.
+Preamble
+The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too.
+When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things.
+To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it.
+For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
+We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software.
+Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations.
+Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all.
+The precise terms and conditions for copying, distribution and modification follow.
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you".
+Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does.
+1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program.
+You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee.
+2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions:
+a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change.
+b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License.
+c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.)
+These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it.
+Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program.
+In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License.
+3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following:
+a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
+b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
+c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.)
+The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable.
+If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code.
+4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance.
+5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it.
+6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License.
+7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program.
+If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances.
+It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice.
+This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License.
+8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License.
+9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
+Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation.
+10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally.
+NO WARRANTY
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+END OF TERMS AND CONDITIONS
+How to Apply These Terms to Your New Programs
+If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
+To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
+One line to give the program's name and a brief idea of what it does.
+Copyright (C) <year> <name of author>
+This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+Also add information on how to contact you by electronic and paper mail.
+If the program is interactive, make it output a short notice like this when it starts in an interactive mode:
+Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
+The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program.
+You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names:
+Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker.
+signature of Ty Coon, 1 April 1989
+Ty Coon, President of Vice
+This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License.

3rdparty/ittnotify/src/ittnotify/disable_warnings.h

@@ -0,0 +1,77 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#include "ittnotify_config.h"
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+
+#pragma warning (disable: 593)   /* parameter "XXXX" was set but never used                 */
+#pragma warning (disable: 344)   /* typedef name has already been declared (with same type) */
+#pragma warning (disable: 174)   /* expression has no effect                                */
+#pragma warning (disable: 4127)  /* conditional expression is constant                      */
+#pragma warning (disable: 4306)  /* conversion from '?' to '?' of greater size              */
+
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if defined __INTEL_COMPILER
+
+#pragma warning (disable: 869)  /* parameter "XXXXX" was never referenced                  */
+#pragma warning (disable: 1418) /* external function definition with no prior declaration  */
+#pragma warning (disable: 1419) /* external declaration in primary source file             */
+
+#endif /* __INTEL_COMPILER */

3rdparty/ittnotify/src/ittnotify/ittnotify_config.h

@@ -0,0 +1,616 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+#ifndef _ITTNOTIFY_CONFIG_H_
+#define _ITTNOTIFY_CONFIG_H_
+
+/** @cond exclude_from_documentation */
+#ifndef ITT_OS_WIN
+#  define ITT_OS_WIN   1
+#endif /* ITT_OS_WIN */
+
+#ifndef ITT_OS_LINUX
+#  define ITT_OS_LINUX 2
+#endif /* ITT_OS_LINUX */
+
+#ifndef ITT_OS_MAC
+#  define ITT_OS_MAC   3
+#endif /* ITT_OS_MAC */
+
+#ifndef ITT_OS_FREEBSD
+#  define ITT_OS_FREEBSD   4
+#endif /* ITT_OS_FREEBSD */
+
+#ifndef ITT_OS
+#  if defined WIN32 || defined _WIN32
+#    define ITT_OS ITT_OS_WIN
+#  elif defined( __APPLE__ ) && defined( __MACH__ )
+#    define ITT_OS ITT_OS_MAC
+#  elif defined( __FreeBSD__ )
+#    define ITT_OS ITT_OS_FREEBSD
+#  else
+#    define ITT_OS ITT_OS_LINUX
+#  endif
+#endif /* ITT_OS */
+
+#ifndef ITT_PLATFORM_WIN
+#  define ITT_PLATFORM_WIN 1
+#endif /* ITT_PLATFORM_WIN */
+
+#ifndef ITT_PLATFORM_POSIX
+#  define ITT_PLATFORM_POSIX 2
+#endif /* ITT_PLATFORM_POSIX */
+
+#ifndef ITT_PLATFORM_MAC
+#  define ITT_PLATFORM_MAC 3
+#endif /* ITT_PLATFORM_MAC */
+
+#ifndef ITT_PLATFORM_FREEBSD
+#  define ITT_PLATFORM_FREEBSD 4
+#endif /* ITT_PLATFORM_FREEBSD */
+
+#ifndef ITT_PLATFORM
+#  if ITT_OS==ITT_OS_WIN
+#    define ITT_PLATFORM ITT_PLATFORM_WIN
+#  elif ITT_OS==ITT_OS_MAC
+#    define ITT_PLATFORM ITT_PLATFORM_MAC
+#  elif ITT_OS==ITT_OS_FREEBSD
+#    define ITT_PLATFORM ITT_PLATFORM_FREEBSD
+#  else
+#    define ITT_PLATFORM ITT_PLATFORM_POSIX
+#  endif
+#endif /* ITT_PLATFORM */
+
+#if defined(_UNICODE) && !defined(UNICODE)
+#define UNICODE
+#endif
+
+#include <stddef.h>
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <tchar.h>
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <stdint.h>
+#if defined(UNICODE) || defined(_UNICODE)
+#include <wchar.h>
+#endif /* UNICODE || _UNICODE */
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#ifndef ITTAPI_CDECL
+#  if ITT_PLATFORM==ITT_PLATFORM_WIN
+#    define ITTAPI_CDECL __cdecl
+#  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#    if defined _M_IX86 || defined __i386__
+#      define ITTAPI_CDECL __attribute__ ((cdecl))
+#    else  /* _M_IX86 || __i386__ */
+#      define ITTAPI_CDECL /* actual only on x86 platform */
+#    endif /* _M_IX86 || __i386__ */
+#  endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* ITTAPI_CDECL */
+
+#ifndef STDCALL
+#  if ITT_PLATFORM==ITT_PLATFORM_WIN
+#    define STDCALL __stdcall
+#  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#    if defined _M_IX86 || defined __i386__
+#      define STDCALL __attribute__ ((stdcall))
+#    else  /* _M_IX86 || __i386__ */
+#      define STDCALL /* supported only on x86 platform */
+#    endif /* _M_IX86 || __i386__ */
+#  endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* STDCALL */
+
+#define ITTAPI    ITTAPI_CDECL
+#define LIBITTAPI ITTAPI_CDECL
+
+/* TODO: Temporary for compatibility! */
+#define ITTAPI_CALL    ITTAPI_CDECL
+#define LIBITTAPI_CALL ITTAPI_CDECL
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+/* use __forceinline (VC++ specific) */
+#define ITT_INLINE           __forceinline
+#define ITT_INLINE_ATTRIBUTE /* nothing */
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+/*
+ * Generally, functions are not inlined unless optimization is specified.
+ * For functions declared inline, this attribute inlines the function even
+ * if no optimization level was specified.
+ */
+#ifdef __STRICT_ANSI__
+#define ITT_INLINE           static
+#define ITT_INLINE_ATTRIBUTE __attribute__((unused))
+#else  /* __STRICT_ANSI__ */
+#define ITT_INLINE           static inline
+#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused))
+#endif /* __STRICT_ANSI__ */
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+/** @endcond */
+
+#ifndef ITT_ARCH_IA32
+#  define ITT_ARCH_IA32  1
+#endif /* ITT_ARCH_IA32 */
+
+#ifndef ITT_ARCH_IA32E
+#  define ITT_ARCH_IA32E 2
+#endif /* ITT_ARCH_IA32E */
+
+#ifndef ITT_ARCH_ARM
+#  define ITT_ARCH_ARM  4
+#endif /* ITT_ARCH_ARM */
+
+#ifndef ITT_ARCH_PPC64
+#  define ITT_ARCH_PPC64  5
+#endif /* ITT_ARCH_PPC64 */
+
+#ifndef ITT_ARCH
+#  if defined _M_IX86 || defined __i386__
+#    define ITT_ARCH ITT_ARCH_IA32
+#  elif defined _M_X64 || defined _M_AMD64 || defined __x86_64__
+#    define ITT_ARCH ITT_ARCH_IA32E
+#  elif defined _M_IA64 || defined __ia64__
+#    define ITT_ARCH ITT_ARCH_IA64
+#  elif defined _M_ARM || defined __arm__
+#    define ITT_ARCH ITT_ARCH_ARM
+#  elif defined __powerpc64__
+#    define ITT_ARCH ITT_ARCH_PPC64
+#  endif
+#endif
+
+#ifdef __cplusplus
+#  define ITT_EXTERN_C extern "C"
+#  define ITT_EXTERN_C_BEGIN extern "C" {
+#  define ITT_EXTERN_C_END }
+#else
+#  define ITT_EXTERN_C /* nothing */
+#  define ITT_EXTERN_C_BEGIN /* nothing */
+#  define ITT_EXTERN_C_END /* nothing */
+#endif /* __cplusplus */
+
+#define ITT_TO_STR_AUX(x) #x
+#define ITT_TO_STR(x)     ITT_TO_STR_AUX(x)
+
+#define __ITT_BUILD_ASSERT(expr, suffix) do { \
+    static char __itt_build_check_##suffix[(expr) ? 1 : -1]; \
+    __itt_build_check_##suffix[0] = 0; \
+} while(0)
+#define _ITT_BUILD_ASSERT(expr, suffix)  __ITT_BUILD_ASSERT((expr), suffix)
+#define ITT_BUILD_ASSERT(expr)           _ITT_BUILD_ASSERT((expr), __LINE__)
+
+#define ITT_MAGIC { 0xED, 0xAB, 0xAB, 0xEC, 0x0D, 0xEE, 0xDA, 0x30 }
+
+/* Replace with snapshot date YYYYMMDD for promotion build. */
+#define API_VERSION_BUILD    20151119
+
+#ifndef API_VERSION_NUM
+#define API_VERSION_NUM 0.0.0
+#endif /* API_VERSION_NUM */
+
+#define API_VERSION "ITT-API-Version " ITT_TO_STR(API_VERSION_NUM) \
+                                " (" ITT_TO_STR(API_VERSION_BUILD) ")"
+
+/* OS communication functions */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <windows.h>
+typedef HMODULE           lib_t;
+typedef DWORD             TIDT;
+typedef CRITICAL_SECTION  mutex_t;
+#define MUTEX_INITIALIZER { 0 }
+#define strong_alias(name, aliasname) /* empty for Windows */
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <dlfcn.h>
+#if defined(UNICODE) || defined(_UNICODE)
+#include <wchar.h>
+#endif /* UNICODE */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1 /* need for PTHREAD_MUTEX_RECURSIVE */
+#endif /* _GNU_SOURCE */
+#ifndef __USE_UNIX98
+#define __USE_UNIX98 1 /* need for PTHREAD_MUTEX_RECURSIVE, on SLES11.1 with gcc 4.3.4 wherein pthread.h missing dependency on __USE_XOPEN2K8 */
+#endif /*__USE_UNIX98*/
+#include <pthread.h>
+typedef void*             lib_t;
+typedef pthread_t         TIDT;
+typedef pthread_mutex_t   mutex_t;
+#define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#define _strong_alias(name, aliasname) \
+            extern __typeof (name) aliasname __attribute__ ((alias (#name)));
+#define strong_alias(name, aliasname) _strong_alias(name, aliasname)
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#define __itt_get_proc(lib, name) GetProcAddress(lib, name)
+#define __itt_mutex_init(mutex)   InitializeCriticalSection(mutex)
+#define __itt_mutex_lock(mutex)   EnterCriticalSection(mutex)
+#define __itt_mutex_unlock(mutex) LeaveCriticalSection(mutex)
+#define __itt_load_lib(name)      LoadLibraryA(name)
+#define __itt_unload_lib(handle)  FreeLibrary(handle)
+#define __itt_system_error()      (int)GetLastError()
+#define __itt_fstrcmp(s1, s2)     lstrcmpA(s1, s2)
+#define __itt_fstrnlen(s, l)      strnlen_s(s, l)
+#define __itt_fstrcpyn(s1, b, s2, l) strncpy_s(s1, b, s2, l)
+#define __itt_fstrdup(s)          _strdup(s)
+#define __itt_thread_id()         GetCurrentThreadId()
+#define __itt_thread_yield()      SwitchToThread()
+#ifndef ITT_SIMPLE_INIT
+ITT_INLINE long
+__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __itt_interlocked_increment(volatile long* ptr)
+{
+    return InterlockedIncrement(ptr);
+}
+#endif /* ITT_SIMPLE_INIT */
+
+#define DL_SYMBOLS (1)
+#define PTHREAD_SYMBOLS (1)
+
+#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+#define __itt_get_proc(lib, name) dlsym(lib, name)
+#define __itt_mutex_init(mutex)   {\
+    pthread_mutexattr_t mutex_attr;                                         \
+    int error_code = pthread_mutexattr_init(&mutex_attr);                   \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_init",    \
+                           error_code);                                     \
+    error_code = pthread_mutexattr_settype(&mutex_attr,                     \
+                                           PTHREAD_MUTEX_RECURSIVE);        \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_settype", \
+                           error_code);                                     \
+    error_code = pthread_mutex_init(mutex, &mutex_attr);                    \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutex_init",        \
+                           error_code);                                     \
+    error_code = pthread_mutexattr_destroy(&mutex_attr);                    \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_destroy", \
+                           error_code);                                     \
+}
+#define __itt_mutex_lock(mutex)   pthread_mutex_lock(mutex)
+#define __itt_mutex_unlock(mutex) pthread_mutex_unlock(mutex)
+#define __itt_load_lib(name)      dlopen(name, RTLD_LAZY)
+#define __itt_unload_lib(handle)  dlclose(handle)
+#define __itt_system_error()      errno
+#define __itt_fstrcmp(s1, s2)     strcmp(s1, s2)
+
+/* makes customer code define safe APIs for SDL_STRNLEN_S and SDL_STRNCPY_S */
+#ifdef SDL_STRNLEN_S
+#define __itt_fstrnlen(s, l)      SDL_STRNLEN_S(s, l)
+#else
+#define __itt_fstrnlen(s, l)      strlen(s)
+#endif /* SDL_STRNLEN_S */
+#ifdef SDL_STRNCPY_S
+#define __itt_fstrcpyn(s1, b, s2, l) SDL_STRNCPY_S(s1, b, s2, l)
+#else
+#define __itt_fstrcpyn(s1, b, s2, l) strncpy(s1, s2, l)
+#endif /* SDL_STRNCPY_S */
+
+#define __itt_fstrdup(s)          strdup(s)
+#define __itt_thread_id()         pthread_self()
+#define __itt_thread_yield()      sched_yield()
+#if ITT_ARCH==ITT_ARCH_IA64
+#ifdef __INTEL_COMPILER
+#define __TBB_machine_fetchadd4(addr, val) __fetchadd4_acq((void *)addr, val)
+#else  /* __INTEL_COMPILER */
+/* TODO: Add Support for not Intel compilers for IA-64 architecture */
+#endif /* __INTEL_COMPILER */
+#elif ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_IA32E /* ITT_ARCH!=ITT_ARCH_IA64 */
+ITT_INLINE long
+__TBB_machine_fetchadd4(volatile void* ptr, long addend) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend)
+{
+    long result;
+    __asm__ __volatile__("lock\nxadd %0,%1"
+                          : "=r"(result),"=m"(*(int*)ptr)
+                          : "0"(addend), "m"(*(int*)ptr)
+                          : "memory");
+    return result;
+}
+#elif ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_PPC64
+#define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val)
+#endif /* ITT_ARCH==ITT_ARCH_IA64 */
+#ifndef ITT_SIMPLE_INIT
+ITT_INLINE long
+__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __itt_interlocked_increment(volatile long* ptr)
+{
+    return __TBB_machine_fetchadd4(ptr, 1) + 1L;
+}
+#endif /* ITT_SIMPLE_INIT */
+
+void* dlopen(const char*, int) __attribute__((weak));
+void* dlsym(void*, const char*) __attribute__((weak));
+int dlclose(void*) __attribute__((weak));
+#define DL_SYMBOLS (dlopen && dlsym && dlclose)
+
+int pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*) __attribute__((weak));
+int pthread_mutex_lock(pthread_mutex_t*) __attribute__((weak));
+int pthread_mutex_unlock(pthread_mutex_t*) __attribute__((weak));
+int pthread_mutex_destroy(pthread_mutex_t*) __attribute__((weak));
+int pthread_mutexattr_init(pthread_mutexattr_t*) __attribute__((weak));
+int pthread_mutexattr_settype(pthread_mutexattr_t*, int) __attribute__((weak));
+int pthread_mutexattr_destroy(pthread_mutexattr_t*) __attribute__((weak));
+pthread_t pthread_self(void) __attribute__((weak));
+#define PTHREAD_SYMBOLS (pthread_mutex_init && pthread_mutex_lock && pthread_mutex_unlock && pthread_mutex_destroy && pthread_mutexattr_init && pthread_mutexattr_settype && pthread_mutexattr_destroy && pthread_self)
+
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+typedef enum {
+    __itt_collection_normal = 0,
+    __itt_collection_paused = 1
+} __itt_collection_state;
+
+typedef enum {
+    __itt_thread_normal  = 0,
+    __itt_thread_ignored = 1
+} __itt_thread_state;
+
+#pragma pack(push, 8)
+
+typedef struct ___itt_thread_info
+{
+    const char* nameA; /*!< Copy of original name in ASCII. */
+#if defined(UNICODE) || defined(_UNICODE)
+    const wchar_t* nameW; /*!< Copy of original name in UNICODE. */
+#else  /* UNICODE || _UNICODE */
+    void* nameW;
+#endif /* UNICODE || _UNICODE */
+    TIDT               tid;
+    __itt_thread_state state;   /*!< Thread state (paused or normal) */
+    int                extra1;  /*!< Reserved to the runtime */
+    void*              extra2;  /*!< Reserved to the runtime */
+    struct ___itt_thread_info* next;
+} __itt_thread_info;
+
+#include "ittnotify_types.h" /* For __itt_group_id definition */
+
+typedef struct ___itt_api_info_20101001
+{
+    const char*    name;
+    void**         func_ptr;
+    void*          init_func;
+    __itt_group_id group;
+}  __itt_api_info_20101001;
+
+typedef struct ___itt_api_info
+{
+    const char*    name;
+    void**         func_ptr;
+    void*          init_func;
+    void*          null_func;
+    __itt_group_id group;
+}  __itt_api_info;
+
+typedef struct __itt_counter_info
+{
+    const char* nameA;  /*!< Copy of original name in ASCII. */
+#if defined(UNICODE) || defined(_UNICODE)
+    const wchar_t* nameW; /*!< Copy of original name in UNICODE. */
+#else  /* UNICODE || _UNICODE */
+    void* nameW;
+#endif /* UNICODE || _UNICODE */
+    const char* domainA;  /*!< Copy of original name in ASCII. */
+#if defined(UNICODE) || defined(_UNICODE)
+    const wchar_t* domainW; /*!< Copy of original name in UNICODE. */
+#else  /* UNICODE || _UNICODE */
+    void* domainW;
+#endif /* UNICODE || _UNICODE */
+    int type;
+    long index;
+    int   extra1; /*!< Reserved to the runtime */
+    void* extra2; /*!< Reserved to the runtime */
+    struct __itt_counter_info* next;
+}  __itt_counter_info_t;
+
+struct ___itt_domain;
+struct ___itt_string_handle;
+
+typedef struct ___itt_global
+{
+    unsigned char          magic[8];
+    unsigned long          version_major;
+    unsigned long          version_minor;
+    unsigned long          version_build;
+    volatile long          api_initialized;
+    volatile long          mutex_initialized;
+    volatile long          atomic_counter;
+    mutex_t                mutex;
+    lib_t                  lib;
+    void*                  error_handler;
+    const char**           dll_path_ptr;
+    __itt_api_info*        api_list_ptr;
+    struct ___itt_global*  next;
+    /* Joinable structures below */
+    __itt_thread_info*     thread_list;
+    struct ___itt_domain*  domain_list;
+    struct ___itt_string_handle* string_list;
+    __itt_collection_state state;
+    __itt_counter_info_t* counter_list;
+} __itt_global;
+
+#pragma pack(pop)
+
+#define NEW_THREAD_INFO_W(gptr,h,h_tail,t,s,n) { \
+    h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \
+    if (h != NULL) { \
+        h->tid    = t; \
+        h->nameA  = NULL; \
+        h->nameW  = n ? _wcsdup(n) : NULL; \
+        h->state  = s; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->thread_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_THREAD_INFO_A(gptr,h,h_tail,t,s,n) { \
+    h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \
+    if (h != NULL) { \
+        h->tid    = t; \
+        h->nameA  = n ? __itt_fstrdup(n) : NULL; \
+        h->nameW  = NULL; \
+        h->state  = s; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->thread_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_DOMAIN_W(gptr,h,h_tail,name) { \
+    h = (__itt_domain*)malloc(sizeof(__itt_domain)); \
+    if (h != NULL) { \
+        h->flags  = 1;    /* domain is enabled by default */ \
+        h->nameA  = NULL; \
+        h->nameW  = name ? _wcsdup(name) : NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->domain_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_DOMAIN_A(gptr,h,h_tail,name) { \
+    h = (__itt_domain*)malloc(sizeof(__itt_domain)); \
+    if (h != NULL) { \
+        h->flags  = 1;    /* domain is enabled by default */ \
+        h->nameA  = name ? __itt_fstrdup(name) : NULL; \
+        h->nameW  = NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->domain_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_STRING_HANDLE_W(gptr,h,h_tail,name) { \
+    h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \
+    if (h != NULL) { \
+        h->strA   = NULL; \
+        h->strW   = name ? _wcsdup(name) : NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->string_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_STRING_HANDLE_A(gptr,h,h_tail,name) { \
+    h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \
+    if (h != NULL) { \
+        h->strA   = name ? __itt_fstrdup(name) : NULL; \
+        h->strW   = NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->string_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_COUNTER_W(gptr,h,h_tail,name,domain,type) { \
+    h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \
+    if (h != NULL) { \
+        h->nameA   = NULL; \
+        h->nameW   = name ? _wcsdup(name) : NULL; \
+        h->domainA   = NULL; \
+        h->domainW   = name ? _wcsdup(domain) : NULL; \
+        h->type = type; \
+        h->index = 0; \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->counter_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_COUNTER_A(gptr,h,h_tail,name,domain,type) { \
+    h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \
+    if (h != NULL) { \
+        h->nameA   = name ? __itt_fstrdup(name) : NULL; \
+        h->nameW   = NULL; \
+        h->domainA   = domain ? __itt_fstrdup(domain) : NULL; \
+        h->domainW   = NULL; \
+        h->type = type; \
+        h->index = 0; \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->counter_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#endif /* _ITTNOTIFY_CONFIG_H_ */

3rdparty/ittnotify/src/ittnotify/ittnotify_static.h

@@ -0,0 +1,389 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#include "ittnotify_config.h"
+
+#ifndef ITT_FORMAT_DEFINED
+#  ifndef ITT_FORMAT
+#    define ITT_FORMAT
+#  endif /* ITT_FORMAT */
+#  ifndef ITT_NO_PARAMS
+#    define ITT_NO_PARAMS
+#  endif /* ITT_NO_PARAMS */
+#endif /* ITT_FORMAT_DEFINED */
+
+/*
+ * parameters for macro expected:
+ * ITT_STUB(api, type, func_name, arguments, params, func_name_in_dll, group, printf_fmt)
+ */
+#ifdef __ITT_INTERNAL_INIT
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char    *name), (ITT_FORMAT name), domain_createA, __itt_group_structure, "\"%s\"")
+ITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name), (ITT_FORMAT name), domain_createW, __itt_group_structure, "\"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_domain*, domain_create,  (const char    *name), (ITT_FORMAT name), domain_create,  __itt_group_structure, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char    *name), (ITT_FORMAT name), string_handle_createA, __itt_group_structure, "\"%s\"")
+ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name), (ITT_FORMAT name), string_handle_createW, __itt_group_structure, "\"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create,  (const char    *name), (ITT_FORMAT name), string_handle_create,  __itt_group_structure, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char    *name, const char    *domain), (ITT_FORMAT name, domain), counter_createA, __itt_group_counter, "\"%s\", \"%s\"")
+ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), counter_createW, __itt_group_counter, "\"%s\", \"%s\"")
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_counter, counter_create,  (const char    *name, const char    *domain), (ITT_FORMAT name, domain), counter_create,  __itt_group_counter, "\"%s\", \"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char    *name, const char    *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedA, __itt_group_counter, "\"%s\", \"%s\", %d")
+ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedW, __itt_group_counter, "\"%s\", \"%s\", %d")
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_counter, counter_create_typed,  (const char    *name, const char    *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typed,  __itt_group_counter, "\"%s\", \"%s\", %d")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+
+ITT_STUBV(ITTAPI, void, pause,  (void), (ITT_NO_PARAMS), pause,  __itt_group_control | __itt_group_legacy, "no args")
+ITT_STUBV(ITTAPI, void, resume, (void), (ITT_NO_PARAMS), resume, __itt_group_control | __itt_group_legacy, "no args")
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, thread_set_nameA, (const char    *name), (ITT_FORMAT name), thread_set_nameA, __itt_group_thread, "\"%s\"")
+ITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name), (ITT_FORMAT name), thread_set_nameW, __itt_group_thread, "\"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, thread_set_name,  (const char    *name), (ITT_FORMAT name), thread_set_name,  __itt_group_thread, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, thread_ignore, (void), (ITT_NO_PARAMS), thread_ignore, __itt_group_thread, "no args")
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(LIBITTAPI, int,  thr_name_setA, (const char    *name, int namelen), (ITT_FORMAT name, namelen), thr_name_setA, __itt_group_thread | __itt_group_legacy, "\"%s\", %d")
+ITT_STUB(LIBITTAPI, int,  thr_name_setW, (const wchar_t *name, int namelen), (ITT_FORMAT name, namelen), thr_name_setW, __itt_group_thread | __itt_group_legacy, "\"%S\", %d")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(LIBITTAPI, int,  thr_name_set,  (const char    *name, int namelen), (ITT_FORMAT name, namelen), thr_name_set,  __itt_group_thread | __itt_group_legacy, "\"%s\", %d")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUBV(LIBITTAPI, void, thr_ignore,   (void),                             (ITT_NO_PARAMS),            thr_ignore,    __itt_group_thread | __itt_group_legacy, "no args")
+#endif /* __ITT_INTERNAL_BODY */
+
+ITT_STUBV(ITTAPI, void, enable_attach, (void), (ITT_NO_PARAMS), enable_attach, __itt_group_all, "no args")
+
+#else  /* __ITT_INTERNAL_INIT */
+
+ITT_STUBV(ITTAPI, void, detach, (void), (ITT_NO_PARAMS), detach, __itt_group_control | __itt_group_legacy, "no args")
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char    *objtype, const char    *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_createA, __itt_group_sync | __itt_group_fsync, "%p, \"%s\", \"%s\", %x")
+ITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_createW, __itt_group_sync | __itt_group_fsync, "%p, \"%S\", \"%S\", %x")
+ITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char    *name), (ITT_FORMAT addr, name), sync_renameA, __itt_group_sync | __itt_group_fsync, "%p, \"%s\"")
+ITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name), (ITT_FORMAT addr, name), sync_renameW, __itt_group_sync | __itt_group_fsync, "%p, \"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, sync_create,  (void *addr, const char    *objtype, const char    *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_create,  __itt_group_sync | __itt_group_fsync, "%p, \"%s\", \"%s\", %x")
+ITT_STUBV(ITTAPI, void, sync_rename,  (void *addr, const char    *name), (ITT_FORMAT addr, name), sync_rename,  __itt_group_sync | __itt_group_fsync, "%p, \"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, sync_destroy,    (void *addr), (ITT_FORMAT addr), sync_destroy,   __itt_group_sync | __itt_group_fsync, "%p")
+
+ITT_STUBV(ITTAPI, void, sync_prepare,    (void* addr), (ITT_FORMAT addr), sync_prepare,   __itt_group_sync,  "%p")
+ITT_STUBV(ITTAPI, void, sync_cancel,     (void *addr), (ITT_FORMAT addr), sync_cancel,    __itt_group_sync,  "%p")
+ITT_STUBV(ITTAPI, void, sync_acquired,   (void *addr), (ITT_FORMAT addr), sync_acquired,  __itt_group_sync,  "%p")
+ITT_STUBV(ITTAPI, void, sync_releasing,  (void* addr), (ITT_FORMAT addr), sync_releasing, __itt_group_sync,  "%p")
+
+ITT_STUBV(ITTAPI, void, suppress_push,       (unsigned int mask),                             (ITT_FORMAT mask), suppress_push,  __itt_group_suppress,  "%p")
+ITT_STUBV(ITTAPI, void, suppress_pop,        (void),                                          (ITT_NO_PARAMS),   suppress_pop,   __itt_group_suppress,  "no args")
+ITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size),(ITT_FORMAT mode, mask, address, size), suppress_mark_range, __itt_group_suppress, "%d, %p, %p, %d")
+ITT_STUBV(ITTAPI, void, suppress_clear_range,(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size),(ITT_FORMAT mode, mask, address, size), suppress_clear_range,__itt_group_suppress, "%d, %p, %p, %d")
+
+ITT_STUBV(ITTAPI, void, fsync_prepare,   (void* addr), (ITT_FORMAT addr), sync_prepare,   __itt_group_fsync, "%p")
+ITT_STUBV(ITTAPI, void, fsync_cancel,    (void *addr), (ITT_FORMAT addr), sync_cancel,    __itt_group_fsync, "%p")
+ITT_STUBV(ITTAPI, void, fsync_acquired,  (void *addr), (ITT_FORMAT addr), sync_acquired,  __itt_group_fsync, "%p")
+ITT_STUBV(ITTAPI, void, fsync_releasing, (void* addr), (ITT_FORMAT addr), sync_releasing, __itt_group_fsync, "%p")
+
+ITT_STUBV(ITTAPI, void, model_site_begin,          (__itt_model_site *site, __itt_model_site_instance *instance, const char *name), (ITT_FORMAT site, instance, name), model_site_begin, __itt_group_model, "%p, %p, \"%s\"")
+ITT_STUBV(ITTAPI, void, model_site_end,            (__itt_model_site *site, __itt_model_site_instance *instance),                   (ITT_FORMAT site, instance),       model_site_end,   __itt_group_model, "%p, %p")
+ITT_STUBV(ITTAPI, void, model_task_begin,          (__itt_model_task *task, __itt_model_task_instance *instance, const char *name), (ITT_FORMAT task, instance, name), model_task_begin, __itt_group_model, "%p, %p, \"%s\"")
+ITT_STUBV(ITTAPI, void, model_task_end,            (__itt_model_task *task, __itt_model_task_instance *instance),                   (ITT_FORMAT task, instance),       model_task_end,   __itt_group_model, "%p, %p")
+ITT_STUBV(ITTAPI, void, model_lock_acquire,        (void *lock), (ITT_FORMAT lock), model_lock_acquire, __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_lock_release,        (void *lock), (ITT_FORMAT lock), model_lock_release, __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_record_allocation,   (void *addr, size_t size), (ITT_FORMAT addr, size), model_record_allocation,   __itt_group_model, "%p, %d")
+ITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr),              (ITT_FORMAT addr),       model_record_deallocation, __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_induction_uses,      (void* addr, size_t size), (ITT_FORMAT addr, size), model_induction_uses,      __itt_group_model, "%p, %d")
+ITT_STUBV(ITTAPI, void, model_reduction_uses,      (void* addr, size_t size), (ITT_FORMAT addr, size), model_reduction_uses,      __itt_group_model, "%p, %d")
+ITT_STUBV(ITTAPI, void, model_observe_uses,        (void* addr, size_t size), (ITT_FORMAT addr, size), model_observe_uses,        __itt_group_model, "%p, %d")
+ITT_STUBV(ITTAPI, void, model_clear_uses,          (void* addr),              (ITT_FORMAT addr),       model_clear_uses,          __itt_group_model, "%p")
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, model_site_beginW,         (const wchar_t *name),     (ITT_FORMAT name),       model_site_beginW,         __itt_group_model, "\"%s\"")
+ITT_STUBV(ITTAPI, void, model_task_beginW,         (const wchar_t *name),     (ITT_FORMAT name),       model_task_beginW,         __itt_group_model, "\"%s\"")
+ITT_STUBV(ITTAPI, void, model_iteration_taskW,     (const wchar_t *name),     (ITT_FORMAT name),       model_iteration_taskW,     __itt_group_model, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, model_site_beginA,         (const char *name),        (ITT_FORMAT name),       model_site_beginA,         __itt_group_model, "\"%s\"")
+ITT_STUBV(ITTAPI, void, model_site_beginAL,        (const char *name, size_t len), (ITT_FORMAT name, len), model_site_beginAL,    __itt_group_model, "\"%s\", %d")
+ITT_STUBV(ITTAPI, void, model_task_beginA,         (const char *name),        (ITT_FORMAT name),       model_task_beginA,         __itt_group_model, "\"%s\"")
+ITT_STUBV(ITTAPI, void, model_task_beginAL,        (const char *name, size_t len), (ITT_FORMAT name, len), model_task_beginAL,    __itt_group_model, "\"%s\", %d")
+ITT_STUBV(ITTAPI, void, model_iteration_taskA,     (const char *name),        (ITT_FORMAT name),       model_iteration_taskA,     __itt_group_model, "\"%s\"")
+ITT_STUBV(ITTAPI, void, model_iteration_taskAL,    (const char *name, size_t len), (ITT_FORMAT name, len), model_iteration_taskAL, __itt_group_model, "\"%s\", %d")
+ITT_STUBV(ITTAPI, void, model_site_end_2,          (void),                    (ITT_NO_PARAMS),         model_site_end_2,          __itt_group_model, "no args")
+ITT_STUBV(ITTAPI, void, model_task_end_2,          (void),                    (ITT_NO_PARAMS),         model_task_end_2,          __itt_group_model, "no args")
+ITT_STUBV(ITTAPI, void, model_lock_acquire_2,      (void *lock),              (ITT_FORMAT lock),       model_lock_acquire_2,      __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_lock_release_2,      (void *lock),              (ITT_FORMAT lock),       model_lock_release_2,      __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_aggregate_task,      (size_t count),            (ITT_FORMAT count),      model_aggregate_task,      __itt_group_model, "%d")
+ITT_STUBV(ITTAPI, void, model_disable_push,        (__itt_model_disable x),   (ITT_FORMAT x),          model_disable_push,        __itt_group_model, "%p")
+ITT_STUBV(ITTAPI, void, model_disable_pop,         (void),                    (ITT_NO_PARAMS),         model_disable_pop,         __itt_group_model, "no args")
+#endif /* __ITT_INTERNAL_BODY */
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char    *name, const char    *domain), (ITT_FORMAT name, domain), heap_function_createA, __itt_group_heap, "\"%s\", \"%s\"")
+ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), heap_function_createW, __itt_group_heap, "\"%s\", \"%s\"")
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_heap_function, heap_function_create,  (const char    *name, const char    *domain), (ITT_FORMAT name, domain), heap_function_create,  __itt_group_heap, "\"%s\", \"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* __ITT_INTERNAL_BODY */
+ITT_STUBV(ITTAPI, void, heap_allocate_begin,   (__itt_heap_function h, size_t size, int initialized),             (ITT_FORMAT h, size, initialized),       heap_allocate_begin, __itt_group_heap, "%p, %lu, %d")
+ITT_STUBV(ITTAPI, void, heap_allocate_end,     (__itt_heap_function h, void** addr, size_t size, int initialized), (ITT_FORMAT h, addr, size, initialized), heap_allocate_end,   __itt_group_heap, "%p, %p, %lu, %d")
+ITT_STUBV(ITTAPI, void, heap_free_begin,       (__itt_heap_function h, void*  addr), (ITT_FORMAT h, addr), heap_free_begin, __itt_group_heap, "%p, %p")
+ITT_STUBV(ITTAPI, void, heap_free_end,         (__itt_heap_function h, void*  addr), (ITT_FORMAT h, addr), heap_free_end,   __itt_group_heap, "%p, %p")
+ITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void*  addr, size_t new_size, int initialized),                  (ITT_FORMAT h, addr, new_size, initialized),           heap_reallocate_begin, __itt_group_heap, "%p, %p, %lu, %d")
+ITT_STUBV(ITTAPI, void, heap_reallocate_end,   (__itt_heap_function h, void*  addr, void** new_addr, size_t new_size, int initialized), (ITT_FORMAT h, addr, new_addr, new_size, initialized), heap_reallocate_end,   __itt_group_heap, "%p, %p, %p, %lu, %d")
+ITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void), (ITT_NO_PARAMS), heap_internal_access_begin, __itt_group_heap, "no args")
+ITT_STUBV(ITTAPI, void, heap_internal_access_end,   (void), (ITT_NO_PARAMS), heap_internal_access_end,   __itt_group_heap, "no args")
+ITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void), (ITT_NO_PARAMS), heap_record_memory_growth_begin, __itt_group_heap, "no args")
+ITT_STUBV(ITTAPI, void, heap_record_memory_growth_end,   (void), (ITT_NO_PARAMS), heap_record_memory_growth_end,   __itt_group_heap, "no args")
+ITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask),  (ITT_FORMAT reset_mask), heap_reset_detection, __itt_group_heap, "%u")
+ITT_STUBV(ITTAPI, void, heap_record,          (unsigned int record_mask), (ITT_FORMAT record_mask),  heap_record,        __itt_group_heap, "%u")
+
+ITT_STUBV(ITTAPI, void, id_create,  (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), id_create,  __itt_group_structure, "%p, %lu")
+ITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), id_destroy, __itt_group_structure, "%p, %lu")
+
+ITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void), (ITT_NO_PARAMS), get_timestamp,  __itt_group_structure, "no args")
+
+ITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), region_begin, __itt_group_structure, "%p, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, region_end,   (const __itt_domain *domain, __itt_id id),                                             (ITT_FORMAT domain, id),               region_end,   __itt_group_structure, "%p, %lu")
+
+#ifndef __ITT_INTERNAL_BODY
+ITT_STUBV(ITTAPI, void, frame_begin_v3,  (const __itt_domain *domain, __itt_id *id),                                             (ITT_FORMAT domain, id),             frame_begin_v3,  __itt_group_structure, "%p, %p")
+ITT_STUBV(ITTAPI, void, frame_end_v3,    (const __itt_domain *domain, __itt_id *id),                                             (ITT_FORMAT domain, id),             frame_end_v3,    __itt_group_structure, "%p, %p")
+ITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end), (ITT_FORMAT domain, id, begin, end), frame_submit_v3, __itt_group_structure, "%p, %p, %lu, %lu")
+#endif /* __ITT_INTERNAL_BODY */
+
+ITT_STUBV(ITTAPI, void, task_group,   (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), task_group,  __itt_group_structure, "%p, %lu, %lu, %p")
+
+ITT_STUBV(ITTAPI, void, task_begin,    (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), task_begin,    __itt_group_structure, "%p, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parent, void* fn),                  (ITT_FORMAT domain, id, parent, fn),   task_begin_fn, __itt_group_structure, "%p, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_end,      (const __itt_domain *domain),                                                          (ITT_FORMAT domain),                   task_end,      __itt_group_structure, "%p")
+
+ITT_STUBV(ITTAPI, void, counter_inc_v3,       (const __itt_domain *domain, __itt_string_handle *name),                           (ITT_FORMAT domain, name),        counter_inc_v3,       __itt_group_structure, "%p, %p")
+ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_inc_delta_v3, __itt_group_structure, "%p, %p, %lu")
+ITT_STUBV(ITTAPI, void, counter_dec_v3,       (const __itt_domain *domain, __itt_string_handle *name),                           (ITT_FORMAT domain, name),        counter_dec_v3,       __itt_group_structure, "%p, %p")
+ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_dec_delta_v3, __itt_group_structure, "%p, %p, %lu")
+
+ITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope), (ITT_FORMAT domain, id, name, scope), marker, __itt_group_structure, "%p, %lu, %p, %d")
+
+ITT_STUBV(ITTAPI, void, metadata_add,      (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data), (ITT_FORMAT domain, id, key, type, count, data), metadata_add, __itt_group_structure, "%p, %lu, %p, %d, %lu, %p")
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char* data, size_t length),    (ITT_FORMAT domain, id, key, data, length), metadata_str_addA, __itt_group_structure, "%p, %lu, %p, %p, %lu")
+ITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t* data, size_t length), (ITT_FORMAT domain, id, key, data, length), metadata_str_addW, __itt_group_structure, "%p, %lu, %p, %p, %lu")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, metadata_str_add,  (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char* data, size_t length),    (ITT_FORMAT domain, id, key, data, length), metadata_str_add,  __itt_group_structure, "%p, %lu, %p, %p, %lu")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+ITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail),                (ITT_FORMAT domain, relation, tail),       relation_add_to_current, __itt_group_structure, "%p, %lu, %p")
+ITT_STUBV(ITTAPI, void, relation_add,            (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, head, relation, tail), relation_add,            __itt_group_structure, "%p, %p, %lu, %p")
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char    *name, int namelen), (ITT_FORMAT name, namelen), event_createA, __itt_group_mark | __itt_group_legacy, "\"%s\", %d")
+ITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen), (ITT_FORMAT name, namelen), event_createW, __itt_group_mark | __itt_group_legacy, "\"%S\", %d")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(LIBITTAPI, __itt_event, event_create,  (const char    *name, int namelen), (ITT_FORMAT name, namelen), event_create,  __itt_group_mark | __itt_group_legacy, "\"%s\", %d")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(LIBITTAPI, int,  event_start,          (__itt_event event),                (ITT_FORMAT event),         event_start,   __itt_group_mark | __itt_group_legacy, "%d")
+ITT_STUB(LIBITTAPI, int,  event_end,            (__itt_event event),                (ITT_FORMAT event),         event_end,     __itt_group_mark | __itt_group_legacy, "%d")
+#endif /* __ITT_INTERNAL_BODY */
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, sync_set_nameA, (void *addr, const char    *objtype, const char    *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_nameA, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p, \"%s\", \"%s\", %x")
+ITT_STUBV(ITTAPI, void, sync_set_nameW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_nameW, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p, \"%S\", \"%S\", %x")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, sync_set_name,  (void *addr, const char    *objtype, const char    *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_name,  __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "p, \"%s\", \"%s\", %x")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(LIBITTAPI, int, notify_sync_nameA, (void *p, const char    *objtype, int typelen, const char    *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_nameA, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p, \"%s\", %d, \"%s\", %d, %x")
+ITT_STUB(LIBITTAPI, int, notify_sync_nameW, (void *p, const wchar_t *objtype, int typelen, const wchar_t *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_nameW, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p, \"%S\", %d, \"%S\", %d, %x")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(LIBITTAPI, int, notify_sync_name,  (void *p, const char    *objtype, int typelen, const char    *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_name,  __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p, \"%s\", %d, \"%s\", %d, %x")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+ITT_STUBV(LIBITTAPI, void, notify_sync_prepare,   (void *p), (ITT_FORMAT p), notify_sync_prepare,   __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p")
+ITT_STUBV(LIBITTAPI, void, notify_sync_cancel,    (void *p), (ITT_FORMAT p), notify_sync_cancel,    __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p")
+ITT_STUBV(LIBITTAPI, void, notify_sync_acquired,  (void *p), (ITT_FORMAT p), notify_sync_acquired,  __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p")
+ITT_STUBV(LIBITTAPI, void, notify_sync_releasing, (void *p), (ITT_FORMAT p), notify_sync_releasing, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, "%p")
+#endif /* __ITT_INTERNAL_BODY */
+
+ITT_STUBV(LIBITTAPI, void, memory_read,   (void *addr, size_t size), (ITT_FORMAT addr, size), memory_read,   __itt_group_legacy, "%p, %lu")
+ITT_STUBV(LIBITTAPI, void, memory_write,  (void *addr, size_t size), (ITT_FORMAT addr, size), memory_write,  __itt_group_legacy, "%p, %lu")
+ITT_STUBV(LIBITTAPI, void, memory_update, (void *addr, size_t size), (ITT_FORMAT addr, size), memory_update, __itt_group_legacy, "%p, %lu")
+
+ITT_STUB(LIBITTAPI, __itt_state_t,     state_get,    (void),                                    (ITT_NO_PARAMS),   state_get,    __itt_group_legacy, "no args")
+ITT_STUB(LIBITTAPI, __itt_state_t,     state_set,    (__itt_state_t s),                         (ITT_FORMAT s),    state_set,    __itt_group_legacy, "%d")
+ITT_STUB(LIBITTAPI, __itt_obj_state_t, obj_mode_set, (__itt_obj_prop_t p, __itt_obj_state_t s), (ITT_FORMAT p, s), obj_mode_set, __itt_group_legacy, "%d, %d")
+ITT_STUB(LIBITTAPI, __itt_thr_state_t, thr_mode_set, (__itt_thr_prop_t p, __itt_thr_state_t s), (ITT_FORMAT p, s), thr_mode_set, __itt_group_legacy, "%d, %d")
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_frame, frame_createA, (const char    *domain), (ITT_FORMAT domain), frame_createA, __itt_group_frame, "\"%s\"")
+ITT_STUB(ITTAPI, __itt_frame, frame_createW, (const wchar_t *domain), (ITT_FORMAT domain), frame_createW, __itt_group_frame, "\"%s\"")
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_frame, frame_create,  (const char    *domain), (ITT_FORMAT domain), frame_create,  __itt_group_frame, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* __ITT_INTERNAL_BODY */
+ITT_STUBV(ITTAPI, void, frame_begin,         (__itt_frame frame),     (ITT_FORMAT frame),  frame_begin,   __itt_group_frame, "%p")
+ITT_STUBV(ITTAPI, void, frame_end,           (__itt_frame frame),     (ITT_FORMAT frame),  frame_end,     __itt_group_frame, "%p")
+
+ITT_STUBV(ITTAPI, void, counter_destroy,      (__itt_counter id),                                                                                  (ITT_FORMAT id),        counter_destroy,   __itt_group_counter, "%p")
+ITT_STUBV(ITTAPI, void, counter_inc,          (__itt_counter id),                                                                                  (ITT_FORMAT id),        counter_inc,       __itt_group_counter, "%p")
+ITT_STUBV(ITTAPI, void, counter_inc_delta,    (__itt_counter id, unsigned long long value),                                                        (ITT_FORMAT id, value), counter_inc_delta, __itt_group_counter, "%p, %lu")
+ITT_STUBV(ITTAPI, void, counter_dec,          (__itt_counter id),                                                                                  (ITT_FORMAT id),        counter_dec,       __itt_group_counter, "%p")
+ITT_STUBV(ITTAPI, void, counter_dec_delta,    (__itt_counter id, unsigned long long value),                                                        (ITT_FORMAT id, value), counter_dec_delta, __itt_group_counter, "%p, %lu")
+ITT_STUBV(ITTAPI, void, counter_set_value,    (__itt_counter id, void *value_ptr),                                                                 (ITT_FORMAT id, value_ptr),                          counter_set_value,    __itt_group_counter, "%p, %p")
+ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr), (ITT_FORMAT id, clock_domain, timestamp, value_ptr), counter_set_value_ex, __itt_group_counter, "%p, %p, %llu, %p")
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char    *name), (ITT_FORMAT name), mark_createA, __itt_group_mark, "\"%s\"")
+ITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name), (ITT_FORMAT name), mark_createW, __itt_group_mark, "\"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, __itt_mark_type, mark_create,  (const char    *name), (ITT_FORMAT name), mark_create,  __itt_group_mark, "\"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* __ITT_INTERNAL_BODY */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, int,  markA,        (__itt_mark_type mt, const char    *parameter), (ITT_FORMAT mt, parameter), markA, __itt_group_mark, "%d, \"%s\"")
+ITT_STUB(ITTAPI, int,  markW,        (__itt_mark_type mt, const wchar_t *parameter), (ITT_FORMAT mt, parameter), markW, __itt_group_mark, "%d, \"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, int,  mark,         (__itt_mark_type mt, const char    *parameter), (ITT_FORMAT mt, parameter), mark,  __itt_group_mark, "%d, \"%s\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, int,  mark_off, (__itt_mark_type mt), (ITT_FORMAT mt), mark_off, __itt_group_mark, "%d")
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, int,  mark_globalA, (__itt_mark_type mt, const char    *parameter), (ITT_FORMAT mt, parameter), mark_globalA, __itt_group_mark, "%d, \"%s\"")
+ITT_STUB(ITTAPI, int,  mark_globalW, (__itt_mark_type mt, const wchar_t *parameter), (ITT_FORMAT mt, parameter), mark_globalW, __itt_group_mark, "%d, \"%S\"")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, int,  mark_global,  (__itt_mark_type mt, const char    *parameter), (ITT_FORMAT mt, parameter), mark_global,  __itt_group_mark, "%d, \"%S\"")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, int,  mark_global_off, (__itt_mark_type mt),                        (ITT_FORMAT mt),            mark_global_off, __itt_group_mark, "%d")
+
+#ifndef __ITT_INTERNAL_BODY
+ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void), (ITT_NO_PARAMS), stack_caller_create,  __itt_group_stitch, "no args")
+#endif /* __ITT_INTERNAL_BODY */
+ITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id), (ITT_FORMAT id), stack_caller_destroy, __itt_group_stitch, "%p")
+ITT_STUBV(ITTAPI, void, stack_callee_enter,   (__itt_caller id), (ITT_FORMAT id), stack_callee_enter,   __itt_group_stitch, "%p")
+ITT_STUBV(ITTAPI, void, stack_callee_leave,   (__itt_caller id), (ITT_FORMAT id), stack_callee_leave,   __itt_group_stitch, "%p")
+
+ITT_STUB(ITTAPI,  __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data), (ITT_FORMAT fn, fn_data), clock_domain_create, __itt_group_structure, "%p, %p")
+ITT_STUBV(ITTAPI, void,                clock_domain_reset,  (void),                                      (ITT_NO_PARAMS),          clock_domain_reset,  __itt_group_structure, "no args")
+ITT_STUBV(ITTAPI, void, id_create_ex,  (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id), (ITT_FORMAT domain, clock_domain, timestamp, id), id_create_ex,  __itt_group_structure, "%p, %p, %lu, %lu")
+ITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id), (ITT_FORMAT domain, clock_domain, timestamp, id), id_destroy_ex, __itt_group_structure, "%p, %p, %lu, %lu")
+ITT_STUBV(ITTAPI, void, task_begin_ex,    (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name), (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, name), task_begin_ex, __itt_group_structure, "%p, %p, %lu, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn),                  (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, fn), task_begin_fn_ex, __itt_group_structure, "%p, %p, %lu, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_end_ex,      (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp),                                                            (ITT_FORMAT domain, clock_domain, timestamp), task_end_ex, __itt_group_structure, "%p, %p, %lu")
+ITT_STUBV(ITTAPI, void, task_begin_overlapped,       (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name),                                                                   (ITT_FORMAT domain, id, parent, name), task_begin_overlapped, __itt_group_structure, "%p, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_begin_overlapped_ex,    (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name), (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, name), task_begin_overlapped_ex, __itt_group_structure, "%p, %p, %lu, %lu, %lu, %p")
+ITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id id),                                                                                                                       (ITT_FORMAT domain, id), task_end_overlapped, __itt_group_structure, "%p, %lu")
+ITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id),                                                    (ITT_FORMAT domain, clock_domain, timestamp, id), task_end_overlapped_ex, __itt_group_structure, "%p, %p, %lu, %lu")
+ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope), (ITT_FORMAT domain, clock_domain, timestamp, id, name, scope), marker_ex, __itt_group_structure, "%p, %p, %lu, %lu, %p, %d")
+ITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data), (ITT_FORMAT domain, scope, key, type, count, data), metadata_add_with_scope, __itt_group_structure, "%p, %d, %p, %d, %lu, %p")
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length),    (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scopeA, __itt_group_structure, "%p, %d, %p, %p, %lu")
+ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length), (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scopeW, __itt_group_structure, "%p, %d, %p, %p, %lu")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, metadata_str_add_with_scope,  (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length),    (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scope,  __itt_group_structure, "%p, %d, %p, %p, %lu")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail),                (ITT_FORMAT domain, clock_domain, timestamp, relation, tail),       relation_add_to_current_ex, __itt_group_structure, "%p, %p, %lu, %d, %lu")
+ITT_STUBV(ITTAPI, void, relation_add_ex,            (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, clock_domain, timestamp, head, relation, tail), relation_add_ex,            __itt_group_structure, "%p, %p, %lu, %lu, %d, %lu")
+ITT_STUB(ITTAPI,  __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type),                    (ITT_FORMAT name, track_group_type),        track_group_create, __itt_group_structure, "%p, %d")
+ITT_STUB(ITTAPI,  __itt_track*,       track_create,       (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type), (ITT_FORMAT track_group, name, track_type), track_create,       __itt_group_structure, "%p, %p, %d")
+ITT_STUBV(ITTAPI, void,               set_track,          (__itt_track *track),                                                                    (ITT_FORMAT track),                         set_track,          __itt_group_structure, "%p")
+
+#ifndef __ITT_INTERNAL_BODY
+ITT_STUB(ITTAPI, const char*, api_version, (void), (ITT_NO_PARAMS), api_version, __itt_group_all & ~__itt_group_legacy, "no args")
+#endif /* __ITT_INTERNAL_BODY */
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_saveA, __itt_group_arrays, "%p, %d, %p, %d, \"%s\", %d")
+ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_saveW, __itt_group_arrays, "%p, %d, %p, %d, \"%S\", %d")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUB(ITTAPI, int, av_save,  (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_save,  __itt_group_arrays, "%p, %d, %p, %d, \"%s\", %d")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* __ITT_INTERNAL_BODY */
+
+#ifndef __ITT_INTERNAL_BODY
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+ITT_STUBV(ITTAPI, void, module_loadA, (void *start_addr, void* end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_loadA, __itt_group_none, "%p, %p, %p")
+ITT_STUBV(ITTAPI, void, module_loadW, (void *start_addr, void* end_addr, const wchar_t *path), (ITT_FORMAT start_addr, end_addr, path), module_loadW, __itt_group_none, "%p, %p, %p")
+#else  /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+ITT_STUBV(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_load, __itt_group_none, "%p, %p, %p")
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* __ITT_INTERNAL_BODY */
+
+
+#endif /* __ITT_INTERNAL_INIT */

3rdparty/ittnotify/src/ittnotify/ittnotify_types.h

@@ -0,0 +1,115 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#ifndef _ITTNOTIFY_TYPES_H_
+#define _ITTNOTIFY_TYPES_H_
+
+typedef enum ___itt_group_id
+{
+    __itt_group_none      = 0,
+    __itt_group_legacy    = 1<<0,
+    __itt_group_control   = 1<<1,
+    __itt_group_thread    = 1<<2,
+    __itt_group_mark      = 1<<3,
+    __itt_group_sync      = 1<<4,
+    __itt_group_fsync     = 1<<5,
+    __itt_group_jit       = 1<<6,
+    __itt_group_model     = 1<<7,
+    __itt_group_splitter_min = 1<<7,
+    __itt_group_counter   = 1<<8,
+    __itt_group_frame     = 1<<9,
+    __itt_group_stitch    = 1<<10,
+    __itt_group_heap      = 1<<11,
+    __itt_group_splitter_max = 1<<12,
+    __itt_group_structure = 1<<12,
+    __itt_group_suppress = 1<<13,
+    __itt_group_arrays    = 1<<14,
+    __itt_group_all       = -1
+} __itt_group_id;
+
+#pragma pack(push, 8)
+
+typedef struct ___itt_group_list
+{
+    __itt_group_id id;
+    const char*    name;
+} __itt_group_list;
+
+#pragma pack(pop)
+
+#define ITT_GROUP_LIST(varname) \
+    static __itt_group_list varname[] = {       \
+        { __itt_group_all,       "all"       }, \
+        { __itt_group_control,   "control"   }, \
+        { __itt_group_thread,    "thread"    }, \
+        { __itt_group_mark,      "mark"      }, \
+        { __itt_group_sync,      "sync"      }, \
+        { __itt_group_fsync,     "fsync"     }, \
+        { __itt_group_jit,       "jit"       }, \
+        { __itt_group_model,     "model"     }, \
+        { __itt_group_counter,   "counter"   }, \
+        { __itt_group_frame,     "frame"     }, \
+        { __itt_group_stitch,    "stitch"    }, \
+        { __itt_group_heap,      "heap"      }, \
+        { __itt_group_structure, "structure" }, \
+        { __itt_group_suppress,  "suppress"  }, \
+        { __itt_group_arrays,    "arrays"    }, \
+        { __itt_group_none,      NULL        }  \
+    }
+
+#endif /* _ITTNOTIFY_TYPES_H_ */

3rdparty/ittnotify/src/ittnotify/jitprofiling.c

@@ -0,0 +1,314 @@
+/* <copyright>
+  This file is provided under a dual BSD/GPLv2 license.  When using or
+  redistributing this file, you may do so under either license.
+
+  GPL LICENSE SUMMARY
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+  The full GNU General Public License is included in this distribution
+  in the file called LICENSE.GPL.
+
+  Contact Information:
+  http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/
+
+  BSD LICENSE
+
+  Copyright (c) 2005-2014 Intel Corporation. All rights reserved.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+</copyright> */
+
+#include "ittnotify_config.h"
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <windows.h>
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#if ITT_PLATFORM != ITT_PLATFORM_MAC && ITT_PLATFORM != ITT_PLATFORM_FREEBSD
+#include <malloc.h>
+#endif
+#include <stdlib.h>
+
+#include "jitprofiling.h"
+
+static const char rcsid[] = "\n@(#) $Revision: 471937 $\n";
+
+#define DLL_ENVIRONMENT_VAR             "VS_PROFILER"
+
+#ifndef NEW_DLL_ENVIRONMENT_VAR
+#if ITT_ARCH==ITT_ARCH_IA32
+#define NEW_DLL_ENVIRONMENT_VAR	        "INTEL_JIT_PROFILER32"
+#else
+#define NEW_DLL_ENVIRONMENT_VAR	        "INTEL_JIT_PROFILER64"
+#endif
+#endif /* NEW_DLL_ENVIRONMENT_VAR */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#define DEFAULT_DLLNAME                 "JitPI.dll"
+HINSTANCE m_libHandle = NULL;
+#elif ITT_PLATFORM==ITT_PLATFORM_MAC
+#define DEFAULT_DLLNAME                 "libJitPI.dylib"
+void* m_libHandle = NULL;
+#else
+#define DEFAULT_DLLNAME                 "libJitPI.so"
+void* m_libHandle = NULL;
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+/* default location of JIT profiling agent on Android */
+#define ANDROID_JIT_AGENT_PATH  "/data/intel/libittnotify.so"
+
+/* the function pointers */
+typedef unsigned int(JITAPI *TPInitialize)(void);
+static TPInitialize FUNC_Initialize=NULL;
+
+typedef unsigned int(JITAPI *TPNotify)(unsigned int, void*);
+static TPNotify FUNC_NotifyEvent=NULL;
+
+static iJIT_IsProfilingActiveFlags executionMode = iJIT_NOTHING_RUNNING;
+
+/* end collector dll part. */
+
+/* loadiJIT_Funcs() : this function is called just in the beginning
+ * and is responsible to load the functions from BistroJavaCollector.dll
+ * result:
+ *  on success: the functions loads, iJIT_DLL_is_missing=0, return value = 1
+ *  on failure: the functions are NULL, iJIT_DLL_is_missing=1, return value = 0
+ */
+static int loadiJIT_Funcs(void);
+
+/* global representing whether the collector can't be loaded */
+static int iJIT_DLL_is_missing = 0;
+
+ITT_EXTERN_C int JITAPI
+iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData)
+{
+    int ReturnValue = 0;
+
+    /* initialization part - the collector has not been loaded yet. */
+    if (!FUNC_NotifyEvent)
+    {
+        if (iJIT_DLL_is_missing)
+            return 0;
+
+        if (!loadiJIT_Funcs())
+            return 0;
+    }
+
+    if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED ||
+        event_type == iJVM_EVENT_TYPE_METHOD_UPDATE)
+    {
+        if (((piJIT_Method_Load)EventSpecificData)->method_id == 0)
+            return 0;
+    }
+    else if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2)
+    {
+        if (((piJIT_Method_Load_V2)EventSpecificData)->method_id == 0)
+            return 0;
+    }
+    else if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3)
+    {
+        if (((piJIT_Method_Load_V3)EventSpecificData)->method_id == 0)
+            return 0;
+    }
+    else if (event_type == iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED)
+    {
+        if (((piJIT_Method_Inline_Load)EventSpecificData)->method_id == 0 ||
+            ((piJIT_Method_Inline_Load)EventSpecificData)->parent_method_id == 0)
+            return 0;
+    }
+
+    ReturnValue = (int)FUNC_NotifyEvent(event_type, EventSpecificData);
+
+    return ReturnValue;
+}
+
+ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive()
+{
+    if (!iJIT_DLL_is_missing)
+    {
+        loadiJIT_Funcs();
+    }
+
+    return executionMode;
+}
+
+/* This function loads the collector dll and the relevant functions.
+ * on success: all functions load,     iJIT_DLL_is_missing = 0, return value = 1
+ * on failure: all functions are NULL, iJIT_DLL_is_missing = 1, return value = 0
+ */
+static int loadiJIT_Funcs()
+{
+    static int bDllWasLoaded = 0;
+    char *dllName = (char*)rcsid; /* !! Just to avoid unused code elimination */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    DWORD dNameLength = 0;
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+    if(bDllWasLoaded)
+    {
+        /* dll was already loaded, no need to do it for the second time */
+        return 1;
+    }
+
+    /* Assumes that the DLL will not be found */
+    iJIT_DLL_is_missing = 1;
+    FUNC_NotifyEvent = NULL;
+
+    if (m_libHandle)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        FreeLibrary(m_libHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        dlclose(m_libHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        m_libHandle = NULL;
+    }
+
+    /* Try to get the dll name from the environment */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    dNameLength = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, NULL, 0);
+    if (dNameLength)
+    {
+        DWORD envret = 0;
+        dllName = (char*)malloc(sizeof(char) * (dNameLength + 1));
+        if(dllName != NULL)
+        {
+            envret = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, 
+                                             dllName, dNameLength);
+            if (envret)
+            {
+                /* Try to load the dll from the PATH... */
+                m_libHandle = LoadLibraryExA(dllName, 
+                                             NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
+            }
+            free(dllName);
+        }
+    } else {
+        /* Try to use old VS_PROFILER variable */
+        dNameLength = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, NULL, 0);
+        if (dNameLength)
+        {
+            DWORD envret = 0;
+            dllName = (char*)malloc(sizeof(char) * (dNameLength + 1));
+            if(dllName != NULL)
+            {
+                envret = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, 
+                                                 dllName, dNameLength);
+                if (envret)
+                {
+                    /* Try to load the dll from the PATH... */
+                    m_libHandle = LoadLibraryA(dllName);
+                }
+                free(dllName);
+            }
+        }
+    }
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    dllName = getenv(NEW_DLL_ENVIRONMENT_VAR);
+    if (!dllName)
+        dllName = getenv(DLL_ENVIRONMENT_VAR);
+#if defined(__ANDROID__) || defined(ANDROID)
+    if (!dllName)
+        dllName = ANDROID_JIT_AGENT_PATH;
+#endif
+    if (dllName)
+    {
+        /* Try to load the dll from the PATH... */
+        m_libHandle = dlopen(dllName, RTLD_LAZY);
+    }
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+    if (!m_libHandle)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        m_libHandle = LoadLibraryA(DEFAULT_DLLNAME);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        m_libHandle = dlopen(DEFAULT_DLLNAME, RTLD_LAZY);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    }
+
+    /* if the dll wasn't loaded - exit. */
+    if (!m_libHandle)
+    {
+        iJIT_DLL_is_missing = 1; /* don't try to initialize
+                                  * JIT agent the second time
+                                  */
+        return 0;
+    }
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    FUNC_NotifyEvent = (TPNotify)GetProcAddress(m_libHandle, "NotifyEvent");
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    FUNC_NotifyEvent = (TPNotify)dlsym(m_libHandle, "NotifyEvent");
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    if (!FUNC_NotifyEvent) 
+    {
+        FUNC_Initialize = NULL;
+        return 0;
+    }
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    FUNC_Initialize = (TPInitialize)GetProcAddress(m_libHandle, "Initialize");
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    FUNC_Initialize = (TPInitialize)dlsym(m_libHandle, "Initialize");
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    if (!FUNC_Initialize) 
+    {
+        FUNC_NotifyEvent = NULL;
+        return 0;
+    }
+
+    executionMode = (iJIT_IsProfilingActiveFlags)FUNC_Initialize();
+
+    bDllWasLoaded = 1;
+    iJIT_DLL_is_missing = 0; /* DLL is ok. */
+
+    return 1;
+}
+
+ITT_EXTERN_C unsigned int JITAPI iJIT_GetNewMethodID()
+{
+    static unsigned int methodID = 1;
+
+    if (methodID == 0)
+        return 0;  /* ERROR : this is not a valid value */
+
+    return methodID++;
+}

3rdparty/jinja2/AUTHORS

@@ -1,33 +0,0 @@
-Jinja is written and maintained by the Jinja Team and various
-contributors:
-
-Lead Developer:
-
-- Armin Ronacher <armin.ronacher@active-4.com>
-
-Developers:
-
-- Christoph Hack
-- Georg Brandl
-
-Contributors:
-
-- Bryan McLemore
-- Mickaël Guérin <kael@crocobox.org>
-- Cameron Knight
-- Lawrence Journal-World.
-- David Cramer
-
-Patches and suggestions:
-
-- Ronny Pfannschmidt
-- Axel Böhm
-- Alexey Melchakov
-- Bryan McLemore
-- Clovis Fabricio (nosklo)
-- Cameron Knight
-- Peter van Dijk (Habbie)
-- Stefan Ebner
-- Rene Leonhardt
-- Thomas Waldmann
-- Cory Benfield (Lukasa)

3rdparty/jinja2/LICENSE

@@ -1,31 +0,0 @@
-Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
-
-Some rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above
-      copyright notice, this list of conditions and the following
-      disclaimer in the documentation and/or other materials provided
-      with the distribution.
-
-    * The names of the contributors may not be used to endorse or
-      promote products derived from this software without specific
-      prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3rdparty/jinja2/__init__.py

@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja2
-    ~~~~~~
-
-    Jinja2 is a template engine written in pure Python.  It provides a
-    Django inspired non-XML syntax but supports inline expressions and
-    an optional sandboxed environment.
-
-    Nutshell
-    --------
-
-    Here a small example of a Jinja2 template::
-
-        {% extends 'base.html' %}
-        {% block title %}Memberlist{% endblock %}
-        {% block content %}
-          <ul>
-          {% for user in users %}
-            <li><a href="{{ user.url }}">{{ user.username }}</a></li>
-          {% endfor %}
-          </ul>
-        {% endblock %}
-
-
-    :copyright: (c) 2010 by the Jinja Team.
-    :license: BSD, see LICENSE for more details.
-"""
-__docformat__ = 'restructuredtext en'
-__version__ = '2.7.1'
-
-# high level interface
-from jinja2.environment import Environment, Template
-
-# loaders
-from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
-     DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \
-     ModuleLoader
-
-# bytecode caches
-from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \
-     MemcachedBytecodeCache
-
-# undefined types
-from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined
-
-# exceptions
-from jinja2.exceptions import TemplateError, UndefinedError, \
-     TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \
-     TemplateAssertionError
-
-# decorators and public utilities
-from jinja2.filters import environmentfilter, contextfilter, \
-     evalcontextfilter
-from jinja2.utils import Markup, escape, clear_caches, \
-     environmentfunction, evalcontextfunction, contextfunction, \
-     is_undefined
-
-__all__ = [
-    'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
-    'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader',
-    'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache',
-    'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined',
-    'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound',
-    'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError',
-    'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape',
-    'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined',
-    'evalcontextfilter', 'evalcontextfunction'
-]

3rdparty/jinja2/_compat.py

@@ -1,150 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja2._compat
-    ~~~~~~~~~~~~~~
-
-    Some py2/py3 compatibility support based on a stripped down
-    version of six so we don't have to depend on a specific version
-    of it.
-
-    :copyright: Copyright 2013 by the Jinja team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-import sys
-
-PY2 = sys.version_info[0] == 2
-PYPY = hasattr(sys, 'pypy_translation_info')
-_identity = lambda x: x
-
-
-if not PY2:
-    unichr = chr
-    range_type = range
-    text_type = str
-    string_types = (str,)
-
-    iterkeys = lambda d: iter(d.keys())
-    itervalues = lambda d: iter(d.values())
-    iteritems = lambda d: iter(d.items())
-
-    import pickle
-    from io import BytesIO, StringIO
-    NativeStringIO = StringIO
-
-    def reraise(tp, value, tb=None):
-        if value.__traceback__ is not tb:
-            raise value.with_traceback(tb)
-        raise value
-
-    ifilter = filter
-    imap = map
-    izip = zip
-    intern = sys.intern
-
-    implements_iterator = _identity
-    implements_to_string = _identity
-    encode_filename = _identity
-    get_next = lambda x: x.__next__
-
-else:
-    unichr = unichr
-    text_type = unicode
-    range_type = xrange
-    string_types = (str, unicode)
-
-    iterkeys = lambda d: d.iterkeys()
-    itervalues = lambda d: d.itervalues()
-    iteritems = lambda d: d.iteritems()
-
-    import cPickle as pickle
-    from cStringIO import StringIO as BytesIO, StringIO
-    NativeStringIO = BytesIO
-
-    exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
-
-    from itertools import imap, izip, ifilter
-    intern = intern
-
-    def implements_iterator(cls):
-        cls.next = cls.__next__
-        del cls.__next__
-        return cls
-
-    def implements_to_string(cls):
-        cls.__unicode__ = cls.__str__
-        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
-        return cls
-
-    get_next = lambda x: x.next
-
-    def encode_filename(filename):
-        if isinstance(filename, unicode):
-            return filename.encode('utf-8')
-        return filename
-
-try:
-    next = next
-except NameError:
-    def next(it):
-        return it.next()
-
-
-def with_metaclass(meta, *bases):
-    # This requires a bit of explanation: the basic idea is to make a
-    # dummy metaclass for one level of class instanciation that replaces
-    # itself with the actual metaclass.  Because of internal type checks
-    # we also need to make sure that we downgrade the custom metaclass
-    # for one level to something closer to type (that's why __call__ and
-    # __init__ comes back from type etc.).
-    #
-    # This has the advantage over six.with_metaclass in that it does not
-    # introduce dummy classes into the final MRO.
-    class metaclass(meta):
-        __call__ = type.__call__
-        __init__ = type.__init__
-        def __new__(cls, name, this_bases, d):
-            if this_bases is None:
-                return type.__new__(cls, name, (), d)
-            return meta(name, bases, d)
-    return metaclass('temporary_class', None, {})
-
-
-try:
-    from collections import Mapping as mapping_types
-except ImportError:
-    import UserDict
-    mapping_types = (UserDict.UserDict, UserDict.DictMixin, dict)
-
-
-# common types.  These do exist in the special types module too which however
-# does not exist in IronPython out of the box.  Also that way we don't have
-# to deal with implementation specific stuff here
-class _C(object):
-    def method(self): pass
-def _func():
-    yield None
-function_type = type(_func)
-generator_type = type(_func())
-method_type = type(_C().method)
-code_type = type(_C.method.__code__)
-try:
-    raise TypeError()
-except TypeError:
-    _tb = sys.exc_info()[2]
-    traceback_type = type(_tb)
-    frame_type = type(_tb.tb_frame)
-
-
-try:
-    from urllib.parse import quote_from_bytes as url_quote
-except ImportError:
-    from urllib import quote as url_quote
-
-
-try:
-    from thread import allocate_lock
-except ImportError:
-    try:
-        from threading import Lock as allocate_lock
-    except ImportError:
-        from dummy_thread import allocate_lock

3rdparty/jinja2/bccache.py

@@ -1,311 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja2.bccache
-    ~~~~~~~~~~~~~~
-
-    This module implements the bytecode cache system Jinja is optionally
-    using.  This is useful if you have very complex template situations and
-    the compiliation of all those templates slow down your application too
-    much.
-
-    Situations where this is useful are often forking web applications that
-    are initialized on the first request.
-
-    :copyright: (c) 2010 by the Jinja Team.
-    :license: BSD.
-"""
-from os import path, listdir
-import sys
-import marshal
-import tempfile
-import fnmatch
-from hashlib import sha1
-from jinja2.utils import open_if_exists
-from jinja2._compat import BytesIO, pickle, PY2, text_type
-
-
-# marshal works better on 3.x, one hack less required
-if not PY2:
-    marshal_dump = marshal.dump
-    marshal_load = marshal.load
-else:
-
-    def marshal_dump(code, f):
-        if isinstance(f, file):
-            marshal.dump(code, f)
-        else:
-            f.write(marshal.dumps(code))
-
-    def marshal_load(f):
-        if isinstance(f, file):
-            return marshal.load(f)
-        return marshal.loads(f.read())
-
-
-bc_version = 2
-
-# magic version used to only change with new jinja versions.  With 2.6
-# we change this to also take Python version changes into account.  The
-# reason for this is that Python tends to segfault if fed earlier bytecode
-# versions because someone thought it would be a good idea to reuse opcodes
-# or make Python incompatible with earlier versions.
-bc_magic = 'j2'.encode('ascii') + \
-    pickle.dumps(bc_version, 2) + \
-    pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
-
-
-class Bucket(object):
-    """Buckets are used to store the bytecode for one template.  It's created
-    and initialized by the bytecode cache and passed to the loading functions.
-
-    The buckets get an internal checksum from the cache assigned and use this
-    to automatically reject outdated cache material.  Individual bytecode
-    cache subclasses don't have to care about cache invalidation.
-    """
-
-    def __init__(self, environment, key, checksum):
-        self.environment = environment
-        self.key = key
-        self.checksum = checksum
-        self.reset()
-
-    def reset(self):
-        """Resets the bucket (unloads the bytecode)."""
-        self.code = None
-
-    def load_bytecode(self, f):
-        """Loads bytecode from a file or file like object."""
-        # make sure the magic header is correct
-        magic = f.read(len(bc_magic))
-        if magic != bc_magic:
-            self.reset()
-            return
-        # the source code of the file changed, we need to reload
-        checksum = pickle.load(f)
-        if self.checksum != checksum:
-            self.reset()
-            return
-        self.code = marshal_load(f)
-
-    def write_bytecode(self, f):
-        """Dump the bytecode into the file or file like object passed."""
-        if self.code is None:
-            raise TypeError('can\'t write empty bucket')
-        f.write(bc_magic)
-        pickle.dump(self.checksum, f, 2)
-        marshal_dump(self.code, f)
-
-    def bytecode_from_string(self, string):
-        """Load bytecode from a string."""
-        self.load_bytecode(BytesIO(string))
-
-    def bytecode_to_string(self):
-        """Return the bytecode as string."""
-        out = BytesIO()
-        self.write_bytecode(out)
-        return out.getvalue()
-
-
-class BytecodeCache(object):
-    """To implement your own bytecode cache you have to subclass this class
-    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
-    these methods are passed a :class:`~jinja2.bccache.Bucket`.
-
-    A very basic bytecode cache that saves the bytecode on the file system::
-
-        from os import path
-
-        class MyCache(BytecodeCache):
-
-            def __init__(self, directory):
-                self.directory = directory
-
-            def load_bytecode(self, bucket):
-                filename = path.join(self.directory, bucket.key)
-                if path.exists(filename):
-                    with open(filename, 'rb') as f:
-                        bucket.load_bytecode(f)
-
-            def dump_bytecode(self, bucket):
-                filename = path.join(self.directory, bucket.key)
-                with open(filename, 'wb') as f:
-                    bucket.write_bytecode(f)
-
-    A more advanced version of a filesystem based bytecode cache is part of
-    Jinja2.
-    """
-
-    def load_bytecode(self, bucket):
-        """Subclasses have to override this method to load bytecode into a
-        bucket.  If they are not able to find code in the cache for the
-        bucket, it must not do anything.
-        """
-        raise NotImplementedError()
-
-    def dump_bytecode(self, bucket):
-        """Subclasses have to override this method to write the bytecode
-        from a bucket back to the cache.  If it unable to do so it must not
-        fail silently but raise an exception.
-        """
-        raise NotImplementedError()
-
-    def clear(self):
-        """Clears the cache.  This method is not used by Jinja2 but should be
-        implemented to allow applications to clear the bytecode cache used
-        by a particular environment.
-        """
-
-    def get_cache_key(self, name, filename=None):
-        """Returns the unique hash key for this template name."""
-        hash = sha1(name.encode('utf-8'))
-        if filename is not None:
-            filename = '|' + filename
-            if isinstance(filename, text_type):
-                filename = filename.encode('utf-8')
-            hash.update(filename)
-        return hash.hexdigest()
-
-    def get_source_checksum(self, source):
-        """Returns a checksum for the source."""
-        return sha1(source.encode('utf-8')).hexdigest()
-
-    def get_bucket(self, environment, name, filename, source):
-        """Return a cache bucket for the given template.  All arguments are
-        mandatory but filename may be `None`.
-        """
-        key = self.get_cache_key(name, filename)
-        checksum = self.get_source_checksum(source)
-        bucket = Bucket(environment, key, checksum)
-        self.load_bytecode(bucket)
-        return bucket
-
-    def set_bucket(self, bucket):
-        """Put the bucket into the cache."""
-        self.dump_bytecode(bucket)
-
-
-class FileSystemBytecodeCache(BytecodeCache):
-    """A bytecode cache that stores bytecode on the filesystem.  It accepts
-    two arguments: The directory where the cache items are stored and a
-    pattern string that is used to build the filename.
-
-    If no directory is specified the system temporary items folder is used.
-
-    The pattern can be used to have multiple separate caches operate on the
-    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
-    is replaced with the cache key.
-
-    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
-
-    This bytecode cache supports clearing of the cache using the clear method.
-    """
-
-    def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
-        if directory is None:
-            directory = tempfile.gettempdir()
-        self.directory = directory
-        self.pattern = pattern
-
-    def _get_cache_filename(self, bucket):
-        return path.join(self.directory, self.pattern % bucket.key)
-
-    def load_bytecode(self, bucket):
-        f = open_if_exists(self._get_cache_filename(bucket), 'rb')
-        if f is not None:
-            try:
-                bucket.load_bytecode(f)
-            finally:
-                f.close()
-
-    def dump_bytecode(self, bucket):
-        f = open(self._get_cache_filename(bucket), 'wb')
-        try:
-            bucket.write_bytecode(f)
-        finally:
-            f.close()
-
-    def clear(self):
-        # imported lazily here because google app-engine doesn't support
-        # write access on the file system and the function does not exist
-        # normally.
-        from os import remove
-        files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
-        for filename in files:
-            try:
-                remove(path.join(self.directory, filename))
-            except OSError:
-                pass
-
-
-class MemcachedBytecodeCache(BytecodeCache):
-    """This class implements a bytecode cache that uses a memcache cache for
-    storing the information.  It does not enforce a specific memcache library
-    (tummy's memcache or cmemcache) but will accept any class that provides
-    the minimal interface required.
-
-    Libraries compatible with this class:
-
-    -   `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
-    -   `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
-    -   `cmemcache <http://gijsbert.org/cmemcache/>`_
-
-    (Unfortunately the django cache interface is not compatible because it
-    does not support storing binary data, only unicode.  You can however pass
-    the underlying cache client to the bytecode cache which is available
-    as `django.core.cache.cache._client`.)
-
-    The minimal interface for the client passed to the constructor is this:
-
-    .. class:: MinimalClientInterface
-
-        .. method:: set(key, value[, timeout])
-
-            Stores the bytecode in the cache.  `value` is a string and
-            `timeout` the timeout of the key.  If timeout is not provided
-            a default timeout or no timeout should be assumed, if it's
-            provided it's an integer with the number of seconds the cache
-            item should exist.
-
-        .. method:: get(key)
-
-            Returns the value for the cache key.  If the item does not
-            exist in the cache the return value must be `None`.
-
-    The other arguments to the constructor are the prefix for all keys that
-    is added before the actual cache key and the timeout for the bytecode in
-    the cache system.  We recommend a high (or no) timeout.
-
-    This bytecode cache does not support clearing of used items in the cache.
-    The clear method is a no-operation function.
-
-    .. versionadded:: 2.7
-       Added support for ignoring memcache errors through the
-       `ignore_memcache_errors` parameter.
-    """
-
-    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
-                 ignore_memcache_errors=True):
-        self.client = client
-        self.prefix = prefix
-        self.timeout = timeout
-        self.ignore_memcache_errors = ignore_memcache_errors
-
-    def load_bytecode(self, bucket):
-        try:
-            code = self.client.get(self.prefix + bucket.key)
-        except Exception:
-            if not self.ignore_memcache_errors:
-                raise
-            code = None
-        if code is not None:
-            bucket.bytecode_from_string(code)
-
-    def dump_bytecode(self, bucket):
-        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
-        if self.timeout is not None:
-            args += (self.timeout,)
-        try:
-            self.client.set(*args)
-        except Exception:
-            if not self.ignore_memcache_errors:
-                raise

3rdparty/jinja2/constants.py

@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja.constants
-    ~~~~~~~~~~~~~~~
-
-    Various constants.
-
-    :copyright: (c) 2010 by the Jinja Team.
-    :license: BSD, see LICENSE for more details.
-"""
-
-
-#: list of lorem ipsum words used by the lipsum() helper function
-LOREM_IPSUM_WORDS = u'''\
-a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at
-auctor augue bibendum blandit class commodo condimentum congue consectetuer
-consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus
-diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend
-elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames
-faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac
-hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum
-justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem
-luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie
-mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non
-nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque
-penatibus per pharetra phasellus placerat platea porta porttitor posuere
-potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus
-ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit
-sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor
-tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices
-ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus
-viverra volutpat vulputate'''

3rdparty/jinja2/debug.py

@@ -1,337 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja2.debug
-    ~~~~~~~~~~~~
-
-    Implements the debug interface for Jinja.  This module does some pretty
-    ugly stuff with the Python traceback system in order to achieve tracebacks
-    with correct line numbers, locals and contents.
-
-    :copyright: (c) 2010 by the Jinja Team.
-    :license: BSD, see LICENSE for more details.
-"""
-import sys
-import traceback
-from types import TracebackType
-from jinja2.utils import missing, internal_code
-from jinja2.exceptions import TemplateSyntaxError
-from jinja2._compat import iteritems, reraise, code_type
-
-# on pypy we can take advantage of transparent proxies
-try:
-    from __pypy__ import tproxy
-except ImportError:
-    tproxy = None
-
-
-# how does the raise helper look like?
-try:
-    exec("raise TypeError, 'foo'")
-except SyntaxError:
-    raise_helper = 'raise __jinja_exception__[1]'
-except TypeError:
-    raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
-
-
-class TracebackFrameProxy(object):
-    """Proxies a traceback frame."""
-
-    def __init__(self, tb):
-        self.tb = tb
-        self._tb_next = None
-
-    @property
-    def tb_next(self):
-        return self._tb_next
-
-    def set_next(self, next):
-        if tb_set_next is not None:
-            try:
-                tb_set_next(self.tb, next and next.tb or None)
-            except Exception:
-                # this function can fail due to all the hackery it does
-                # on various python implementations.  We just catch errors
-                # down and ignore them if necessary.
-                pass
-        self._tb_next = next
-
-    @property
-    def is_jinja_frame(self):
-        return '__jinja_template__' in self.tb.tb_frame.f_globals
-
-    def __getattr__(self, name):
-        return getattr(self.tb, name)
-
-
-def make_frame_proxy(frame):
-    proxy = TracebackFrameProxy(frame)
-    if tproxy is None:
-        return proxy
-    def operation_handler(operation, *args, **kwargs):
-        if operation in ('__getattribute__', '__getattr__'):
-            return getattr(proxy, args[0])
-        elif operation == '__setattr__':
-            proxy.__setattr__(*args, **kwargs)
-        else:
-            return getattr(proxy, operation)(*args, **kwargs)
-    return tproxy(TracebackType, operation_handler)
-
-
-class ProcessedTraceback(object):
-    """Holds a Jinja preprocessed traceback for printing or reraising."""
-
-    def __init__(self, exc_type, exc_value, frames):
-        assert frames, 'no frames for this traceback?'
-        self.exc_type = exc_type
-        self.exc_value = exc_value
-        self.frames = frames
-
-        # newly concatenate the frames (which are proxies)
-        prev_tb = None
-        for tb in self.frames:
-            if prev_tb is not None:
-                prev_tb.set_next(tb)
-            prev_tb = tb
-        prev_tb.set_next(None)
-
-    def render_as_text(self, limit=None):
-        """Return a string with the traceback."""
-        lines = traceback.format_exception(self.exc_type, self.exc_value,
-                                           self.frames[0], limit=limit)
-        return ''.join(lines).rstrip()
-
-    def render_as_html(self, full=False):
-        """Return a unicode string with the traceback as rendered HTML."""
-        from jinja2.debugrenderer import render_traceback
-        return u'%s\n\n<!--\n%s\n-->' % (
-            render_traceback(self, full=full),
-            self.render_as_text().decode('utf-8', 'replace')
-        )
-
-    @property
-    def is_template_syntax_error(self):
-        """`True` if this is a template syntax error."""
-        return isinstance(self.exc_value, TemplateSyntaxError)
-
-    @property
-    def exc_info(self):
-        """Exception info tuple with a proxy around the frame objects."""
-        return self.exc_type, self.exc_value, self.frames[0]
-
-    @property
-    def standard_exc_info(self):
-        """Standard python exc_info for re-raising"""
-        tb = self.frames[0]
-        # the frame will be an actual traceback (or transparent proxy) if
-        # we are on pypy or a python implementation with support for tproxy
-        if type(tb) is not TracebackType:
-            tb = tb.tb
-        return self.exc_type, self.exc_value, tb
-
-
-def make_traceback(exc_info, source_hint=None):
-    """Creates a processed traceback object from the exc_info."""
-    exc_type, exc_value, tb = exc_info
-    if isinstance(exc_value, TemplateSyntaxError):
-        exc_info = translate_syntax_error(exc_value, source_hint)
-        initial_skip = 0
-    else:
-        initial_skip = 1
-    return translate_exception(exc_info, initial_skip)
-
-
-def translate_syntax_error(error, source=None):
-    """Rewrites a syntax error to please traceback systems."""
-    error.source = source
-    error.translated = True
-    exc_info = (error.__class__, error, None)
-    filename = error.filename
-    if filename is None:
-        filename = '<unknown>'
-    return fake_exc_info(exc_info, filename, error.lineno)
-
-
-def translate_exception(exc_info, initial_skip=0):
-    """If passed an exc_info it will automatically rewrite the exceptions
-    all the way down to the correct line numbers and frames.
-    """
-    tb = exc_info[2]
-    frames = []
-
-    # skip some internal frames if wanted
-    for x in range(initial_skip):
-        if tb is not None:
-            tb = tb.tb_next
-    initial_tb = tb
-
-    while tb is not None:
-        # skip frames decorated with @internalcode.  These are internal
-        # calls we can't avoid and that are useless in template debugging
-        # output.
-        if tb.tb_frame.f_code in internal_code:
-            tb = tb.tb_next
-            continue
-
-        # save a reference to the next frame if we override the current
-        # one with a faked one.
-        next = tb.tb_next
-
-        # fake template exceptions
-        template = tb.tb_frame.f_globals.get('__jinja_template__')
-        if template is not None:
-            lineno = template.get_corresponding_lineno(tb.tb_lineno)
-            tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
-                               lineno)[2]
-
-        frames.append(make_frame_proxy(tb))
-        tb = next
-
-    # if we don't have any exceptions in the frames left, we have to
-    # reraise it unchanged.
-    # XXX: can we backup here?  when could this happen?
-    if not frames:
-        reraise(exc_info[0], exc_info[1], exc_info[2])
-
-    return ProcessedTraceback(exc_info[0], exc_info[1], frames)
-
-
-def fake_exc_info(exc_info, filename, lineno):
-    """Helper for `translate_exception`."""
-    exc_type, exc_value, tb = exc_info
-
-    # figure the real context out
-    if tb is not None:
-        real_locals = tb.tb_frame.f_locals.copy()
-        ctx = real_locals.get('context')
-        if ctx:
-            locals = ctx.get_all()
-        else:
-            locals = {}
-        for name, value in iteritems(real_locals):
-            if name.startswith('l_') and value is not missing:
-                locals[name[2:]] = value
-
-        # if there is a local called __jinja_exception__, we get
-        # rid of it to not break the debug functionality.
-        locals.pop('__jinja_exception__', None)
-    else:
-        locals = {}
-
-    # assamble fake globals we need
-    globals = {
-        '__name__':             filename,
-        '__file__':             filename,
-        '__jinja_exception__':  exc_info[:2],
-
-        # we don't want to keep the reference to the template around
-        # to not cause circular dependencies, but we mark it as Jinja
-        # frame for the ProcessedTraceback
-        '__jinja_template__':   None
-    }
-
-    # and fake the exception
-    code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
-
-    # if it's possible, change the name of the code.  This won't work
-    # on some python environments such as google appengine
-    try:
-        if tb is None:
-            location = 'template'
-        else:
-            function = tb.tb_frame.f_code.co_name
-            if function == 'root':
-                location = 'top-level template code'
-            elif function.startswith('block_'):
-                location = 'block "%s"' % function[6:]
-            else:
-                location = 'template'
-        code = code_type(0, code.co_nlocals, code.co_stacksize,
-                         code.co_flags, code.co_code, code.co_consts,
-                         code.co_names, code.co_varnames, filename,
-                         location, code.co_firstlineno,
-                         code.co_lnotab, (), ())
-    except:
-        pass
-
-    # execute the code and catch the new traceback
-    try:
-        exec(code, globals, locals)
-    except:
-        exc_info = sys.exc_info()
-        new_tb = exc_info[2].tb_next
-
-    # return without this frame
-    return exc_info[:2] + (new_tb,)
-
-
-def _init_ugly_crap():
-    """This function implements a few ugly things so that we can patch the
-    traceback objects.  The function returned allows resetting `tb_next` on
-    any python traceback object.  Do not attempt to use this on non cpython
-    interpreters
-    """
-    import ctypes
-    from types import TracebackType
-
-    # figure out side of _Py_ssize_t
-    if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
-        _Py_ssize_t = ctypes.c_int64
-    else:
-        _Py_ssize_t = ctypes.c_int
-
-    # regular python
-    class _PyObject(ctypes.Structure):
-        pass
-    _PyObject._fields_ = [
-        ('ob_refcnt', _Py_ssize_t),
-        ('ob_type', ctypes.POINTER(_PyObject))
-    ]
-
-    # python with trace
-    if hasattr(sys, 'getobjects'):
-        class _PyObject(ctypes.Structure):
-            pass
-        _PyObject._fields_ = [
-            ('_ob_next', ctypes.POINTER(_PyObject)),
-            ('_ob_prev', ctypes.POINTER(_PyObject)),
-            ('ob_refcnt', _Py_ssize_t),
-            ('ob_type', ctypes.POINTER(_PyObject))
-        ]
-
-    class _Traceback(_PyObject):
-        pass
-    _Traceback._fields_ = [
-        ('tb_next', ctypes.POINTER(_Traceback)),
-        ('tb_frame', ctypes.POINTER(_PyObject)),
-        ('tb_lasti', ctypes.c_int),
-        ('tb_lineno', ctypes.c_int)
-    ]
-
-    def tb_set_next(tb, next):
-        """Set the tb_next attribute of a traceback object."""
-        if not (isinstance(tb, TracebackType) and
-                (next is None or isinstance(next, TracebackType))):
-            raise TypeError('tb_set_next arguments must be traceback objects')
-        obj = _Traceback.from_address(id(tb))
-        if tb.tb_next is not None:
-            old = _Traceback.from_address(id(tb.tb_next))
-            old.ob_refcnt -= 1
-        if next is None:
-            obj.tb_next = ctypes.POINTER(_Traceback)()
-        else:
-            next = _Traceback.from_address(id(next))
-            next.ob_refcnt += 1
-            obj.tb_next = ctypes.pointer(next)
-
-    return tb_set_next
-
-
-# try to get a tb_set_next implementation if we don't have transparent
-# proxies.
-tb_set_next = None
-if tproxy is None:
-    try:
-        tb_set_next = _init_ugly_crap()
-    except:
-        pass
-    del _init_ugly_crap