potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 07:43:03 +04:00
Code Packages Releases Wiki Activity
Files
next
opencv/3rdparty/mlas/lib/mlasi.h
T
Abhishek Gola bf0cf34963 Merge pull request #29126 from abhishek-gola:flash_attention 
Attention graph fusion with MLAS FlashAttention #29126

Performance numbers for Owl-v2 model on intel i9:
```
ORT: Average inference time over 10 runs: 1411.55 ms (min 1399.75, max 1438.89)
NEW: Average inference time over 10 runs: 1078 ms (min 1048.04, max 1110.61)
```
### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2026-05-27 09:32:27 +03:00

3103 lines
89 KiB
C++
Raw Permalink Blame History

/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
mlasi.h
Abstract:
This module contains the private data structures and procedure prototypes
for the Microsoft Machine Learning algebra subprogram library.
--*/
#pragma once
#include <algorithm>
#include <atomic>
#include <cmath>
#include <functional>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <type_traits>
#ifdef MLAS_NO_EXCEPTION
#if defined(__ANDROID__)
#include <android/log.h>
#else
#include <iostream>
#endif
#endif // MLAS_NO_EXCEPTION
// Vendored under 3rdparty/mlas/. The ORT path "core/mlas/inc/mlas.h" only
// works when MLAS is part of the ORT source tree.
#include "../inc/mlas.h"
#if defined(_WIN32)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <intrin.h>
#else
#if defined(__arm__) || defined(__aarch64__)
#include <arm_neon.h>
#endif
#if defined(__x86_64__) || defined(__i386__)
#if !defined(signature_VORTEX_ebx) && !defined(signature_NEXGEN_ebx) && !defined(signature_AMD_ebx)//workaround for Bug 96238 - [i386] cpuid.h header needs include guards
#include <cpuid.h>
#endif
#if defined(__GNUC__) && __GNUC__ >= 12
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" // GCC 12 warns about uninitialized variables in immintrin.h.
#include <immintrin.h>
#pragma GCC diagnostic pop
#else
#include <immintrin.h>
#endif
#endif
#if defined(__VSX__)
#include <altivec.h>
// Undefine unwanted aliases from altivec.h.
#undef vector
#undef pixel
#undef bool
#endif
#if defined(__s390x__)
#include <vecintrin.h>
#endif
#if defined(__loongarch64)
#include <lsxintrin.h>
#endif
#if defined(MLAS_TARGET_WASM_SIMD)
#include <wasm_simd128.h>
#endif
#endif
//
// Macro to place variables at a specified alignment.
//
#ifdef _WIN32
#define MLAS_DECLSPEC_ALIGN(variable, alignment) DECLSPEC_ALIGN(alignment) variable
#else
#define MLAS_DECLSPEC_ALIGN(variable, alignment) variable __attribute__ ((aligned(alignment)))
#endif
//
// Macro to force inline expansion of a function.
//
#if defined(_MSC_VER)
#define MLAS_FORCEINLINE __forceinline
#else
#define MLAS_FORCEINLINE __attribute__ ((always_inline)) inline
#endif
//
// Macro to tag globals as internal data shared with kernels written in
// assembly. These globals are marked with having hidden visibility to avoid
// needing to access the data through the global object table.
//
#if defined(_MSC_VER)
#define MLAS_INTERNAL_DATA extern "C"
#else
#define MLAS_INTERNAL_DATA extern "C" __attribute ((visibility("hidden")))
#endif
//
// Macro to suppress unreferenced parameter warnings.
//
#define MLAS_UNREFERENCED_PARAMETER(parameter) ((void)(parameter))
#ifdef MLAS_NO_EXCEPTION
MLAS_FORCEINLINE void
MlasPrintFinalMessage(const std::string& msg)
{
#if defined(__ANDROID__)
__android_log_print(ANDROID_LOG_ERROR, "mlas", "%s", msg.c_str());
#else
// TODO, consider changing the output of the error message from std::cerr to logging when the
// exceptions are disabled, since using std::cerr might increase binary size, and std::cerr
// output might not be easily accesible on some systems such as mobile
// TODO, see if we need to change the output of the error message from std::cerr to NSLog for
// iOS
std::cerr << msg << std::endl;
#endif
}
#define MLAS_THROW_EX(ex, what) \
do { \
std::string msg = #ex; \
msg.append(what); \
MlasPrintFinalMessage(msg); \
abort(); \
} while (false)
#else
#define MLAS_THROW_EX(ex, ...) throw ex(__VA_ARGS__)
#endif // MLAS_NO_EXCEPTION
//
// Select the threading model.
//
// N.B. BUILD_MLAS_NO_ONNXRUNTIME is used to build MLAS test code outside
// of the ONNX Runtime source tree. OpenMP may or may not be enabled in this
// configuration.
//
#if !defined(BUILD_MLAS_NO_ONNXRUNTIME)
#include "core/platform/threadpool.h"
#include "core/common/cpuid_info.h"
using MLAS_CPUIDINFO = onnxruntime::CPUIDInfo;
#include "core/common/float16.h"
#else // BUILD_MLAS_NO_ONNXRUNTIME
class MLASCPUIDInfo
{
public:
static const MLASCPUIDInfo& GetCPUIDInfo()
{
static MLASCPUIDInfo cpuid_info;
return cpuid_info;
}
// ARM
bool HasArmNeonDot() const { return has_arm_neon_dot_; }
bool HasFp16VectorAcceleration() const { return has_fp16_; }
uint32_t GetCurrentCoreIdx() const { return 0xFFFFFFFF; }
int32_t GetCurrentUarch() const { return -1; }
int32_t GetCoreUarch(uint32_t coreId) const { return -1; }
bool IsCoreArmv8NarrowLd(uint32_t coreId) const { return false; }
bool IsCurrentCoreArmv8NarrowLd() const { return false; }
bool HasArmNeon_I8MM() const { return has_arm_neon_i8mm_; }
bool HasArmSVE() const { return has_arm_sve_; }
bool HasArmSVE_I8MM() const { return has_arm_sve_i8mm_; }
bool HasArmNeon_BF16() const { return has_arm_neon_bf16_; }
private:
MLASCPUIDInfo();
bool has_arm_neon_dot_{false};
bool has_fp16_{false};
bool has_arm_neon_i8mm_{false};
bool has_arm_sve_{false};
bool has_arm_sve_i8mm_{false};
bool has_arm_neon_bf16_{false};
};
using MLAS_CPUIDINFO = MLASCPUIDInfo;
#if defined(MLAS_TARGET_ARM64)
/**
* @brief IDs for cpu microarchitectures.
*
* Copied from python cpuinfo package. Can't use the definition
* from cpuinfo directly as it causes lots of compilation issues
* in many platforms that we support.
*/
enum MlasUArch {
cpuinfo_uarch_unknown = 0,
/** ARM Cortex-A32. */
cpuinfo_uarch_cortex_a32 = 0x00300332,
/** ARM Cortex-A35. */
cpuinfo_uarch_cortex_a35 = 0x00300335,
/** ARM Cortex-A53. */
cpuinfo_uarch_cortex_a53 = 0x00300353,
/** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */
cpuinfo_uarch_cortex_a55r0 = 0x00300354,
/** ARM Cortex-A55. */
cpuinfo_uarch_cortex_a55 = 0x00300355,
/** ARM Cortex-A57. */
cpuinfo_uarch_cortex_a57 = 0x00300357,
/** ARM Cortex-A65. */
cpuinfo_uarch_cortex_a65 = 0x00300365,
/** ARM Cortex-A72. */
cpuinfo_uarch_cortex_a72 = 0x00300372,
/** ARM Cortex-A73. */
cpuinfo_uarch_cortex_a73 = 0x00300373,
/** ARM Cortex-A75. */
cpuinfo_uarch_cortex_a75 = 0x00300375,
/** ARM Cortex-A76. */
cpuinfo_uarch_cortex_a76 = 0x00300376,
/** ARM Cortex-A77. */
cpuinfo_uarch_cortex_a77 = 0x00300377,
/** ARM Cortex-A78. */
cpuinfo_uarch_cortex_a78 = 0x00300378,
};
#endif // MLAS_TARGET_ARM64
//
// Define MLAS_FP16
//
#include "mlas_float16.h"
namespace onnxruntime
{
struct MLFloat16 {
uint16_t val{0};
MLFloat16() = default;
explicit constexpr MLFloat16(uint16_t x) : val(x) {}
explicit MLFloat16(float ff) : val(MLAS_Float2Half(ff)) {}
constexpr static MLFloat16 FromBits(uint16_t x) noexcept { return MLFloat16(x); }
MLFloat16 Abs() const noexcept {
return MLFloat16(static_cast<uint16_t>(val & ~kSignMask));
}
bool IsNaN() const noexcept {
return Abs().val > kPositiveInfinityBits;
}
bool IsNegative() const noexcept {
return static_cast<int16_t>(val) < 0;
}
MLFloat16 Negate() const {
return MLFloat16(IsNaN() ? val : static_cast<uint16_t>(val ^ kSignMask));
}
static constexpr uint16_t kSignMask = 0x8000U;
static constexpr uint16_t kPositiveInfinityBits = 0x7C00U;
float ToFloat() const { return MLAS_Half2Float(val); }
operator float() const { return ToFloat(); }
MLFloat16& operator=(float ff)
{
val = MLAS_Float2Half(ff);
return *this;
}
};
inline bool
operator==(const MLFloat16& left, const MLFloat16& right)
{
return left.val == right.val;
}
inline bool
operator!=(const MLFloat16& left, const MLFloat16& right)
{
return left.val != right.val;
}
}
#endif // BUILD_MLAS_NO_ONNXRUNTIME
static_assert(sizeof(MLAS_FP16) == FP16_SIZE, "");
//
// Define the maximum number of threads supported by this implementation.
//
#define MLAS_MAXIMUM_THREAD_COUNT 16
//
// Define the default strides to step through slices of the input matrices.
//
#define MLAS_HGEMM_STRIDEN 128
#define MLAS_HGEMM_STRIDEK 128
#define MLAS_SGEMM_STRIDEN 128
#define MLAS_SGEMM_STRIDEK 128
#define MLAS_SGEMM_PACKED_STRIDEN 128
#define MLAS_SGEMM_PACKED_STRIDEK 256
#define MLAS_DGEMM_STRIDEN 64
#define MLAS_DGEMM_STRIDEK 128
//
// Define the alignment for segmenting a GEMM operation across multiple
// threads.
//
// All of the SGEMM kernels can efficiently handle 16 elements. AVX512F can
// efficiently handle 32 elements, but making this value dynamic is not worth
// the effort at this time.
//
#define MLAS_HGEMM_STRIDEN_THREAD_ALIGN 32
#define MLAS_SGEMM_STRIDEN_THREAD_ALIGN 16
#define MLAS_DGEMM_STRIDEN_THREAD_ALIGN 8
#define MLAS_QGEMM_STRIDEN_THREAD_ALIGN 16
//
// Define the prototypes of the platform optimized routines.
//
#if defined(MLAS_TARGET_AMD64_IX86) || defined(MLAS_TARGET_POWER) || \
defined(MLAS_TARGET_LARCH64) || defined(MLAS_TARGET_S390X) || \
defined(MLAS_TARGET_RISCV64)
typedef
size_t
(MLASCALL MLAS_GEMM_FLOAT_KERNEL)(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
float alpha,
bool ZeroMode
);
typedef
size_t
(MLASCALL MLAS_GEMM_DOUBLE_KERNEL)(
const double* A,
const double* B,
double* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
double alpha,
bool ZeroMode
);
#ifdef FORCE_GENERIC_ALGORITHMS
typedef
size_t
(MLASCALL MLAS_GEMM_FLOAT_KERNEL_GENERIC)(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
float alpha
);
#endif
#else
#if defined(__aarch64__) && defined(__linux__)
typedef size_t(MLASCALL MLAS_SBGEMM_FLOAT_KERNEL)(
const float* A,
const void* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
const float* Bias
);
#endif
typedef
size_t
(MLASCALL MLAS_GEMM_FLOAT_KERNEL)(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
float alpha
);
typedef
size_t
(MLASCALL MLAS_GEMM_DOUBLE_KERNEL)(
const double* A,
const double* B,
double* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
double alpha
);
#endif
typedef
void
(MLASCALL MLAS_GEMV_FLOAT_KERNEL)(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountN,
size_t ldb,
bool ZeroMode
);
typedef
void
(MLASCALL MLAS_SGEMM_KERNEL_M1_ROUTINE)(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountN,
size_t ldb,
float beta
);
typedef
void
(MLASCALL MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE)(
float* D,
const float* B,
size_t ldb
);
typedef
size_t
(MLASCALL MLAS_GEMM_U8S8_KERNEL)(
const uint8_t* A,
const uint8_t* B,
int32_t* C,
size_t PackedCountK,
size_t CountM,
size_t CountN,
size_t ldc,
const int32_t* RowSumVector,
const int32_t* ColumnSumVector,
const int32_t* ZeroPointB,
bool ZeroMode
);
typedef
size_t
(MLASCALL MLAS_GEMV_U8S8_KERNEL)(
const uint8_t* A,
const uint8_t* B,
int32_t* C,
size_t CountK,
size_t CountN,
size_t ldb
);
typedef
size_t
(MLASCALL MLAS_GEMM_U8U8_KERNEL)(
const int16_t* A,
const uint8_t* B,
int32_t* C,
size_t PackedCountK,
size_t CountM,
size_t CountN,
size_t ldc,
const int32_t* RowSumVector,
const int32_t* ColumnSumVector,
const int32_t* ZeroPointB,
bool ZeroMode
);
typedef
void
(MLASCALL MLAS_CONV_FLOAT_KERNEL)(
const float* Input,
const float* Filter,
float* Output,
size_t StrideWidth,
size_t DilationWidth,
size_t FilterCount,
size_t InputStride,
size_t FilterStride,
size_t OutputStride,
size_t KernelHeight,
size_t KernelWidth,
const float* InputBase,
size_t InputWidth,
size_t DilatedInputWidth,
size_t OutputCountLeftPad,
size_t OutputCount,
size_t OutputCountRightPad,
const float* Bias,
unsigned KernelFlags
);
typedef
void
(MLASCALL MLAS_CONV_DEPTHWISE_FLOAT_KERNEL)(
const float* Input,
const float* Filter,
float* Output,
size_t StrideWidth,
size_t DilationWidth,
size_t InputStride,
size_t KernelHeight,
size_t KernelWidth,
const float* InputBase,
size_t InputWidth,
size_t DilatedInputWidth,
size_t OutputCountLeftPad,
size_t OutputCount,
size_t OutputCountRightPad,
const float* Bias,
unsigned KernelFlags
);
typedef
void
(MLASCALL MLAS_CONV_POINTWISE_FLOAT_KERNEL)(
const float* Input,
const float* Filter,
float* Output,
size_t StrideWidth,
size_t InputChannels,
size_t FilterCount,
size_t InputStride,
size_t FilterStride,
size_t OutputStride,
size_t OutputCount,
const float* Bias,
unsigned KernelFlags
);
typedef
void
(MLASCALL MLAS_POOL_FLOAT_KERNEL)(
const float* Input,
float* Output,
size_t StrideWidth,
size_t DilationWidth,
size_t InputStride,
size_t ActualKernelSize,
size_t KernelHeight,
size_t KernelWidth,
const float* InputBase,
size_t InputWidth,
size_t DilatedInputWidth,
size_t OutputCountLeftPad,
size_t OutputCount,
size_t OutputCountRightPad
);
typedef
void
(MLASCALL MLAS_COMPUTE_UNARY_FLOAT_KERNEL)(
const float* Input,
float* Output,
size_t N
);
typedef
void
(MLASCALL MLAS_COMPUTE_ERF_FP16_KERNEL)(
const MLAS_FP16* Input,
MLAS_FP16* Output,
size_t N
);
typedef
void
(MLASCALL MLAS_COMPUTE_GELU_FP16_KERNEL)(
const MLAS_FP16* Input,
MLAS_FP16* Output,
MLAS_FP16* Temp,
size_t N,
MLAS_GELU_ALGORITHM Algo
);
typedef void
(MLASCALL MLAS_COMPUTE_TANH_FP16_KERNEL)(
const MLAS_FP16* Input,
MLAS_FP16* Output,
size_t N
);
typedef
float
(MLASCALL MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL)(
const float* Input,
float* Output,
size_t N,
const float* NegativeMaximum
);
typedef
void
(MLASCALL MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL)(
float* Output,
size_t N,
const float* Parameters
);
typedef
void
(MLASCALL MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL)(
const float* Input,
float* Output,
size_t N,
const float* Parameters
);
typedef
float
(MLASCALL MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL)(
const float* Input,
size_t N
);
typedef
void
(MLASCALL MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL)(
const float* Input,
float* Min,
float* Max,
size_t N
);
typedef
void(MLASCALL MLAS_CAST_F16_TO_F32_KERNEL)(
const unsigned short* Source,
float* Destination,
size_t Count
);
typedef void(MLASCALL MLAS_CAST_F32_TO_F16_KERNEL)(
const float* Source,
unsigned short* Destination,
size_t Count
);
typedef
void
(MLASCALL MLAS_QLINEAR_BINARY_OP_S8_KERNEL)(
const int8_t* InputA,
float ScaleA,
int32_t ZeroPointA,
const int8_t* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
int8_t* OutputC,
size_t N,
bool IsScalarB
);
typedef
void
(MLASCALL MLAS_QLINEAR_BINARY_OP_U8_KERNEL)(
const uint8_t* InputA,
float ScaleA,
int32_t ZeroPointA,
const uint8_t* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
uint8_t* OutputC,
size_t N,
bool IsScalarB
);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_U8_KERNEL)(
const float* Input,
uint8_t* Output,
size_t N,
float Scale,
uint8_t ZeroPoint
);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_S8_KERNEL)(
const float* Input,
int8_t* Output,
size_t N,
float Scale,
int8_t ZeroPoint
);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_U16_KERNEL)(
const float* Input,
uint16_t* Output,
size_t N,
float Scale,
uint16_t ZeroPoint);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_S16_KERNEL)(
const float* Input,
int16_t* Output,
size_t N,
float Scale,
int16_t ZeroPoint);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_U4_KERNEL)(
const float* Input,
uint8_t* Output,
size_t N,
float Scale,
int8_t ZeroPoint);
typedef
void
(MLASCALL MLAS_QUANTIZE_LINEAR_S4_KERNEL)(
const float* Input,
uint8_t* Output,
size_t N,
float Scale,
int8_t ZeroPoint);
typedef
void
(MLASCALL MLAS_DEQUANTIZE_LINEAR_U8_KERNEL)(
const uint8_t* Input,
float* Output,
size_t N,
float Scale,
uint8_t ZeroPoint);
typedef
void
(MLASCALL MLAS_DEQUANTIZE_LINEAR_S8_KERNEL)(
const int8_t* Input,
float* Output,
size_t N,
float Scale,
int8_t ZeroPoint);
template<typename InputType, typename FilterType>
struct MLAS_QUANT_KERNEL
{
typedef
void
(MLASCALL DepthwiseKernel)(
const InputType* const* Input,
InputType InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
};
typedef
void
(MLASCALL MLAS_CONV_FLOAT_FN)(
const MLAS_CONV_PARAMETERS* Parameters,
const float* Input,
const float* Filter,
const float* Bias,
float* WorkingBuffer,
float* Output,
MLAS_THREADPOOL* ThreadPool
);
typedef
bool
(MLASCALL MLAS_CONV_FLOAT_OVERRIDE)(
const MLAS_CONV_PARAMETERS* Parameters,
const float* Input,
const float* Filter,
const float* Bias,
float* WorkingBuffer,
float* Output,
MLAS_THREADPOOL* ThreadPool
);
// TODO: Investigate if overridden typedefs can be removed
typedef
void
(MLASCALL MLAS_CONV_PREPARE_FLOAT_FN)(
MLAS_CONV_PARAMETERS* Parameters,
size_t Dimensions,
size_t BatchCount,
size_t GroupCount,
size_t InputChannels,
const int64_t* InputShape,
const int64_t* KernelShape,
const int64_t* DilationShape,
const int64_t* Padding,
const int64_t* StrideShape,
const int64_t* OutputShape,
size_t FilterCount,
const MLAS_ACTIVATION* Activation,
size_t* WorkingBufferSize,
float Beta,
MLAS_THREADPOOL* ThreadPool
);
typedef
bool
(MLASCALL MLAS_CONV_PREPARE_FLOAT_OVERRIDE)(
MLAS_CONV_PARAMETERS* Parameters,
size_t Dimensions,
size_t BatchCount,
size_t GroupCount,
size_t InputChannels,
const int64_t* InputShape,
const int64_t* KernelShape,
const int64_t* DilationShape,
const int64_t* Padding,
const int64_t* StrideShape,
const int64_t* OutputShape,
size_t FilterCount,
const MLAS_ACTIVATION* Activation,
size_t* WorkingBufferSize,
float Beta,
MLAS_THREADPOOL* ThreadPool
);
typedef
bool
(MLASCALL MLAS_SGEMM_BATCH_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t M,
size_t N,
size_t K,
const MLAS_SGEMM_DATA_PARAMS* Data,
size_t BatchSize,
MLAS_THREADPOOL* ThreadPool);
typedef
size_t
(MLASCALL MLAS_SGEMM_PACK_B_SIZE_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t N,
size_t K);
typedef
bool
(MLASCALL MLAS_SGEMM_PACK_B_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t N,
size_t K,
const float* B,
size_t ldb,
void* PackedB);
typedef
void
(MLASCALL MLAS_DYNAMIC_QGEMM_BATCH_OVERRIDE)(
const MLAS_GEMM_DYN_QUANT_SHAPE_PARAMS& Shape,
const MLAS_GEMM_DYN_QUANT_DATA_PARAMS* DataParams,
const size_t BatchN,
MLAS_THREADPOOL* ThreadPool);
typedef
size_t
(MLASCALL MLAS_DYNAMIC_QGEMM_PACK_B_SIZE_OVERRIDE)(
size_t N,
size_t K);
typedef
void
(MLASCALL MLAS_DYNAMIC_QGEMM_PACK_B_OVERRIDE)(
size_t N,
size_t K,
const int8_t* B,
const float* Scales,
const float* Bias,
void* PackedB);
#if defined(__aarch64__) && defined(__linux__)
typedef
bool
(MLASCALL MLAS_SBGEMM_BATCH_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t M,
size_t N,
size_t K,
const MLAS_SBGEMM_DATA_PARAMS* Data,
size_t BatchSize,
MLAS_THREADPOOL* ThreadPool);
typedef
size_t
(MLASCALL MLAS_SBGEMM_PACK_B_SIZE_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t N,
size_t K);
typedef
bool
(MLASCALL MLAS_SBGEMM_PACK_B_OVERRIDE)(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t N,
size_t K,
const float* B,
size_t ldb,
void* PackedB);
#endif
extern "C" {
#if defined(MLAS_TARGET_AMD64_IX86)
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelSse;
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelAvx;
#ifdef FORCE_GENERIC_ALGORITHMS
MLAS_GEMM_FLOAT_KERNEL_GENERIC MlasSgemmKernelZero;
MLAS_GEMM_FLOAT_KERNEL_GENERIC MlasSgemmKernelAdd;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelFma3;
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelAvx512F;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelSse;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelAvx;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelFma3;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelAvx512F;
#endif
#elif defined(MLAS_TARGET_POWER)
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernel;
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernelPOWER10;
MLAS_GEMM_DOUBLE_KERNEL MlasDgemmKernel;
MLAS_GEMM_DOUBLE_KERNEL MlasDgemmKernelPOWER10;
MLAS_QUANTIZE_LINEAR_S8_KERNEL MlasQuantizeLinearS8KernelVSX;
MLAS_QUANTIZE_LINEAR_U8_KERNEL MlasQuantizeLinearU8KernelVSX;
#elif defined(MLAS_TARGET_S390X)
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernel;
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernelZVECTOR;
MLAS_GEMM_DOUBLE_KERNEL MlasDgemmKernel;
MLAS_QUANTIZE_LINEAR_S8_KERNEL MlasQuantizeLinearS8KernelZVECTOR;
MLAS_QUANTIZE_LINEAR_U8_KERNEL MlasQuantizeLinearU8KernelZVECTOR;
#elif defined(MLAS_TARGET_LARCH64)
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelLSX;
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelLasx;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelLSX;
MLAS_GEMM_DOUBLE_KERNEL MlasGemmDoubleKernelLasx;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelLSX;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelLSX;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelLSX;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelLSX;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelLasx;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelLasx;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelLasx;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelLasx;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelLSX;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelLSX;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelLSX;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelLasx;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelLasx;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelLasx;
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4LSX;
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4Lasx;
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32KernelLasx;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeSoftmaxOutputF32KernelLasx;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeLogSoftmaxOutputF32KernelLasx;
#elif defined(MLAS_TARGET_RISCV64)
#if defined(MLAS_USE_RVV)
MLAS_GEMM_FLOAT_KERNEL MlasGemmFloatKernelRvv;
void MlasSgemmCopyPackBRvv(
float* D,
const float* B,
size_t ldb,
size_t CountX,
size_t CountY);
#endif
size_t MLASCALL MlasSgemmKernelZero(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
float alpha);
size_t MLASCALL MlasSgemmKernelAdd(
const float* A,
const float* B,
float* C,
size_t CountK,
size_t CountM,
size_t CountN,
size_t lda,
size_t ldc,
float alpha);
#else
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernelZero;
MLAS_GEMM_FLOAT_KERNEL MlasSgemmKernelAdd;
#if defined(__aarch64__) && defined(__linux__)
MLAS_SBGEMM_FLOAT_KERNEL MlasSbgemmKernelZero;
MLAS_SBGEMM_FLOAT_KERNEL MlasSbgemmKernelAdd;
#endif
#if defined(MLAS_TARGET_ARM64) && defined(MLAS_USE_ARM_NEON_NCHWC)
// Intrinsics kernel for direct NCHW convolution
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelNeon;
#if !defined(_WIN32)
// AArch64 assembly micro-kernel for direct NCHW convolution
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelNeonAsm;
#endif
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelNeon;
#if !defined(_WIN32)
// AArch64 assembly micro-kernel for direct NCHWc convolution
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelNeonAsm;
#endif
// Intrinsics kernel for depthwise NCHWc convolution
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelNeon;
#if !defined(_WIN32)
// AArch64 assembly micro-kernel for depthwise NCHWc convolution
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelNeonAsm;
#endif
// Intrinsics kernel for pointwise NCHWc convolution
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelNeon;
#if !defined(_WIN32)
// AArch64 assembly micro-kernel for pointwise NCHWc convolution
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelNeonAsm;
#endif
#if defined(__linux__)
// AArch64 assembly fast-math micro-kernels
MLAS_CONV_FLOAT_KERNEL MlasConvNchwBf16KernelNeon;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseBf16KernelNeon;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseBf16KernelNeon;
#endif
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelNeon;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelNeon;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelNeon;
#endif
MLAS_GEMM_DOUBLE_KERNEL MlasDgemmKernelZero;
MLAS_GEMM_DOUBLE_KERNEL MlasDgemmKernelAdd;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_SGEMM_KERNEL_M1_ROUTINE MlasSgemmKernelM1Avx;
MLAS_SGEMM_KERNEL_M1_ROUTINE MlasSgemmKernelM1TransposeBAvx;
#elif defined(MLAS_TARGET_ARM64) || defined(MLAS_TARGET_WASM)
MLAS_GEMV_FLOAT_KERNEL MlasGemvFloatKernel;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4Sse;
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4Avx;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_GEMM_U8S8_KERNEL MlasGemmU8S8KernelAvx2;
MLAS_GEMV_U8S8_KERNEL MlasGemvU8S8KernelAvx2;
MLAS_GEMM_U8S8_KERNEL MlasGemmU8S8KernelAvx512Core;
MLAS_GEMV_U8S8_KERNEL MlasGemvU8S8KernelAvx512Core;
MLAS_GEMM_U8S8_KERNEL MlasGemmU8S8KernelAvx512Vnni;
MLAS_GEMV_U8S8_KERNEL MlasGemvU8S8KernelAvx512Vnni;
MLAS_GEMM_U8S8_KERNEL MlasGemmU8S8KernelAvxVnni;
MLAS_GEMV_U8S8_KERNEL MlasGemvU8S8KernelAvxVnni;
MLAS_GEMM_U8S8_KERNEL MlasGemmU8U8KernelAvx2Vnni;
MLAS_GEMM_U8S8_KERNEL MlasGemmS8S8KernelAvx2Vnni;
MLAS_GEMM_U8S8_KERNEL MlasGemmS8U8KernelAvx2Vnni;
MLAS_GEMM_U8U8_KERNEL MlasGemmU8U8KernelAvx2;
MLAS_GEMM_U8U8_KERNEL MlasGemmU8U8KernelAvx512Core;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelSse;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelSse;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelSse;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelSse;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelAvx;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelAvx;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelAvx;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelAvx;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelFma3;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelFma3;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelFma3;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelFma3;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernelAvx512F;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernelAvx512F;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernelAvx512F;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernelAvx512F;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelSse;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelAvx;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernelAvx512F;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelSse;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelAvx;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernelAvx512F;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelSse;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelAvx;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernelAvx512F;
#else
MLAS_CONV_FLOAT_KERNEL MlasConvNchwFloatKernel;
MLAS_CONV_FLOAT_KERNEL MlasConvNchwcFloatKernel;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL MlasConvDepthwiseFloatKernel;
MLAS_CONV_POINTWISE_FLOAT_KERNEL MlasConvPointwiseFloatKernel;
MLAS_POOL_FLOAT_KERNEL MlasPoolMaximumFloatKernel;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageExcludePadFloatKernel;
MLAS_POOL_FLOAT_KERNEL MlasPoolAverageIncludePadFloatKernel;
#endif
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasErfKernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasGeluErfKernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasSiluKernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasComputeExpF32Kernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasLogisticKernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasTanhKernel;
MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL MlasComputeSumExpF32Kernel;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeSoftmaxOutputF32Kernel;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeLogSoftmaxOutputF32Kernel;
MLAS_QLINEAR_BINARY_OP_S8_KERNEL MlasQLinearAddS8Kernel;
MLAS_QLINEAR_BINARY_OP_U8_KERNEL MlasQLinearAddU8Kernel;
MLAS_QUANTIZE_LINEAR_S8_KERNEL MlasQuantizeLinearS8Kernel;
MLAS_QUANTIZE_LINEAR_U8_KERNEL MlasQuantizeLinearU8Kernel;
MLAS_QUANTIZE_LINEAR_S16_KERNEL MlasQuantizeLinearS16Kernel;
MLAS_QUANTIZE_LINEAR_U16_KERNEL MlasQuantizeLinearU16Kernel;
MLAS_QUANTIZE_LINEAR_S4_KERNEL MlasQuantizeLinearS4Kernel;
MLAS_QUANTIZE_LINEAR_U4_KERNEL MlasQuantizeLinearU4Kernel;
#if defined(MLAS_TARGET_AMD64)
MLAS_DEQUANTIZE_LINEAR_S8_KERNEL MlasDequantizeLinearS8Kernel;
MLAS_DEQUANTIZE_LINEAR_U8_KERNEL MlasDequantizeLinearU8Kernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasErfKernelFma3;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasComputeExpF32KernelFma3;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasComputeExpF32KernelAvx512F;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasComputeLogisticF32KernelFma3;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasComputeTanhF32KernelFma3;
MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL MlasComputeSumExpF32KernelFma3;
MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL MlasComputeSumExpF32KernelAvx512F;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeSoftmaxOutputF32KernelAvx;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeLogSoftmaxOutputF32KernelAvx;
MLAS_QLINEAR_BINARY_OP_S8_KERNEL MlasQLinearAddS8KernelAvx2;
MLAS_QLINEAR_BINARY_OP_U8_KERNEL MlasQLinearAddU8KernelAvx2;
MLAS_QUANTIZE_LINEAR_S8_KERNEL MlasQuantizeLinearS8KernelAvx512F;
MLAS_QUANTIZE_LINEAR_U8_KERNEL MlasQuantizeLinearU8KernelAvx512F;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasGeluErfKernelAvx512F;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasSiluKernelAvx512F;
#endif
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32Kernel;
MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL MlasReduceMinimumMaximumF32Kernel;
#if defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV)
MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL MlasComputeSumExpF32KernelRvv;
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32KernelRvv;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeSoftmaxOutputF32KernelRvv;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL MlasComputeLogSoftmaxOutputF32KernelRvv;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32KernelAvx;
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32KernelAvx512F;
MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL MlasReduceMinimumMaximumF32KernelAvx;
#endif
#if defined(MLAS_TARGET_AMD64)
MLAS_CAST_F16_TO_F32_KERNEL MlasCastF16ToF32KernelSse;
MLAS_CAST_F16_TO_F32_KERNEL MlasCastF16ToF32KernelAvx;
MLAS_CAST_F16_TO_F32_KERNEL MlasCastF16ToF32KernelAvx2;
MLAS_CAST_F32_TO_F16_KERNEL MlasCastF32ToF16KernelAvx2;
#endif
#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) && defined(MLAS_TARGET_ARM64)
MLAS_CAST_F16_TO_F32_KERNEL MlasCastF16ToF32KernelNeon;
MLAS_CAST_F32_TO_F16_KERNEL MlasCastF32ToF16KernelNeon;
#endif
}
//
// Define the default preferred byte alignment for buffers.
//
// MLAS_TARGET_AMD64_IX86: The typical architecture uses AVX instructions
// accessing 256-bit vectors. MLAS_TARGET_AMD64 returns a larger value if the
// platform supports 512-bit vectors to ensure that vectors are not split.
//
// MLAS_TARGET_ARM64: The kernels use "load pair" instructions to access 128-bit
// vectors, so this value keeps both vectors in the same cache line.
//
// MLAS_TARGET_ARM: Using 16 for a single 128-bit vector may be sufficient for
// this architecture, but the ONNX Runtime has historically used this larger
// value.
//
#define MLAS_DEFAULT_PREFERRED_BUFFER_ALIGNMENT 64
//
// Define the target number of per-thread multiplies before using another
// thread to perform additional work.
//
#define MLAS_SGEMM_THREAD_COMPLEXITY (size_t(64) * size_t(1024))
#define MLAS_DGEMM_THREAD_COMPLEXITY (size_t(64) * size_t(1024))
#define MLAS_QGEMM_THREAD_COMPLEXITY 65536
#define MLAS_HGEMM_THREAD_COMPLEXITY 65536
#if defined(__aarch64__) && defined(__linux__)
#define MLAS_SBGEMM_THREAD_COMPLEXITY (size_t(64) * size_t(1024))
#endif
//
// Single-threaded single precision matrix/matrix multiply operation.
//
void
MlasSgemmOperation(
CBLAS_TRANSPOSE TransA,
CBLAS_TRANSPOSE TransB,
size_t M,
size_t N,
size_t K,
float alpha,
const float* A,
size_t lda,
const float* B,
size_t ldb,
float beta,
float* C,
size_t ldc
);
//
// Quantized integer matrix/matrix dispatch structure.
//
struct MLAS_GEMM_QUANT_DISPATCH;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchSse;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchLSX;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchLSX;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8U8DispatchLSX;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8S8DispatchSse41;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8S8DispatchAvx2;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8U8DispatchAvx2;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8U8DispatchAvx2Vnni;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchAvx2Vnni;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8U8DispatchAvx2Vnni;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8S8DispatchAmx;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchNeon;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmX8S8DispatchNeon;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchUdot;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchSdot;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchUmmla;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchSmmla;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchWasmSimd;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchWasmRelaxedSimd;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmQuantDispatchDefault;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemm8X8DispatchPOWER10;
extern const MLAS_GEMM_QUANT_DISPATCH MlasGemm8X8DispatchZVECTOR;
#if defined(MLAS_TARGET_WASM_RELAXED_SIMD)
extern bool HasUSDot();
#endif
//
// Symmetric quantized qgemm dispatch structure
//
struct MLAS_SYMM_QGEMM_DISPATCH;
extern const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchNeon;
extern const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchSdot;
//
// Symmetric quantized integer convolution dispatch structure.
//
struct MLAS_CONV_SYM_DISPATCH;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymDispatchAvx2;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymDispatchAvxVnni;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymDispatchAvx512Core;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymDispatchAvx512Vnni;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymU8DispatchNeon;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymS8DispatchNeon;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymU8DispatchDot;
extern const MLAS_CONV_SYM_DISPATCH MlasConvSymS8DispatchDot;
//
// Quantized 8-bit integer/quantized 4-bit integer matrix/matrix multiply dispatch structure.
//
struct MLAS_Q8Q4GEMM_DISPATCH;
extern const MLAS_Q8Q4GEMM_DISPATCH MlasQ8Q4GemmDispatchAvx512vnni;
//
// Float/quantized 4-bit integer matrix/matrix multiply dispatch structure.
//
struct MLAS_FPQ4GEMM_DISPATCH;
extern const MLAS_FPQ4GEMM_DISPATCH MlasFpQ4GemmDispatchAvx512;
//
// Float/quantized n-bit integer matrix/matrix multiply dispatch structure.
//
struct MLAS_QNBIT_GEMM_DISPATCH;
const MLAS_QNBIT_GEMM_DISPATCH&
GetMlasQNBitGemmDispatchNeon(
bool InitializeWithDotSupport,
bool InitializeWithI8MMSupport
);
extern const MLAS_QNBIT_GEMM_DISPATCH MlasSQNBitGemmDispatchAvx2;
extern const MLAS_QNBIT_GEMM_DISPATCH MlasSQNBitGemmDispatchAvx2vnni;
extern const MLAS_QNBIT_GEMM_DISPATCH MlasSQNBitGemmDispatchAvx512;
extern const MLAS_QNBIT_GEMM_DISPATCH MlasSQNBitGemmDispatchAvx512vnni;
extern const MLAS_QNBIT_GEMM_DISPATCH MlasSQNBitGemmDispatchLasx;
struct MLAS_QNBIT_LUT_GEMM_DISPATCH;
extern const MLAS_QNBIT_LUT_GEMM_DISPATCH MlasLutGenKernelAvx2;
//
// Rotary embedding dispatch structure.
//
struct MLAS_ROPE_DISPATCH;
extern const MLAS_ROPE_DISPATCH MlasRopeDispatchNeon;
extern const MLAS_ROPE_DISPATCH MlasRopeDispatchAvx2;
//
// half gemm dispatch structure
//
struct MLAS_HGEMM_DISPATCH;
extern const MLAS_HGEMM_DISPATCH MlasHGemmDispatchNeon;
// softmax dispatch structure
struct MLAS_SOFTMAX_DISPATCH;
extern const MLAS_SOFTMAX_DISPATCH MlasSoftmaxDispatchNeon;
// eltwise dispatch structure
struct MLAS_ELTWISE_DISPATCH;
extern const MLAS_ELTWISE_DISPATCH MlasEltwiseDispatchNeon;
//
// Quantized depthwise convolution kernels.
//
template<typename InputType, typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernel(
const InputType* const* Input,
InputType InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template <typename InputType, typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernelAvx2(
const InputType* const* Input,
InputType InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
//
// Define the kernel flags for conv sym
//
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 0x00000001
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 0x00000002
//
// Define the post-processing parameters for conv sym: bias and re-quant params
//
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias;
const float* Scale;
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
};
//
// Environment information class.
//
enum MlasCoreType { mlas_core_unknown = 0, mlas_core_little = 2, mlas_core_big = 3 };
struct MLAS_PLATFORM {
MLAS_PLATFORM(void);
// TODO: move to cpuinfo
bool Avx2Supported_ = false;
bool Avx512Supported_ = false;
bool ArmNeonIsQuantActivationsUnsigned = false;
// MLAS SGemm overrides
MLAS_SGEMM_BATCH_OVERRIDE* MlasSGemmBatchOverride = nullptr;
MLAS_SGEMM_PACK_B_SIZE_OVERRIDE* MlasSGemmPackBSizeOverride = nullptr;
MLAS_SGEMM_PACK_B_OVERRIDE* MlasSGemmPackBOverride = nullptr;
// MLAS Dynamic QGemm overrides
MLAS_DYNAMIC_QGEMM_BATCH_OVERRIDE* MlasDynamicQGemmBatchOverride = nullptr;
MLAS_DYNAMIC_QGEMM_PACK_B_SIZE_OVERRIDE* MlasDynamicQGemmPackBSizeOverride = nullptr;
MLAS_DYNAMIC_QGEMM_PACK_B_OVERRIDE* MlasDynamicQGemmPackBOverride = nullptr;
// MLAS Conv overrides
MLAS_CONV_PREPARE_FLOAT_OVERRIDE* MlasConvPrepareOverride = nullptr;
MLAS_CONV_FLOAT_OVERRIDE* MlasConvOverride = nullptr;
#if defined(__aarch64__) && defined(__linux__)
// SBGemm overrides
MLAS_SBGEMM_BATCH_OVERRIDE* MlasSBGemmBatchOverride = nullptr;
MLAS_SBGEMM_PACK_B_SIZE_OVERRIDE* MlasSBGemmPackBSizeOverride = nullptr;
MLAS_SBGEMM_PACK_B_OVERRIDE* MlasSBGemmPackBOverride = nullptr;
#endif
#if defined(MLAS_TARGET_AMD64_IX86) || defined(MLAS_TARGET_POWER) || defined(MLAS_TARGET_S390X) || defined(MLAS_TARGET_RISCV64)
MLAS_GEMM_FLOAT_KERNEL* GemmFloatKernel;
#endif
#if defined(MLAS_TARGET_LARCH64)
const MLAS_GEMM_QUANT_DISPATCH* GemmU8S8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmU8U8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmS8S8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmS8U8Dispatch;
MLAS_GEMM_FLOAT_KERNEL* GemmFloatKernel;
MLAS_GEMM_DOUBLE_KERNEL* GemmDoubleKernel;
MLAS_CONV_FLOAT_KERNEL* ConvNchwFloatKernel;
MLAS_CONV_FLOAT_KERNEL* ConvNchwcFloatKernel;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* ConvDepthwiseFloatKernel;
MLAS_CONV_POINTWISE_FLOAT_KERNEL* ConvPointwiseFloatKernel;
MLAS_POOL_FLOAT_KERNEL* PoolFloatKernel[MlasPoolingKindCount];
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE* TransposePackB16x4Routine;
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL* ReduceMaximumF32Kernel;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeSoftmaxOutputF32Kernel;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeLogSoftmaxOutputF32Kernel;
uint32_t NchwcBlockSize;
#endif
#if defined(MLAS_TARGET_AMD64_IX86)
const MLAS_GEMM_QUANT_DISPATCH* GemmU8S8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmU8U8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmS8S8Dispatch{&MlasGemmQuantDispatchDefault};
const MLAS_GEMM_QUANT_DISPATCH* GemmS8U8Dispatch{&MlasGemmQuantDispatchDefault};
#elif defined(MLAS_TARGET_ARM64)
const MLAS_GEMM_QUANT_DISPATCH* GemmU8U8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmU8S8Dispatch;
const MLAS_GEMM_QUANT_DISPATCH* GemmS8S8Dispatch;
#if defined(MLAS_USE_ARM_NEON_NCHWC)
MLAS_CONV_FLOAT_KERNEL* ConvNchwFloatKernel;
MLAS_CONV_FLOAT_KERNEL* ConvNchwcFloatKernel;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* ConvDepthwiseFloatKernel;
MLAS_CONV_POINTWISE_FLOAT_KERNEL* ConvPointwiseFloatKernel;
#if defined(__linux__)
MLAS_CONV_FLOAT_KERNEL* ConvNchwBf16Kernel;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* ConvDepthwiseBf16Kernel;
MLAS_CONV_POINTWISE_FLOAT_KERNEL* ConvPointwiseBf16Kernel;
#endif
MLAS_POOL_FLOAT_KERNEL* PoolFloatKernel[MlasPoolingKindCount];
uint32_t NchwcBlockSize;
#endif
#endif
const MLAS_SYMM_QGEMM_DISPATCH* SymmQgemmDispatch{nullptr};
const MLAS_CONV_SYM_DISPATCH* ConvSymU8S8Dispatch{nullptr};
const MLAS_CONV_SYM_DISPATCH* ConvSymS8S8Dispatch{nullptr};
MLAS_QUANT_KERNEL<uint8_t, int8_t>::DepthwiseKernel* ConvDepthwiseU8S8Kernel;
MLAS_QUANT_KERNEL<uint8_t, uint8_t>::DepthwiseKernel* ConvDepthwiseU8U8Kernel;
MLAS_QUANT_KERNEL<int8_t, int8_t>::DepthwiseKernel* ConvDepthwiseS8S8Kernel;
MLAS_QUANT_KERNEL<int8_t, uint8_t>::DepthwiseKernel* ConvDepthwiseS8U8Kernel;
#if defined(MLAS_TARGET_POWER) || defined(MLAS_TARGET_S390X)
MLAS_GEMM_DOUBLE_KERNEL* GemmDoubleKernel;
const MLAS_GEMM_QUANT_DISPATCH* GemmU8X8Dispatch;
MLAS_QUANTIZE_LINEAR_S8_KERNEL* QuantizeLinearS8Kernel;
MLAS_QUANTIZE_LINEAR_U8_KERNEL* QuantizeLinearU8Kernel;
MLAS_QUANTIZE_LINEAR_S16_KERNEL* QuantizeLinearS16Kernel;
MLAS_QUANTIZE_LINEAR_U16_KERNEL* QuantizeLinearU16Kernel;
MLAS_QUANTIZE_LINEAR_S4_KERNEL* QuantizeLinearS4Kernel;
MLAS_QUANTIZE_LINEAR_U4_KERNEL* QuantizeLinearU4Kernel;
#endif
#if defined(MLAS_USE_SVE) || defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64)
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* ErfKernelRoutine;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* LogisticKernelRoutine;
MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL* ReduceMaximumF32Kernel;
MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL* ComputeSumExpF32Kernel;
MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeLogSoftmaxOutputF32Kernel;
MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeSoftmaxOutputF32Kernel;
#endif
MLAS_COMPUTE_ERF_FP16_KERNEL* ErfFP16KernelRoutine = nullptr;
MLAS_COMPUTE_GELU_FP16_KERNEL* GeluFP16KernelRoutine = nullptr;
MLAS_COMPUTE_TANH_FP16_KERNEL* TanhFP16KernelRoutine = nullptr;
#if defined(MLAS_TARGET_AMD64)
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* GeluErfKernelRoutine;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* SiluKernelRoutine;
MLAS_SGEMM_KERNEL_M1_ROUTINE* KernelM1Routine;
MLAS_SGEMM_KERNEL_M1_ROUTINE* KernelM1TransposeBRoutine;
MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE* TransposePackB16x4Routine;
MLAS_GEMM_DOUBLE_KERNEL* GemmDoubleKernel;
MLAS_GEMM_U8S8_KERNEL* GemmU8S8Kernel;
MLAS_GEMM_U8S8_KERNEL* GemmS8S8Kernel;
MLAS_GEMM_U8S8_KERNEL* GemmS8U8Kernel;
MLAS_GEMV_U8S8_KERNEL* GemvU8S8Kernel;
MLAS_GEMM_U8U8_KERNEL* GemmU8U8Kernel;
MLAS_CONV_FLOAT_KERNEL* ConvNchwFloatKernel;
MLAS_CONV_FLOAT_KERNEL* ConvNchwcFloatKernel;
MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* ConvDepthwiseFloatKernel;
MLAS_CONV_POINTWISE_FLOAT_KERNEL* ConvPointwiseFloatKernel;
MLAS_POOL_FLOAT_KERNEL* PoolFloatKernel[MlasPoolingKindCount];
MLAS_QLINEAR_BINARY_OP_S8_KERNEL* QLinearAddS8Kernel;
MLAS_QLINEAR_BINARY_OP_U8_KERNEL* QLinearAddU8Kernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* ComputeExpF32Kernel;
MLAS_COMPUTE_UNARY_FLOAT_KERNEL* TanhKernelRoutine;
MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL* ReduceMinimumMaximumF32Kernel;
MLAS_QUANTIZE_LINEAR_S8_KERNEL* QuantizeLinearS8Kernel;
MLAS_QUANTIZE_LINEAR_U8_KERNEL* QuantizeLinearU8Kernel;
MLAS_QUANTIZE_LINEAR_S16_KERNEL* QuantizeLinearS16Kernel;
MLAS_QUANTIZE_LINEAR_U16_KERNEL* QuantizeLinearU16Kernel;
MLAS_QUANTIZE_LINEAR_S4_KERNEL* QuantizeLinearS4Kernel;
MLAS_QUANTIZE_LINEAR_U4_KERNEL* QuantizeLinearU4Kernel;
MLAS_DEQUANTIZE_LINEAR_S8_KERNEL* DequantizeLinearS8Kernel;
MLAS_DEQUANTIZE_LINEAR_U8_KERNEL* DequantizeLinearU8Kernel;
uint32_t NchwcBlockSize;
uint32_t PreferredBufferAlignment;
int32_t MaximumThreadCount;
#elif defined(MLAS_TARGET_ARM64)
static constexpr int32_t MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT * 4;
static constexpr size_t MLAS_NEON_NCHWC_BLOCK_SIZE = 16;
#else
static constexpr int32_t MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT;
#endif
const MLAS_FPQ4GEMM_DISPATCH* FpQ4GemmDispatch{nullptr};
const MLAS_Q8Q4GEMM_DISPATCH* Q8Q4GemmDispatch{nullptr};
const MLAS_QNBIT_GEMM_DISPATCH* QNBitGemmDispatch{nullptr};
const MLAS_QNBIT_LUT_GEMM_DISPATCH* LutGenKernel{nullptr};
MLAS_CAST_F16_TO_F32_KERNEL* CastF16ToF32Kernel;
MLAS_CAST_F32_TO_F16_KERNEL* CastF32ToF16Kernel;
const MLAS_ROPE_DISPATCH* RopeDispatch{nullptr};
const MLAS_HGEMM_DISPATCH* HGemmDispatch{nullptr};
const MLAS_SOFTMAX_DISPATCH* SoftmaxDispatch{nullptr};
const MLAS_ELTWISE_DISPATCH* EltwiseDispatch{nullptr};
};
inline
MLAS_PLATFORM& GetMlasPlatform(){
static MLAS_PLATFORM MlasPlatform;
return MlasPlatform;
}
//
// Threading support.
//
typedef
void
(MLAS_THREADED_ROUTINE)(
void* Context,
ptrdiff_t Index
);
void
MlasExecuteThreaded(
MLAS_THREADED_ROUTINE* ThreadedRoutine,
void* Context,
ptrdiff_t Iterations,
MLAS_THREADPOOL* ThreadPool
);
constexpr
size_t
MlasDivRoundup(size_t up, size_t down)
{
return (up + down - 1) / down;
}
/**
* @brief Distribute multiple iterations of work over a thread pool if supported
*
* @param ThreadPool [IN] Optional thread pool. Ignored when using OpenMP
* @param Iterations [IN] Total number of iterations
* @param Work [IN] Logic for computing a range of iterations [begin, end)
*/
void
MlasTrySimpleParallel(
MLAS_THREADPOOL* ThreadPool,
const std::ptrdiff_t Iterations,
const std::function<void(std::ptrdiff_t tid)>& Work
);
/**
* @brief Distribute many iterations of work over a thread pool if supported.
* This function is for small workloads in non-performance critical situation.
*
* @param ThreadPool [IN] Optional thread pool. Ignored when using OpenMP
* @param Iterations [IN] Total number of iterations
* @param Work [IN] Logic for computing a range of iterations [begin, end)
*/
void
MlasTryBatchParallel(
MLAS_THREADPOOL * ThreadPool,
const std::ptrdiff_t Iterations,
const std::function<void(std::ptrdiff_t tid)>& Work
);
#if defined(MLAS_OPENCV_THREADING)
// Defined in 3rdparty/mlas/threading_opencv.cpp. Returns
// cv::getNumThreads(). Hidden behind a free function so this header doesn't
// need to pull <opencv2/core/utility.hpp> into every MLAS translation unit.
extern "C" int opencv_dnn_mlas_max_threads();
#endif
inline
ptrdiff_t
MlasGetMaximumThreadCount(
MLAS_THREADPOOL* ThreadPool
)
{
#if defined(MLAS_OPENCV_THREADING)
MLAS_UNREFERENCED_PARAMETER(ThreadPool);
return static_cast<ptrdiff_t>(opencv_dnn_mlas_max_threads());
#elif defined(BUILD_MLAS_NO_ONNXRUNTIME)
MLAS_UNREFERENCED_PARAMETER(ThreadPool);
return 1;
#else
return onnxruntime::concurrency::ThreadPool::DegreeOfParallelism(ThreadPool);
#endif
}
inline
void
MlasPartitionWork(
ptrdiff_t ThreadId,
ptrdiff_t ThreadCount,
size_t TotalWork,
size_t* WorkIndex,
size_t* WorkRemaining
)
{
const size_t WorkPerThread = TotalWork / ThreadCount;
const size_t WorkPerThreadExtra = TotalWork % ThreadCount;
if (size_t(ThreadId) < WorkPerThreadExtra) {
*WorkIndex = (WorkPerThread + 1) * ThreadId;
*WorkRemaining = WorkPerThread + 1;
} else {
*WorkIndex = WorkPerThread * ThreadId + WorkPerThreadExtra;
*WorkRemaining = WorkPerThread;
}
}
//
// Define the minimum floating point value (and its bit value equivalent) that
// has no fractional bits. This number can be used for fast rounding of floating
// point numbers to integers.
//
#define MLAS_ROUNDING_BIAS_MAGIC 12582912.f
#define MLAS_ROUNDING_BIAS_MAGIC_BITS 0x4B400000
//
// Helpers to cast a floating point type to and from an integer bit format.
//
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(push)
// VC++ suggests we can attempt to make 'MlasBitsOfFp32' constexpr, but it is not valid.
#pragma warning(disable:26497)
#endif
MLAS_FORCEINLINE
uint32_t
MlasBitsOfFp32(
float FloatValue
)
{
union {
uint32_t IntegerValue;
float FloatValue;
} u;
u.FloatValue = FloatValue;
return u.IntegerValue;
}
MLAS_FORCEINLINE
float
MlasFp32FromBits(
uint32_t IntegerValue
)
{
union {
uint32_t IntegerValue;
float FloatValue;
} u;
u.IntegerValue = IntegerValue;
return u.FloatValue;
}
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(pop)
#endif
#if defined(MLAS_TARGET_WASM_SCALAR) || defined(MLAS_TARGET_ARM64)
void
MLASCALL
MlasConvDepthwiseFloat_CHW(
const MLAS_CONV_PARAMETERS* Parameters,
const float* Input,
const float* Filter,
float* Output,
const float* Zeros
);
#endif
void
MlasConvDepthwiseWithMultiplierFloat_CHW(
const MLAS_CONV_PARAMETERS* Parameters,
const float* Input,
const float* Filter,
float* Output,
const float* Zeros
);
#if defined(MLAS_TARGET_AMD64)
void
MlasConvDepthwiseMultiplier2CHWKernel7x7S2Avx512F(
const float* Input,
size_t InputHeight,
size_t InputWidth,
const float* Filter,
float* Output,
size_t OutputHeight,
size_t OutputWidth,
float Beta
);
#endif
//
// Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable
// cross-compiler support.
//
// Also define additional standard NEON intrinsics using the MSVC aliases.
//
#if defined(_M_ARM64)
#ifndef vmaxvq_f32
#define vmaxvq_f32(src) neon_fmaxv(src)
#endif
#ifndef vminvq_f32
#define vminvq_f32(src) neon_fminv(src)
#endif
#endif
//
// Cross-platform wrappers for 32-bit vector intrinsics.
//
#if defined(MLAS_TARGET_ARM)
#define MLAS_NEON_INTRINSICS
#define MLAS_NEON32_INTRINSICS
#elif defined(MLAS_TARGET_ARM64) || defined(MLAS_TARGET_ARM64EC)
#define MLAS_NEON_INTRINSICS
#define MLAS_NEON64_INTRINSICS
#elif defined(MLAS_TARGET_POWER)
#define MLAS_VSX_INTRINSICS
#elif defined(MLAS_TARGET_S390X)
#define MLAS_ZVECTOR_INTRINSICS
#elif defined(MLAS_TARGET_AMD64_IX86)
#define MLAS_SSE2_INTRINSICS
#if defined(__SSE4_1__) || (defined(_MSC_VER) && defined(__AVX__))
#define MLAS_SSE41_INTRINSICS
#endif
#if defined(__AVX__)
#define MLAS_AVX_INTRINSICS
#endif
#if defined(__AVX2__)
#define MLAS_AVX2_INTRINSICS
#endif
#if defined(__FMA__) || (defined(_MSC_VER) && defined(__AVX2__))
#define MLAS_FMA3_INTRINSICS
#endif
#elif defined(MLAS_TARGET_WASM_SIMD)
#define MLAS_WASM_SIMD_INTRINSICS
#if defined(MLAS_TARGET_WASM_RELAXED_SIMD)
#define MLAS_WASM_RELAXED_SIMD_INTRINSICS
#endif
#elif defined(MLAS_TARGET_LARCH64)
#define MLAS_LSX_INTRINSICS
#endif
#if defined(MLAS_NEON_INTRINSICS)
typedef float32x4_t MLAS_FLOAT32X4;
typedef int32x4_t MLAS_INT32X4;
#elif defined(MLAS_SSE2_INTRINSICS)
typedef __m128 MLAS_FLOAT32X4;
typedef __m128i MLAS_INT32X4;
#elif defined(MLAS_VSX_INTRINSICS)
typedef __vector float MLAS_FLOAT32X4;
typedef __vector int MLAS_INT32X4;
typedef __vector unsigned MLAS_UINT32X4;
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
typedef v128_t MLAS_FLOAT32X4;
typedef v128_t MLAS_INT32X4;
#elif defined(MLAS_LSX_INTRINSICS)
typedef __m128 MLAS_FLOAT32X4;
typedef __m128i MLAS_INT32X4;
#else
typedef float MLAS_FLOAT32X4 __attribute__ ((vector_size(16)));
typedef int32_t MLAS_INT32X4 __attribute__ ((vector_size(16)));
#endif
MLAS_FORCEINLINE
MLAS_INT32X4
MlasReinterpretAsInt32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vreinterpretq_s32_f32(Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_castps_si128(Vector);
#elif defined(MLAS_LSX_INTRINSICS)
return (MLAS_INT32X4)Vector;
#else
return MLAS_INT32X4(Vector);
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasCastToInt32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vcvtq_s32_f32(Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_cvttps_epi32(Vector);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_cts(Vector, 0);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_signed(Vector);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vftint_w_s(Vector);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return (MLAS_INT32X4)__builtin_convertvector((__f32x4)Vector, __i32x4);
#else
return MLAS_INT32X4{int32_t(Vector[0]), int32_t(Vector[1]), int32_t(Vector[2]), int32_t(Vector[3])};
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasCastToFloat32x4(MLAS_INT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vcvtq_f32_s32(Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_cvtepi32_ps(Vector);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_ctf(Vector, 0);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_float(Vector);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_convert_i32x4(Vector);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vffint_s_w(Vector);
#else
return MLAS_FLOAT32X4{float(Vector[0]), float(Vector[1]), float(Vector[2]), float(Vector[3])};
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasBroadcastInt32x4(int32_t Value)
{
#if defined(MLAS_NEON_INTRINSICS)
return vdupq_n_s32(Value);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_set1_epi32(Value);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_splat(Value);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return vec_splats(Value);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vreplgr2vr_w(Value);
#else
return MLAS_INT32X4{Value, Value, Value, Value};
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasLoadInt32x4(const int32_t* Buffer)
{
#if defined(MLAS_NEON_INTRINSICS)
return vld1q_s32(Buffer);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_loadu_si128((const __m128i*)Buffer);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_vsx_ld(0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_xl(0, Buffer);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_load(Buffer);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vld((const MLAS_INT32X4*)Buffer, 0);
#else
return *((MLAS_INT32X4*)Buffer);
#endif
}
MLAS_FORCEINLINE
void
MlasStoreInt32x4(int32_t* Buffer, MLAS_INT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
vst1q_s32(Buffer, Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
_mm_storeu_si128((__m128i*)Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS)
vec_vsx_st(Vector, 0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
vec_xst(Vector, 0, Buffer);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
wasm_v128_store(Buffer, Vector);
#elif defined(MLAS_LSX_INTRINSICS)
__lsx_vst(Vector, (MLAS_INT32X4 *)Buffer, 0);
#else
*((MLAS_INT32X4*)Buffer) = Vector;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasAddInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vaddq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_add_epi32(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_add(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_add(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vadd_w(Vector1, Vector2);
#else
return Vector1 + Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasSubtractInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vsubq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_sub_epi32(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_sub(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vsub_w(Vector1, Vector2);
#else
return Vector1 - Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasAndInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vandq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_and_si128(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_and(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vand_v(Vector1, Vector2);
#else
return Vector1 & Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasOrInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vorrq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_or_si128(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_or(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vor_v(Vector1, Vector2);
#else
return Vector1 | Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasAndNotInt32x4(MLAS_INT32X4 VectorNot, MLAS_INT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vandq_s32(vmvnq_s32(VectorNot), Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_andnot_si128(VectorNot, Vector);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_andnot(Vector, VectorNot);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vandn_v(VectorNot, Vector);
#else
return (~VectorNot) & Vector;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasXorInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return veorq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_xor_si128(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_xor(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return vec_xor(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vxor_v(Vector1, Vector2);
#else
return Vector1 ^ Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasBlendInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2, MLAS_INT32X4 Selection)
{
return MlasOrInt32x4(MlasAndInt32x4(Vector2, Selection), MlasAndNotInt32x4(Selection, Vector1));
}
template<unsigned ShiftCount>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftLeftInt32x4(MLAS_INT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vshlq_n_s32(Vector, ShiftCount);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_slli_epi32(Vector, ShiftCount);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_shl(Vector, ShiftCount);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vslli_w(Vector, ShiftCount);
#else
return Vector << ShiftCount;
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasMaximumInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vmaxq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE41_INTRINSICS)
return _mm_max_epi32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return MlasBlendInt32x4(Vector2, Vector1, _mm_cmpgt_epi32(Vector1, Vector2));
#elif defined(MLAS_VSX_INTRINSICS)
return vec_vmaxsw(Vector1, Vector2);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_max(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_max(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vmax_w(Vector1, Vector2);
#else
return MlasBlendInt32x4(Vector2, Vector1, Vector1 > Vector2);
#endif
}
MLAS_FORCEINLINE
MLAS_INT32X4
MlasMinimumInt32x4(MLAS_INT32X4 Vector1, MLAS_INT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vminq_s32(Vector1, Vector2);
#elif defined(MLAS_SSE41_INTRINSICS)
return _mm_min_epi32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return MlasBlendInt32x4(Vector2, Vector1, _mm_cmpgt_epi32(Vector2, Vector1));
#elif defined(MLAS_VSX_INTRINSICS)
return vec_vminsw(Vector1, Vector2);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_min(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_min(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vmin_w(Vector1, Vector2);
#else
return MlasBlendInt32x4(Vector2, Vector1, Vector2 > Vector1);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasReinterpretAsFloat32x4(MLAS_INT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vreinterpretq_f32_s32(Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_castsi128_ps(Vector);
#elif defined(MLAS_LSX_INTRINSICS)
return MLAS_FLOAT32X4(Vector);
#else
return MLAS_FLOAT32X4(Vector);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasBroadcastFloat32x4(float Value)
{
#if defined(MLAS_NEON_INTRINSICS)
return vdupq_n_f32(Value);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_set1_ps(Value);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_splat(Value);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
// Suppress wrong GCC warnings
MLAS_UNREFERENCED_PARAMETER(Value);
return vec_splats(Value);
#elif defined(MLAS_LSX_INTRINSICS)
return MLAS_FLOAT32X4{Value, Value, Value, Value};
#else
return MLAS_FLOAT32X4{Value, Value, Value, Value};
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasBroadcastFloat32x4(const float* Value)
{
#if defined(MLAS_NEON_INTRINSICS)
return vld1q_dup_f32(Value);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_load_ps1(Value);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_load32_splat(Value);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return vec_splats(*Value);
#elif defined(MLAS_LSX_INTRINSICS)
return MLAS_FLOAT32X4{*Value, *Value, *Value, *Value};
#else
return MLAS_FLOAT32X4{*Value, *Value, *Value, *Value};
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasZeroFloat32x4(void)
{
#if defined(MLAS_NEON_INTRINSICS)
return vdupq_n_f32(0.0f);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_setzero_ps();
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_const(0.0f, 0.0f, 0.0f, 0.0f);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasBroadcastFloat32x4(0.0f);
#else
return MlasBroadcastFloat32x4(0.0f);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasLoadFloat32x4(const float* Buffer)
{
#if defined(MLAS_NEON_INTRINSICS)
return vld1q_f32(Buffer);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_loadu_ps(Buffer);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_vsx_ld(0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_xl(0, Buffer);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_load(Buffer);
#elif defined(MLAS_LSX_INTRINSICS)
// return MlasReinterpretAsFloat32x4(__lsx_vld((const MLAS_INT32X4 *)Buffer, 0));
return (MLAS_FLOAT32X4)__lsx_vld((const MLAS_INT32X4 *)Buffer, 0);
#else
return *((MLAS_FLOAT32X4*)Buffer);
#endif
}
MLAS_FORCEINLINE
void
MlasStoreFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
vst1q_f32(Buffer, Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
_mm_storeu_ps(Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS)
vec_vsx_st(Vector, 0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
vec_xst(Vector, 0, Buffer);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
wasm_v128_store(Buffer, Vector);
#elif defined(MLAS_LSX_INTRINSICS)
__lsx_vst(MlasReinterpretAsInt32x4(Vector), Buffer, 0);
#else
*((MLAS_FLOAT32X4*)Buffer) = Vector;
#endif
}
MLAS_FORCEINLINE
void
MlasStoreAlignedFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
vst1q_f32(Buffer, Vector);
#elif defined(MLAS_SSE2_INTRINSICS)
_mm_store_ps(Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS)
// Workaround for bad GCC warning that these parameters are set but not used.
MLAS_UNREFERENCED_PARAMETER(Buffer);
MLAS_UNREFERENCED_PARAMETER(Vector);
vec_st(Vector, 0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
vec_xst(Vector, 0, Buffer);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
wasm_v128_store(Buffer, Vector);
#elif defined(MLAS_LSX_INTRINSICS)
MlasStoreFloat32x4(Buffer, Vector);
#else
MlasStoreFloat32x4(Buffer, Vector);
#endif
}
template<unsigned Lane>
MLAS_FORCEINLINE
void
MlasStoreLaneFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
vst1q_lane_f32(Buffer, Vector, Lane);
#elif defined(MLAS_SSE2_INTRINSICS)
// N.B. When building with AVX instructions, compilers optimize the following
// to a single vextractps instruction.
_mm_store_ss(Buffer, _mm_shuffle_ps(Vector, Vector, _MM_SHUFFLE(Lane, Lane, Lane, Lane)));
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
*Buffer = ((__f32x4)(Vector))[Lane];
#elif defined(MLAS_LSX_INTRINSICS)
*Buffer = Vector[Lane];
#else
*Buffer = Vector[Lane];
#endif
}
MLAS_FORCEINLINE
void
MlasStoreLowHalfFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
vst1_f32(Buffer, vget_low_f32(Vector));
#elif defined(MLAS_SSE2_INTRINSICS)
_mm_storel_pi((__m64*)Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS)
*((long long*)Buffer) = ((__vector long long)Vector)[0];
#elif defined(MLAS_LSX_INTRINSICS)
MlasStoreLaneFloat32x4<0>(&Buffer[0], Vector);
MlasStoreLaneFloat32x4<1>(&Buffer[1], Vector);
#else
MlasStoreLaneFloat32x4<0>(&Buffer[0], Vector);
MlasStoreLaneFloat32x4<1>(&Buffer[1], Vector);
#endif
}
template<unsigned Lane>
MLAS_FORCEINLINE
float
MlasExtractLaneFloat32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON_INTRINSICS)
return vgetq_lane_f32(Vector, Lane);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_cvtss_f32(_mm_shuffle_ps(Vector, Vector, _MM_SHUFFLE(Lane, Lane, Lane, Lane)));
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_extract_lane(Vector, Lane);
#elif defined(MLAS_LSX_INTRINSICS)
return Vector[Lane];
#else
return Vector[Lane];
#endif
}
#if defined(MLAS_SSE2_INTRINSICS)
template<>
MLAS_FORCEINLINE
void
MlasStoreLaneFloat32x4<0>(float* Buffer, MLAS_FLOAT32X4 Vector)
{
_mm_store_ss(Buffer, Vector);
}
template<>
MLAS_FORCEINLINE
float
MlasExtractLaneFloat32x4<0>(MLAS_FLOAT32X4 Vector)
{
return _mm_cvtss_f32(Vector);
}
template<unsigned Index0, unsigned Index1, unsigned Index2, unsigned Index3>
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasShuffleFloat32x4(MLAS_FLOAT32X4 Vector)
{
return _mm_shuffle_ps(Vector, Vector, _MM_SHUFFLE(Index3, Index2, Index1, Index0));
}
#endif
#if !defined(MLAS_SSE2_INTRINSICS) && !defined(_MSC_VER)
template<unsigned Index0, unsigned Index1, unsigned Index2, unsigned Index3>
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasShuffleFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_i32x4_shuffle(Vector1, Vector2, Index0, Index1, Index2, Index3);
#elif defined(__clang__)
return __builtin_shufflevector(Vector1, Vector2, Index0, Index1, Index2, Index3);
#elif defined(MLAS_LSX_INTRINSICS)
typedef int32_t GEN_INT32X4 __attribute__ ((vector_size(16)));
return __builtin_shuffle(Vector1, Vector2, GEN_INT32X4{Index0, Index1, Index2, Index3});
#else
return __builtin_shuffle(Vector1, Vector2, MLAS_INT32X4{Index0, Index1, Index2, Index3});
#endif
}
template<unsigned Index0, unsigned Index1, unsigned Index2, unsigned Index3>
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasShuffleFloat32x4(MLAS_FLOAT32X4 Vector)
{
return MlasShuffleFloat32x4<Index0, Index1, Index2, Index3>(Vector, Vector);
}
#endif
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasInterleaveLowFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vzip1q_f32(Vector1, Vector2);
#elif defined(MLAS_NEON32_INTRINSICS)
float32x4x2_t zipped = vzipq_f32(Vector1, Vector2);
return zipped.val[0];
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_unpacklo_ps(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return vec_mergeh(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return (MLAS_FLOAT32X4)__lsx_vilvl_w(MlasReinterpretAsInt32x4(Vector2), MlasReinterpretAsInt32x4(Vector1));
#else
return MlasShuffleFloat32x4<0, 4, 1, 5>(Vector1, Vector2);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasInterleaveHighFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vzip2q_f32(Vector1, Vector2);
#elif defined(MLAS_NEON32_INTRINSICS)
float32x4x2_t zipped = vzipq_f32(Vector1, Vector2);
return zipped.val[1];
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_unpackhi_ps(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return vec_mergel(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return (MLAS_FLOAT32X4)__lsx_vilvh_w(MlasReinterpretAsInt32x4(Vector2), MlasReinterpretAsInt32x4(Vector1));
#else
return MlasShuffleFloat32x4<2, 6, 3, 7>(Vector1, Vector2);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasAddFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vaddq_f32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_add_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_add(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_add(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfadd_s(Vector1, Vector2);
#else
return Vector1 + Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasSubtractFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vsubq_f32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_sub_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_sub(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_sub(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfsub_s(Vector1, Vector2);
#else
return Vector1 - Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMultiplyFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vmulq_f32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_mul_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_mul(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS)
// Suppress wrong GCC warnings
MLAS_UNREFERENCED_PARAMETER(Vector1);
MLAS_UNREFERENCED_PARAMETER(Vector2);
return vec_mul(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfmul_s(Vector1, Vector2);
#else
return Vector1 * Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMultiplyAddFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2, MLAS_FLOAT32X4 Vector3)
{
#if defined(MLAS_NEON_INTRINSICS)
#if defined(MLAS_TARGET_ARM)
// ARMv7 NEON doesn't have vfmaq_f32()
return vmlaq_f32(Vector3, Vector1, Vector2);
#else
return vfmaq_f32(Vector3, Vector1, Vector2);
#endif
#elif defined(MLAS_FMA3_INTRINSICS)
return _mm_fmadd_ps(Vector1, Vector2, Vector3);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_add_ps(_mm_mul_ps(Vector1, Vector2), Vector3);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_madd(Vector1, Vector2, Vector3);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return __builtin_s390_vfmasb(Vector1, Vector2, Vector3);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_add(wasm_f32x4_mul(Vector1, Vector2), Vector3);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfmadd_s(Vector1, Vector2, Vector3);
#else
return Vector1 * Vector2 + Vector3;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMultiplyAddFloat32x4(MLAS_FLOAT32X4 Vector1, float Scalar2, MLAS_FLOAT32X4 Vector3)
{
return MlasMultiplyAddFloat32x4(Vector1, MlasBroadcastFloat32x4(Scalar2), Vector3);
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMultiplyAddFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2, float Scalar3)
{
return MlasMultiplyAddFloat32x4(Vector1, Vector2, MlasBroadcastFloat32x4(Scalar3));
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasDivideFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vdivq_f32(Vector1, Vector2);
#elif defined(MLAS_NEON32_INTRINSICS)
Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 0) / vgetq_lane_f32(Vector2, 0), Vector1, 0);
Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 1) / vgetq_lane_f32(Vector2, 1), Vector1, 1);
Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 2) / vgetq_lane_f32(Vector2, 2), Vector1, 2);
Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 3) / vgetq_lane_f32(Vector2, 3), Vector1, 3);
return Vector1;
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_div_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_div(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfdiv_s(Vector1, Vector2);
#else
return Vector1 / Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasGreaterThanFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vreinterpretq_f32_u32(vcgtq_f32(Vector1, Vector2));
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_cmpgt_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_gt(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return MLAS_FLOAT32X4(vec_cmpgt(Vector1, Vector2));
#elif defined(MLAS_LSX_INTRINSICS)
return (MLAS_FLOAT32X4)__lsx_vfcmp_clt_s(Vector2, Vector1);
#else
return Vector1 > Vector2;
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasAndFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_and_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_and(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasReinterpretAsFloat32x4(MlasAndInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#else
return MlasReinterpretAsFloat32x4(MlasAndInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasOrFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_or_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_or(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasReinterpretAsFloat32x4(MlasOrInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#else
return MlasReinterpretAsFloat32x4(MlasOrInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasAndNotFloat32x4(MLAS_FLOAT32X4 VectorNot, MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_andnot_ps(VectorNot, Vector);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_andnot(Vector, VectorNot);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasReinterpretAsFloat32x4(MlasAndNotInt32x4(MlasReinterpretAsInt32x4(VectorNot), MlasReinterpretAsInt32x4(Vector)));
#else
return MlasReinterpretAsFloat32x4(MlasAndNotInt32x4(MlasReinterpretAsInt32x4(VectorNot), MlasReinterpretAsInt32x4(Vector)));
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasXorFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_xor_ps(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_v128_xor(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasReinterpretAsFloat32x4(MlasXorInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#else
return MlasReinterpretAsFloat32x4(MlasXorInt32x4(MlasReinterpretAsInt32x4(Vector1), MlasReinterpretAsInt32x4(Vector2)));
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasBlendFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2, MLAS_FLOAT32X4 Selection)
{
return MlasOrFloat32x4(MlasAndFloat32x4(Vector2, Selection), MlasAndNotFloat32x4(Selection, Vector1));
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMaximumFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vmaxq_f32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_max_ps(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
// Don't use vec_max to avoid undefined behavior if NAN
return vec_sel(Vector2, Vector1, vec_cmpgt(Vector1, Vector2));
#elif defined(MLAS_WASM_RELAXED_SIMD_INTRINSICS)
return wasm_f32x4_relaxed_max(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_max(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfmax_s(Vector1, Vector2);
#else
return MlasBlendFloat32x4(Vector2, Vector1, Vector1 > Vector2);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasMinimumFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2)
{
#if defined(MLAS_NEON_INTRINSICS)
return vminq_f32(Vector1, Vector2);
#elif defined(MLAS_SSE2_INTRINSICS)
return _mm_min_ps(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
// Don't use vec_min to avoid undefined behavior if NAN
return vec_sel(Vector2, Vector1, vec_cmpgt(Vector2, Vector1));
#elif defined(MLAS_WASM_RELAXED_SIMD_INTRINSICS)
return wasm_f32x4_relaxed_min(Vector1, Vector2);
#elif defined(MLAS_WASM_SIMD_INTRINSICS)
return wasm_f32x4_min(Vector1, Vector2);
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfmin_s(Vector1, Vector2);
#else
return MlasBlendFloat32x4(Vector2, Vector1, Vector2 > Vector1);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasClampFloat32x4(MLAS_FLOAT32X4 Value, float LowerRange, float UpperRange)
{
#if defined(MLAS_SSE2_INTRINSICS)
// N.B. MINPS and MAXPS propagates the value from the second vector if the
// value is a NaN.
#endif
Value = MlasMaximumFloat32x4(MlasBroadcastFloat32x4(LowerRange), Value);
Value = MlasMinimumFloat32x4(MlasBroadcastFloat32x4(UpperRange), Value);
return Value;
}
MLAS_FORCEINLINE
float
MlasReduceAddFloat32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON64_INTRINSICS)
Vector = vpaddq_f32(Vector, Vector);
Vector = vpaddq_f32(Vector, Vector);
return vgetq_lane_f32(Vector, 0);
#elif defined(MLAS_NEON32_INTRINSICS)
float32x2_t VectorLow = vget_low_f32(Vector);
float32x2_t VectorHigh = vget_high_f32(Vector);
VectorLow = vpadd_f32(VectorLow, VectorHigh);
VectorLow = vpadd_f32(VectorLow, VectorHigh);
return vget_lane_f32(VectorLow, 0);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
Vector = MlasAddFloat32x4(Vector, MLAS_FLOAT32X4(vec_splat((__vector long long)Vector, 1)));
Vector = MlasAddFloat32x4(Vector, vec_splat(Vector, 1));
return Vector[0];
#else
Vector = MlasAddFloat32x4(Vector, MlasShuffleFloat32x4<2, 3, 2, 3>(Vector));
Vector = MlasAddFloat32x4(Vector, MlasShuffleFloat32x4<1, 1, 1, 1>(Vector));
return MlasExtractLaneFloat32x4<0>(Vector);
#endif
}
MLAS_FORCEINLINE
float
MlasReduceMaximumFloat32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vmaxvq_f32(Vector);
#elif defined(MLAS_NEON32_INTRINSICS)
float32x2_t VectorLow = vget_low_f32(Vector);
float32x2_t VectorHigh = vget_high_f32(Vector);
VectorLow = vpmax_f32(VectorLow, VectorHigh);
VectorLow = vpmax_f32(VectorLow, VectorHigh);
return vget_lane_f32(VectorLow, 0);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
Vector = MlasMaximumFloat32x4(Vector, MLAS_FLOAT32X4(vec_splat((__vector long long)Vector, 1)));
Vector = MlasMaximumFloat32x4(Vector, vec_splat(Vector, 1));
return Vector[0];
#else
Vector = MlasMaximumFloat32x4(Vector, MlasShuffleFloat32x4<2, 3, 2, 3>(Vector));
Vector = MlasMaximumFloat32x4(Vector, MlasShuffleFloat32x4<1, 1, 1, 1>(Vector));
return MlasExtractLaneFloat32x4<0>(Vector);
#endif
}
MLAS_FORCEINLINE
float
MlasReduceMinimumFloat32x4(MLAS_FLOAT32X4 Vector)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vminvq_f32(Vector);
#elif defined(MLAS_NEON32_INTRINSICS)
float32x2_t VectorLow = vget_low_f32(Vector);
float32x2_t VectorHigh = vget_high_f32(Vector);
VectorLow = vpmin_f32(VectorLow, VectorHigh);
VectorLow = vpmin_f32(VectorLow, VectorHigh);
return vget_lane_f32(VectorLow, 0);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
Vector = MlasMinimumFloat32x4(Vector, MLAS_FLOAT32X4(vec_splat((__vector long long)Vector, 1)));
Vector = MlasMinimumFloat32x4(Vector, vec_splat(Vector, 1));
return Vector[0];
#else
Vector = MlasMinimumFloat32x4(Vector, MlasShuffleFloat32x4<2, 3, 2, 3>(Vector));
Vector = MlasMinimumFloat32x4(Vector, MlasShuffleFloat32x4<1, 1, 1, 1>(Vector));
return MlasExtractLaneFloat32x4<0>(Vector);
#endif
}
// calc 2^int(N)
MLAS_FORCEINLINE
MLAS_FLOAT32X4
MlasPowerOf2Float32x4(MLAS_FLOAT32X4 Vector)
{
MLAS_INT32X4 emm0 = MlasAddInt32x4(MlasCastToInt32x4(Vector), MlasBroadcastInt32x4(127));
return MlasReinterpretAsFloat32x4(MlasShiftLeftInt32x4<23>(emm0));
}
//
// Cross-platform wrappers for 64-bit vector intrinsics.
//
#if defined(MLAS_SSE2_INTRINSICS)
typedef __m128d MLAS_FLOAT64X2;
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
typedef __vector double MLAS_FLOAT64X2;
#elif defined(MLAS_LSX_INTRINSICS)
typedef __m128d MLAS_FLOAT64X2;
#else
#define MLAS_FLOAT64X2_UNSUPPORTED
#endif
#ifndef MLAS_FLOAT64X2_UNSUPPORTED
#if defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
template<unsigned Lane>
MLAS_FORCEINLINE
double
MlasExtractLaneFloat64x2(MLAS_FLOAT64X2 Vector)
{
return Vector[Lane];
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasMultiplyAddFloat64x2(MLAS_FLOAT64X2 Vector1, MLAS_FLOAT64X2 Vector2, MLAS_FLOAT64X2 Vector3)
{
return vec_madd(Vector1, Vector2, Vector3);
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasBroadcastFloat64x2(const double *Value)
{
return MLAS_FLOAT64X2{*Value, *Value};
}
#elif defined(MLAS_LSX_INTRINSICS)
template<unsigned Lane>
MLAS_FORCEINLINE
double
MlasExtractLaneFloat64x2(MLAS_FLOAT64X2 Vector)
{
return Vector[Lane];
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasMultiplyAddFloat64x2(MLAS_FLOAT64X2 Vector1, MLAS_FLOAT64X2 Vector2, MLAS_FLOAT64X2 Vector3)
{
return __lsx_vfmadd_d(Vector1, Vector2, Vector3);
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasBroadcastFloat64x2(const double *Value)
{
return MLAS_FLOAT64X2{*Value, *Value};
}
#endif
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasBroadcastFloat64x2(double Value)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_set1_pd(Value);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return MLAS_FLOAT64X2{Value, Value};
#elif defined(MLAS_LSX_INTRINSICS)
return MLAS_FLOAT64X2{Value, Value};
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasZeroFloat64x2(void)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_setzero_pd();
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return MlasBroadcastFloat64x2(0.0f);
#elif defined(MLAS_LSX_INTRINSICS)
return MlasBroadcastFloat64x2(0.0f);
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasLoadFloat64x2(const double* Buffer)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_loadu_pd(Buffer);
#elif defined(MLAS_VSX_INTRINSICS)
return vec_vsx_ld(0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
return vec_xl(0, Buffer);
#elif defined(MLAS_LSX_INTRINSICS)
return MLAS_FLOAT64X2(__lsx_vld((const MLAS_INT32X4 *)Buffer, 0));
#endif
}
MLAS_FORCEINLINE
void
MlasStoreFloat64x2(double* Buffer, MLAS_FLOAT64X2 Vector)
{
#if defined(MLAS_SSE2_INTRINSICS)
_mm_storeu_pd(Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS)
vec_vsx_st(Vector, 0, Buffer);
#elif defined(MLAS_ZVECTOR_INTRINSICS)
vec_xst(Vector, 0, Buffer);
#elif defined(MLAS_LSX_INTRINSICS)
(__lsx_vst(MLAS_INT32X4(Vector), Buffer, 0));
#endif
}
MLAS_FORCEINLINE
void
MlasStoreAlignedFloat64x2(double* Buffer, MLAS_FLOAT64X2 Vector)
{
#if defined(MLAS_SSE2_INTRINSICS)
_mm_store_pd(Buffer, Vector);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
*((MLAS_FLOAT64X2*)Buffer) = Vector;
#elif defined(MLAS_LSX_INTRINSICS)
(__lsx_vst(MLAS_INT32X4(Vector), Buffer, 0));
#endif
}
MLAS_FORCEINLINE
MLAS_FLOAT64X2
MlasMultiplyFloat64x2(MLAS_FLOAT64X2 Vector1, MLAS_FLOAT64X2 Vector2)
{
#if defined(MLAS_SSE2_INTRINSICS)
return _mm_mul_pd(Vector1, Vector2);
#elif defined(MLAS_VSX_INTRINSICS) || defined(MLAS_ZVECTOR_INTRINSICS)
return Vector1 * Vector2;
#elif defined(MLAS_LSX_INTRINSICS)
return __lsx_vfmul_d(Vector1, Vector2);
#endif
}
#endif // !MLAS_FLOAT64X2_UNSUPPORTED
//
// Reads a platform specific time stamp counter.
//
MLAS_FORCEINLINE
uint64_t
MlasReadTimeStampCounter(void)
{
#ifdef _WIN32
#if defined(MLAS_TARGET_AMD64_IX86)
return ReadTimeStampCounter();
#else
LARGE_INTEGER PerformanceCounter;
QueryPerformanceCounter(&PerformanceCounter);
return (ULONG64)PerformanceCounter.QuadPart;
#endif
#else
#if defined(MLAS_TARGET_AMD64)
uint32_t eax, edx;
__asm__ __volatile__
(
"rdtsc"
: "=a" (eax), "=d" (edx)
);
return ((uint64_t)edx << 32) | eax;
#elif defined(MLAS_TARGET_LARCH64)
uint64_t time_cnt, id;
__asm__ __volatile__
(
"rdtime.d %0, %1\n\t"
: "=r" (time_cnt), "=r" (id)
::
);
return time_cnt;
#else
return 0;
#endif
#endif
}
//
// Aligned buffer for GEMM packing, etc.
//
constexpr size_t ThreadedBufAlignment = 64;
extern thread_local size_t ThreadedBufSize;
#ifdef _MSC_VER
extern thread_local std::unique_ptr<uint8_t, decltype(&_aligned_free)> ThreadedBufHolder;
#else
extern thread_local std::unique_ptr<uint8_t, decltype(&free)> ThreadedBufHolder;
#endif
MLAS_FORCEINLINE
constexpr size_t
UpAlignSize(size_t size)
{
size = (size + ThreadedBufAlignment - 1) / ThreadedBufAlignment;
return size * ThreadedBufAlignment;
}
MLAS_FORCEINLINE
void
MlasThreadedBufAlloc(size_t size)
{
if (size > ThreadedBufSize) {
#ifdef _MSC_VER
ThreadedBufHolder.reset(
reinterpret_cast<uint8_t*>(_aligned_malloc(size, ThreadedBufAlignment)));
#elif (__STDC_VERSION__ >= 201112L) && !defined(__APPLE__)
ThreadedBufHolder.reset(
reinterpret_cast<uint8_t*>(aligned_alloc(ThreadedBufAlignment, size)));
#else
// aligned_alloc unavailable macos 10.14 or earlier
void* ptr;
int err = posix_memalign(&ptr, ThreadedBufAlignment, size);
if (err != 0) {
ptr = nullptr;
}
ThreadedBufHolder.reset(reinterpret_cast<uint8_t*>(ptr));
#endif
ThreadedBufSize = size;
}
}
//
// Utilities for INT4 quantization.
//
template<bool Signed>
struct Int4Traits;
template<>
struct Int4Traits<true> {
using UnpackedType = int8_t;
static constexpr int8_t Min = -8;
static constexpr int8_t Max = 7;
};
template<>
struct Int4Traits<false> {
using UnpackedType = uint8_t;
static constexpr int8_t Min = 0;
static constexpr int8_t Max = 15;
};
template<typename UnpackedType>
MLAS_FORCEINLINE
void
MlasSetInt4Element(uint8_t* Output, size_t ElemIndex, UnpackedType Value)
{
static_assert(std::is_same<UnpackedType, uint8_t>::value || std::is_same<UnpackedType, int8_t>::value, "");
const size_t OutputIndex = ElemIndex >> 1; // which byte
const size_t NibbleIndex = ElemIndex & 0x1; // which 4-bit elem in the byte
const uint8_t Shift = static_cast<uint8_t>(NibbleIndex << 2); // Either 0 or 4
const uint8_t Mask = static_cast<uint8_t>(0xF0 >> Shift);
uint8_t* Dst = &Output[OutputIndex];
*Dst &= Mask; // Clear 4-bit lane
*Dst |= static_cast<uint8_t>((Value & 0xF) << Shift); // Set 4-bit lane
}
template<typename UnpackedType>
MLAS_FORCEINLINE
void
MlasPackInt4Elements(uint8_t* Output, UnpackedType ValueLow, UnpackedType ValueHigh)
{
static_assert(std::is_same<UnpackedType, uint8_t>::value || std::is_same<UnpackedType, int8_t>::value, "");
*Output = static_cast<uint8_t>(((ValueHigh & 0xF) << 4) | (ValueLow & 0xF));
}
View Git Blame Copy Permalink