| // This file is part of Eigen, a lightweight C++ template library |
| // for linear algebra. |
| // |
| // Copyright (C) 2025 Rasmus Munk Larsen |
| // |
| // This Source Code Form is subject to the terms of the Mozilla |
| // Public License v. 2.0. If a copy of the MPL was not distributed |
| // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| // SPDX-License-Identifier: MPL-2.0 |
| |
| #ifndef EIGEN_PACKET_MATH_CLANG_H |
| #define EIGEN_PACKET_MATH_CLANG_H |
| |
| // IWYU pragma: private |
| #include "../../InternalHeaderCheck.h" |
| |
| namespace Eigen { |
| namespace internal { |
| |
| namespace detail { |
| // namespace detail contains implementation details specific to this |
| // file, while namespace internal contains internal APIs used elsewhere |
| // in Eigen. |
| template <typename ScalarT, int n> |
| using VectorType = ScalarT __attribute__((ext_vector_type(n), aligned(n * sizeof(ScalarT)))); |
| } // namespace detail |
| |
| // --- Naming Convention --- |
| // This backend uses size-independent type aliases so the same code works |
| // for EIGEN_GENERIC_VECTOR_SIZE_BYTES in {16, 32, 64}: |
| // |
| // PacketXf - float vector (4, 8, or 16 elements) |
| // PacketXd - double vector (2, 4, or 8 elements) |
| // PacketXi - int32_t vector (4, 8, or 16 elements) |
| // PacketXl - int64_t vector (2, 4, or 8 elements) |
| // PacketXcf - complex<float> vector (2, 4, or 8 elements) [in Complex.h] |
| // PacketXcd - complex<double> vector (1, 2, or 4 elements) [in Complex.h] |
| // |
| // The "X" suffix indicates the element count is determined by the macro |
| // EIGEN_GENERIC_VECTOR_SIZE_BYTES at compile time. Operations that require |
| // compile-time constant indices (e.g. __builtin_shufflevector) obtain them by |
| // expanding detail::vector_indices<Packet>, so they need no per-size code. |
| |
| static_assert(EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16 || EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32 || |
| EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64, |
| "EIGEN_GENERIC_VECTOR_SIZE_BYTES must be 16, 32, or 64"); |
| |
| constexpr int kFloatPacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES / sizeof(float); |
| constexpr int kDoublePacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES / sizeof(double); |
| using PacketXf = detail::VectorType<float, kFloatPacketSize>; |
| using PacketXd = detail::VectorType<double, kDoublePacketSize>; |
| using PacketXi = detail::VectorType<int32_t, kFloatPacketSize>; |
| using PacketXl = detail::VectorType<int64_t, kDoublePacketSize>; |
| |
| // --- packet_traits specializations --- |
| struct generic_float_packet_traits : default_packet_traits { |
| enum { |
| Vectorizable = 1, |
| AlignedOnScalar = 1, |
| HasAdd = 1, |
| HasSub = 1, |
| HasMul = 1, |
| HasDiv = 1, |
| HasNegate = 1, |
| HasAbs = 1, |
| HasRound = 1, |
| HasMin = 1, |
| HasMax = 1, |
| HasCmp = 1, |
| HasSet1 = 1, |
| HasCast = 1, |
| HasBitwise = 1, |
| HasRedux = 1, |
| HasSign = 1, |
| HasArg = 0, |
| HasConj = 1, |
| // Math functions |
| HasReciprocal = 1, |
| HasSin = 1, |
| HasCos = 1, |
| HasTan = 1, |
| HasACos = 1, |
| HasASin = 1, |
| HasATan = 1, |
| HasATanh = 1, |
| HasLog = 1, |
| HasLog1p = 1, |
| HasExpm1 = 1, |
| HasExp = 1, |
| HasPow = 1, |
| HasNdtri = 1, |
| HasBessel = 1, |
| HasSqrt = 1, |
| HasRsqrt = 1, |
| HasCbrt = 1, |
| HasTanh = 1, |
| HasErf = 1, |
| HasErfc = 1 |
| }; |
| }; |
| |
| template <> |
| struct packet_traits<float> : generic_float_packet_traits { |
| using type = PacketXf; |
| using half = PacketXf; |
| enum { |
| size = kFloatPacketSize, |
| }; |
| }; |
| |
| template <> |
| struct packet_traits<double> : generic_float_packet_traits { |
| using type = PacketXd; |
| using half = PacketXd; |
| // Generic double-precision acos/asin are not yet implemented in |
| // GenericPacketMathFunctions.h (only float versions exist). |
| enum { size = kDoublePacketSize, HasACos = 0, HasASin = 0 }; |
| }; |
| |
| struct generic_integer_packet_traits : default_packet_traits { |
| enum { |
| Vectorizable = 1, |
| AlignedOnScalar = 1, |
| HasAdd = 1, |
| HasSub = 1, |
| HasMul = 1, |
| HasDiv = 1, |
| HasNegate = 1, |
| HasAbs = 1, |
| HasMin = 1, |
| HasMax = 1, |
| HasCmp = 1, |
| HasSet1 = 1, |
| HasCast = 1, |
| HasBitwise = 1, |
| HasRedux = 1, |
| // Set remaining to 0 |
| HasRound = 1, |
| HasSqrt = 0, |
| HasRsqrt = 0, |
| HasReciprocal = 0, |
| HasArg = 0, |
| HasConj = 1, |
| HasExp = 0, |
| HasLog = 0, |
| HasSin = 0, |
| HasCos = 0, |
| }; |
| }; |
| |
| template <> |
| struct packet_traits<int32_t> : generic_integer_packet_traits { |
| using type = PacketXi; |
| using half = PacketXi; |
| enum { |
| size = kFloatPacketSize, |
| }; |
| }; |
| |
| template <> |
| struct packet_traits<int64_t> : generic_integer_packet_traits { |
| using type = PacketXl; |
| using half = PacketXl; |
| enum { |
| size = kDoublePacketSize, |
| }; |
| }; |
| |
| // --- unpacket_traits specializations --- |
| struct generic_unpacket_traits : default_unpacket_traits { |
| enum { |
| alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, |
| vectorizable = true, |
| }; |
| }; |
| |
| template <> |
| struct unpacket_traits<PacketXf> : generic_unpacket_traits { |
| using type = float; |
| using half = PacketXf; |
| using integer_packet = PacketXi; |
| enum { |
| size = kFloatPacketSize, |
| }; |
| }; |
| template <> |
| struct unpacket_traits<PacketXd> : generic_unpacket_traits { |
| using type = double; |
| using half = PacketXd; |
| using integer_packet = PacketXl; |
| enum { |
| size = kDoublePacketSize, |
| }; |
| }; |
| template <> |
| struct unpacket_traits<PacketXi> : generic_unpacket_traits { |
| using type = int32_t; |
| using half = PacketXi; |
| enum { |
| size = kFloatPacketSize, |
| }; |
| }; |
| template <> |
| struct unpacket_traits<PacketXl> : generic_unpacket_traits { |
| using type = int64_t; |
| using half = PacketXl; |
| enum { |
| size = kDoublePacketSize, |
| }; |
| }; |
| |
| namespace detail { |
| // --- vector type helpers --- |
| template <typename VectorT> |
| struct ScalarTypeOfVector { |
| using type = std::remove_all_extents_t<std::remove_reference_t<decltype(VectorT()[0])>>; |
| }; |
| |
| template <typename VectorT> |
| using scalar_type_of_vector_t = typename ScalarTypeOfVector<VectorT>::type; |
| |
| template <typename VectorType> |
| struct UnsignedVectorHelper { |
| static VectorType v; |
| static constexpr int n = __builtin_vectorelements(v); |
| using UnsignedScalar = std::make_unsigned_t<scalar_type_of_vector_t<VectorType>>; |
| using type = UnsignedScalar __attribute__((ext_vector_type(n), aligned(n * sizeof(UnsignedScalar)))); |
| }; |
| |
| template <typename VectorT> |
| using unsigned_vector_t = typename UnsignedVectorHelper<VectorT>::type; |
| |
| template <typename VectorT> |
| constexpr int vector_elements() { |
| return static_cast<int>(sizeof(VectorT) / sizeof(scalar_type_of_vector_t<VectorT>)); |
| } |
| |
| // Signed integer vector with the same lane count and width, for sign-bit |
| // tests and bitwise manipulation of floating-point packets. |
| template <typename VectorT> |
| struct SignedVectorHelper { |
| using SignedScalar = std::conditional_t<sizeof(scalar_type_of_vector_t<VectorT>) == 4, int32_t, int64_t>; |
| using type = VectorType<SignedScalar, vector_elements<VectorT>()>; |
| }; |
| |
| template <typename VectorT> |
| using signed_vector_t = typename SignedVectorHelper<VectorT>::type; |
| |
| template <typename VectorT> |
| using half_vector_t = VectorType<scalar_type_of_vector_t<VectorT>, vector_elements<VectorT>() / 2>; |
| |
| template <typename VectorT> |
| using quarter_vector_t = VectorType<scalar_type_of_vector_t<VectorT>, vector_elements<VectorT>() / 4>; |
| |
| template <typename VectorT> |
| using scalar_pair_t = std::pair<scalar_type_of_vector_t<VectorT>, scalar_type_of_vector_t<VectorT>>; |
| |
| // Index sequence covering every element of VectorT. Expanding it inside a |
| // __builtin_shufflevector index list or a braced initializer is what keeps the |
| // operations below independent of EIGEN_GENERIC_VECTOR_SIZE_BYTES. |
| template <typename VectorT> |
| using vector_indices = std::make_index_sequence<vector_elements<VectorT>()>; |
| |
| // load and store helpers. |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT load_vector_unaligned(const scalar_type_of_vector_t<VectorT>* from) { |
| VectorT to; |
| __builtin_memcpy(&to, from, sizeof(VectorT)); |
| return to; |
| } |
| |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT load_vector_aligned(const scalar_type_of_vector_t<VectorT>* from) { |
| eigen_assert((std::uintptr_t(from) % alignof(VectorT) == 0) && "load_vector_aligned"); |
| return *reinterpret_cast<const VectorT*>(assume_aligned<alignof(VectorT)>(from)); |
| } |
| |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE void store_vector_unaligned(scalar_type_of_vector_t<VectorT>* to, const VectorT& from) { |
| __builtin_memcpy(to, &from, sizeof(VectorT)); |
| } |
| |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE void store_vector_aligned(scalar_type_of_vector_t<VectorT>* to, const VectorT& from) { |
| eigen_assert((std::uintptr_t(to) % alignof(VectorT) == 0) && "store_vector_aligned"); |
| *reinterpret_cast<VectorT*>(assume_aligned<alignof(VectorT)>(to)) = from; |
| } |
| |
| } // namespace detail |
| |
| // --- Intrinsic-like specializations --- |
| |
| // --- Load/Store operations --- |
| #define EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE ploadu<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) { \ |
| return detail::load_vector_unaligned<PACKET_TYPE>(from); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pload<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) { \ |
| return detail::load_vector_aligned<PACKET_TYPE>(from); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE void pstoreu<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>( \ |
| detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) { \ |
| detail::store_vector_unaligned<PACKET_TYPE>(to, from); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE void pstore<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>( \ |
| detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) { \ |
| detail::store_vector_aligned<PACKET_TYPE>(to, from); \ |
| } |
| |
| EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXf) |
| EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXd) |
| EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXi) |
| EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXl) |
| #undef EIGEN_CLANG_PACKET_LOAD_STORE_PACKET |
| |
| // --- Broadcast operation --- |
| template <> |
| EIGEN_STRONG_INLINE PacketXf pset1frombits<PacketXf>(uint32_t from) { |
| return PacketXf(numext::bit_cast<float>(from)); |
| } |
| |
| template <> |
| EIGEN_STRONG_INLINE PacketXd pset1frombits<PacketXd>(uint64_t from) { |
| return PacketXd(numext::bit_cast<double>(from)); |
| } |
| |
| #define EIGEN_CLANG_PACKET_SET1(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pset1<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type& from) { \ |
| return PACKET_TYPE(from); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type pfirst<PACKET_TYPE>(const PACKET_TYPE& from) { \ |
| return from[0]; \ |
| } |
| |
| EIGEN_CLANG_PACKET_SET1(PacketXf) |
| EIGEN_CLANG_PACKET_SET1(PacketXd) |
| EIGEN_CLANG_PACKET_SET1(PacketXi) |
| EIGEN_CLANG_PACKET_SET1(PacketXl) |
| #undef EIGEN_CLANG_PACKET_SET1 |
| |
| // --- Arithmetic operations --- |
| #define EIGEN_CLANG_PACKET_ARITHMETIC(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pisnan<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return reinterpret_cast<PACKET_TYPE>(a != a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pnegate<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return -a; \ |
| } |
| |
| EIGEN_CLANG_PACKET_ARITHMETIC(PacketXf) |
| EIGEN_CLANG_PACKET_ARITHMETIC(PacketXd) |
| EIGEN_CLANG_PACKET_ARITHMETIC(PacketXi) |
| EIGEN_CLANG_PACKET_ARITHMETIC(PacketXl) |
| #undef EIGEN_CLANG_PACKET_ARITHMETIC |
| |
| // --- Bitwise operations (via casting) --- |
| |
| namespace detail { |
| |
| // Reinterpret-cast helpers, equivalent to preinterpret<> but defined here |
| // because PacketMath.h is included before TypeCasting.h. |
| EIGEN_STRONG_INLINE PacketXi preinterpret_float_to_int(const PacketXf& a) { return reinterpret_cast<PacketXi>(a); } |
| EIGEN_STRONG_INLINE PacketXf preinterpret_int_to_float(const PacketXi& a) { return reinterpret_cast<PacketXf>(a); } |
| EIGEN_STRONG_INLINE PacketXl preinterpret_double_to_long(const PacketXd& a) { return reinterpret_cast<PacketXl>(a); } |
| EIGEN_STRONG_INLINE PacketXd preinterpret_long_to_double(const PacketXl& a) { return reinterpret_cast<PacketXd>(a); } |
| |
| } // namespace detail |
| |
| // Bitwise ops for integer packets |
| #define EIGEN_CLANG_PACKET_BITWISE_INT(PACKET_TYPE) \ |
| template <> \ |
| constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) { \ |
| return PACKET_TYPE(0); \ |
| } \ |
| template <> \ |
| constexpr EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) { \ |
| return numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(0) == PACKET_TYPE(0)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return a & b; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return a | b; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return a ^ b; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return a & ~b; \ |
| } \ |
| template <int N> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE parithmetic_shift_right(const PACKET_TYPE& a) { \ |
| return a >> N; \ |
| } \ |
| template <int N> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_right(const PACKET_TYPE& a) { \ |
| using UnsignedT = detail::unsigned_vector_t<PACKET_TYPE>; \ |
| return reinterpret_cast<PACKET_TYPE>(reinterpret_cast<UnsignedT>(a) >> N); \ |
| } \ |
| template <int N> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_left(const PACKET_TYPE& a) { \ |
| return a << N; \ |
| } |
| |
| EIGEN_CLANG_PACKET_BITWISE_INT(PacketXi) |
| EIGEN_CLANG_PACKET_BITWISE_INT(PacketXl) |
| #undef EIGEN_CLANG_PACKET_BITWISE_INT |
| |
| // Bitwise ops for floating point packets |
| #define EIGEN_CLANG_PACKET_BITWISE_FLOAT(PACKET_TYPE, CAST_TO_INT, CAST_FROM_INT) \ |
| template <> \ |
| constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) { \ |
| using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>; \ |
| return PACKET_TYPE(Scalar(0)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& /* unused */) { \ |
| using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>; \ |
| PACKET_TYPE r = numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(Scalar(0)) == PACKET_TYPE(Scalar(0))); \ |
| EIGEN_FAST_MATH_CONSTANT_BARRIER(r); \ |
| return r; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return CAST_FROM_INT(CAST_TO_INT(a) & CAST_TO_INT(b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return CAST_FROM_INT(CAST_TO_INT(a) | CAST_TO_INT(b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return CAST_FROM_INT(CAST_TO_INT(a) ^ CAST_TO_INT(b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return CAST_FROM_INT(CAST_TO_INT(a) & ~CAST_TO_INT(b)); \ |
| } |
| |
| EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXf, detail::preinterpret_float_to_int, detail::preinterpret_int_to_float) |
| EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXd, detail::preinterpret_double_to_long, detail::preinterpret_long_to_double) |
| #undef EIGEN_CLANG_PACKET_BITWISE_FLOAT |
| |
| // --- Comparison operations --- |
| // Clang vector extensions perform comparisons in the original type (float/double), |
| // returning an int vector with all-ones (-1) for true and all-zeros for false. |
| // The bit_cast reinterprets those int bitmasks as float packets, which is the |
| // format expected by pselect and other Eigen packet operations. |
| #define EIGEN_CLANG_PACKET_CMP(PACKET_TYPE, INT_PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pcmp_eq<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a == b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a < b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pcmp_le<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a <= b)); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt_or_nan<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(!(a >= b))); \ |
| } |
| |
| EIGEN_CLANG_PACKET_CMP(PacketXf, PacketXi) |
| EIGEN_CLANG_PACKET_CMP(PacketXd, PacketXl) |
| #undef EIGEN_CLANG_PACKET_CMP |
| |
| // --- Min/Max/select operations --- |
| namespace detail { |
| // Functors usable at any vector width; the min/max reduction trees in |
| // Reductions.h reuse them on progressively narrower vectors. The |
| // compare-select forms compile to a single min/max instruction on targets |
| // whose min/max returns the second operand when the inputs are unordered |
| // (e.g. x86), and they spell out the NaN propagation of std::min/std::max: |
| // the first argument is returned if either input is NaN. |
| struct pmin_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| return b < a ? b : a; |
| } |
| }; |
| struct pmax_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| return b > a ? b : a; |
| } |
| }; |
| // IEEE 754-2008 minNum/maxNum semantics: return the other operand if one input |
| // is NaN. Floating-point support in __builtin_elementwise_{min,max} is |
| // deprecated because the name does not say which of the several IEEE min/max |
| // flavors is meant; __builtin_elementwise_{minnum,maxnum} spell out the same |
| // semantics the deprecated builtins provided for floats, and additionally pin |
| // down +0.0 > -0.0. The elementwise_{min,max} fallback (always available |
| // under this backend's clang >= 16 gate) has the same NaN semantics but |
| // leaves the zero-sign tie unspecified. |
| struct pmin_num_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| #if EIGEN_HAS_BUILTIN(__builtin_elementwise_minnum) |
| return __builtin_elementwise_minnum(a, b); |
| #else |
| return __builtin_elementwise_min(a, b); |
| #endif |
| } |
| }; |
| struct pmax_num_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| #if EIGEN_HAS_BUILTIN(__builtin_elementwise_maxnum) |
| return __builtin_elementwise_maxnum(a, b); |
| #else |
| return __builtin_elementwise_max(a, b); |
| #endif |
| } |
| }; |
| // Return NaN if either input is NaN, otherwise the min/max. When a is NaN the |
| // plain compare-select form already returns a, so only b needs an explicit |
| // test. |
| struct pmin_nan_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| return b != b ? b : pmin_op()(a, b); |
| } |
| }; |
| struct pmax_nan_op { |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE VectorT operator()(const VectorT& a, const VectorT& b) const { |
| return b != b ? b : pmax_op()(a, b); |
| } |
| }; |
| } // namespace detail |
| |
| // pmin/pmax/pselect are pure compare-select code and apply to all packet types. |
| #define EIGEN_CLANG_PACKET_MINMAX_SELECT(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmin<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmin_op()(a, b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmax<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmax_op()(a, b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pselect<PACKET_TYPE>(const PACKET_TYPE& mask, const PACKET_TYPE& a, \ |
| const PACKET_TYPE& b) { \ |
| /* The mask is all-ones or all-zeros per lane, so testing the sign of the */ \ |
| /* signed integer view suffices and maps to a single blend instruction. */ \ |
| /* Unlike a floating-point `mask != 0` test it also survives -ffast-math, */ \ |
| /* which may assume the all-ones NaN bit pattern cannot occur in a float. */ \ |
| return reinterpret_cast<detail::signed_vector_t<PACKET_TYPE>>(mask) < 0 ? a : b; \ |
| } |
| |
| EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXf) |
| EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXd) |
| EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXi) |
| EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXl) |
| #undef EIGEN_CLANG_PACKET_MINMAX_SELECT |
| |
| // NaN-propagation variants for the floating-point packets. |
| #define EIGEN_CLANG_PACKET_MINMAX_FLOAT(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmin_num_op()(a, b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmax_num_op()(a, b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmin_nan_op()(a, b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ |
| return detail::pmax_nan_op()(a, b); \ |
| } |
| |
| EIGEN_CLANG_PACKET_MINMAX_FLOAT(PacketXf) |
| EIGEN_CLANG_PACKET_MINMAX_FLOAT(PacketXd) |
| #undef EIGEN_CLANG_PACKET_MINMAX_FLOAT |
| |
| #if EIGEN_HAS_BUILTIN(__builtin_elementwise_abs) |
| #define EIGEN_CLANG_PACKET_ABS(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pabs<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_abs(a); \ |
| } |
| |
| EIGEN_CLANG_PACKET_ABS(PacketXf) |
| EIGEN_CLANG_PACKET_ABS(PacketXd) |
| EIGEN_CLANG_PACKET_ABS(PacketXi) |
| EIGEN_CLANG_PACKET_ABS(PacketXl) |
| #undef EIGEN_CLANG_PACKET_ABS |
| #endif |
| |
| // psignbit: a signed compare of the integer view is a single instruction, |
| // unlike the generic floating-point fallback. |
| template <> |
| EIGEN_STRONG_INLINE PacketXf psignbit(const PacketXf& a) { |
| return reinterpret_cast<PacketXf>(reinterpret_cast<PacketXi>(a) < 0); |
| } |
| template <> |
| EIGEN_STRONG_INLINE PacketXd psignbit(const PacketXd& a) { |
| return reinterpret_cast<PacketXd>(reinterpret_cast<PacketXl>(a) < 0); |
| } |
| |
| // --- Math functions (float/double only) --- |
| |
| #if EIGEN_HAS_BUILTIN(__builtin_elementwise_floor) && EIGEN_HAS_BUILTIN(__builtin_elementwise_ceil) && \ |
| EIGEN_HAS_BUILTIN(__builtin_elementwise_round) && EIGEN_HAS_BUILTIN(__builtin_elementwise_roundeven) && \ |
| EIGEN_HAS_BUILTIN(__builtin_elementwise_trunc) && EIGEN_HAS_BUILTIN(__builtin_elementwise_sqrt) |
| #define EIGEN_CLANG_PACKET_MATH_FLOAT(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pfloor<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_floor(a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pceil<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_ceil(a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pround<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_round(a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE print<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_roundeven(a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE ptrunc<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_trunc(a); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE psqrt<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return __builtin_elementwise_sqrt(a); \ |
| } |
| |
| EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXf) |
| EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXd) |
| #undef EIGEN_CLANG_PACKET_MATH_FLOAT |
| #endif |
| |
| // --- Fused Multiply-Add (MADD) --- |
| #if defined(__FMA__) && EIGEN_HAS_BUILTIN(__builtin_elementwise_fma) |
| #define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return __builtin_elementwise_fma(a, b, c); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return __builtin_elementwise_fma(a, b, -c); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pnmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return __builtin_elementwise_fma(-a, b, c); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pnmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return -(__builtin_elementwise_fma(a, b, c)); \ |
| } |
| #else |
| // Fallback if FMA builtin is not available |
| #define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return (a * b) + c; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return (a * b) - c; \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pnmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return c - (a * b); \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pnmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ |
| const PACKET_TYPE& c) { \ |
| return -((a * b) + c); \ |
| } |
| #endif |
| |
| EIGEN_CLANG_PACKET_MADD(PacketXf) |
| EIGEN_CLANG_PACKET_MADD(PacketXd) |
| #undef EIGEN_CLANG_PACKET_MADD |
| |
| #define EIGEN_CLANG_PACKET_SCATTER_GATHER(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE void pscatter(unpacket_traits<PACKET_TYPE>::type* to, const PACKET_TYPE& from, Index stride) { \ |
| constexpr int size = unpacket_traits<PACKET_TYPE>::size; \ |
| for (int i = 0; i < size; ++i) { \ |
| to[i * stride] = from[i]; \ |
| } \ |
| } \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE pgather<typename unpacket_traits<PACKET_TYPE>::type, PACKET_TYPE>( \ |
| const unpacket_traits<PACKET_TYPE>::type* from, Index stride) { \ |
| constexpr int size = unpacket_traits<PACKET_TYPE>::size; \ |
| PACKET_TYPE result; \ |
| for (int i = 0; i < size; ++i) { \ |
| result[i] = from[i * stride]; \ |
| } \ |
| return result; \ |
| } |
| |
| EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXf) |
| EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXd) |
| EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXi) |
| EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXl) |
| |
| #undef EIGEN_CLANG_PACKET_SCATTER_GATHER |
| |
| // ---- Various operations that depend on __builtin_shufflevector. |
| #if EIGEN_HAS_BUILTIN(__builtin_shufflevector) |
| namespace detail { |
| |
| // --- Half / whole vector helpers --- |
| template <typename VectorT, std::size_t... Is> |
| EIGEN_STRONG_INLINE half_vector_t<VectorT> lower_half_impl(const VectorT& a, std::index_sequence<Is...>) { |
| return __builtin_shufflevector(a, a, Is...); |
| } |
| |
| template <typename VectorT, std::size_t... Is> |
| EIGEN_STRONG_INLINE half_vector_t<VectorT> upper_half_impl(const VectorT& a, std::index_sequence<Is...>) { |
| return __builtin_shufflevector(a, a, (sizeof...(Is) + Is)...); |
| } |
| |
| template <typename VectorT, typename HalfT, std::size_t... Is> |
| EIGEN_STRONG_INLINE VectorT concat_halves_impl(const HalfT& lo, const HalfT& hi, std::index_sequence<Is...>) { |
| return __builtin_shufflevector(lo, hi, Is...); |
| } |
| |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE half_vector_t<VectorT> lower_half(const VectorT& a) { |
| return lower_half_impl(a, vector_indices<half_vector_t<VectorT>>{}); |
| } |
| |
| template <typename VectorT> |
| EIGEN_STRONG_INLINE half_vector_t<VectorT> upper_half(const VectorT& a) { |
| return upper_half_impl(a, vector_indices<half_vector_t<VectorT>>{}); |
| } |
| |
| template <typename VectorT, typename HalfT> |
| EIGEN_STRONG_INLINE VectorT concat_halves(const HalfT& lo, const HalfT& hi) { |
| return concat_halves_impl<VectorT>(lo, hi, vector_indices<VectorT>{}); |
| } |
| |
| // --- Width-generic bodies for the packet operations below --- |
| template <typename Packet, std::size_t... Is> |
| EIGEN_STRONG_INLINE Packet preverse_impl(const Packet& a, std::index_sequence<Is...>) { |
| return __builtin_shufflevector(a, a, (sizeof...(Is) - 1 - Is)...); |
| } |
| |
| // Loads half a packet worth of scalars and repeats each of them twice. |
| template <typename Packet, std::size_t... Is> |
| EIGEN_STRONG_INLINE Packet ploaddup_impl(const typename unpacket_traits<Packet>::type* from, |
| std::index_sequence<Is...>) { |
| static_assert((unpacket_traits<Packet>::size) % 2 == 0, "Packet size must be a multiple of 2"); |
| using HalfT = half_vector_t<Packet>; |
| const HalfT a = load_vector_unaligned<HalfT>(from); |
| return __builtin_shufflevector(a, a, (Is / 2)...); |
| } |
| |
| // Loads a quarter of a packet worth of scalars and repeats each of them four times. |
| template <typename Packet, std::size_t... Is> |
| EIGEN_STRONG_INLINE Packet ploadquad_impl(const typename unpacket_traits<Packet>::type* from, |
| std::index_sequence<Is...>) { |
| static_assert((unpacket_traits<Packet>::size) % 4 == 0, "Packet size must be a multiple of 4"); |
| using QuarterT = quarter_vector_t<Packet>; |
| const QuarterT a = load_vector_unaligned<QuarterT>(from); |
| return __builtin_shufflevector(a, a, (Is / 4)...); |
| } |
| |
| template <typename Packet, std::size_t... Is> |
| EIGEN_STRONG_INLINE Packet plset_impl(const typename unpacket_traits<Packet>::type& a, std::index_sequence<Is...>) { |
| using Scalar = typename unpacket_traits<Packet>::type; |
| return Packet{(a + Scalar(Is))...}; |
| } |
| |
| // All ones in the even lanes, all zeros in the odd ones. Return the integer representation so finite fast-math cannot |
| // make the all-ones lanes poison before the caller applies EIGEN_FAST_MATH_CONSTANT_BARRIER. |
| template <typename Packet, std::size_t... Is> |
| EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::integer_packet peven_mask_impl(std::index_sequence<Is...>) { |
| using IntegerPacket = typename unpacket_traits<Packet>::integer_packet; |
| using Bits = scalar_type_of_vector_t<IntegerPacket>; |
| return IntegerPacket{(Is % 2 == 0 ? Bits(-1) : Bits(0))...}; |
| } |
| |
| } // namespace detail |
| |
| #define EIGEN_CLANG_PACKET_PREVERSE(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE preverse<PACKET_TYPE>(const PACKET_TYPE& a) { \ |
| return detail::preverse_impl(a, detail::vector_indices<PACKET_TYPE>{}); \ |
| } |
| |
| EIGEN_CLANG_PACKET_PREVERSE(PacketXf) |
| EIGEN_CLANG_PACKET_PREVERSE(PacketXd) |
| EIGEN_CLANG_PACKET_PREVERSE(PacketXi) |
| EIGEN_CLANG_PACKET_PREVERSE(PacketXl) |
| #undef EIGEN_CLANG_PACKET_PREVERSE |
| |
| #define EIGEN_CLANG_PACKET_LOADDUP(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE ploaddup<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type* from) { \ |
| return detail::ploaddup_impl<PACKET_TYPE>(from, detail::vector_indices<PACKET_TYPE>{}); \ |
| } |
| |
| EIGEN_CLANG_PACKET_LOADDUP(PacketXf) |
| EIGEN_CLANG_PACKET_LOADDUP(PacketXd) |
| EIGEN_CLANG_PACKET_LOADDUP(PacketXi) |
| EIGEN_CLANG_PACKET_LOADDUP(PacketXl) |
| #undef EIGEN_CLANG_PACKET_LOADDUP |
| |
| #define EIGEN_CLANG_PACKET_LOADQUAD(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE ploadquad<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type* from) { \ |
| return detail::ploadquad_impl<PACKET_TYPE>(from, detail::vector_indices<PACKET_TYPE>{}); \ |
| } |
| |
| EIGEN_CLANG_PACKET_LOADQUAD(PacketXf) |
| EIGEN_CLANG_PACKET_LOADQUAD(PacketXi) |
| #if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 32 |
| // PacketXd and PacketXl hold only two elements at 16 bytes, so they have no quarter packet to load from. |
| EIGEN_CLANG_PACKET_LOADQUAD(PacketXd) |
| EIGEN_CLANG_PACKET_LOADQUAD(PacketXl) |
| #endif |
| #undef EIGEN_CLANG_PACKET_LOADQUAD |
| |
| #define EIGEN_CLANG_PACKET_PLSET(PACKET_TYPE) \ |
| template <> \ |
| EIGEN_STRONG_INLINE PACKET_TYPE plset<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type& a) { \ |
| return detail::plset_impl<PACKET_TYPE>(a, detail::vector_indices<PACKET_TYPE>{}); \ |
| } |
| |
| EIGEN_CLANG_PACKET_PLSET(PacketXf) |
| EIGEN_CLANG_PACKET_PLSET(PacketXd) |
| EIGEN_CLANG_PACKET_PLSET(PacketXi) |
| EIGEN_CLANG_PACKET_PLSET(PacketXl) |
| #undef EIGEN_CLANG_PACKET_PLSET |
| |
| // --- peven_mask --- |
| template <> |
| EIGEN_STRONG_INLINE PacketXf peven_mask(const PacketXf& /* unused */) { |
| PacketXf r = numext::bit_cast<PacketXf>(detail::peven_mask_impl<PacketXf>(detail::vector_indices<PacketXf>{})); |
| EIGEN_FAST_MATH_CONSTANT_BARRIER(r); |
| return r; |
| } |
| template <> |
| EIGEN_STRONG_INLINE PacketXd peven_mask(const PacketXd& /* unused */) { |
| PacketXd r = numext::bit_cast<PacketXd>(detail::peven_mask_impl<PacketXd>(detail::vector_indices<PacketXd>{})); |
| EIGEN_FAST_MATH_CONSTANT_BARRIER(r); |
| return r; |
| } |
| |
| // Helpers for ptranspose. |
| namespace detail { |
| |
| // Shuffle index of output element `i` when interleaving two vectors of `Size` |
| // elements each. `Group` adjacent elements move together: 1 for scalar packets, |
| // 2 for the complex packets in Complex.h, whose real and imaginary parts must |
| // stay adjacent. Output groups alternate between the two inputs, taking group |
| // `first_group` of each first. |
| template <std::size_t Group, std::size_t Size> |
| constexpr std::size_t zip_index(std::size_t i, std::size_t first_group) { |
| return Size * ((i / Group) % 2) + Group * (first_group + i / Group / 2) + i % Group; |
| } |
| |
| // Interleaves p1 and p2 in place, leaving the low half of the result in p1 and |
| // the high half in p2. |
| template <std::size_t Group, typename VectorT, std::size_t... Is> |
| EIGEN_ALWAYS_INLINE void zip_in_place_impl(VectorT& p1, VectorT& p2, std::index_sequence<Is...>) { |
| constexpr std::size_t kSize = sizeof...(Is); |
| // With a single lane group per vector both output shuffles would pick group |
| // 0 and silently duplicate p1; such packets must not reach this code. |
| static_assert(kSize >= 2 * Group, "zip_in_place needs at least two lane groups per vector"); |
| const VectorT tmp = __builtin_shufflevector(p1, p2, zip_index<Group, kSize>(Is, 0)...); |
| p2 = __builtin_shufflevector(p1, p2, zip_index<Group, kSize>(Is, kSize / (2 * Group))...); |
| p1 = tmp; |
| } |
| |
| // Complex.h specializes this for its packet types, which zip whole complex |
| // values rather than individual reals. |
| template <typename Packet> |
| EIGEN_ALWAYS_INLINE void zip_in_place(Packet& p1, Packet& p2) { |
| zip_in_place_impl<1>(p1, p2, vector_indices<Packet>{}); |
| } |
| |
| template <typename Packet> |
| EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 2>& kernel) { |
| zip_in_place(kernel.packet[0], kernel.packet[1]); |
| } |
| |
| template <typename Packet> |
| EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 4>& kernel) { |
| zip_in_place(kernel.packet[0], kernel.packet[2]); |
| zip_in_place(kernel.packet[1], kernel.packet[3]); |
| zip_in_place(kernel.packet[0], kernel.packet[1]); |
| zip_in_place(kernel.packet[2], kernel.packet[3]); |
| } |
| |
| template <typename Packet> |
| EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 8>& kernel) { |
| zip_in_place(kernel.packet[0], kernel.packet[4]); |
| zip_in_place(kernel.packet[1], kernel.packet[5]); |
| zip_in_place(kernel.packet[2], kernel.packet[6]); |
| zip_in_place(kernel.packet[3], kernel.packet[7]); |
| |
| zip_in_place(kernel.packet[0], kernel.packet[2]); |
| zip_in_place(kernel.packet[1], kernel.packet[3]); |
| zip_in_place(kernel.packet[4], kernel.packet[6]); |
| zip_in_place(kernel.packet[5], kernel.packet[7]); |
| |
| zip_in_place(kernel.packet[0], kernel.packet[1]); |
| zip_in_place(kernel.packet[2], kernel.packet[3]); |
| zip_in_place(kernel.packet[4], kernel.packet[5]); |
| zip_in_place(kernel.packet[6], kernel.packet[7]); |
| } |
| |
| template <typename Packet> |
| EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 16>& kernel) { |
| EIGEN_UNROLL_LOOP |
| for (int i = 0; i < 4; ++i) { |
| const int m = (1 << i); |
| EIGEN_UNROLL_LOOP |
| for (int j = 0; j < m; ++j) { |
| const int n = (1 << (3 - i)); |
| EIGEN_UNROLL_LOOP |
| for (int k = 0; k < n; ++k) { |
| const int idx = 2 * j * n + k; |
| zip_in_place(kernel.packet[idx], kernel.packet[idx + n]); |
| } |
| } |
| } |
| } |
| |
| } // namespace detail |
| |
| // ptranspose overloads: only emit valid block sizes per vector size. |
| // At 16 bytes: float has 4 elems, double has 2 elems. |
| // At 32 bytes: float has 8 elems, double has 4 elems. |
| // At 64 bytes: float has 16 elems, double has 8 elems. |
| |
| // All sizes support PacketBlock<PacketXf, 2> and PacketBlock<PacketXf, 4>. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 4>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 2>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| |
| // All sizes support PacketBlock<PacketXd, 2>. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 2>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| |
| // All sizes support PacketBlock<PacketXi, 2> and PacketBlock<PacketXi, 4>. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 4>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 2>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| |
| // All sizes support PacketBlock<PacketXl, 2>. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 2>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| |
| #if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 32 |
| // 32+ bytes: float has 8+ elems, double has 4+ elems. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 8>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 4>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 8>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 4>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| #endif |
| |
| #if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 64 |
| // 64 bytes: float has 16 elems, double has 8 elems. |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 16>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 8>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 16>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 8>& kernel) { |
| detail::ptranspose_impl(kernel); |
| } |
| #endif |
| #endif |
| |
| } // end namespace internal |
| } // end namespace Eigen |
| |
| #endif // EIGEN_PACKET_MATH_CLANG_H |
