NEON: Add Packet8bf as the full bfloat16 packet libeigen/eigen!3414
diff --git a/Eigen/Core b/Eigen/Core index 2330b76..f9d9293 100644 --- a/Eigen/Core +++ b/Eigen/Core
@@ -275,12 +275,14 @@ #include "src/Core/arch/NEON/PacketMath.h" #include "src/Core/arch/NEON/TypeCasting.h" #include "src/Core/arch/NEON/MathFunctions.h" +#include "src/Core/arch/NEON/PacketMathBF16.h" #include "src/Core/arch/NEON/Complex.h" #elif defined EIGEN_VECTORIZE_SME // Keep NEON packet types outside the specialized streaming kernels. #include "src/Core/arch/NEON/PacketMath.h" #include "src/Core/arch/NEON/TypeCasting.h" #include "src/Core/arch/NEON/MathFunctions.h" +#include "src/Core/arch/NEON/PacketMathBF16.h" #include "src/Core/arch/NEON/Complex.h" #include "src/Core/arch/SME/PacketMath.h" #elif defined EIGEN_VECTORIZE_LSX
diff --git a/Eigen/src/Core/arch/Default/BFloat16.h b/Eigen/src/Core/arch/Default/BFloat16.h index d142f2b..70f348e 100644 --- a/Eigen/src/Core/arch/Default/BFloat16.h +++ b/Eigen/src/Core/arch/Default/BFloat16.h
@@ -39,23 +39,40 @@ return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32(_x))); \ } +// BF16_PACKET_FUNCTION for a bfloat16 packet that spans two PACKET_F, through the backend's +// Bf16ToF32Low, Bf16ToF32High and two-argument F32ToBf16. +// METHOD is a template name, which cannot be parenthesized. +// NOLINTBEGIN(bugprone-macro-parentheses) +#define BF16_PACKET_FUNCTION_HALVES(PACKET_F, PACKET_BF16, METHOD) \ + template <> \ + EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \ + const PACKET_BF16& _x) { \ + return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32Low(_x)), METHOD<PACKET_F>(Bf16ToF32High(_x))); \ + } +// NOLINTEND(bugprone-macro-parentheses) + +// The _WITH forms take the macro that defines each function, BF16_PACKET_FUNCTION or +// BF16_PACKET_FUNCTION_HALVES, so that every backend instantiates the same list. +#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \ + FUNCTION(PACKET_F, PACKET_BF16, pcos) \ + FUNCTION(PACKET_F, PACKET_BF16, psin) \ + FUNCTION(PACKET_F, PACKET_BF16, psinh) \ + FUNCTION(PACKET_F, PACKET_BF16, pcosh) \ + FUNCTION(PACKET_F, PACKET_BF16, pasinh) \ + FUNCTION(PACKET_F, PACKET_BF16, pacosh) \ + FUNCTION(PACKET_F, PACKET_BF16, pexp) \ + FUNCTION(PACKET_F, PACKET_BF16, pexp2) \ + FUNCTION(PACKET_F, PACKET_BF16, pexpm1) \ + FUNCTION(PACKET_F, PACKET_BF16, plog) \ + FUNCTION(PACKET_F, PACKET_BF16, plog1p) \ + FUNCTION(PACKET_F, PACKET_BF16, plog2) \ + FUNCTION(PACKET_F, PACKET_BF16, plog10) \ + FUNCTION(PACKET_F, PACKET_BF16, preciprocal) \ + FUNCTION(PACKET_F, PACKET_BF16, pcbrt) \ + FUNCTION(PACKET_F, PACKET_BF16, ptanh) + #define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(PACKET_F, PACKET_BF16) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pcos) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, psin) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, psinh) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pcosh) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pasinh) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pacosh) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pexp) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pexp2) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pexpm1) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, plog) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, plog1p) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, plog2) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, plog10) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, preciprocal) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pcbrt) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, ptanh) + EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16) // psqrt and prsqrt through the float round trip, for backends without their own bfloat16 kernels. #define EIGEN_INSTANTIATE_GENERIC_SQRT_FUNCS_BF16(PACKET_F, PACKET_BF16) \ @@ -63,23 +80,29 @@ BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, prsqrt) // BF16 wrappers for contrib/SpecialFunctions. +#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \ + FUNCTION(PACKET_F, PACKET_BF16, perf) \ + FUNCTION(PACKET_F, PACKET_BF16, pndtri) + #define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16(PACKET_F, PACKET_BF16) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, perf) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pndtri) + EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16) + +#define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0e) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1e) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_j0) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_j1) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0e) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1e) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_y0) \ + FUNCTION(PACKET_F, PACKET_BF16, pbessel_y1) #define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16(PACKET_F, PACKET_BF16) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0e) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1e) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_j0) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_j1) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0e) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1e) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_y0) \ - BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, pbessel_y1) + EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16) // Only use HIP GPU bf16 in kernels #if defined(EIGEN_HAS_HIP_BF16) && defined(EIGEN_GPU_COMPILE_PHASE)
diff --git a/Eigen/src/Core/arch/NEON/MathFunctions.h b/Eigen/src/Core/arch/NEON/MathFunctions.h index 8d028be..63b2571 100644 --- a/Eigen/src/Core/arch/NEON/MathFunctions.h +++ b/Eigen/src/Core/arch/NEON/MathFunctions.h
@@ -35,74 +35,6 @@ } #endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16 -// psqrt and prsqrt have their own kernels below. -EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet4f, Packet4bf) - -template <> -EIGEN_STRONG_INLINE Packet4bf psqrt<Packet4bf>(const Packet4bf& a) { - // y = x r with r = FRSQRTE(x), then one FRSQRTS step y <- y (3 - y r) / 2. FRSQRTE is specified - // bit for bit, and rounding y to bfloat16 equals bfloat16(sqrtf(x)) for all 65536 inputs - // (packetmath checks them all); the raw estimate alone misrounds 7366 of them. RSHRN adds half a - // bfloat16 ulp and truncates. That is round half up, which differs from F32ToBf16 only on an - // exact float midpoint, and no input produces one; NaNs stay NaN since their low half is zero. - Packet4f x = Bf16ToF32(a); - Packet4f r = vrsqrteq_f32(x); - Packet4f y = pmul(x, r); - y = pmul(y, vrsqrtsq_f32(y, r)); - Packet4bf ybf = vrshrn_n_u32(vreinterpretq_u32_f32(y), 16); - // y is NaN where r is +-inf (x is +-0, or a subnormal the FPU flushes) or +0 (x is +inf), and - // the answer there is the sign and exponent of a. Reading r rather than a keeps that right under - // FPCR.FZ and on ARMv7. The high half of r plus 0x80 is 0x80, 0 or 0x8000 on exactly those - // lanes; every other r, a positive normal or a NaN, has a bit set under 0x7f7f. - uint16x4_t u = vadd_u16(vshrn_n_u32(vreinterpretq_u32_f32(r), 16), vdup_n_u16(0x80)); - uint16x4_t step_lane = vtst_u16(u, vdup_n_u16(0x7f7f)); - return vbsl_u16(step_lane, ybf, vand_u16(a, vdup_n_u16(0xff80))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf prsqrt<Packet4bf>(const Packet4bf& a) { - // r = FRSQRTE(x) refined by one FRSQRTS step, r1 = r (3 - x r r) / 2, then the same RSHRN - // narrowing as psqrt. r1 rounds to the correctly rounded bfloat16 rsqrt for all 65536 inputs - // (packetmath checks them all). FRSQRTE has 8 fraction bits, so the raw estimate always lands on - // a bfloat16 value or midpoint and misrounds 1270 of them. - Packet4f x = Bf16ToF32(a); - Packet4f r = vrsqrteq_f32(x); -#if EIGEN_ARCH_ARM64 - // FMULX gives 2 for 0 * inf where FMUL gives NaN, so no lane needs a select. Where the FPU reads - // x as +-0 (a subnormal too under FPCR.FZ), r is +-inf, y = 2 and the step is -+inf, and |step| - // restores r1 = +-inf. Where x is +inf, r is 0, y = 2 and r1 = 0 * 1.5. Negative and NaN x stay - // NaN. On every other lane FMULX is FMUL and the step is positive, so r1 is the plain step. - Packet4f y = vmulxq_f32(x, r); - Packet4f r1 = pmul(r, vabsq_f32(vrsqrtsq_f32(y, r))); - return vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16); -#else - // ARMv7 has no FMULX. r1 is NaN where x is +-0, +inf or a subnormal, which ARMv7 NEON flushes, - // and there r is already the answer (+-inf, 0, +-inf) with a zero low half, so those lanes take - // r truncated. They are the bit patterns with a - 0x80 >= 0x7f00 unsigned: the exponent-0 range - // wraps around, and +inf, NaN and every negative value sit above. - Packet4f r1 = pmul(r, vrsqrtsq_f32(pmul(x, r), r)); - Packet4bf refined = vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16); - Packet4bf raw = vshrn_n_u32(vreinterpretq_u32_f32(r), 16); - uint16x4_t special = vcge_u16(vsub_u16(a, vdup_n_u16(0x80)), vdup_n_u16(0x7f00)); - return vbsl_u16(special, raw, refined); -#endif -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pfrexp(const Packet4bf& a, Packet4bf& exponent) { - // Both results are exact: the mantissa keeps the input's significand, and the exponent is a - // small integer. - Packet4f fexponent; - const Packet4bf out = F32ToBf16Truncate(pfrexp<Packet4f>(Bf16ToF32(a), fexponent)); - exponent = F32ToBf16Truncate(fexponent); - return out; -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& exponent) { - return F32ToBf16(pldexp<Packet4f>(Bf16ToF32(a), Bf16ToF32(exponent))); -} - //---------- double ---------- #if EIGEN_ARCH_ARM64
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 9b1340f..2159449 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -4744,374 +4744,6 @@ } #endif -//---------- bfloat16 ---------- -// TODO: Add support for native armv8.6-a bfloat16_t - -// TODO: Guard if we have native bfloat16 support -typedef eigen_packet_wrapper<uint16x4_t, 19> Packet4bf; - -template <> -struct is_arithmetic<Packet4bf> : std::true_type {}; - -template <> -struct packet_traits<bfloat16> : default_packet_traits { - typedef Packet4bf type; - typedef Packet4bf half; - enum { - Vectorizable = 1, - AlignedOnScalar = 1, - size = 4, - - HasCmp = 1, - HasAdd = 1, - HasSub = 1, - HasShift = 1, - HasMul = 1, - HasNegate = 1, - HasAbs = 1, - HasArg = 0, - HasAbsDiff = 1, - HasMin = 1, - HasMax = 1, - HasConj = 1, - HasSetLinear = 1, - HasDiv = 1, - HasSin = EIGEN_FAST_MATH, - HasCos = EIGEN_FAST_MATH, - HasLog = 1, - HasExp = 1, - HasSqrt = 1, - HasRsqrt = 1, - HasTanh = EIGEN_FAST_MATH, - HasErf = EIGEN_FAST_MATH, - HasBessel = 0, // Issues with accuracy. - HasNdtri = 0 - }; -}; - -template <> -struct unpacket_traits<Packet4bf> : neon_unpacket_default<Packet4bf, bfloat16> {}; - -namespace detail { -template <> -EIGEN_ALWAYS_INLINE void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) { - const uint16x4x2_t tmp = vzip_u16(p1, p2); - p1 = tmp.val[0]; - p2 = tmp.val[1]; -} -} // namespace detail - -// Round to nearest even on the bfloat16 boundary. With lsb the lowest bit kept, the scalar -// formula is (p + 0x7fff + lsb) >> 16, and RSHRN computes (v + 0x8000) >> 16, so v = p - 1 + lsb, -// that is p minus one where lsb is clear. The caller guarantees that every NaN lane has a zero low -// half, which the FPU's NaN propagation does for bfloat16 operands: the rounding then leaves the -// NaN's high half alone. A NaN with a nonzero low half can instead round to +-inf or carry into a -// zero. -EIGEN_STRONG_INLINE Packet4bf F32ToBf16KeepNaN(const Packet4f& p) { - Packet4ui bits = vreinterpretq_u32_f32(p); - Packet4ui round_down = vceqq_u32(vandq_u32(bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0)); - return vrshrn_n_u32(vaddq_u32(bits, round_down), 16); -} - -// F32ToBf16KeepNaN for any float: NaN lanes become the bfloat16 NaN 0x7fc0, as on AVX. The scalar -// conversion keeps the NaN's sign; doing that here costs an op on the critical path of every round -// trip, 25% of a cast loop and 6% of exp on a Cortex-A72, for a bit that carries no value. -EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) { - Packet4ui bits = vbslq_u32(vceqq_f32(p, p), vreinterpretq_u32_f32(p), vdupq_n_u32(0x7fc00000)); - return F32ToBf16KeepNaN(vreinterpretq_f32_u32(bits)); -} - -// Discard the low 16 bits of each float. Only valid when every lane already holds an exact -// bfloat16 value, in which case this agrees with F32ToBf16 but skips its rounding and NaN -// canonicalization. pmin/pmax qualify: they return one of their operands bit for bit. -EIGEN_STRONG_INLINE Packet4bf F32ToBf16Truncate(const Packet4f& p) { return vshrn_n_u32(vreinterpretq_u32_f32(p), 16); } - -EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p) { - return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16))); -} - -EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { return vmovn_u32(vreinterpretq_u32_f32(p)); } - -template <> -EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(const bfloat16& from) { - return Packet4bf(pset1<Packet4us>(from.value)); -} - -template <> -EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(const Packet4bf& from) { - return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(Packet4us(from)))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from) { - return Packet4bf( - pload<Packet4us>(reinterpret_cast<const uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(from)))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(const bfloat16* from) { - return Packet4bf(ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from))); -} - -template <> -EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from) { - EIGEN_DEBUG_ALIGNED_STORE vst1_u16( - reinterpret_cast<uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(to)), from); -} - -template <> -EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet4bf& from) { - EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(const bfloat16* from) { - return Packet4bf(ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) { - return Packet4bf(vand_u16(a, vdup_n_u16(0x7fff))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} -template <> -EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} -template <> -EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(const bfloat16& a) { - return F32ToBf16(plset<Packet4f>(static_cast<float>(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a, const Packet4bf& b) { - return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a, const Packet4bf& b) { - return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a, const Packet4bf& b) { - return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a, const Packet4bf& b) { - return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pnot(const Packet4bf& a) { - return Packet4bf(pnot<Packet4us>(Packet4us(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf ptrue(const Packet4bf& /*a*/) { - return Packet4bf(vdup_n_u16(0xffff)); -} - -template <> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) { - return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(const Packet4bf& a) { - return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(const Packet4bf& a) { - return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(const Packet4bf& a) { - return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pround<Packet4bf>(const Packet4bf& a) { - return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf ptrunc<Packet4bf>(const Packet4bf& a) { - return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { - return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { - return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pnmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { - return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pnmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { - return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(const bfloat16* from, Index stride) { - return Packet4bf(pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride)); -} - -template <> -EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet4bf>(bfloat16* to, const Packet4bf& from, Index stride) { - pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), Packet4us(from), stride); -} - -template <> -EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(const Packet4bf& a) { - return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(const Packet4bf& a) { - return bfloat16_impl::exact_float_to_bfloat16(predux_max<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(const Packet4bf& a) { - return bfloat16_impl::exact_float_to_bfloat16(predux_min<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(const Packet4bf& a) { - return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a))); -} - -template <> -EIGEN_STRONG_INLINE bool predux_any(const Packet4bf& a) { - return vget_lane_u64(vreinterpret_u64_u16(Packet4us(a)), 0) != 0; -} - -template <> -EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(const Packet4bf& a) { - return Packet4bf(preverse<Packet4us>(Packet4us(a))); -} - -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4bf, 4>& kernel) { - detail::ptranspose_impl(kernel); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -// Classify on the raw bits, as the scalar isinf/isnan/isfinite do: |a| ==, >, < 0x7f80. -template <> -EIGEN_STRONG_INLINE Packet4bf pisinf<Packet4bf>(const Packet4bf& a) { - constexpr uint16_t kInf = ((1 << 8) - 1) << 7; - constexpr uint16_t kAbsMask = (1 << 15) - 1; - return vceq_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pisnan<Packet4bf>(const Packet4bf& a) { - constexpr uint16_t kInf = ((1 << 8) - 1) << 7; - constexpr uint16_t kAbsMask = (1 << 15) - 1; - return vcgt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); -} - -// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. -template <> -EIGEN_STRONG_INLINE Packet4bf psign<Packet4bf>(const Packet4bf& a) { - const uint16x4_t abs_mask = vdup_n_u16(0x7fff); - const uint16x4_t is_nan = vcgt_u16(vand_u16(a, abs_mask), vdup_n_u16(0x7f80)); - const uint16x4_t keep = vorr_u16(is_nan, vdup_n_u16(0x8000)); - return vand_u16(vbsl_u16(keep, a, vdup_n_u16(0x3f80)), vtst_u16(a, abs_mask)); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pisfinite<Packet4bf>(const Packet4bf& a) { - constexpr uint16_t kInf = ((1 << 8) - 1) << 7; - constexpr uint16_t kAbsMask = (1 << 15) - 1; - return vclt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { - return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(const Packet4bf& a) { - return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(static_cast<uint16_t>(0x8000)))); -} - //---------- double ---------- #if EIGEN_ARCH_ARM64
diff --git a/Eigen/src/Core/arch/NEON/PacketMathBF16.h b/Eigen/src/Core/arch/NEON/PacketMathBF16.h new file mode 100644 index 0000000..3f64f87 --- /dev/null +++ b/Eigen/src/Core/arch/NEON/PacketMathBF16.h
@@ -0,0 +1,935 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#ifndef EIGEN_PACKET_MATH_BF16_NEON_H +#define EIGEN_PACKET_MATH_BF16_NEON_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +// TODO: Add support for native armv8.6-a bfloat16_t + +// TODO: Guard if we have native bfloat16 support +typedef eigen_packet_wrapper<uint16x4_t, 19> Packet4bf; +using Packet8bf = eigen_packet_wrapper<uint16x8_t, 20>; + +template <> +struct is_arithmetic<Packet4bf> : std::true_type {}; +template <> +struct is_arithmetic<Packet8bf> : std::true_type {}; + +// Packet8bf is the full packet: every bfloat16 op widens to float, and the float work of two +// Packet4f halves overlaps where one Packet4bf's chain would leave the pipes idle. +template <> +struct packet_traits<bfloat16> : default_packet_traits { + using type = Packet8bf; + using half = Packet4bf; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + + HasCmp = 1, + HasAdd = 1, + HasSub = 1, + HasShift = 1, + HasMul = 1, + HasNegate = 1, + HasAbs = 1, + HasArg = 0, + HasAbsDiff = 1, + HasMin = 1, + HasMax = 1, + HasConj = 1, + HasSetLinear = 1, + HasDiv = 1, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasLog = 1, + HasExp = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasTanh = EIGEN_FAST_MATH, + HasErf = EIGEN_FAST_MATH, + HasBessel = 0, // Issues with accuracy. + HasNdtri = 0 + }; +}; + +template <> +struct unpacket_traits<Packet4bf> : neon_unpacket_default<Packet4bf, bfloat16> {}; +template <> +struct unpacket_traits<Packet8bf> : neon_unpacket_default<Packet8bf, bfloat16> { + using half = Packet4bf; +}; + +namespace detail { +template <> +EIGEN_ALWAYS_INLINE void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) { + const uint16x4x2_t tmp = vzip_u16(p1, p2); + p1 = tmp.val[0]; + p2 = tmp.val[1]; +} +template <> +EIGEN_ALWAYS_INLINE void zip_in_place<Packet8bf>(Packet8bf& p1, Packet8bf& p2) { + const uint16x8x2_t tmp = vzipq_u16(p1, p2); + p1 = tmp.val[0]; + p2 = tmp.val[1]; +} +} // namespace detail + +// Round to nearest even on the bfloat16 boundary. With lsb the lowest bit kept, the scalar +// formula is (p + 0x7fff + lsb) >> 16, and RADDHN computes (p + d + 0x8000) >> 16, so d = lsb - 1, +// that is -1 where lsb is clear. Written as an add and RSHRN, the add stays a separate op in +// loops with GCC and clang on both targets. The caller guarantees that every NaN lane has a zero low +// half, which the FPU's NaN propagation does for bfloat16 operands: the rounding then leaves the +// NaN's high half alone. A NaN with a nonzero low half can instead round to +-inf or carry into a +// zero. +EIGEN_STRONG_INLINE Packet4bf F32ToBf16KeepNaN(const Packet4f& p) { + Packet4ui bits = vreinterpretq_u32_f32(p); + Packet4ui round_down = vceqq_u32(vandq_u32(bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0)); + return vraddhn_u32(bits, round_down); +} + +// F32ToBf16KeepNaN for any float: NaN lanes become the bfloat16 NaN 0x7fc0, as on AVX. The scalar +// conversion keeps the NaN's sign; doing that here costs an op on the critical path of every round +// trip, 25% of a cast loop and 6% of exp on a Cortex-A72, for a bit that carries no value. +// |p| <= inf is false only for NaN. The NaN lanes are selected after narrowing, so the rounding +// reads the raw bits. Selecting first costs clang an extra AND and MOV. +EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) { + uint16x4_t not_nan = vmovn_u32(vcageq_f32(pinf<Packet4f>(), p)); + return vbsl_u16(not_nan, F32ToBf16KeepNaN(p), vdup_n_u16(0x7fc0)); +} + +// Discard the low 16 bits of each float. Only valid when every lane already holds an exact +// bfloat16 value, in which case this agrees with F32ToBf16 but skips its rounding and NaN +// canonicalization. pmin/pmax qualify: they return one of their operands bit for bit. +EIGEN_STRONG_INLINE Packet4bf F32ToBf16Truncate(const Packet4f& p) { return vshrn_n_u32(vreinterpretq_u32_f32(p), 16); } + +EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p) { + return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16))); +} + +EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { return vmovn_u32(vreinterpretq_u32_f32(p)); } + +// Packet8bf conversions work on two Packet4f halves, lanes 0-3 and 4-7, with the same rounding as +// the Packet4bf ones. +EIGEN_STRONG_INLINE Packet4f Bf16ToF32Low(const Packet8bf& p) { + return vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(p), 16)); +} + +EIGEN_STRONG_INLINE Packet4f Bf16ToF32High(const Packet8bf& p) { +#if EIGEN_ARCH_ARM64 + return vreinterpretq_f32_u32(vshll_high_n_u16(p, 16)); +#else + return vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(p), 16)); +#endif +} + +// (v + 0x8000) >> 16 on each lane of lo and hi. +EIGEN_STRONG_INLINE Packet8bf RoundingNarrowBf16(const Packet4ui& lo, const Packet4ui& hi) { +#if EIGEN_ARCH_ARM64 + return vrshrn_high_n_u32(vrshrn_n_u32(lo, 16), hi, 16); +#else + return vcombine_u16(vrshrn_n_u32(lo, 16), vrshrn_n_u32(hi, 16)); +#endif +} + +EIGEN_STRONG_INLINE Packet8bf F32ToBf16KeepNaN(const Packet4f& lo, const Packet4f& hi) { + Packet4ui lo_bits = vreinterpretq_u32_f32(lo); + Packet4ui hi_bits = vreinterpretq_u32_f32(hi); + Packet4ui lo_round_down = vceqq_u32(vandq_u32(lo_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0)); + Packet4ui hi_round_down = vceqq_u32(vandq_u32(hi_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0)); +#if EIGEN_ARCH_ARM64 + return vraddhn_high_u32(vraddhn_u32(lo_bits, lo_round_down), hi_bits, hi_round_down); +#else + return vcombine_u16(vraddhn_u32(lo_bits, lo_round_down), vraddhn_u32(hi_bits, hi_round_down)); +#endif +} + +// As the Packet4bf F32ToBf16, with one UZP1 narrowing both NaN masks. +EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet4f& lo, const Packet4f& hi) { + Packet4f inf = pinf<Packet4f>(); + uint16x8_t not_nan = + vuzpq_u16(vreinterpretq_u16_u32(vcageq_f32(inf, lo)), vreinterpretq_u16_u32(vcageq_f32(inf, hi))).val[0]; + return vbslq_u16(not_nan, F32ToBf16KeepNaN(lo, hi), vdupq_n_u16(0x7fc0)); +} + +// The high and low 16 bits of every float: one UZP2 or UZP1 on AArch64, one VUZP on ARMv7. +EIGEN_STRONG_INLINE Packet8bf F32ToBf16Truncate(const Packet4f& lo, const Packet4f& hi) { + return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[1]; +} + +EIGEN_STRONG_INLINE Packet8bf F32MaskToBf16Mask(const Packet4f& lo, const Packet4f& hi) { + return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(const bfloat16& from) { + return Packet4bf(pset1<Packet4us>(from.value)); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(const Packet4bf& from) { + return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(Packet4us(from)))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from) { + return Packet4bf( + pload<Packet4us>(reinterpret_cast<const uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(from)))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(const bfloat16* from) { + return Packet4bf(ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from))); +} + +template <> +EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from) { + EIGEN_DEBUG_ALIGNED_STORE vst1_u16( + reinterpret_cast<uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(to)), from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet4bf& from) { + EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(const bfloat16* from) { + return Packet4bf(ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) { + return Packet4bf(vand_u16(a, vdup_n_u16(0x7fff))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} +template <> +EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} +template <> +EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(const bfloat16& a) { + return F32ToBf16(plset<Packet4f>(static_cast<float>(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a, const Packet4bf& b) { + return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a, const Packet4bf& b) { + return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a, const Packet4bf& b) { + return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a, const Packet4bf& b) { + return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pnot(const Packet4bf& a) { + return Packet4bf(pnot<Packet4us>(Packet4us(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf ptrue(const Packet4bf& /*a*/) { + return Packet4bf(vdup_n_u16(0xffff)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) { + return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(const Packet4bf& a) { + return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(const Packet4bf& a) { + return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(const Packet4bf& a) { + return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pround<Packet4bf>(const Packet4bf& a) { + return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf ptrunc<Packet4bf>(const Packet4bf& a) { + return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { + return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { + return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pnmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { + return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pnmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) { + return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(const bfloat16* from, Index stride) { + return Packet4bf(pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride)); +} + +template <> +EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet4bf>(bfloat16* to, const Packet4bf& from, Index stride) { + pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), Packet4us(from), stride); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(const Packet4bf& a) { + return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(const Packet4bf& a) { + return bfloat16_impl::exact_float_to_bfloat16(predux_max<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(const Packet4bf& a) { + return bfloat16_impl::exact_float_to_bfloat16(predux_min<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(const Packet4bf& a) { + return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4bf& a) { + return vget_lane_u64(vreinterpret_u64_u16(Packet4us(a)), 0) != 0; +} + +template <> +EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(const Packet4bf& a) { + return Packet4bf(preverse<Packet4us>(Packet4us(a))); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4bf, 4>& kernel) { + detail::ptranspose_impl(kernel); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +// Classify on the raw bits, as the scalar isinf/isnan/isfinite do: |a| ==, >, < 0x7f80. +template <> +EIGEN_STRONG_INLINE Packet4bf pisinf<Packet4bf>(const Packet4bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vceq_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pisnan<Packet4bf>(const Packet4bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vcgt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); +} + +// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. +template <> +EIGEN_STRONG_INLINE Packet4bf psign<Packet4bf>(const Packet4bf& a) { + const uint16x4_t abs_mask = vdup_n_u16(0x7fff); + const uint16x4_t is_nan = vcgt_u16(vand_u16(a, abs_mask), vdup_n_u16(0x7f80)); + const uint16x4_t keep = vorr_u16(is_nan, vdup_n_u16(0x8000)); + return vand_u16(vbsl_u16(keep, a, vdup_n_u16(0x3f80)), vtst_u16(a, abs_mask)); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pisfinite<Packet4bf>(const Packet4bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vclt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(const Packet4bf& a, const Packet4bf& b) { + return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(const Packet4bf& a) { + return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(static_cast<uint16_t>(0x8000)))); +} + +//---------- Packet8bf ---------- + +template <> +EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(const bfloat16& from) { + return Packet8bf(pset1<Packet8us>(from.value)); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 pfirst<Packet8bf>(const Packet8bf& from) { + return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet8us>(Packet8us(from)))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(const bfloat16* from) { + return Packet8bf( + pload<Packet8us>(reinterpret_cast<const uint16_t*>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(from)))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(const bfloat16* from) { + return Packet8bf(ploadu<Packet8us>(reinterpret_cast<const uint16_t*>(from))); +} + +template <> +EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet8bf& from) { + EIGEN_DEBUG_ALIGNED_STORE vst1q_u16( + reinterpret_cast<uint16_t*>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(to)), from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet8bf& from) { + EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(reinterpret_cast<uint16_t*>(to), from); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(const bfloat16* from) { + return Packet8bf(ploaddup<Packet8us>(reinterpret_cast<const uint16_t*>(from))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(const bfloat16* from) { + return Packet8bf(ploadquad<Packet8us>(reinterpret_cast<const uint16_t*>(from))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) { + return Packet8bf(vandq_u16(a, vdupq_n_u16(0x7fff))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNumbers, Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmin<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} +template <> +EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNaN, Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmin<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmin<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNumbers, Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmax<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} +template <> +EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNaN, Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmax<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmax<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(const bfloat16& a) { + Packet4f lo = plset<Packet4f>(static_cast<float>(a)); + return F32ToBf16(lo, padd(lo, pset1<Packet4f>(4.0f))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a, const Packet8bf& b) { + return Packet8bf(por<Packet8us>(Packet8us(a), Packet8us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a, const Packet8bf& b) { + return Packet8bf(pxor<Packet8us>(Packet8us(a), Packet8us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pand(const Packet8bf& a, const Packet8bf& b) { + return Packet8bf(pand<Packet8us>(Packet8us(a), Packet8us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pandnot(const Packet8bf& a, const Packet8bf& b) { + return Packet8bf(pandnot<Packet8us>(Packet8us(a), Packet8us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pnot(const Packet8bf& a) { + return Packet8bf(pnot<Packet8us>(Packet8us(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf ptrue(const Packet8bf& /*a*/) { + return Packet8bf(vdupq_n_u16(0xffff)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8bf pselect(const Packet8bf& mask, const Packet8bf& a, const Packet8bf& b) { + return Packet8bf(pselect<Packet8us>(Packet8us(mask), Packet8us(a), Packet8us(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(const Packet8bf& a) { + return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32Low(a)), print<Packet4f>(Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(const Packet8bf& a) { + return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32Low(a)), pfloor<Packet4f>(Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(const Packet8bf& a) { + return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32Low(a)), pceil<Packet4f>(Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(const Packet8bf& a) { + return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32Low(a)), pround<Packet4f>(Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) { + return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32Low(a)), ptrunc<Packet4f>(Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + padd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + psub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pmul<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { + return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)), + pmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { + return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)), + pmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pnmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { + return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)), + pnmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pnmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { + return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)), + pnmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pdiv<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pgather<bfloat16, Packet8bf>(const bfloat16* from, Index stride) { + return Packet8bf(pgather<uint16_t, Packet8us>(reinterpret_cast<const uint16_t*>(from), stride)); +} + +template <> +EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from, Index stride) { + pscatter<uint16_t, Packet8us>(reinterpret_cast<uint16_t*>(to), Packet8us(from), stride); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux<Packet8bf>(const Packet8bf& a) { + return static_cast<bfloat16>(predux<Packet4f>(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max<Packet8bf>(const Packet8bf& a) { + return bfloat16_impl::exact_float_to_bfloat16( + predux_max<Packet4f>(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min<Packet8bf>(const Packet8bf& a) { + return bfloat16_impl::exact_float_to_bfloat16( + predux_min<Packet4f>(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet8bf>(const Packet8bf& a) { + return static_cast<bfloat16>(predux_mul<Packet4f>(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf predux_half<Packet8bf>(const Packet8bf& a) { + return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8bf& a) { + return predux_any(Packet8us(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf preverse<Packet8bf>(const Packet8bf& a) { + return Packet8bf(preverse<Packet8us>(Packet8us(a))); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 4>& kernel) { + detail::ptranspose_impl(kernel); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 8>& kernel) { + detail::ptranspose_impl(kernel); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pabsdiff<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pabsdiff<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcmp_eq<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pcmp_eq<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcmp_lt<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pcmp_lt<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + // Negate a >= b once on the narrowed mask rather than on each float half. + return vmvnq_u16(F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(b), Bf16ToF32Low(a)), + pcmp_le<Packet4f>(Bf16ToF32High(b), Bf16ToF32High(a)))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcmp_le<Packet8bf>(const Packet8bf& a, const Packet8bf& b) { + return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)), + pcmp_le<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pisinf<Packet8bf>(const Packet8bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vceqq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pisnan<Packet8bf>(const Packet8bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vcgtq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pisfinite<Packet8bf>(const Packet8bf& a) { + constexpr uint16_t kInf = ((1 << 8) - 1) << 7; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return vcltq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) { + uint16x8_t abs_mask = vdupq_n_u16(0x7fff); + uint16x8_t is_nan = vcgtq_u16(vandq_u16(a, abs_mask), vdupq_n_u16(0x7f80)); + uint16x8_t keep = vorrq_u16(is_nan, vdupq_n_u16(0x8000)); + return vandq_u16(vbslq_u16(keep, a, vdupq_n_u16(0x3f80)), vtstq_u16(a, abs_mask)); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pnegate<Packet8bf>(const Packet8bf& a) { + return Packet8bf(pxor<Packet8us>(Packet8us(a), pset1<Packet8us>(static_cast<uint16_t>(0x8000)))); +} + +// psqrt and prsqrt have their own kernels below. +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet4f, Packet4bf) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION_HALVES, Packet4f, Packet8bf) + +template <> +EIGEN_STRONG_INLINE Packet4bf psqrt<Packet4bf>(const Packet4bf& a) { + // y = x r with r = FRSQRTE(x), then one FRSQRTS step y <- y (3 - y r) / 2. FRSQRTE is specified + // bit for bit, and rounding y to bfloat16 equals bfloat16(sqrtf(x)) for all 65536 inputs + // (packetmath checks them all); the raw estimate alone misrounds 7366 of them. RSHRN adds half a + // bfloat16 ulp and truncates. That is round half up, which differs from F32ToBf16 only on an + // exact float midpoint, and no input produces one; NaNs stay NaN since their low half is zero. + Packet4f x = Bf16ToF32(a); + Packet4f r = vrsqrteq_f32(x); + Packet4f y = pmul(x, r); + y = pmul(y, vrsqrtsq_f32(y, r)); + Packet4bf ybf = vrshrn_n_u32(vreinterpretq_u32_f32(y), 16); + // y is NaN where r is +-inf (x is +-0, or a subnormal the FPU flushes) or +0 (x is +inf), and + // the answer there is the sign and exponent of a. Reading r rather than a keeps that right under + // FPCR.FZ and on ARMv7. The high half of r plus 0x80 is 0x80, 0 or 0x8000 on exactly those + // lanes; every other r, a positive normal or a NaN, has a bit set under 0x7f7f. + uint16x4_t u = vadd_u16(vshrn_n_u32(vreinterpretq_u32_f32(r), 16), vdup_n_u16(0x80)); + uint16x4_t step_lane = vtst_u16(u, vdup_n_u16(0x7f7f)); + return vbsl_u16(step_lane, ybf, vand_u16(a, vdup_n_u16(0xff80))); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf prsqrt<Packet4bf>(const Packet4bf& a) { + // r = FRSQRTE(x) refined by one FRSQRTS step, r1 = r (3 - x r r) / 2, then the same RSHRN + // narrowing as psqrt. r1 rounds to the correctly rounded bfloat16 rsqrt for all 65536 inputs + // (packetmath checks them all). FRSQRTE has 8 fraction bits, so the raw estimate always lands on + // a bfloat16 value or midpoint and misrounds 1270 of them. + Packet4f x = Bf16ToF32(a); + Packet4f r = vrsqrteq_f32(x); +#if EIGEN_ARCH_ARM64 + // FMULX gives 2 for 0 * inf where FMUL gives NaN, so no lane needs a select. Where the FPU reads + // x as +-0 (a subnormal too under FPCR.FZ), r is +-inf, y = 2 and the step is -+inf, and |step| + // restores r1 = +-inf. Where x is +inf, r is 0, y = 2 and r1 = 0 * 1.5. Negative and NaN x stay + // NaN. On every other lane FMULX is FMUL and the step is positive, so r1 is the plain step. + Packet4f y = vmulxq_f32(x, r); + Packet4f r1 = pmul(r, vabsq_f32(vrsqrtsq_f32(y, r))); + return vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16); +#else + // ARMv7 has no FMULX. r1 is NaN where x is +-0, +inf or a subnormal, which ARMv7 NEON flushes, + // and there r is already the answer (+-inf, 0, +-inf) with a zero low half, so those lanes take + // r truncated. They are the bit patterns with a - 0x80 >= 0x7f00 unsigned: the exponent-0 range + // wraps around, and +inf, NaN and every negative value sit above. + Packet4f r1 = pmul(r, vrsqrtsq_f32(pmul(x, r), r)); + Packet4bf refined = vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16); + Packet4bf raw = vshrn_n_u32(vreinterpretq_u32_f32(r), 16); + uint16x4_t special = vcge_u16(vsub_u16(a, vdup_n_u16(0x80)), vdup_n_u16(0x7f00)); + return vbsl_u16(special, raw, refined); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pfrexp(const Packet4bf& a, Packet4bf& exponent) { + // Both results are exact: the mantissa keeps the input's significand, and the exponent is a + // small integer. + Packet4f fexponent; + const Packet4bf out = F32ToBf16Truncate(pfrexp<Packet4f>(Bf16ToF32(a), fexponent)); + exponent = F32ToBf16Truncate(fexponent); + return out; +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& exponent) { + return F32ToBf16(pldexp<Packet4f>(Bf16ToF32(a), Bf16ToF32(exponent))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf psqrt<Packet8bf>(const Packet8bf& a) { + // psqrt<Packet4bf> on each float half, narrowed and selected once over all 8 lanes. + Packet4f x_lo = Bf16ToF32Low(a); + Packet4f x_hi = Bf16ToF32High(a); + Packet4f r_lo = vrsqrteq_f32(x_lo); + Packet4f r_hi = vrsqrteq_f32(x_hi); + Packet4f y_lo = pmul(x_lo, r_lo); + Packet4f y_hi = pmul(x_hi, r_hi); + y_lo = pmul(y_lo, vrsqrtsq_f32(y_lo, r_lo)); + y_hi = pmul(y_hi, vrsqrtsq_f32(y_hi, r_hi)); + Packet8bf ybf = RoundingNarrowBf16(vreinterpretq_u32_f32(y_lo), vreinterpretq_u32_f32(y_hi)); + uint16x8_t u = vaddq_u16(F32ToBf16Truncate(r_lo, r_hi), vdupq_n_u16(0x80)); + uint16x8_t step_lane = vtstq_u16(u, vdupq_n_u16(0x7f7f)); + return vbslq_u16(step_lane, ybf, vandq_u16(a, vdupq_n_u16(0xff80))); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf prsqrt<Packet8bf>(const Packet8bf& a) { + // prsqrt<Packet4bf> on each float half. + Packet4f x_lo = Bf16ToF32Low(a); + Packet4f x_hi = Bf16ToF32High(a); + Packet4f r_lo = vrsqrteq_f32(x_lo); + Packet4f r_hi = vrsqrteq_f32(x_hi); +#if EIGEN_ARCH_ARM64 + Packet4f r1_lo = pmul(r_lo, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_lo, r_lo), r_lo))); + Packet4f r1_hi = pmul(r_hi, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_hi, r_hi), r_hi))); + return RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi)); +#else + Packet4f r1_lo = pmul(r_lo, vrsqrtsq_f32(pmul(x_lo, r_lo), r_lo)); + Packet4f r1_hi = pmul(r_hi, vrsqrtsq_f32(pmul(x_hi, r_hi), r_hi)); + Packet8bf refined = RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi)); + Packet8bf raw = F32ToBf16Truncate(r_lo, r_hi); + uint16x8_t special = vcgeq_u16(vsubq_u16(a, vdupq_n_u16(0x80)), vdupq_n_u16(0x7f00)); + return vbslq_u16(special, raw, refined); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pfrexp(const Packet8bf& a, Packet8bf& exponent) { + Packet4f fexponent_lo; + Packet4f fexponent_hi; + Packet4f lo = pfrexp<Packet4f>(Bf16ToF32Low(a), fexponent_lo); + Packet4f hi = pfrexp<Packet4f>(Bf16ToF32High(a), fexponent_hi); + exponent = F32ToBf16Truncate(fexponent_lo, fexponent_hi); + return F32ToBf16Truncate(lo, hi); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pldexp(const Packet8bf& a, const Packet8bf& exponent) { + return F32ToBf16(pldexp<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(exponent)), + pldexp<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(exponent))); +} + +template <> +struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {}; +template <> +struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {}; + +template <> +EIGEN_STRONG_INLINE Packet4f pcast<Packet4bf, Packet4f>(const Packet4bf& a) { + return Bf16ToF32(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcast<Packet4f, Packet4bf>(const Packet4f& a) { + return F32ToBf16(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast<Packet8bf, Packet4f>(const Packet8bf& a) { + return Bf16ToF32Low(a); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcast<Packet4f, Packet8bf>(const Packet4f& a, const Packet4f& b) { + return F32ToBf16(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2f pcast<Packet4bf, Packet2f>(const Packet4bf& a) { + return vget_low_f32(Bf16ToF32(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet4bf pcast<Packet2f, Packet4bf>(const Packet2f& a, const Packet2f& b) { + return F32ToBf16(vcombine_f32(a, b)); +} +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_BF16_NEON_H
diff --git a/Eigen/src/Core/arch/NEON/TypeCasting.h b/Eigen/src/Core/arch/NEON/TypeCasting.h index 5ab6bb7..5ea977e 100644 --- a/Eigen/src/Core/arch/NEON/TypeCasting.h +++ b/Eigen/src/Core/arch/NEON/TypeCasting.h
@@ -1740,21 +1740,6 @@ #endif // EIGEN_ARCH_ARM64 -template <> -struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {}; -template <> -struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {}; - -template <> -EIGEN_STRONG_INLINE Packet4f pcast<Packet4bf, Packet4f>(const Packet4bf& a) { - return Bf16ToF32(a); -} - -template <> -EIGEN_STRONG_INLINE Packet4bf pcast<Packet4f, Packet4bf>(const Packet4f& a) { - return F32ToBf16(a); -} - } // end namespace internal } // end namespace Eigen
diff --git a/contrib/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h b/contrib/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h index cfaa48d..b0d4306 100644 --- a/contrib/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h +++ b/contrib/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h
@@ -39,6 +39,7 @@ #endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16 EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16(Packet4f, Packet4bf) +EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION_HALVES, Packet4f, Packet8bf) } // namespace internal } // namespace Eigen
diff --git a/contrib/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h b/contrib/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h index 3b27b9c..a7307af 100644 --- a/contrib/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h +++ b/contrib/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h
@@ -65,6 +65,7 @@ #endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16 EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16(Packet4f, Packet4bf) +EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION_HALVES, Packet4f, Packet8bf) } // namespace internal } // namespace Eigen
diff --git a/contrib/test/special_packetmath.cpp b/contrib/test/special_packetmath.cpp index 516e52b..1806975 100644 --- a/contrib/test/special_packetmath.cpp +++ b/contrib/test/special_packetmath.cpp
@@ -93,6 +93,7 @@ void neon_erf_subnormals_bfloat16() { packet_erf_subnormals<bfloat16, internal::Packet4bf>(); + packet_erf_subnormals<bfloat16, internal::Packet8bf>(); // Exhaustive bfloat16 subnormals, with both signs, checked without floating-point subnormal conversions. for (unsigned int n = 1; n < 128; ++n) { const numext::uint16_t expectedBits =
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index c576868..7ee2610 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -3127,20 +3127,26 @@ // the float packet op, over every encoding of one operand and a fixed set for the other that puts sums and products // on exact midpoints: 1 and 3 against [256, 512), 1 + 2^-7 and 2^-7 against values near 1, plus the range ends and // the special values. NaN lanes compare as NaN, since a kernel may keep the operand's payload. +template <typename Packet> void packetmath_bfloat16_arithmetic_rounding() { - using Packet = internal::packet_traits<bfloat16>::type; - using FloatPacket = internal::packet_traits<float>::type; constexpr int packet_size = internal::unpacket_traits<Packet>::size; - static_assert(internal::unpacket_traits<FloatPacket>::size == packet_size, - "the bfloat16 packet and the float packet must have the same size"); + using FloatPacket = + typename internal::find_packet_by_size<float, internal::plain_enum_min( + packet_size, internal::packet_traits<float>::size)>::type; + constexpr int float_size = internal::unpacket_traits<FloatPacket>::size; + static_assert(packet_size % float_size == 0, "the bfloat16 packet must span whole float packets"); const numext::uint16_t others[] = {0x0000, 0x8000, 0x3f80, 0x4040, 0x3f81, 0x3c00, 0x4380, 0xbf81, 0x7f7f, 0x0080, 0x0001, 0x7f80, 0xff80, 0x7fc0, 0x7f81, 0x3f7f}; constexpr int count = sizeof(others) / sizeof(others[0]); EIGEN_ALIGN_MAX bfloat16 a[packet_size], b[packet_size], c[packet_size], out[packet_size]; EIGEN_ALIGN_MAX float fa[packet_size], fb[packet_size], fc[packet_size], ref[packet_size]; - auto verify = [&](const Packet& result, const FloatPacket& expected) { + // op is applied to the float operands one float packet at a time. + auto verify = [&](const Packet& result, auto op) { internal::pstore(out, result); - internal::pstore(ref, expected); + for (int k = 0; k < packet_size; k += float_size) { + internal::pstore(ref + k, op(internal::pload<FloatPacket>(fa + k), internal::pload<FloatPacket>(fb + k), + internal::pload<FloatPacket>(fc + k))); + } for (int i = 0; i < packet_size; ++i) { numext::uint16_t got = numext::bit_cast<numext::uint16_t>(out[i]); numext::uint16_t want = numext::bit_cast<numext::uint16_t>(static_cast<bfloat16>(ref[i])); @@ -3159,23 +3165,22 @@ fc[i] = static_cast<float>(c[i]); } Packet qb = internal::pload<Packet>(b), qc = internal::pload<Packet>(c); - FloatPacket pb = internal::pload<FloatPacket>(fb), pc = internal::pload<FloatPacket>(fc); for (unsigned first = 0; first < 65536; first += packet_size) { for (int i = 0; i < packet_size; ++i) { a[i] = numext::bit_cast<bfloat16>(static_cast<numext::uint16_t>(first + i)); fa[i] = static_cast<float>(a[i]); } Packet qa = internal::pload<Packet>(a); - FloatPacket pa = internal::pload<FloatPacket>(fa); - verify(internal::padd(qa, qb), internal::padd(pa, pb)); - verify(internal::psub(qa, qb), internal::psub(pa, pb)); - verify(internal::pmul(qa, qb), internal::pmul(pa, pb)); - verify(internal::pdiv(qa, qb), internal::pdiv(pa, pb)); - verify(internal::pabsdiff(qa, qb), internal::pabsdiff(pa, pb)); - verify(internal::pmadd(qa, qb, qc), internal::pmadd(pa, pb, pc)); - verify(internal::pmsub(qa, qb, qc), internal::pmsub(pa, pb, pc)); - verify(internal::pnmadd(qa, qb, qc), internal::pnmadd(pa, pb, pc)); - verify(internal::pnmsub(qa, qb, qc), internal::pnmsub(pa, pb, pc)); + using P = const FloatPacket&; + verify(internal::padd(qa, qb), [](P x, P y, P) { return internal::padd(x, y); }); + verify(internal::psub(qa, qb), [](P x, P y, P) { return internal::psub(x, y); }); + verify(internal::pmul(qa, qb), [](P x, P y, P) { return internal::pmul(x, y); }); + verify(internal::pdiv(qa, qb), [](P x, P y, P) { return internal::pdiv(x, y); }); + verify(internal::pabsdiff(qa, qb), [](P x, P y, P) { return internal::pabsdiff(x, y); }); + verify(internal::pmadd(qa, qb, qc), [](P x, P y, P z) { return internal::pmadd(x, y, z); }); + verify(internal::pmsub(qa, qb, qc), [](P x, P y, P z) { return internal::pmsub(x, y, z); }); + verify(internal::pnmadd(qa, qb, qc), [](P x, P y, P z) { return internal::pnmadd(x, y, z); }); + verify(internal::pnmsub(qa, qb, qc), [](P x, P y, P z) { return internal::pnmsub(x, y, z); }); } } } @@ -3203,8 +3208,8 @@ // enable this claim correctly rounded results. With flushed set, the FPU reads a subnormal input as a zero of its // sign, and so does the reference; a kernel that passes such a lane through unchanged, as the AVX-512 fast-math // psqrt does under DAZ, is accepted too. +template <typename Packet> void packetmath_bfloat16_sqrt_exhaustive(bool flushed) { - using Packet = internal::packet_traits<bfloat16>::type; constexpr int packet_size = internal::unpacket_traits<Packet>::size; EIGEN_ALIGN_MAX bfloat16 input[packet_size], sqrt_output[packet_size], rsqrt_output[packet_size]; for (unsigned first = 0; first < 65536; first += packet_size) { @@ -3234,10 +3239,11 @@ } } +template <typename Packet> void packetmath_bfloat16_sqrt_exhaustive_flushed() { ScopedFlushToZero flush_to_zero(FlushToZeroMode::Both); if (flush_to_zero.mode() != FlushToZeroMode::Both) return; - packetmath_bfloat16_sqrt_exhaustive(true); + packetmath_bfloat16_sqrt_exhaustive<Packet>(true); } #endif @@ -3322,10 +3328,15 @@ #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) || defined(EIGEN_VECTORIZE_NEON) || \ defined(EIGEN_VECTORIZE_SME) - CALL_SUBTEST_15(packetmath_bfloat16_arithmetic_rounding()); + using Bf16Packet = internal::packet_traits<bfloat16>::type; + using Bf16HalfPacket = internal::packet_traits<bfloat16>::half; + CALL_SUBTEST_15(packetmath_bfloat16_arithmetic_rounding<Bf16Packet>()); + CALL_SUBTEST_15(packetmath_bfloat16_arithmetic_rounding<Bf16HalfPacket>()); // 32-bit ARM NEON flushes subnormals whatever FPSCR says. - CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive(EIGEN_ARCH_ARM != 0)); - CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive_flushed()); + CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive<Bf16Packet>(EIGEN_ARCH_ARM != 0)); + CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive<Bf16HalfPacket>(EIGEN_ARCH_ARM != 0)); + CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive_flushed<Bf16Packet>()); + CALL_SUBTEST_15(packetmath_bfloat16_sqrt_exhaustive_flushed<Bf16HalfPacket>()); #endif #if defined(EIGEN_VECTORIZE_RVV10)
diff --git a/test/packetmath_fastmath.cpp b/test/packetmath_fastmath.cpp index bd3b055..c64f8e7 100644 --- a/test/packetmath_fastmath.cpp +++ b/test/packetmath_fastmath.cpp
@@ -387,6 +387,7 @@ CALL_SUBTEST((verify_mask_reduction<float, Eigen::internal::Packet2f>())); CALL_SUBTEST((verify_low_bit_mask_any_ftz<float, Eigen::internal::Packet2f>())); CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::bfloat16, Eigen::internal::Packet4bf>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::bfloat16, Eigen::internal::Packet8bf>())); #if EIGEN_ARCH_ARM64 CALL_SUBTEST((verify_low_bit_mask_any_ftz<double, Eigen::internal::Packet2d>())); #endif