Core: Reduce packet truth masks through integer bits libeigen/eigen!3007
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index 4df0e31..98ab1a2 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -1571,10 +1571,33 @@ #undef EIGEN_BINARY_OP_NAN_PROPAGATION -/** \internal \returns true if all coeffs of \a a means "true" - * It is supposed to be called on values returned by pcmp_*. - */ -// TODO: implement predux_all when needed. +template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type, bool>::value> +struct predux_count_impl { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index run(const Packet& a) { + using Scalar = typename unpacket_traits<Packet>::type; + const Packet true_values = pandnot(pset1<Packet>(Scalar(1)), pcmp_eq(a, pzero(a))); + return static_cast<Index>(numext::real(predux(true_values))); + } +}; + +template <typename Packet> +struct predux_count_impl<Packet, true> { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index run(const Packet& a) { + using Scalar = typename unpacket_traits<Packet>::type; + constexpr int PacketSize = unpacket_traits<Packet>::size; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[PacketSize]; + pstoreu<Scalar>(values, a); + Index result = 0; + for (int i = 0; i < PacketSize; ++i) result += values[i] ? 1 : 0; + return result; + } +}; + +/** \internal \returns the number of nonzero coefficients in \a a. */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index predux_count(const Packet& a) { + return predux_count_impl<Packet>::run(a); +} /** \internal \returns true if any coeffs of \a a means "true" * It is supposed to be called on values returned by pcmp_*. @@ -1586,11 +1609,28 @@ // - Scalar(1) // - bits full of ones (NaN for floats), // - or first bit equals to 1 (1 for ints, smallest denormal for floats). - // For all these cases, taking the sum is just fine, and this boils down to a no-op for scalars. + // This arithmetic fallback boils down to a no-op for scalars. Vector backends whose masks use floating-point bit + // patterns must specialize this with an integer-bit reduction because fast-math or FTZ can discard those values. using Scalar = typename unpacket_traits<Packet>::type; return numext::not_equal_strict(predux(a), Scalar(0)); } +template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type, bool>::value> +struct predux_all_impl { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(const Packet& a) { return !predux_any(pcmp_eq(a, pzero(a))); } +}; + +template <typename Packet> +struct predux_all_impl<Packet, true> { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(const Packet& a) { return predux_mul(a); } +}; + +/** \internal \returns true if every coefficient in \a a is nonzero. */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_all(const Packet& a) { + return predux_all_impl<Packet>::run(a); +} + /*************************************************************************** * The following functions might not have to be overwritten for vectorized types ***************************************************************************/
diff --git a/Eigen/src/Core/Visitor.h b/Eigen/src/Core/Visitor.h index c0cc168..43a9311 100644 --- a/Eigen/src/Core/Visitor.h +++ b/Eigen/src/Core/Visitor.h
@@ -423,7 +423,7 @@ using Packet = typename packet_traits<Scalar>::type; EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); } EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); } - EIGEN_DEVICE_FUNC inline bool all_predux(const Packet& p) const { return !predux_any(pcmp_eq(p, pzero(p))); } + EIGEN_DEVICE_FUNC inline bool all_predux(const Packet& p) const { return predux_all(p); } EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = all_predux(p); } EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = all_predux(p); } EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res && (value != Scalar(0)); } @@ -433,12 +433,6 @@ EIGEN_DEVICE_FUNC inline bool done() const { return !res; } bool res = true; }; -// Bool packets already contain the truth values that all()/any() need, so avoid constructing an equivalent comparison -// mask before reducing them. -template <> -EIGEN_DEVICE_FUNC inline bool all_visitor<bool>::all_predux(const Packet& p) const { - return predux_mul(p); -} template <typename Scalar> struct functor_traits<all_visitor<Scalar>> { enum { Cost = NumTraits<Scalar>::ReadCost, LinearAccess = true, PacketAccess = packet_traits<Scalar>::HasCmp }; @@ -477,12 +471,7 @@ using Packet = typename packet_traits<Scalar>::type; EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = value != Scalar(0) ? 1 : 0; } EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = value != Scalar(0) ? 1 : 0; } - EIGEN_DEVICE_FUNC inline Index count_redux(const Packet& p) const { - const Packet cst_one = pset1<Packet>(Scalar(1)); - Packet true_vals = pandnot(cst_one, pcmp_eq(p, pzero(p))); - Scalar num_true = predux(true_vals); - return static_cast<Index>(num_true); - } + EIGEN_DEVICE_FUNC inline Index count_redux(const Packet& p) const { return predux_count(p); } EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = count_redux(p); } EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = count_redux(p); } EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { @@ -501,8 +490,7 @@ enum { Cost = NumTraits<Scalar>::AddCost, LinearAccess = true, - // predux is problematic for bool - PacketAccess = packet_traits<Scalar>::HasCmp && packet_traits<Scalar>::HasAdd && !std::is_same<Scalar, bool>::value + PacketAccess = packet_traits<Scalar>::HasCmp && packet_traits<Scalar>::HasAdd }; };
diff --git a/Eigen/src/Core/arch/AVX/Reductions.h b/Eigen/src/Core/arch/AVX/Reductions.h index f1fa8e1..38b7e9b 100644 --- a/Eigen/src/Core/arch/AVX/Reductions.h +++ b/Eigen/src/Core/arch/AVX/Reductions.h
@@ -189,6 +189,13 @@ return _mm256_movemask_ps(a) != 0x0; } +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet8f& a) { + const unsigned int mask = + static_cast<unsigned int>(_mm256_movemask_ps(_mm256_cmp_ps(a, _mm256_setzero_ps(), _CMP_NEQ_UQ))); + return Index(popcount(mask)); +} + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet4d -- -- -- -- -- -- -- -- -- -- -- -- */ template <> @@ -252,6 +259,13 @@ return _mm256_movemask_pd(a) != 0x0; } +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet4d& a) { + const unsigned int mask = + static_cast<unsigned int>(_mm256_movemask_pd(_mm256_cmp_pd(a, _mm256_setzero_pd(), _CMP_NEQ_UQ))); + return Index(popcount(mask)); +} + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet8h -- -- -- -- -- -- -- -- -- -- -- -- */ #ifndef EIGEN_VECTORIZE_AVX512FP16
diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h index ac87973..2b1e2fc 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h
@@ -736,6 +736,38 @@ return half(_mm_reduce_add_ph(a)); } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet32h& a) { + return avx512_predux_any(_mm512_castph_si512(a)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16h& a) { + const __m256i bits = _mm256_castph_si256(a); + return _mm256_testz_si256(bits, bits) == 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8h& a) { + const __m128i bits = _mm_castph_si128(a); + return _mm_testz_si128(bits, bits) == 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet32h& a) { + return _mm512_cmp_ph_mask(a, _mm512_setzero_ph(), _CMP_EQ_OQ) == 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet16h& a) { + return _mm256_cmp_ph_mask(a, _mm256_setzero_ph(), _CMP_EQ_OQ) == 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet8h& a) { + return _mm_cmp_ph_mask(a, _mm_setzero_ph(), _CMP_EQ_OQ) == 0; +} + // predux_half template <> EIGEN_STRONG_INLINE Packet16h predux_half<Packet32h>(const Packet32h& a) {
diff --git a/Eigen/src/Core/arch/AVX512/Reductions.h b/Eigen/src/Core/arch/AVX512/Reductions.h index 5871f04..9ba423a 100644 --- a/Eigen/src/Core/arch/AVX512/Reductions.h +++ b/Eigen/src/Core/arch/AVX512/Reductions.h
@@ -48,6 +48,11 @@ return avx512_predux_any(a); } +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet16i& a) { + return _mm512_cmp_epi32_mask(a, _mm512_setzero_epi32(), _MM_CMPINT_EQ) == 0; +} + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet8l -- -- -- -- -- -- -- -- -- -- -- -- */ template <> @@ -89,6 +94,11 @@ return avx512_predux_any(a); } +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet8l& a) { + return _mm512_cmp_epi64_mask(a, _mm512_setzero_si512(), _MM_CMPINT_EQ) == 0; +} + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet16f -- -- -- -- -- -- -- -- -- -- -- -- */ template <> @@ -144,6 +154,16 @@ return avx512_predux_any(_mm512_castps_si512(a)); } +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet16f& a) { + return _mm512_cmp_ps_mask(a, _mm512_setzero_ps(), _CMP_EQ_OQ) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet16f& a) { + return Index(popcount(static_cast<unsigned int>(_mm512_cmp_ps_mask(a, _mm512_setzero_ps(), _CMP_NEQ_UQ)))); +} + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet8d -- -- -- -- -- -- -- -- -- -- -- -- */ template <> @@ -199,6 +219,16 @@ return avx512_predux_any(_mm512_castpd_si512(a)); } +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet8d& a) { + return _mm512_cmp_pd_mask(a, _mm512_setzero_pd(), _CMP_EQ_OQ) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet8d& a) { + return Index(popcount(static_cast<unsigned int>(_mm512_cmp_pd_mask(a, _mm512_setzero_pd(), _CMP_NEQ_UQ)))); +} + #ifndef EIGEN_VECTORIZE_AVX512FP16 /* -- -- -- -- -- -- -- -- -- -- -- -- Packet16h -- -- -- -- -- -- -- -- -- -- -- -- */ @@ -246,6 +276,11 @@ EIGEN_STRONG_INLINE bool predux_any(const Packet16h& a) { return predux_any<Packet8i>(a.m_val); } + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet16h& a) { + return predux_all(half2float(a)); +} #endif /* -- -- -- -- -- -- -- -- -- -- -- -- Packet16bf -- -- -- -- -- -- -- -- -- -- -- -- */ @@ -295,6 +330,11 @@ return predux_any<Packet8i>(a.m_val); } +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet16bf& a) { + return predux_all(Bf16ToF32(a)); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AltiVec/PacketMath.h b/Eigen/src/Core/arch/AltiVec/PacketMath.h index 65040a4..6c65669 100644 --- a/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/Eigen/src/Core/arch/AltiVec/PacketMath.h
@@ -2767,7 +2767,14 @@ template <> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) { - return vec_any_ne(x, pzero(x)); + const Packet4ui zero = {0, 0, 0, 0}; + return vec_any_ne(reinterpret_cast<Packet4ui>(x), zero); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8bf& x) { + const Packet8us zero = {0, 0, 0, 0, 0, 0, 0, 0}; + return vec_any_ne(x.m_val, zero); } template <typename T> @@ -3641,6 +3648,12 @@ return pfirst<Packet2d>(sum); } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + const Packet2ul zero = {0, 0}; + return vec_any_ne(reinterpret_cast<Packet2ul>(a), zero); +} + // Other reduction functions: // mul template <>
diff --git a/Eigen/src/Core/arch/GPU/PacketMath.h b/Eigen/src/Core/arch/GPU/PacketMath.h index c66bbe0..7f23760 100644 --- a/Eigen/src/Core/arch/GPU/PacketMath.h +++ b/Eigen/src/Core/arch/GPU/PacketMath.h
@@ -244,6 +244,16 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_le<double2>(const double2& a, const double2& b) { return make_double2(le_mask(a.x, b.x), le_mask(a.y, b.y)); } + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const float4& a) { + return (__float_as_int(a.x) | __float_as_int(a.y) | __float_as_int(a.z) | __float_as_int(a.w)) != 0; +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const double2& a) { + return (__double_as_longlong(a.x) | __double_as_longlong(a.y)) != 0; +} #endif // EIGEN_HAS_GPU_DEVICE_FUNCTIONS template <> @@ -841,6 +851,12 @@ return __hadd(__low2half(a), __high2half(a)); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const half2& a) { + const Eigen::half low(__low2half(a)); + const Eigen::half high(__high2half(a)); + return (half_impl::raw_half_as_uint16(low) | half_impl::raw_half_as_uint16(high)) != 0; +} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max(const half2& a) { __half first = __low2half(a); __half second = __high2half(a); @@ -1357,6 +1373,12 @@ } template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const Packet4h2& a) { + const half2* a_alias = reinterpret_cast<const half2*>(&a); + return predux_any(a_alias[0]) | predux_any(a_alias[1]) | predux_any(a_alias[2]) | predux_any(a_alias[3]); +} + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4h2>(const Packet4h2& a) { const half2* a_alias = reinterpret_cast<const half2*>(&a); half2 m0 = __halves2half2(predux_max(a_alias[0]), predux_max(a_alias[1]));
diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h index 6a55221..f2789b9 100644 --- a/Eigen/src/Core/arch/HVX/PacketMath.h +++ b/Eigen/src/Core/arch/HVX/PacketMath.h
@@ -1007,17 +1007,27 @@ return predux_generic(a, pmin<Packet8f>); } +template <HVXPacketSize T> +EIGEN_STRONG_INLINE bool predux_any_hvx(const HVXPacket<T>& a) { + const Index packet_size = unpacket_traits<HVXPacket<T>>::size; + HVX_Vector reduced = a.Get(); + for (int i = 1; i < packet_size; i <<= 1) { + reduced |= Q6_V_vror_VR(reduced, i * sizeof(float)); + } + return Q6_R_vextract_VR(reduced, 0) != 0; +} + template <> EIGEN_STRONG_INLINE bool predux_any(const Packet32f& a) { - return predux_generic(a, por<Packet32f>) != 0.0f; + return predux_any_hvx(a); } template <> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& a) { - return predux_generic(a, por<Packet16f>) != 0.0f; + return predux_any_hvx(a); } template <> EIGEN_STRONG_INLINE bool predux_any(const Packet8f& a) { - return predux_generic(a, por<Packet8f>) != 0.0f; + return predux_any_hvx(a); } static const float index_vsf[32]
diff --git a/Eigen/src/Core/arch/LSX/PacketMath.h b/Eigen/src/Core/arch/LSX/PacketMath.h index bde4f8c..643c9b6 100644 --- a/Eigen/src/Core/arch/LSX/PacketMath.h +++ b/Eigen/src/Core/arch/LSX/PacketMath.h
@@ -1894,6 +1894,62 @@ EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst<Packet2d>(__lsx_vfadd_d(a, preverse(a))); } + +template <typename Packet> +EIGEN_STRONG_INLINE bool predux_any_lsx(const Packet& a) { + return __lsx_bnz_v((__m128i)a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16c& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8s& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2l& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16uc& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8us& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& a) { + return predux_any_lsx(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2ul& a) { + return predux_any_lsx(a); +} + template <> EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) { Packet8s tmp1 = __lsx_vhaddw_h_b(a, a);
diff --git a/Eigen/src/Core/arch/MSA/PacketMath.h b/Eigen/src/Core/arch/MSA/PacketMath.h index c7c844a..cce44b9 100644 --- a/Eigen/src/Core/arch/MSA/PacketMath.h +++ b/Eigen/src/Core/arch/MSA/PacketMath.h
@@ -554,6 +554,11 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) { + return __builtin_msa_bnz_v((v16u8)a); +} + +template <> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) { EIGEN_MSA_DEBUG; @@ -562,6 +567,11 @@ return s[0]; } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) { + return __builtin_msa_bnz_v((v16u8)a); +} + // Other reduction functions: // mul template <> @@ -1073,6 +1083,11 @@ return s[0]; } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + return __builtin_msa_bnz_v((v16u8)a); +} + // Other reduction functions: // mul template <>
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index f150d05..2973a85 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -3245,6 +3245,12 @@ EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { return vaddvq_f32(a); } + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet4f& a) { + const uint32x4_t nonzero = vbicq_u32(vdupq_n_u32(1), vceqq_f32(a, vdupq_n_f32(0.0f))); + return static_cast<Index>(vaddvq_u32(nonzero)); +} #else template <> EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) { @@ -3878,6 +3884,11 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2f& x) { + return vget_lane_u64(vreinterpret_u64_f32(x), 0) != 0; +} + +template <> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) { uint32x4_t u = vreinterpretq_u32_f32(x); #if EIGEN_ARCH_ARM64 @@ -4793,6 +4804,11 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4bf& a) { + return vget_lane_u64(vreinterpret_u64_u16(Packet4us(a)), 0) != 0; +} + +template <> EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(const Packet4bf& a) { return Packet4bf(preverse<Packet4us>(Packet4us(a))); } @@ -5155,6 +5171,18 @@ return vaddvq_f64(a); } +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet2d& a) { + // Each zero lane contributes UINT64_MAX, so unsigned 2 + sum(mask) is the number of nonzero lanes. + const uint64_t zeroMaskSum = vaddvq_u64(vceqq_f64(a, vdupq_n_f64(0.0))); + return static_cast<Index>(zeroMaskSum + 2); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + return vmaxvq_u32(vreinterpretq_u32_f64(a)) != 0; +} + // Other reduction functions: // mul #if EIGEN_COMP_CLANGAPPLE
diff --git a/Eigen/src/Core/arch/RVV10/PacketMath.h b/Eigen/src/Core/arch/RVV10/PacketMath.h index 01f16c3..8bbee9e 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMath.h +++ b/Eigen/src/Core/arch/RVV10/PacketMath.h
@@ -237,6 +237,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xi& a) { + const PacketMask32 mask = __riscv_vmseq_vx_i32m1_b32(a, 0, unpacket_traits<Packet1Xi>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xi>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet1Xi>(const Packet1Xi& a) { // Multiply the vector by its reverse Packet1Xi prod = __riscv_vmul_vv_i32m1(preverse(a), a, unpacket_traits<Packet1Xi>::size); @@ -600,6 +606,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet1Xf& a) { + const PacketMask32 mask = + __riscv_vmsne_vx_u32m1_b32(__riscv_vreinterpret_v_f32m1_u32m1(a), 0, unpacket_traits<Packet1Xf>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xf& a) { + const PacketMask32 mask = __riscv_vmfeq_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet1Xf& a) { + const PacketMask32 mask = __riscv_vmfne_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size); + return static_cast<Index>(__riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size)); +} + +template <> EIGEN_STRONG_INLINE float predux_mul<Packet1Xf>(const Packet1Xf& a) { // Multiply the vector by its reverse Packet1Xf prod = __riscv_vfmul_vv_f32m1(preverse(a), a, unpacket_traits<Packet1Xf>::size); @@ -914,6 +939,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xl& a) { + const PacketMask64 mask = __riscv_vmseq_vx_i64m1_b64(a, 0, unpacket_traits<Packet1Xl>::size); + return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xl>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet1Xl>(const Packet1Xl& a) { // Multiply the vector by its reverse Packet1Xl prod = __riscv_vmul_vv_i64m1(preverse(a), a, unpacket_traits<Packet1Xl>::size); @@ -1276,6 +1307,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet1Xd& a) { + const PacketMask64 mask = + __riscv_vmsne_vx_u64m1_b64(__riscv_vreinterpret_v_f64m1_u64m1(a), 0, unpacket_traits<Packet1Xd>::size); + return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xd& a) { + const PacketMask64 mask = __riscv_vmfeq_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size); + return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet1Xd& a) { + const PacketMask64 mask = __riscv_vmfne_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size); + return static_cast<Index>(__riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size)); +} + +template <> EIGEN_STRONG_INLINE double predux_mul<Packet1Xd>(const Packet1Xd& a) { // Multiply the vector by its reverse Packet1Xd prod = __riscv_vfmul_vv_f64m1(preverse(a), a, unpacket_traits<Packet1Xd>::size); @@ -1590,6 +1640,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xs& a) { + const PacketMask16 mask = __riscv_vmseq_vx_i16m1_b16(a, 0, unpacket_traits<Packet1Xs>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xs>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet1Xs>(const Packet1Xs& a) { // Multiply the vector by its reverse Packet1Xs prod = __riscv_vmul_vv_i16m1(preverse(a), a, unpacket_traits<Packet1Xs>::size);
diff --git a/Eigen/src/Core/arch/RVV10/PacketMath2.h b/Eigen/src/Core/arch/RVV10/PacketMath2.h index 1bb28bd..4af0751 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMath2.h +++ b/Eigen/src/Core/arch/RVV10/PacketMath2.h
@@ -237,6 +237,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xi& a) { + const PacketMask16 mask = __riscv_vmseq_vx_i32m2_b16(a, 0, unpacket_traits<Packet2Xi>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet2Xi>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet2Xi>(const Packet2Xi& a) { return predux_mul<Packet1Xi>(__riscv_vmul_vv_i32m1(__riscv_vget_v_i32m2_i32m1(a, 0), __riscv_vget_v_i32m2_i32m1(a, 1), unpacket_traits<Packet1Xi>::size)); @@ -594,6 +600,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2Xf& a) { + const PacketMask16 mask = + __riscv_vmsne_vx_u32m2_b16(__riscv_vreinterpret_v_f32m2_u32m2(a), 0, unpacket_traits<Packet2Xf>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet2Xf>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xf& a) { + const PacketMask16 mask = __riscv_vmfeq_vf_f32m2_b16(a, 0.0f, unpacket_traits<Packet2Xf>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet2Xf>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet2Xf& a) { + const PacketMask16 mask = __riscv_vmfne_vf_f32m2_b16(a, 0.0f, unpacket_traits<Packet2Xf>::size); + return static_cast<Index>(__riscv_vcpop_m_b16(mask, unpacket_traits<Packet2Xf>::size)); +} + +template <> EIGEN_STRONG_INLINE float predux_mul<Packet2Xf>(const Packet2Xf& a) { return predux_mul<Packet1Xf>(__riscv_vfmul_vv_f32m1( __riscv_vget_v_f32m2_f32m1(a, 0), __riscv_vget_v_f32m2_f32m1(a, 1), unpacket_traits<Packet1Xf>::size)); @@ -861,6 +886,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xl& a) { + const PacketMask32 mask = __riscv_vmseq_vx_i64m2_b32(a, 0, unpacket_traits<Packet2Xl>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet2Xl>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet2Xl>(const Packet2Xl& a) { return predux_mul<Packet1Xl>(__riscv_vmul_vv_i64m1(__riscv_vget_v_i64m2_i64m1(a, 0), __riscv_vget_v_i64m2_i64m1(a, 1), unpacket_traits<Packet1Xl>::size)); @@ -1221,6 +1252,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2Xd& a) { + const PacketMask32 mask = + __riscv_vmsne_vx_u64m2_b32(__riscv_vreinterpret_v_f64m2_u64m2(a), 0, unpacket_traits<Packet2Xd>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet2Xd>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xd& a) { + const PacketMask32 mask = __riscv_vmfeq_vf_f64m2_b32(a, 0.0, unpacket_traits<Packet2Xd>::size); + return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet2Xd>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet2Xd& a) { + const PacketMask32 mask = __riscv_vmfne_vf_f64m2_b32(a, 0.0, unpacket_traits<Packet2Xd>::size); + return static_cast<Index>(__riscv_vcpop_m_b32(mask, unpacket_traits<Packet2Xd>::size)); +} + +template <> EIGEN_STRONG_INLINE double predux_mul<Packet2Xd>(const Packet2Xd& a) { return predux_mul<Packet1Xd>(__riscv_vfmul_vv_f64m1( __riscv_vget_v_f64m2_f64m1(a, 0), __riscv_vget_v_f64m2_f64m1(a, 1), unpacket_traits<Packet1Xd>::size)); @@ -1493,6 +1543,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xs& a) { + const PacketMask8 mask = __riscv_vmseq_vx_i16m2_b8(a, 0, unpacket_traits<Packet2Xs>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet2Xs>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet2Xs>(const Packet2Xs& a) { return predux_mul<Packet1Xs>(__riscv_vmul_vv_i16m1(__riscv_vget_v_i16m2_i16m1(a, 0), __riscv_vget_v_i16m2_i16m1(a, 1), unpacket_traits<Packet1Xs>::size));
diff --git a/Eigen/src/Core/arch/RVV10/PacketMath4.h b/Eigen/src/Core/arch/RVV10/PacketMath4.h index 7363d55..0cec0de 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMath4.h +++ b/Eigen/src/Core/arch/RVV10/PacketMath4.h
@@ -238,6 +238,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet4Xi& a) { + const PacketMask8 mask = __riscv_vmseq_vx_i32m4_b8(a, 0, unpacket_traits<Packet4Xi>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xi>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet4Xi>(const Packet4Xi& a) { Packet1Xi half1 = __riscv_vmul_vv_i32m1(__riscv_vget_v_i32m4_i32m1(a, 0), __riscv_vget_v_i32m4_i32m1(a, 1), unpacket_traits<Packet1Xi>::size); @@ -594,6 +600,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4Xf& a) { + const PacketMask8 mask = + __riscv_vmsne_vx_u32m4_b8(__riscv_vreinterpret_v_f32m4_u32m4(a), 0, unpacket_traits<Packet4Xf>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet4Xf& a) { + const PacketMask8 mask = __riscv_vmfeq_vf_f32m4_b8(a, 0.0f, unpacket_traits<Packet4Xf>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet4Xf& a) { + const PacketMask8 mask = __riscv_vmfne_vf_f32m4_b8(a, 0.0f, unpacket_traits<Packet4Xf>::size); + return static_cast<Index>(__riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size)); +} + +template <> EIGEN_STRONG_INLINE float predux_mul<Packet4Xf>(const Packet4Xf& a) { Packet1Xf half1 = __riscv_vfmul_vv_f32m1(__riscv_vget_v_f32m4_f32m1(a, 0), __riscv_vget_v_f32m4_f32m1(a, 1), unpacket_traits<Packet1Xf>::size); @@ -866,6 +891,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet4Xl& a) { + const PacketMask16 mask = __riscv_vmseq_vx_i64m4_b16(a, 0, unpacket_traits<Packet4Xl>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xl>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet4Xl>(const Packet4Xl& a) { Packet1Xl half1 = __riscv_vmul_vv_i64m1(__riscv_vget_v_i64m4_i64m1(a, 0), __riscv_vget_v_i64m4_i64m1(a, 1), unpacket_traits<Packet1Xl>::size); @@ -1223,6 +1254,25 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4Xd& a) { + const PacketMask16 mask = + __riscv_vmsne_vx_u64m4_b16(__riscv_vreinterpret_v_f64m4_u64m4(a), 0, unpacket_traits<Packet4Xd>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet4Xd& a) { + const PacketMask16 mask = __riscv_vmfeq_vf_f64m4_b16(a, 0.0, unpacket_traits<Packet4Xd>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size) == 0; +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet4Xd& a) { + const PacketMask16 mask = __riscv_vmfne_vf_f64m4_b16(a, 0.0, unpacket_traits<Packet4Xd>::size); + return static_cast<Index>(__riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size)); +} + +template <> EIGEN_STRONG_INLINE double predux_mul<Packet4Xd>(const Packet4Xd& a) { Packet1Xd half1 = __riscv_vfmul_vv_f64m1(__riscv_vget_v_f64m4_f64m1(a, 0), __riscv_vget_v_f64m4_f64m1(a, 1), unpacket_traits<Packet1Xd>::size); @@ -1492,6 +1542,12 @@ } template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet4Xs& a) { + const PacketMask4 mask = __riscv_vmseq_vx_i16m4_b4(a, 0, unpacket_traits<Packet4Xs>::size); + return __riscv_vcpop_m_b4(mask, unpacket_traits<Packet4Xs>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet4Xs>(const Packet4Xs& a) { Packet1Xs half1 = __riscv_vmul_vv_i16m1(__riscv_vget_v_i16m4_i16m1(a, 0), __riscv_vget_v_i16m4_i16m1(a, 1), unpacket_traits<Packet1Xs>::size);
diff --git a/Eigen/src/Core/arch/RVV10/PacketMathBF16.h b/Eigen/src/Core/arch/RVV10/PacketMathBF16.h index 1d36e35..202f83f 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMathBF16.h +++ b/Eigen/src/Core/arch/RVV10/PacketMathBF16.h
@@ -396,6 +396,19 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet1Xbf& a) { + const PacketMask16 mask = + __riscv_vmsne_vx_u16m1_b16(__riscv_vreinterpret_v_bf16m1_u16m1(a), 0, unpacket_traits<Packet1Xbf>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xbf>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xbf& a) { + const PacketMask16 mask = __riscv_vmfeq_vf_f32m2_b16(Bf16ToF32(a), 0.0f, unpacket_traits<Packet1Xbf>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xbf>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet1Xbf>(const Packet1Xbf& a) { return static_cast<bfloat16>(predux_mul<Packet2Xf>(Bf16ToF32(a))); } @@ -744,6 +757,19 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2Xbf& a) { + const PacketMask8 mask = + __riscv_vmsne_vx_u16m2_b8(__riscv_vreinterpret_v_bf16m2_u16m2(a), 0, unpacket_traits<Packet2Xbf>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet2Xbf>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xbf& a) { + const PacketMask8 mask = __riscv_vmfeq_vf_f32m4_b8(Bf16ToF32(a), 0.0f, unpacket_traits<Packet2Xbf>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet2Xbf>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet2Xbf>(const Packet2Xbf& a) { return static_cast<bfloat16>(predux_mul<Packet4Xf>(Bf16ToF32(a))); }
diff --git a/Eigen/src/Core/arch/RVV10/PacketMathFP16.h b/Eigen/src/Core/arch/RVV10/PacketMathFP16.h index 646c66a..0b6fdd5 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMathFP16.h +++ b/Eigen/src/Core/arch/RVV10/PacketMathFP16.h
@@ -381,6 +381,19 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet1Xh& a) { + const PacketMask16 mask = + __riscv_vmsne_vx_u16m1_b16(__riscv_vreinterpret_v_f16m1_u16m1(a), 0, unpacket_traits<Packet1Xh>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xh>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet1Xh& a) { + const PacketMask16 mask = __riscv_vmfeq_vf_f16m1_b16(a, static_cast<_Float16>(0.0), unpacket_traits<Packet1Xh>::size); + return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xh>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet1Xh>(const Packet1Xh& a) { // Multiply the vector by its reverse Packet1Xh prod = __riscv_vfmul_vv_f16m1(preverse(a), a, unpacket_traits<Packet1Xh>::size); @@ -751,6 +764,19 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2Xh& a) { + const PacketMask8 mask = + __riscv_vmsne_vx_u16m2_b8(__riscv_vreinterpret_v_f16m2_u16m2(a), 0, unpacket_traits<Packet2Xh>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet2Xh>::size) != 0; +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const Packet2Xh& a) { + const PacketMask8 mask = __riscv_vmfeq_vf_f16m2_b8(a, static_cast<_Float16>(0.0), unpacket_traits<Packet2Xh>::size); + return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet2Xh>::size) == 0; +} + +template <> EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet2Xh>(const Packet2Xh& a) { return predux_mul<Packet1Xh>(__riscv_vfmul_vv_f16m1( __riscv_vget_v_f16m2_f16m1(a, 0), __riscv_vget_v_f16m2_f16m1(a, 1), unpacket_traits<Packet1Xh>::size));
diff --git a/Eigen/src/Core/arch/SSE/Reductions.h b/Eigen/src/Core/arch/SSE/Reductions.h index 9324c72..e0126de 100644 --- a/Eigen/src/Core/arch/SSE/Reductions.h +++ b/Eigen/src/Core/arch/SSE/Reductions.h
@@ -34,6 +34,14 @@ // Packet16b stores one bool per byte. Reduce the byte-wise zero mask rather than extracting and short-circuiting two // scalar halves. This also treats every nonzero byte as true, matching the scalar reduction for non-canonical inputs. template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet16b& a) { + const __m128i normalized = _mm_min_epu8(a, _mm_set1_epi8(1)); + const __m128i sums = _mm_sad_epu8(normalized, _mm_setzero_si128()); + return static_cast<Index>(_mm_cvtsi128_si32(sums)) + + static_cast<Index>(_mm_cvtsi128_si32(_mm_unpackhi_epi64(sums, sums))); +} + +template <> EIGEN_STRONG_INLINE bool predux(const Packet16b& a) { return _mm_movemask_epi8(_mm_cmpeq_epi8(a, _mm_setzero_si128())) != 0xffff; } @@ -210,6 +218,14 @@ return _mm_movemask_ps(a) != 0x0; } +#ifdef EIGEN_VECTORIZE_SSE4_2 +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet4f& a) { + const unsigned int mask = static_cast<unsigned int>(_mm_movemask_ps(_mm_cmpneq_ps(a, _mm_setzero_ps()))); + return Index(popcount(mask)); +} +#endif + /* -- -- -- -- -- -- -- -- -- -- -- -- Packet2d -- -- -- -- -- -- -- -- -- -- -- -- */ // The 2->1 step is not packed: a packed step pins the result in a vector register, so @@ -290,6 +306,14 @@ return _mm_movemask_pd(a) != 0x0; } +#ifdef EIGEN_VECTORIZE_SSE4_2 +template <> +EIGEN_STRONG_INLINE Index predux_count(const Packet2d& a) { + const unsigned int mask = static_cast<unsigned int>(_mm_movemask_pd(_mm_cmpneq_pd(a, _mm_setzero_pd()))); + return Index(popcount(mask)); +} +#endif + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/SVE/PacketMath.h b/Eigen/src/Core/arch/SVE/PacketMath.h index 15f4ab5..88bcc01 100644 --- a/Eigen/src/Core/arch/SVE/PacketMath.h +++ b/Eigen/src/Core/arch/SVE/PacketMath.h
@@ -298,6 +298,11 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const PacketXi& a) { + return svptest_any(svptrue_b32(), svcmpne_n_s32(svptrue_b32(), a, 0)); +} + +template <> EIGEN_STRONG_INLINE numext::int32_t predux_mul<PacketXi>(const PacketXi& a) { // Multiply the vector by its reverse. svint32_t prod = svmul_s32_x(svptrue_b32(), a, svrev_s32(a)); @@ -918,6 +923,24 @@ return svaddv_f32(svptrue_b32(), a); } +template <> +EIGEN_STRONG_INLINE bool predux_any(const PacketXf& a) { + const svuint32_t bits = svreinterpret_u32_f32(a); + return svptest_any(svptrue_b32(), svcmpne_n_u32(svptrue_b32(), bits, 0)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const PacketXf& a) { + const svbool_t all = svptrue_b32(); + return !svptest_any(all, svcmpeq_n_f32(all, a, 0.0f)); +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const PacketXf& a) { + const svbool_t all = svptrue_b32(); + return static_cast<Index>(svcntp_b32(all, svcmpne_n_f32(all, a, 0.0f))); +} + // Other reduction functions: // mul template <> @@ -1271,6 +1294,24 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const PacketXd& a) { + const svuint64_t bits = svreinterpret_u64_f64(a); + return svptest_any(svptrue_b64(), svcmpne_n_u64(svptrue_b64(), bits, 0)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_all(const PacketXd& a) { + const svbool_t all = svptrue_b64(); + return !svptest_any(all, svcmpeq_n_f64(all, a, 0.0)); +} + +template <> +EIGEN_STRONG_INLINE Index predux_count(const PacketXd& a) { + const svbool_t all = svptrue_b64(); + return static_cast<Index>(svcntp_b64(all, svcmpne_n_f64(all, a, 0.0))); +} + +template <> EIGEN_STRONG_INLINE double predux_mul<PacketXd>(const PacketXd& a) { // Multiply the vector by its reverse. svfloat64_t prod = svmul_f64_x(svptrue_b64(), a, svrev_f64(a));
diff --git a/Eigen/src/Core/arch/SYCL/PacketMath.h b/Eigen/src/Core/arch/SYCL/PacketMath.h index ba730de..0c1bba8 100644 --- a/Eigen/src/Core/arch/SYCL/PacketMath.h +++ b/Eigen/src/Core/arch/SYCL/PacketMath.h
@@ -310,6 +310,21 @@ } template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predux_any(const cl::sycl::cl_half8& a) { + return cl::sycl::any(a.template as<cl::sycl::cl_short8>() != cl::sycl::cl_short8(0)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predux_any(const cl::sycl::cl_float4& a) { + return cl::sycl::any(a.template as<cl::sycl::cl_int4>() != cl::sycl::cl_int4(0)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predux_any(const cl::sycl::cl_double2& a) { + return cl::sycl::any(a.template as<cl::sycl::cl_long2>() != cl::sycl::cl_long2(0)); +} + +template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half predux_max<cl::sycl::cl_half8>(const cl::sycl::cl_half8& a) { return Eigen::half(cl::sycl::fmax(cl::sycl::fmax(cl::sycl::fmax(a.s0(), a.s1()), cl::sycl::fmax(a.s2(), a.s3())), cl::sycl::fmax(cl::sycl::fmax(a.s4(), a.s5()), cl::sycl::fmax(a.s6(), a.s7()))));
diff --git a/Eigen/src/Core/arch/ZVector/PacketMath.h b/Eigen/src/Core/arch/ZVector/PacketMath.h index 984a8a0..19ece0f 100644 --- a/Eigen/src/Core/arch/ZVector/PacketMath.h +++ b/Eigen/src/Core/arch/ZVector/PacketMath.h
@@ -667,6 +667,12 @@ return pfirst(sum); } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + const Packet2ul zero = {0, 0}; + return vec_any_ne(reinterpret_cast<Packet2ul>(a), zero); +} + // Other reduction functions: // mul template <> @@ -990,6 +996,11 @@ } template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) { + return predux_any(a.v4f[0]) | predux_any(a.v4f[1]); +} + +template <> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) { // Return predux_mul<Packet2d> of the subvectors product return static_cast<float>(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1])))); @@ -1234,6 +1245,12 @@ return pfirst(sum); } +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) { + const Packet4ui zero = {0, 0, 0, 0}; + return vec_any_ne(reinterpret_cast<Packet4ui>(a), zero); +} + // Other reduction functions: // mul template <>
diff --git a/benchmarks/Core/bench_boolean_reductions.cpp b/benchmarks/Core/bench_boolean_reductions.cpp index f2b1a4d..4c79c4f 100644 --- a/benchmarks/Core/bench_boolean_reductions.cpp +++ b/benchmarks/Core/bench_boolean_reductions.cpp
@@ -143,6 +143,12 @@ EIGEN_BENCH_AVX512_PACKET(internal::Packet16f); EIGEN_BENCH_AVX512_PACKET(internal::Packet8d); +#ifdef EIGEN_VECTORIZE_AVX512FP16 +EIGEN_BENCH_AVX512_PACKET(internal::Packet32h); +EIGEN_BENCH_AVX512_PACKET(internal::Packet16h); +EIGEN_BENCH_AVX512_PACKET(internal::Packet8h); +#endif + #undef EIGEN_BENCH_AVX512_PACKET #endif
diff --git a/test/gpu_basic.cu b/test/gpu_basic.cu index 0ac9876..06d60a4 100644 --- a/test/gpu_basic.cu +++ b/test/gpu_basic.cu
@@ -552,6 +552,25 @@ } }; +template <typename Scalar> +struct packet_any_test { + EIGEN_DEVICE_FUNC void operator()(int i, const float* /*in*/, float* out) const { +#if defined(EIGEN_GPU_COMPILE_PHASE) + using Packet = typename Eigen::internal::packet_traits<Scalar>::type; + const Packet zero = Eigen::internal::pzero(Packet()); + const Packet sequence = Eigen::internal::plset<Packet>(Scalar(0)); + const Packet mixed = Eigen::internal::pcmp_eq(sequence, Eigen::internal::pset1<Packet>(Scalar(1))); + out[3 * i] = float(Eigen::internal::predux_any(zero)); + out[3 * i + 1] = float(Eigen::internal::predux_any(Eigen::internal::ptrue(zero))); + out[3 * i + 2] = float(Eigen::internal::predux_any(mixed)); +#else + out[3 * i] = 0; + out[3 * i + 1] = 1; + out[3 * i + 2] = 1; +#endif + } +}; + template <typename T> void test_reverse() { typedef typename T::Scalar Scalar; @@ -649,6 +668,9 @@ CALL_SUBTEST((test_reverse<Eigen::Array<float, 32, 1>>())); CALL_SUBTEST((test_reverse<Eigen::Array<double, 32, 1>>())); CALL_SUBTEST((test_reverse<Eigen::Array<Eigen::half, 32, 1>>())); + CALL_SUBTEST(run_and_compare_to_gpu(packet_any_test<float>(), nthreads, in, out)); + CALL_SUBTEST(run_and_compare_to_gpu(packet_any_test<double>(), nthreads, in, out)); + CALL_SUBTEST(run_and_compare_to_gpu(packet_any_test<Eigen::half>(), nthreads, in, out)); typedef Matrix<float, 6, 6> Matrix6f; CALL_SUBTEST(run_and_compare_to_gpu(selfadjoint_rank2_update<Matrix4f, Lower>(), nthreads, in, out));
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index cba4161..f445f01 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -1716,6 +1716,45 @@ VERIFY_IS_EQUAL(internal::predux_max(negative_infinity), -infinity); } +#if defined(EIGEN_VECTORIZE_SSE2) +void packetmath_packet16b_reductions() { + const internal::Packet16b packet(_mm_setr_epi8(0, 1, -1, 2, 0, -128, 127, 0, 0, 0, 3, -2, 0, 42, 0, -1)); + VERIFY_IS_EQUAL(internal::predux_count(packet), 9); + VERIFY(!internal::predux_all(packet)); + + const internal::Packet16b all_nonzero(_mm_set1_epi8(-1)); + VERIFY(internal::predux_all(all_nonzero)); + + EIGEN_ALIGN16 bool values[16]; + for (int i = 0; i < 16; ++i) values[i] = i % 3 != 0; + const internal::Packet16b canonical_packet = internal::pload<internal::Packet16b>(values); + VERIFY_IS_EQUAL(internal::predux_count_impl<internal::Packet16b>::run(canonical_packet), 10); +} + +void packetmath_packet16b_select() { + EIGEN_ALIGN16 bool condition[16]; + EIGEN_ALIGN16 bool then_values[16]; + EIGEN_ALIGN16 bool else_values[16]; + EIGEN_ALIGN16 bool actual[16]; + + for (int i = 0; i < 16; ++i) { + condition[i] = (i % 3) == 0; + then_values[i] = (i % 2) == 0; + else_values[i] = (i % 5) == 0; + } + + const internal::Packet16b condition_packet = internal::pload<internal::Packet16b>(condition); + const internal::Packet16b mask = internal::pcmp_eq(condition_packet, internal::pzero(condition_packet)); + const internal::Packet16b selected = internal::pselect(mask, internal::pload<internal::Packet16b>(else_values), + internal::pload<internal::Packet16b>(then_values)); + internal::pstore(actual, selected); + + for (int i = 0; i < 16; ++i) { + VERIFY_IS_EQUAL(actual[i], condition[i] ? then_values[i] : else_values[i]); + } +} +#endif + template <typename Scalar, typename Packet> void packetmath_notcomplex() { packetmath_ieee_special_values<Scalar, Packet>(); @@ -1762,15 +1801,13 @@ { unsigned char* data1_bits = reinterpret_cast<unsigned char*>(data1); - // predux_all - not needed yet - // for (unsigned int i=0; i<PacketSize*sizeof(Scalar); ++i) data1_bits[i] = 0xff; - // VERIFY(internal::predux_all(internal::pload<Packet>(data1)) && "internal::predux_all(1111)"); - // for(int k=0; k<PacketSize; ++k) - // { - // for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0x0; - // VERIFY( (!internal::predux_all(internal::pload<Packet>(data1))) && "internal::predux_all(0101)"); - // for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0xff; - // } + for (unsigned int i = 0; i < PacketSize * sizeof(Scalar); ++i) data1_bits[i] = 0xff; + VERIFY(internal::predux_all(internal::pload<Packet>(data1)) && "internal::predux_all(1111)"); + for (int k = 0; k < PacketSize; ++k) { + for (unsigned int i = 0; i < sizeof(Scalar); ++i) data1_bits[k * sizeof(Scalar) + i] = 0x0; + VERIFY((!internal::predux_all(internal::pload<Packet>(data1))) && "internal::predux_all(0101)"); + for (unsigned int i = 0; i < sizeof(Scalar); ++i) data1_bits[k * sizeof(Scalar) + i] = 0xff; + } // predux_any for (unsigned int i = 0; i < PacketSize * sizeof(Scalar); ++i) data1_bits[i] = 0x0; @@ -1778,8 +1815,24 @@ for (int k = 0; k < PacketSize; ++k) { for (unsigned int i = 0; i < sizeof(Scalar); ++i) data1_bits[k * sizeof(Scalar) + i] = 0xff; VERIFY(internal::predux_any(internal::pload<Packet>(data1)) && "internal::predux_any(0101)"); + VERIFY_IS_EQUAL(internal::predux_count(internal::pload<Packet>(data1)), k + 1); + } + for (int k = 0; k < PacketSize; ++k) { for (unsigned int i = 0; i < sizeof(Scalar); ++i) data1_bits[k * sizeof(Scalar) + i] = 0x00; } + data1[0] = Scalar(-0.0); + VERIFY_IS_EQUAL(internal::predux_count(internal::pload<Packet>(data1)), 0); + + for (int k = 0; k < PacketSize; ++k) { + data1[k] = Scalar(1); + VERIFY_IS_EQUAL(internal::predux_count(internal::pload<Packet>(data1)), k + 1); + } + + if (!NumTraits<Scalar>::IsInteger) { + for (int k = 0; k < PacketSize; ++k) data2[k] = Scalar(0); + data2[PacketSize - 1] = NumTraits<Scalar>::quiet_NaN(); + VERIFY_IS_EQUAL(internal::predux_count(internal::pload<Packet>(data2)), 1); + } } // Test NaN propagation. @@ -2286,6 +2339,10 @@ CALL_SUBTEST_13(test::runner<half>::run()); CALL_SUBTEST_14((packetmath<bool, internal::packet_traits<bool>::type>())); CALL_SUBTEST_14((packetmath_scatter_gather<bool, internal::packet_traits<bool>::type>())); +#if defined(EIGEN_VECTORIZE_SSE2) + CALL_SUBTEST_14(packetmath_packet16b_reductions()); + CALL_SUBTEST_14(packetmath_packet16b_select()); +#endif CALL_SUBTEST_15(test::runner<bfloat16>::run()); g_first_pass = false; }
diff --git a/test/packetmath_fastmath.cpp b/test/packetmath_fastmath.cpp index 0bdeac5..148f80d 100644 --- a/test/packetmath_fastmath.cpp +++ b/test/packetmath_fastmath.cpp
@@ -4,6 +4,7 @@ #include <cstring> #include "main.h" +#include "fp_control.h" template <typename Scalar, typename Packet> EIGEN_DONT_INLINE void store_ptrue(Scalar* output) { @@ -26,6 +27,63 @@ return Eigen::internal::predux_any(Eigen::internal::ploadu<Packet>(mask)); } +template <typename Scalar, typename Packet> +EIGEN_DONT_INLINE bool mask_all(const Scalar* mask) { + return Eigen::internal::predux_all(Eigen::internal::ploadu<Packet>(mask)); +} + +template <typename Scalar, typename Packet> +void verify_mask_reduction_impl() { + constexpr int packet_size = Eigen::internal::unpacket_traits<Packet>::size; + Scalar mask[packet_size]; + + std::memset(static_cast<void*>(mask), 0, sizeof(mask)); + VERIFY(!(mask_any<Scalar, Packet>(mask))); + VERIFY(!(mask_all<Scalar, Packet>(mask))); + VERIFY_IS_EQUAL(Eigen::internal::predux_count(Eigen::internal::ploadu<Packet>(mask)), 0); + + for (int lane = 0; lane < packet_size; ++lane) { + std::memset(static_cast<void*>(mask), 0, sizeof(mask)); + std::memset(static_cast<void*>(mask + lane), 0xff, sizeof(Scalar)); + VERIFY((mask_any<Scalar, Packet>(mask))); + VERIFY_IS_EQUAL((mask_all<Scalar, Packet>(mask)), packet_size == 1); + VERIFY_IS_EQUAL(Eigen::internal::predux_count(Eigen::internal::ploadu<Packet>(mask)), 1); + } + + std::memset(static_cast<void*>(mask), 0xff, sizeof(mask)); + VERIFY((mask_all<Scalar, Packet>(mask))); +} + +template <typename Scalar, typename Packet> +void verify_mask_reduction() { + verify_mask_reduction_impl<Scalar, Packet>(); + const Eigen::ScopedFlushToZero flush_to_zero; + if (flush_to_zero.isSupported()) verify_mask_reduction_impl<Scalar, Packet>(); +} + +// For bit-test backends, a low-bit truth mask must survive FTZ even though its floating-point encoding is subnormal. +// Do not apply this to backends that consume only the sign bits of canonical comparison masks. +template <typename Scalar, typename Packet> +void verify_low_bit_mask_any_ftz() { + const Eigen::ScopedFlushToZero flush_to_zero; + VERIFY(flush_to_zero.isSupported()); + constexpr int packet_size = Eigen::internal::unpacket_traits<Packet>::size; + using Bits = typename Eigen::numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type; + const Bits true_bits = 1; + Scalar mask[packet_size]; + std::memset(static_cast<void*>(mask), 0, sizeof(mask)); + VERIFY(!(mask_any<Scalar, Packet>(mask))); + for (int lane = 0; lane < packet_size; ++lane) { + std::memset(static_cast<void*>(mask), 0, sizeof(mask)); + std::memcpy(static_cast<void*>(mask + lane), &true_bits, sizeof(Scalar)); + VERIFY((mask_any<Scalar, Packet>(mask))); + } + for (int lane = 0; lane < packet_size; ++lane) { + std::memcpy(static_cast<void*>(mask + lane), &true_bits, sizeof(Scalar)); + } + VERIFY((mask_any<Scalar, Packet>(mask))); +} + // Keep the finite inputs opaque while compiling the reduction itself with fast-math optimizations. template <typename Scalar, typename Packet> EIGEN_DONT_INLINE void reduce_minmax(const Scalar* input, Scalar* output) { @@ -149,6 +207,7 @@ VERIFY((mask_any<Scalar, Packet>(mask))); } + verify_mask_reduction<Scalar, Packet>(); verify_minmax_reduction<Scalar, Packet>(); } }; @@ -186,6 +245,12 @@ CALL_SUBTEST(extended_scalar_constant_runner<long double>::run()); #if defined(EIGEN_VECTORIZE_RVV10) + CALL_SUBTEST((verify_mask_reduction<float, Eigen::internal::Packet1Xf>())); + CALL_SUBTEST((verify_mask_reduction<float, Eigen::internal::Packet2Xf>())); + CALL_SUBTEST((verify_mask_reduction<float, Eigen::internal::Packet4Xf>())); + CALL_SUBTEST((verify_mask_reduction<double, Eigen::internal::Packet1Xd>())); + CALL_SUBTEST((verify_mask_reduction<double, Eigen::internal::Packet2Xd>())); + CALL_SUBTEST((verify_mask_reduction<double, Eigen::internal::Packet4Xd>())); CALL_SUBTEST((verify_minmax_reduction<float, Eigen::internal::Packet1Xf>())); CALL_SUBTEST((verify_minmax_reduction<float, Eigen::internal::Packet2Xf>())); CALL_SUBTEST((verify_minmax_reduction<float, Eigen::internal::Packet4Xf>())); @@ -195,7 +260,33 @@ #endif #if defined(EIGEN_VECTORIZE_RVV10FP16) + CALL_SUBTEST((verify_mask_reduction<Eigen::half, Eigen::internal::Packet1Xh>())); + CALL_SUBTEST((verify_mask_reduction<Eigen::half, Eigen::internal::Packet2Xh>())); CALL_SUBTEST((verify_minmax_reduction<Eigen::half, Eigen::internal::Packet1Xh>())); CALL_SUBTEST((verify_minmax_reduction<Eigen::half, Eigen::internal::Packet2Xh>())); #endif + +#if defined(EIGEN_VECTORIZE_RVV10BF16) + CALL_SUBTEST((verify_mask_reduction<Eigen::bfloat16, Eigen::internal::Packet1Xbf>())); + CALL_SUBTEST((verify_mask_reduction<Eigen::bfloat16, Eigen::internal::Packet2Xbf>())); +#endif + +#if defined(EIGEN_VECTORIZE_NEON) + CALL_SUBTEST((verify_mask_reduction<float, Eigen::internal::Packet2f>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<float, Eigen::internal::Packet2f>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::bfloat16, Eigen::internal::Packet4bf>())); +#if EIGEN_ARCH_ARM64 + CALL_SUBTEST((verify_low_bit_mask_any_ftz<double, Eigen::internal::Packet2d>())); +#endif +#endif + +#if defined(EIGEN_VECTORIZE_AVX512FP16) + CALL_SUBTEST((verify_mask_reduction<Eigen::half, Eigen::internal::Packet16h>())); + CALL_SUBTEST((verify_mask_reduction<Eigen::half, Eigen::internal::Packet8h>())); + VERIFY((ptrue_mask_any<Eigen::half, Eigen::internal::Packet16h>())); + VERIFY((ptrue_mask_any<Eigen::half, Eigen::internal::Packet8h>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::half, Eigen::internal::Packet32h>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::half, Eigen::internal::Packet16h>())); + CALL_SUBTEST((verify_low_bit_mask_any_ftz<Eigen::half, Eigen::internal::Packet8h>())); +#endif }
diff --git a/test/sycl_basic.cpp b/test/sycl_basic.cpp index 40dcfd2..ad9ba04 100644 --- a/test/sycl_basic.cpp +++ b/test/sycl_basic.cpp
@@ -339,6 +339,73 @@ run_and_verify<true, true>(operation, 1, in, out); } +template <typename Operation> +void test_predux_any(Operation& operation) { + VectorXf in(1); + Matrix<int, 3, 1> out; + in.setZero(); + out.setZero(); + + run_and_verify<false, true>(operation, 1, in, out); +} + +void test_predux_any_float() { + auto operation = [](const float* in, int* out) { +#ifdef SYCL_DEVICE_ONLY + const cl::sycl::cl_int4 int_none(0); + const cl::sycl::cl_int4 int_some(0, -1, 0, 0); + const cl::sycl::cl_int4 int_all(-1); + out[0] = internal::predux_any(int_none.template as<cl::sycl::cl_float4>()); + out[1] = internal::predux_any(int_some.template as<cl::sycl::cl_float4>()); + out[2] = internal::predux_any(int_all.template as<cl::sycl::cl_float4>()); +#else + EIGEN_UNUSED_VARIABLE(in); + out[0] = 0; + out[1] = 1; + out[2] = 1; +#endif + }; + test_predux_any(operation); +} + +void test_predux_any_half() { + auto operation = [](const float* in, int* out) { +#ifdef SYCL_DEVICE_ONLY + const cl::sycl::cl_short8 short_none(0); + const cl::sycl::cl_short8 short_some(0, -1, 0, 0, 0, 0, 0, 0); + const cl::sycl::cl_short8 short_all(-1); + out[0] = internal::predux_any(short_none.template as<cl::sycl::cl_half8>()); + out[1] = internal::predux_any(short_some.template as<cl::sycl::cl_half8>()); + out[2] = internal::predux_any(short_all.template as<cl::sycl::cl_half8>()); +#else + EIGEN_UNUSED_VARIABLE(in); + out[0] = 0; + out[1] = 1; + out[2] = 1; +#endif + }; + test_predux_any(operation); +} + +void test_predux_any_double() { + auto operation = [](const float* in, int* out) { +#ifdef SYCL_DEVICE_ONLY + const cl::sycl::cl_long2 long_none(0); + const cl::sycl::cl_long2 long_some(0, -1); + const cl::sycl::cl_long2 long_all(-1); + out[0] = internal::predux_any(long_none.template as<cl::sycl::cl_double2>()); + out[1] = internal::predux_any(long_some.template as<cl::sycl::cl_double2>()); + out[2] = internal::predux_any(long_all.template as<cl::sycl::cl_double2>()); +#else + EIGEN_UNUSED_VARIABLE(in); + out[0] = 0; + out[1] = 1; + out[2] = 1; +#endif + }; + test_predux_any(operation); +} + EIGEN_DECLARE_TEST(sycl_basic) { Eigen::VectorXf in, out; Eigen::VectorXcf cfin, cfout; @@ -380,4 +447,8 @@ CALL_SUBTEST(test_matrix_inverse<Matrix4f>(num_elements, in, out)); CALL_SUBTEST(test_numeric_limits<Vector3f>(in, out)); + CALL_SUBTEST(test_predux_any_float()); + const sycl::device device{sycl::default_selector_v}; + if (device.has(sycl::aspect::fp16)) CALL_SUBTEST(test_predux_any_half()); + if (device.has(sycl::aspect::fp64)) CALL_SUBTEST(test_predux_any_double()); }