Core bug fixes: scalar math, NumTraits and half/bfloat16 libeigen/eigen!3345 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index dbda19f..edc917c 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -1425,15 +1425,10 @@ } }; -// Scale both parts by the real factor: the complex product with (log2(e), 0) is NaN where log(a) is infinite. +// Defined with plog_complex in GenericPacketMathComplex.h; declared here so that plog2 on a complex packet cannot +// silently take the primary template before that header. template <typename Packet> -struct plog2_impl<Packet, true> { - static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { - using RealPacket = typename unpacket_traits<Packet>::as_real; - using RealScalar = typename unpacket_traits<RealPacket>::type; - return Packet(pmul(pset1<RealPacket>(RealScalar(EIGEN_LOG2E)), plog(a).v)); - } -}; +struct plog2_impl<Packet, true>; /** \internal \returns the log2 of \a a (coeff-wise) */ template <typename Packet>
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 3bf3d6e..cf0976b 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -545,15 +545,10 @@ } }; -// Specialization for complex types that are not supported by std::log1p. +// Specialization for complex types that are not supported by std::log1p; defined after the numext functions it is +// built from. template <typename RealScalar> -struct log1p_impl<std::complex<RealScalar>> { - EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) - - EIGEN_DEVICE_FUNC static inline std::complex<RealScalar> run(const std::complex<RealScalar>& x) { - return std_fallback::log1p(x); - } -}; +struct log1p_impl<std::complex<RealScalar>>; // numext::exp2 of a complex argument; defined after the numext functions it is built from. template <typename RealScalar> @@ -2277,16 +2272,18 @@ } }; +// Compares moduli, not their squares: abs2 overflows above sqrt(highest) and underflows below sqrt(min), where unequal +// values then compared as approximately equal. template <typename Scalar> struct scalar_fuzzy_default_impl<Scalar, true, false> { using RealScalar = typename NumTraits<Scalar>::Real; template <typename OtherScalar> EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const Scalar& x, const OtherScalar& y, const RealScalar& prec) { - return numext::abs2(x) <= numext::abs2(y) * prec * prec; + return numext::abs(x) <= numext::abs(y) * prec; } EIGEN_DEVICE_FUNC static inline bool isApprox(const Scalar& x, const Scalar& y, const RealScalar& prec) { - return numext::abs2(x - y) <= numext::mini(numext::abs2(x), numext::abs2(y)) * prec * prec; + return numext::abs(x - y) <= numext::mini(numext::abs(x), numext::abs(y)) * prec; } }; @@ -2387,6 +2384,23 @@ } }; +template <typename RealScalar> +struct log1p_impl<std::complex<RealScalar>> { + EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) + + EIGEN_DEVICE_FUNC static inline std::complex<RealScalar> run(const std::complex<RealScalar>& z) { + const RealScalar x = z.real(); + const RealScalar y = z.imag(); + // log|1 + z| = log1p(x (2 + x) + y^2) / 2. Rounding |1 + z| instead loses the real part to cancellation near + // |1 + z| = 1, and with it the relative accuracy of log1p(z) for small z off the real axis. + if (numext::abs(x) < RealScalar(0.5) && numext::abs(y) < RealScalar(0.5) && !numext::is_exactly_zero(y)) { + return std::complex<RealScalar>(numext::log1p(x * (RealScalar(2) + x) + y * y) / RealScalar(2), + numext::atan2(y, RealScalar(1) + x)); + } + return std_fallback::log1p(z); + } +}; + // 2^(a + ib) = 2^a (cos(t) + i sin(t)) with t = b ln(2). The modulus is the real exp2, so 2^n is exact. Rounding t // would cost |t| eps in the phase, so b ln(2) = t_hi + t_lo is kept in double-word precision, ln(2) = ln2_hi + ln2_lo: // cos(t) = cos(t_hi) + (cos(t_hi) (cos(t_lo) - 1) - sin(t_hi) sin(t_lo)), @@ -2509,7 +2523,13 @@ // guards against over/under-flow. const bool scale_imag = numext::abs(b_imag) <= numext::abs(b_real); const T rscale = scale_imag ? T(1) : b_real / b_imag; - const T iscale = scale_imag ? b_imag / b_real : T(1); + // d / c = +-1 for |c| = |d|, which the quotient misses as 0 / 0 or inf / inf when b = 0 or both parts of b are + // infinite; a zero divisor then gives an infinity and an infinite one zero, as in pdiv_complex. The sign is not taken + // from b_real * b_imag, which can overflow or underflow. + const T iscale = !scale_imag ? T(1) + : numext::abs(b_imag) == numext::abs(b_real) + ? numext::copysign(T(1), b_real) * numext::copysign(T(1), b_imag) + : b_imag / b_real; const T denominator = b_real * rscale + b_imag * iscale; return std::complex<T>((a_real * rscale + a_imag * iscale) / denominator, (a_imag * rscale - a_real * iscale) / denominator);
diff --git a/Eigen/src/Core/MathFunctionsImpl.h b/Eigen/src/Core/MathFunctionsImpl.h index 3ecd3fa..49cc703 100644 --- a/Eigen/src/Core/MathFunctionsImpl.h +++ b/Eigen/src/Core/MathFunctionsImpl.h
@@ -286,6 +286,25 @@ return ComplexT(numext::log(a), b); } +// log2|z| + i arg(z) log2(e), scaled as in complex_log. The real part comes from log2, which is exact where |z| is a +// power of two; log2(e) log|z| can be an ulp off there. +template <typename ComplexT> +EIGEN_DEVICE_FUNC ComplexT complex_log2(const ComplexT& z) { + using T = typename NumTraits<ComplexT>::Real; + const T a = numext::abs(z); + EIGEN_USING_STD(atan2); + const T b = T(EIGEN_LOG2E) * atan2(z.imag(), z.real()); + if (EIGEN_PREDICT_FALSE((numext::isinf)(a) && (numext::isfinite)(z.real()) && (numext::isfinite)(z.imag()))) { + const T half(0.5); + return ComplexT(numext::log2(numext::abs(ComplexT(z.real() * half, z.imag() * half))) + T(1), b); + } + if (EIGEN_PREDICT_FALSE(a < (numext::numeric_limits<T>::min)() && !numext::is_exactly_zero(a))) { + const T eps = NumTraits<T>::epsilon(); + return ComplexT(numext::log2(numext::abs(ComplexT(z.real() / eps, z.imag() / eps))) + numext::log2(eps), b); + } + return ComplexT(numext::log2(a), b); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/NumTraits.h b/Eigen/src/Core/NumTraits.h index 3be727a..75c3789 100644 --- a/Eigen/src/Core/NumTraits.h +++ b/Eigen/src/Core/NumTraits.h
@@ -19,7 +19,7 @@ namespace internal { // default implementation of digits(), based on numeric_limits if specialized, -// 0 for integer types, and log2(epsilon()) otherwise. +// 0 for integer types, and 1 - log2(epsilon()) otherwise, as epsilon() = 2^(1 - digits). template <typename T, bool use_numeric_limits = std::numeric_limits<T>::is_specialized, bool is_integer = NumTraits<T>::IsInteger> struct default_digits_impl { @@ -33,7 +33,7 @@ using std::ceil; using std::log2; using Real = typename NumTraits<T>::Real; - return int(ceil(-log2(NumTraits<Real>::epsilon()))); + return int(ceil(-log2(NumTraits<Real>::epsilon()))) + 1; } };
diff --git a/Eigen/src/Core/arch/Default/BFloat16.h b/Eigen/src/Core/arch/Default/BFloat16.h index bf766c1..5fb7659 100644 --- a/Eigen/src/Core/arch/Default/BFloat16.h +++ b/Eigen/src/Core/arch/Default/BFloat16.h
@@ -778,8 +778,12 @@ } EIGEN_DEVICE_FUNC inline bfloat16 fma(const bfloat16& a, const bfloat16& b, const bfloat16& c) { - // Emulate FMA via float. +#if defined(EIGEN_GPU_COMPILE_PHASE) || defined(SYCL_DEVICE_ONLY) + // Rounds twice; see half_impl::fma_round_to_odd. return bfloat16(numext::fma(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c))); +#else + return bfloat16(half_impl::fma_round_to_odd(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c))); +#endif } #ifndef EIGEN_NO_IO @@ -911,7 +915,9 @@ template <> struct hash<Eigen::bfloat16> { EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::bfloat16& a) const { - return static_cast<std::size_t>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a)); + // -0 == +0, so both must hash alike. + const Eigen::numext::uint16_t bits = Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a); + return static_cast<std::size_t>((bits & 0x7fff) == 0 ? 0 : bits); } }; } // namespace std
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h b/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h index 4fa8e5a..4499cc8 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h
@@ -82,7 +82,7 @@ return padd(tmp_im, tmp_re); // a*c - b*d, a*d + b*c } -template <typename Packet> +template <typename Packet, bool Base2> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex(const Packet& x) { using RealPacket = typename unpacket_traits<Packet>::as_real; using RealScalar = typename unpacket_traits<RealPacket>::type; @@ -107,12 +107,14 @@ const RealPacket k = pselect(is_large, k_large, pand(is_small, k_small)); const RealPacket scale = pselect(is_large, pldexp_fast(cst_one, pnegate(k_large)), pselect(is_small, pldexp_fast(cst_one, pnegate(k_small)), cst_one)); - // log(sqrt(a^2 + b^2)), log(sqrt(a^2 + b^2)) - const RealPacket xlogr = - pmadd(k, pset1<RealPacket>(RealScalar(EIGEN_LN2)), plog(phypot_complex(Packet(pmul(x.v, scale))).v)); + // log(sqrt(a^2 + b^2)), log(sqrt(a^2 + b^2)); in base 2 the real part comes from plog2, which is exact where |x| is a + // power of two, rather than from log2(e) log|x|. + const RealPacket r = phypot_complex(Packet(pmul(x.v, scale))).v; + const RealPacket xlogr = Base2 ? padd(k, plog2(r)) : pmadd(k, pset1<RealPacket>(RealScalar(EIGEN_LN2)), plog(r)); // Imag part RealPacket ximg = patan2(x.v, x_flip); // atan2(a, b), atan2(b, a) + if (Base2) ximg = pmul(pset1<RealPacket>(RealScalar(EIGEN_LOG2E)), ximg); const RealPacket cst_pos_inf = pinf<RealPacket>(); RealPacket is_x_pos_inf = pcmp_eq(x_abs, cst_pos_inf); @@ -124,6 +126,11 @@ } template <typename Packet> +struct plog2_impl<Packet, true> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return plog_complex<Packet, true>(a); } +}; + +template <typename Packet> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_complex(const Packet& a) { using RealPacket = typename unpacket_traits<Packet>::as_real; using Scalar = typename unpacket_traits<Packet>::type;
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h b/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h index ae970aa..784035f 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h
@@ -247,8 +247,8 @@ template <typename Packet, int N> struct ppolevl; -/** \internal \returns log(x) for complex types */ -template <typename Packet> +/** \internal \returns log(x), or log2(x) for Base2, for complex types */ +template <typename Packet, bool Base2 = false> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex(const Packet& x); /** \internal \returns exp(x) for complex types */
diff --git a/Eigen/src/Core/arch/Default/Half.h b/Eigen/src/Core/arch/Default/Half.h index 8139d5e..9ac7184 100644 --- a/Eigen/src/Core/arch/Default/Half.h +++ b/Eigen/src/Core/arch/Default/Half.h
@@ -958,15 +958,39 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half(max)(const half& a, const half& b) { return a < b ? b : a; } +// a * b + c for half or bfloat16 a, b and c, rounded to odd in double: narrowing the result to half or bfloat16, which +// rounds to odd in float and then to nearest, then rounds a * b + c once. A float fma rounds twice instead: for half, +// fma(33/32, 65/64, 2^-24) rounds to the midpoint 2145/2048 in float, then to even, 1 + 3/64, not up to 1 + 49/1024. +// a * b is exact in double, and TwoSum (Knuth) gives the error e of s = a * b + c exactly. Device code does not use +// it: GPUs run double at a fraction of the float rate, and SYCL devices need not support it. +EIGEN_DEVICE_FUNC inline double fma_round_to_odd(float a, float b, float c) { + const double p = static_cast<double>(a) * static_cast<double>(b); + const double c_wide = static_cast<double>(c); + const double s = p + c_wide; + // p is exact, so s is nonfinite only for a nonfinite operand; TwoSum would then evaluate inf - inf (FE_INVALID). + if (!(numext::isfinite)(s)) return s; + const double p_part = s - c_wide; + const double e = (p - p_part) + (c_wide - (s - p_part)); + if (e == 0) return s; + // Inexact: truncate toward zero, then set the last bit. + const bool away = (s > 0) != (e > 0); + return numext::bit_cast<double>((numext::bit_cast<numext::uint64_t>(s) - (away ? 1u : 0u)) | 1u); +} + EIGEN_DEVICE_FUNC inline half fma(const half& a, const half& b, const half& c) { -#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) +#if defined(EIGEN_HAS_NATIVE_GPU_FP16) + // Rounded once in hardware. + return __hfma(::__half(a), ::__half(b), ::__half(c)); +#elif defined(SYCL_DEVICE_ONLY) + // Rounds twice; see fma_round_to_odd. + return half(numext::fma(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c))); +#elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return half_from_rep(vfmah_f16(c.x, a.x, b.x)); #elif defined(EIGEN_VECTORIZE_AVX512FP16) // Reduces to vfmadd213sh. return half(_mm_cvtsh_h(_mm_fmadd_ph(_mm_set_sh(a.x), _mm_set_sh(b.x), _mm_set_sh(c.x)))); #else - // Emulate FMA via float. - return half(numext::fma(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c))); + return half(fma_round_to_odd(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c))); #endif } @@ -1167,7 +1191,9 @@ template <> struct hash<Eigen::half> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::half& a) const { - return static_cast<std::size_t>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a)); + // -0 == +0, so both must hash alike. + const Eigen::numext::uint16_t bits = Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a); + return static_cast<std::size_t>((bits & 0x7fff) == 0 ? 0 : bits); } }; } // end namespace std
diff --git a/Eigen/src/Core/functors/NullaryFunctors.h b/Eigen/src/Core/functors/NullaryFunctors.h index 2719318..7c1fd10 100644 --- a/Eigen/src/Core/functors/NullaryFunctors.h +++ b/Eigen/src/Core/functors/NullaryFunctors.h
@@ -110,31 +110,49 @@ const bool m_flip; }; +// Built-in integers compute in the unsigned type of Scalar's width, widened to Index's, which holds num_steps, the +// index i, |high - low| and the offset from low where high - low overflows a signed Scalar or wraps an unsigned one; +// low +- offset then wraps to the coefficient. Other scalars, custom integers included, compute in Scalar. +template <typename Scalar, bool IsBuiltin = std::is_integral<Scalar>::value> +struct linspaced_integer_traits { + using Wide = std::make_unsigned_t<std::conditional_t<(sizeof(Scalar) < sizeof(Index)), Index, Scalar>>; + EIGEN_DEVICE_FUNC static constexpr Scalar narrow(const Wide& x) { + return static_cast<Scalar>(numext::bit_cast<std::make_signed_t<Wide>>(x)); + } +}; + +template <typename Scalar> +struct linspaced_integer_traits<Scalar, false> { + using Wide = Scalar; + EIGEN_DEVICE_FUNC static constexpr Scalar narrow(const Wide& x) { return x; } +}; + template <typename Scalar> struct linspaced_op_impl<Scalar, /*IsInteger*/ true> { - // Built-in integers narrower than Index compute in Index, as num_steps, the index i and the offset from low need not - // fit in Scalar while the coefficient does. Other scalars, custom integers included, compute in Scalar. - using Wide = std::conditional_t<std::is_integral<Scalar>::value && (sizeof(Scalar) < sizeof(Index)), Index, Scalar>; + using Traits = linspaced_integer_traits<Scalar>; + using Wide = typename Traits::Wide; + // Coefficient i is low +- i * m_step with m_step = |high - low| / (num_steps - 1), or, where values repeat because + // |high - low| + 1 < num_steps, low +- i / m_step with m_step = (num_steps + |high - low|) / (|high - low| + 1). EIGEN_DEVICE_FUNC constexpr linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : m_low(low), - m_multiplier((Wide(high) - Wide(low)) / convert_index<Wide>(num_steps <= 1 ? 1 : num_steps - 1)), - m_divisor(convert_index<Wide>((high >= low ? num_steps : -num_steps) + (Wide(high) - Wide(low))) / - ((numext::abs(Wide(high) - Wide(low)) + 1) == 0 ? 1 : (numext::abs(Wide(high) - Wide(low)) + 1))), - m_use_divisor(num_steps > 1 && (numext::abs(Wide(high) - Wide(low)) + 1) < num_steps) {} + m_decreasing(high < low), + m_range(high < low ? Wide(Wide(low) - Wide(high)) : Wide(Wide(high) - Wide(low))), + m_use_divisor(num_steps > 1 && m_range < convert_index<Wide>(num_steps - 1)), + m_step(m_use_divisor ? (convert_index<Wide>(num_steps) + m_range) / (m_range + Wide(1)) + : m_range / convert_index<Wide>(num_steps <= 1 ? 1 : num_steps - 1)) {} template <typename IndexType> EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Scalar operator()(IndexType i) const { - if (m_use_divisor) - return static_cast<Scalar>(m_low + convert_index<Wide>(i) / m_divisor); - else - return static_cast<Scalar>(m_low + convert_index<Wide>(i) * m_multiplier); + const Wide offset = m_use_divisor ? Wide(convert_index<Wide>(i) / m_step) : Wide(convert_index<Wide>(i) * m_step); + return Traits::narrow(m_decreasing ? Wide(Wide(m_low) - offset) : Wide(Wide(m_low) + offset)); } - const Wide m_low; - const Wide m_multiplier; - const Wide m_divisor; + const Scalar m_low; + const bool m_decreasing; + const Wide m_range; const bool m_use_divisor; + const Wide m_step; }; // ----- Linspace functor ----------------------------------------------------------------
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index 0173d30..822cf9c 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -187,8 +187,10 @@ using RealScalar = typename NumTraits<Scalar>::Real; enum { Cost = functor_traits<scalar_atan2_op<RealScalar>>::Cost, - // The generic pcarg lowers to patan2, whose quotient-based reduction needs pdiv. - PacketAccess = packet_traits<RealScalar>::HasATan && packet_traits<RealScalar>::HasDiv + // The generic pcarg lowers to patan2, whose quotient-based reduction needs pdiv. It takes complex packets only; a + // real argument is 0 or pi, which the scalar path returns. + PacketAccess = + NumTraits<Scalar>::IsComplex && packet_traits<RealScalar>::HasATan && packet_traits<RealScalar>::HasDiv }; }; @@ -562,12 +564,28 @@ struct scalar_log2_op { using RealScalar = typename NumTraits<Scalar>::Real; EIGEN_DEVICE_FUNC constexpr inline Scalar operator()(const Scalar& a) const { - return internal::mul(RealScalar(EIGEN_LOG2E), numext::log(a)); + return run(a, std::integral_constant<int, kKind>()); } template <typename Packet> EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog2(a); } + + private: + // 0: float, double or long double; 1: std::complex of one of them; 2: any other scalar, user types included. + static constexpr int kKind = + std::is_floating_point<Scalar>::value ? 0 + : std::is_floating_point<RealScalar>::value && std::is_same<Scalar, std::complex<RealScalar>>::value ? 1 + : 2; + // std::log2 is exact at powers of two, like plog2; log2(e) * log(a) can be an ulp off there. complex_log2 takes the + // real part log2|a| from std::log2, like plog2 on complex packets. + EIGEN_DEVICE_FUNC static Scalar run(const Scalar& a, std::integral_constant<int, 0>) { return numext::log2(a); } + EIGEN_DEVICE_FUNC static Scalar run(const Scalar& a, std::integral_constant<int, 1>) { + return internal::complex_log2(a); + } + EIGEN_DEVICE_FUNC static Scalar run(const Scalar& a, std::integral_constant<int, 2>) { + return internal::mul(RealScalar(EIGEN_LOG2E), numext::log(a)); + } }; template <typename Scalar> struct functor_traits<scalar_log2_op<Scalar>> {
diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index 57dc0f1..e427d8e 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp
@@ -961,6 +961,8 @@ VERIFY_IS_APPROX(m1.logistic(), logistic(m1)); VERIFY_IS_APPROX(m1.arg(), arg(m1)); + // carg of a real array is its arg as a real Scalar, on the vectorized path too. + VERIFY_IS_CWISE_EQUAL(m1.carg(), m1.arg()); VERIFY_IS_APPROX(m1.round(), round(m1)); VERIFY_IS_APPROX(m1.rint(), rint(m1)); VERIFY_IS_APPROX(m1.floor(), floor(m1));
diff --git a/test/bfloat16_float.cpp b/test/bfloat16_float.cpp index 1d04cc6..7e9edf1 100644 --- a/test/bfloat16_float.cpp +++ b/test/bfloat16_float.cpp
@@ -252,6 +252,12 @@ VERIFY_IS_APPROX(static_cast<float>(bfloat16(1.0f) / bfloat16(3.0f)), 0.3339f); VERIFY_IS_EQUAL(static_cast<float>(-bfloat16(4096.0f)), -4096.0f); VERIFY_IS_EQUAL(static_cast<float>(-bfloat16(-4096.0f)), 4096.0f); + + // fma rounds once: a * b = 289/256 is a midpoint of bfloat16, and c = 2^-30 rounds the sum up, not to even. A float + // fma rounds the sum to that midpoint first. + const bfloat16 tiny = bfloat16(std::ldexp(1.0f, -30)); + VERIFY_IS_EQUAL(static_cast<float>(numext::fma(bfloat16(17.f / 16), bfloat16(17.f / 16), tiny)), 145.f / 128); + VERIFY_IS_EQUAL(static_cast<float>(numext::fma(bfloat16(17.f / 16), bfloat16(-17.f / 16), -tiny)), -145.f / 128); } void test_comparison() { @@ -267,6 +273,10 @@ VERIFY(!(bfloat16(-0.0f) < bfloat16(0.0f))); VERIFY(!(bfloat16(0.0f) > bfloat16(-0.0f))); VERIFY(!(bfloat16(-0.0f) > bfloat16(0.0f))); +#if EIGEN_HAS_STD_HASH + // Equal values hash alike. + VERIFY_IS_EQUAL(std::hash<bfloat16>()(bfloat16(0.0f)), std::hash<bfloat16>()(bfloat16(-0.0f))); +#endif VERIFY(bfloat16(0.2f) > bfloat16(-1.0f)); VERIFY(bfloat16(-1.0f) < bfloat16(0.2f));
diff --git a/test/half_float.cpp b/test/half_float.cpp index 92da5c5..819c1e8 100644 --- a/test/half_float.cpp +++ b/test/half_float.cpp
@@ -194,6 +194,12 @@ VERIFY_IS_EQUAL(float(-half(4096.0f)), -4096.0f); VERIFY_IS_EQUAL(float(-half(-4096.0f)), 4096.0f); + // fma rounds once: a * b = 2145/2048 is a midpoint of half, and c = 2^-24 rounds the sum up, not to even. A float + // fma rounds the sum to that midpoint first. + const half tiny = half(std::ldexp(1.0f, -24)); + VERIFY_IS_EQUAL(float(numext::fma(half(33.f / 32), half(65.f / 64), tiny)), 1.f + 49.f / 1024); + VERIFY_IS_EQUAL(float(numext::fma(half(-33.f / 32), half(65.f / 64), -tiny)), -1.f - 49.f / 1024); + half x(3); half y = ++x; VERIFY_IS_EQUAL(x, half(4)); @@ -222,6 +228,10 @@ VERIFY(!(half(-0.0f) < half(0.0f))); VERIFY(!(half(0.0f) > half(-0.0f))); VERIFY(!(half(-0.0f) > half(0.0f))); +#if EIGEN_HAS_STD_HASH + // Equal values hash alike. + VERIFY_IS_EQUAL(std::hash<half>()(half(0.0f)), std::hash<half>()(half(-0.0f))); +#endif VERIFY(half(0.2f) > half(-1.0f)); VERIFY(half(-1.0f) < half(0.2f));
diff --git a/test/nullary.cpp b/test/nullary.cpp index a3ab9d1..808eb55 100644 --- a/test/nullary.cpp +++ b/test/nullary.cpp
@@ -460,6 +460,27 @@ for (Index k = 0; k < 256; ++k) VERIFY_IS_EQUAL(int(u(k)), 255 - int(k)); } + // 64-bit integers, where high - low wraps for unsigned decreasing sequences or overflows the signed type. + { + using VecU64 = Matrix<uint64_t, Dynamic, 1>; + VecU64 u = VecU64::LinSpaced(4, 9, 0), w = VecU64::LinSpaced(10, 3, 0); + for (Index k = 0; k < 4; ++k) VERIFY_IS_EQUAL(u(k), uint64_t(9 - 3 * k)); + for (Index k = 0; k < 10; ++k) VERIFY_IS_EQUAL(w(k), uint64_t(3 - k / 3)); + const uint64_t umax = (std::numeric_limits<uint64_t>::max)(); + VecU64 x = VecU64::LinSpaced(2, umax, 0); + VERIFY_IS_EQUAL(x(0), umax); + VERIFY_IS_EQUAL(x(1), uint64_t(0)); + using VecI64 = Matrix<int64_t, Dynamic, 1>; + const int64_t imin = (std::numeric_limits<int64_t>::min)(), imax = (std::numeric_limits<int64_t>::max)(); + VecI64 y = VecI64::LinSpaced(2, imin, imax), z = VecI64::LinSpaced(3, imax, imin); + VERIFY_IS_EQUAL(y(0), imin); + VERIFY_IS_EQUAL(y(1), imax); + // The step is (2^64 - 1) / 2 = imax. + VERIFY_IS_EQUAL(z(0), imax); + VERIFY_IS_EQUAL(z(1), int64_t(0)); + VERIFY_IS_EQUAL(z(2), -imax); + } + // Custom integer scalars. { using VecC = Matrix<CustomInt, Dynamic, 1>;
diff --git a/test/numext.cpp b/test/numext.cpp index 4ca0869..5098c83 100644 --- a/test/numext.cpp +++ b/test/numext.cpp
@@ -396,6 +396,225 @@ std::numeric_limits<T>::infinity()); } +// Complex fuzzy comparisons at moduli whose squares overflow or underflow. +template <typename T> +void check_complex_fuzzy() { + using Complex = std::complex<T>; + const T prec = NumTraits<T>::dummy_precision(); + for (const T s : + {numext::sqrt((std::numeric_limits<T>::min)()) / T(16), numext::sqrt((std::numeric_limits<T>::max)()) * T(16)}) { + VERIFY(!internal::isApprox(Complex(s, T(0)), Complex(T(3) * s, T(0)), prec)); + VERIFY(!internal::isApprox(Complex(s, -s), Complex(-s, s), prec)); + VERIFY(internal::isApprox(Complex(s, -s), Complex(s * (T(1) + NumTraits<T>::epsilon()), -s), prec)); + VERIFY(!internal::isMuchSmallerThan(Complex(T(3) * s, T(0)), Complex(s, T(0)), prec)); + VERIFY(internal::isMuchSmallerThan(Complex(s * prec / T(4), T(0)), Complex(s, s), prec)); + Matrix<Complex, 3, 1> v = Matrix<Complex, 3, 1>::Constant(Complex(s, T(0))); + VERIFY(!v.isApproxToConstant(Complex(T(3) * s, T(0)), prec)); + } +} + +// log1p(z) to a few ulps of |log1p(z)| for small z off the real axis, where rounding |1 + z| to 1 loses log|1 + z|. +// W is wider than T; log|1 + z| = log1p(x (2 + x) + y^2) / 2. +template <typename T, typename W> +void check_complex_log1p_accuracy() { + const T eps = NumTraits<T>::epsilon(); + const T tiny_x[] = {T(0), T(4) * eps, -T(3) * eps, T(1e-3) * numext::sqrt(eps)}; + const T tiny_y[] = {numext::sqrt(eps), -T(2) * numext::sqrt(eps), T(1e-3), T(64) * eps}; + for (const T x : tiny_x) { + for (const T y : tiny_y) { + const std::complex<T> got = numext::log1p(std::complex<T>(x, y)); + const W ref_re = std::log1p(W(x) * (W(2) + W(x)) + W(y) * W(y)) / W(2); + const W ref_im = std::atan2(W(y), W(1) + W(x)); + const W err = numext::hypot(W(got.real()) - ref_re, W(got.imag()) - ref_im); + VERIFY(err <= W(4) * W(eps) * numext::hypot(ref_re, ref_im)); + } + } +} + +// C99 Annex G: a nonzero finite numerator over a zero divisor gives an infinity, and a finite one over an infinite +// divisor gives zero, also where the divisor's parts have equal magnitudes. scalar_quotient_op is the coefficient-wise +// quotient's scalar path. +template <typename T> +void check_complex_divide() { + using Complex = std::complex<T>; + const T zero(0), one(1), inf = std::numeric_limits<T>::infinity(); + for (const Complex a : {Complex(one, zero), Complex(-T(3), T(0.5)), Complex(one, one)}) { + for (const Complex b : {Complex(zero, zero), Complex(-zero, zero), Complex(zero, -zero)}) { + for (const Complex q : {numext::divide(a, b), internal::scalar_quotient_op<Complex, Complex>()(a, b)}) { + VERIFY((numext::isinf)(q.real()) || (numext::isinf)(q.imag())); + } + } + for (const Complex b : {Complex(inf, inf), Complex(-inf, inf), Complex(inf, zero)}) { + for (const Complex q : {numext::divide(a, b), internal::scalar_quotient_op<Complex, Complex>()(a, b)}) { + VERIFY_IS_EQUAL(q.real(), zero); + VERIFY_IS_EQUAL(q.imag(), zero); + } + } + } +} + +#if EIGEN_COMP_MSVC +#pragma float_control(precise, on, push) +#pragma fenv_access(on) +#endif + +// The exceptions numext::divide(a, b) raises. Volatile operands and results keep the division between the flag +// calls, where the compiler can neither fold it nor move it. +template <typename T> +EIGEN_DONT_INLINE int complex_divide_exceptions(const std::complex<T>& a, const std::complex<T>& b) { + volatile T parts[4] = {a.real(), a.imag(), b.real(), b.imag()}; + std::fenv_t environment; + if (std::feholdexcept(&environment) != 0) return -1; + const std::complex<T> q = numext::divide(std::complex<T>(parts[0], parts[1]), std::complex<T>(parts[2], parts[3])); + volatile T result[2] = {q.real(), q.imag()}; + EIGEN_UNUSED_VARIABLE(result); + const int exceptions = std::fetestexcept(FE_OVERFLOW | FE_UNDERFLOW | FE_INVALID | FE_DIVBYZERO); + std::fesetenv(&environment); + return exceptions; +} + +// Smith's ratio d / c is +-1 for |c| = |d|: its sign must not come from c * d, which overflows, or underflows +// inexactly, for divisors whose quotient does neither. For a divisor on an axis, the ratio Smith does not take is +// x / 0. +template <typename T> +void check_complex_divide_exceptions() { + using Complex = std::complex<T>; + const T big = numext::sqrt((std::numeric_limits<T>::max)()) * T(3); + const T small = numext::sqrt((std::numeric_limits<T>::min)()) * T(0.3); + const T two(2), zero(0); + for (const Complex b : + {Complex(big, big), Complex(-big, big), Complex(big, -big), Complex(-big, -big), Complex(small, small), + Complex(-small, small), Complex(small, -small), Complex(-small, -small), Complex(two, zero), + Complex(-two, zero), Complex(zero, two), Complex(zero, -two)}) { + const int exceptions = complex_divide_exceptions(Complex(T(1), T(0)), b); + if (exceptions == -1) { + std::cout << "SKIP: complex division exception check: feholdexcept failed.\n"; + return; + } + VERIFY_IS_EQUAL(exceptions, 0); + } +} + +// numext::fma(a, b, c) for half or bfloat16, and whether it raised FE_INVALID. The operands and the result pass +// through volatile bit patterns, which keeps the fma between the flag calls. +template <typename T> +EIGEN_DONT_INLINE int narrow_fma_invalid(const T& a, const T& b, const T& c, T& result) { + using Bits = numext::uint16_t; + volatile Bits bits[3] = {numext::bit_cast<Bits>(a), numext::bit_cast<Bits>(b), numext::bit_cast<Bits>(c)}; + std::fenv_t environment; + if (std::feholdexcept(&environment) != 0) return -1; + volatile Bits result_bits = numext::bit_cast<Bits>(numext::fma( + numext::bit_cast<T>(Bits(bits[0])), numext::bit_cast<T>(Bits(bits[1])), numext::bit_cast<T>(Bits(bits[2])))); + const int invalid = std::fetestexcept(FE_INVALID); + std::fesetenv(&environment); + result = numext::bit_cast<T>(Bits(result_bits)); + return invalid; +} + +// An infinite or NaN operand gives the IEEE fma without FE_INVALID, except for inf * 0 and inf - inf. +template <typename T> +void check_narrow_fma_nonfinite() { + const T inf = NumTraits<T>::infinity(), nan = NumTraits<T>::quiet_NaN(), zero(0), one(1), two(2); + const T quiet[][4] = {{inf, one, zero, inf}, {-inf, one, zero, -inf}, {one, one, inf, inf}, {one, one, -inf, -inf}, + {inf, two, inf, inf}, {two, -inf, -inf, -inf}, {nan, one, zero, nan}, {one, one, nan, nan}, + {inf, one, nan, nan}, {zero, nan, inf, nan}}; + for (const auto& operands : quiet) { + T result; + const int invalid = narrow_fma_invalid(operands[0], operands[1], operands[2], result); + if (invalid == -1) { + std::cout << "SKIP: fma exception check: feholdexcept failed.\n"; + return; + } + VERIFY_IS_EQUAL(invalid, 0); + VERIFY(check_if_equal_or_nans(result, operands[3])); + } + const T invalid_operands[][3] = {{inf, zero, one}, {zero, -inf, one}, {inf, one, -inf}, {-inf, two, inf}}; + for (const auto& operands : invalid_operands) { + T result; + if (narrow_fma_invalid(operands[0], operands[1], operands[2], result) == -1) return; + VERIFY((numext::isnan)(result)); + } +} + +#if EIGEN_COMP_MSVC +#pragma fenv_access(off) +#pragma float_control(pop) +#endif + +// log2(2^k) = k on the scalar path, as on the vectorized one. A size-1 array takes the scalar path. +template <typename T> +void check_log2_powers_of_two() { + for (int k = std::numeric_limits<T>::min_exponent - std::numeric_limits<T>::digits; + k < std::numeric_limits<T>::max_exponent; ++k) { + const T x = numext::ldexp(T(1), k); + VERIFY_IS_EQUAL(internal::scalar_log2_op<T>()(x), T(k)); + VERIFY_IS_EQUAL((Array<T, Dynamic, 1>::Constant(1, x).log2()(0)), T(k)); + } +} + +// log2|z| = k for z = 2^k (+-1, +-0 i) and +-2^k i, on the scalar path and in vectorized lanes, the imaginary part +// arg(z) log2(e) with the sign of a zero. Padding to a multiple of 64 elements puts every z in a full packet. +template <typename T> +void check_complex_log2_powers_of_two() { + using Complex = std::complex<T>; +#if EIGEN_ARCH_ARM // 32-bit ARM flushes subnormals. + const int k_min = std::numeric_limits<T>::min_exponent - 1; +#else + const int k_min = std::numeric_limits<T>::min_exponent - std::numeric_limits<T>::digits; +#endif + const int k_max = std::numeric_limits<T>::max_exponent - 1; + constexpr int kPerExponent = 6; + const Index count = kPerExponent * (k_max - k_min + 1); + Array<Complex, Dynamic, 1> z = Array<Complex, Dynamic, 1>::Ones((count + 63) / 64 * 64); + for (int k = k_min; k <= k_max; ++k) { + const T x = numext::ldexp(T(1), k), zero(0); + z.template segment<kPerExponent>(kPerExponent * (k - k_min)) << Complex(x, zero), Complex(x, -zero), + Complex(-x, zero), Complex(-x, -zero), Complex(zero, x), Complex(zero, -x); + } + const Array<Complex, Dynamic, 1> vectorized = z.log2(); + for (int k = k_min; k <= k_max; ++k) { + for (Index i = kPerExponent * (k - k_min); i < kPerExponent * (k - k_min + 1); ++i) { + const T arg_log2 = T(EIGEN_LOG2E) * std::atan2(z(i).imag(), z(i).real()); + const Complex scalar = internal::scalar_log2_op<Complex>()(z(i)); + VERIFY_IS_EQUAL(scalar.real(), T(k)); + VERIFY_IS_EQUAL(scalar.imag(), arg_log2); + VERIFY_IS_EQUAL(std::signbit(scalar.imag()), std::signbit(arg_log2)); + VERIFY_IS_EQUAL(vectorized(i).real(), T(k)); + VERIFY_IS_APPROX(vectorized(i).imag(), arg_log2); + VERIFY_IS_EQUAL(std::signbit(vectorized(i).imag()), std::signbit(arg_log2)); + } + } +} + +// The scalar complex log2 where |z| overflows or is subnormal, against log2|z| in the wider type W: within 2 ulps. +template <typename T, typename W> +void check_complex_log2_extreme_modulus() { + // W must hold |z| for the largest z. + if (std::numeric_limits<W>::max_exponent <= std::numeric_limits<T>::max_exponent) return; + using Complex = std::complex<T>; + const T highest = (std::numeric_limits<T>::max)(); + std::vector<Complex> zs = {Complex(highest, highest), Complex(-highest, highest / 3)}; +#if !EIGEN_ARCH_ARM // 32-bit ARM flushes subnormals. + const T denorm_min = std::numeric_limits<T>::denorm_min(); + zs.push_back(Complex(3 * denorm_min, 4 * denorm_min)); + zs.push_back(Complex((std::numeric_limits<T>::min)() / 3, -denorm_min)); +#endif + for (const Complex& z : zs) { + const W ref = std::log2(std::hypot(W(z.real()), W(z.imag()))); + const T got = internal::scalar_log2_op<Complex>()(z).real(); + VERIFY(numext::abs(W(got) - ref) <= W(2) * W(NumTraits<T>::epsilon()) * numext::abs(ref)); + } +} + +// NumTraits<T>::digits() counts significand bits like numeric_limits<T>::digits; std::complex<T>, for which +// numeric_limits is not specialized, derives it from epsilon() = 2^(1 - digits). +template <typename T> +void check_complex_digits() { + VERIFY_IS_EQUAL(NumTraits<std::complex<T>>::digits(), std::numeric_limits<T>::digits); + VERIFY_IS_EQUAL(NumTraits<std::complex<T>>::digits10(), std::numeric_limits<T>::digits10); + VERIFY_IS_EQUAL(NumTraits<std::complex<T>>::max_digits10(), std::numeric_limits<T>::max_digits10); +} + // The FMA implementation must retain the product's rounding error even when scalar madd is unfused. template <typename T> void check_twoprod_pair(const T& x, const T& y) { @@ -938,6 +1157,32 @@ CALL_SUBTEST(check_complex_log<float>()); CALL_SUBTEST(check_complex_log<double>()); + CALL_SUBTEST(check_complex_divide<float>()); + CALL_SUBTEST(check_complex_divide<double>()); + CALL_SUBTEST(check_complex_divide_exceptions<float>()); + CALL_SUBTEST(check_complex_divide_exceptions<double>()); + CALL_SUBTEST(check_narrow_fma_nonfinite<half>()); + CALL_SUBTEST(check_narrow_fma_nonfinite<bfloat16>()); + + CALL_SUBTEST(check_complex_fuzzy<float>()); + CALL_SUBTEST(check_complex_fuzzy<double>()); + + CALL_SUBTEST((check_complex_log1p_accuracy<float, double>())); + if (std::numeric_limits<long double>::digits > std::numeric_limits<double>::digits) { + CALL_SUBTEST((check_complex_log1p_accuracy<double, long double>())); + } + + CALL_SUBTEST(check_complex_digits<float>()); + CALL_SUBTEST(check_complex_digits<double>()); + CALL_SUBTEST(check_complex_digits<long double>()); + + CALL_SUBTEST(check_log2_powers_of_two<float>()); + CALL_SUBTEST(check_log2_powers_of_two<double>()); + CALL_SUBTEST(check_complex_log2_powers_of_two<float>()); + CALL_SUBTEST(check_complex_log2_powers_of_two<double>()); + CALL_SUBTEST((check_complex_log2_extreme_modulus<float, double>())); + CALL_SUBTEST((check_complex_log2_extreme_modulus<double, long double>())); + CALL_SUBTEST(check_twoprod<float>()); CALL_SUBTEST(check_twoprod<double>()); CALL_SUBTEST(check_twoprod<long double>());
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 59c80f3..d66ae69 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -1064,6 +1064,16 @@ CHECK_CWISE1_IF(PacketTraits::HasCbrt, numext::cbrt, internal::pcbrt); } +// A part of a complex logarithm: NaN for NaN, the same infinity, a zero of the same sign, or within 8 eps. +template <typename RealScalar> +bool log_part_matches(const RealScalar& actual, const RealScalar& expected) { + if ((numext::isnan)(expected)) return (numext::isnan)(actual); + if ((numext::isinf)(expected) || expected == RealScalar(0)) { + return actual == expected && std::signbit(actual) == std::signbit(expected); + } + return numext::abs(actual - expected) <= RealScalar(8) * NumTraits<RealScalar>::epsilon() * numext::abs(expected); +} + // Notice that this definition works for complex types as well. // std::log2 only supports real types, not complex. template <typename Scalar> @@ -2784,6 +2794,24 @@ data1[2] = Scalar(nan, inf); data1[3] = Scalar(nan, -inf); CHECK_CWISE1_IM1ULP_N(numext::log, internal::plog, 4); + + // plog2 against the scalar functor, with each input alone among ordinary lanes: signed zeros, an infinity beside a + // NaN, and a modulus that overflows or is subnormal. + const internal::scalar_log2_op<Scalar> log2_op; + for (const Scalar& special : + {Scalar(-zero, -zero), Scalar(-one, -zero), Scalar(one, -zero), Scalar(inf, nan), Scalar(-inf, nan), + Scalar(nan, inf), Scalar(nan, -inf), Scalar(highest, highest), Scalar(smallest, smallest)}) { + for (int k = 0; k < PacketSize; ++k) { + for (int i = 0; i < PacketSize; ++i) data1[i] = Scalar(RealScalar(3), RealScalar(-4)); + data1[k] = special; + internal::pstore(data2, internal::plog2(internal::pload<Packet>(data1))); + for (int i = 0; i < PacketSize; ++i) { + const Scalar expected = log2_op(data1[i]); + VERIFY(log_part_matches(data2[i].real(), expected.real()) && "internal::plog2"); + VERIFY(log_part_matches(data2[i].imag(), expected.imag()) && "internal::plog2"); + } + } + } } exp_complex_test<Scalar, Packet>(data1, data2, ref, size); }