Core bug fixes: vectorized math functions libeigen/eigen!3347 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h b/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h index 4c237f8..4fa8e5a 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathComplex.h
@@ -36,12 +36,37 @@ const RealPacket abs_y_flip = pcplxflip(Packet(abs_y)).v; const RealPacket mask = pcmp_lt(abs_y, abs_y_flip); // |c| < |d| - RealPacket y_scaled = pselect(mask, pdiv(abs_y, abs_y_flip), one); + // Elements that do not use the ratio divide 1 / 1: |c| / |d| would raise FE_DIVBYZERO for d = 0. + RealPacket y_scaled = pdiv(pselect(mask, abs_y, one), pselect(mask, abs_y_flip, one)); y_scaled = por(y_scaled, pandnot(y.v, abs_y)); // copy signs in case |c| == |d| RealPacket denom = pmul(y.v, y_scaled); denom = padd(denom, pcplxflip(Packet(denom)).v); // c * c' + d * d' Packet num = pmul(x, pconj(Packet(y_scaled))); // a * c' + b * d', -a * d + b * c - return Packet(pdiv(num.v, denom)); + RealPacket result = pdiv(num.v, denom); + // y_scaled = y / max(|c|, |d|), so the denominator lies in [m, 2 m] for m = max(|c|, |d|) and the numerator's + // components are at most |a| + |b|. Either can overflow where x / y is finite, which needs a component above + // highest / 4. Elements whose numerator or denominator overflowed are recomputed with x (y) scaled by 1/4 where it + // has such a component, and the quotient by the inverse scale. The scaling can lose a subnormal component, so + // elements that did not overflow keep the unscaled quotient. + const RealPacket quarter_highest = pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4)); + if (EIGEN_PREDICT_FALSE(predux_any(pcmp_lt(quarter_highest, pmax(pabs(x.v), abs_y))))) { + const RealPacket cst_quarter = pset1<RealPacket>(RealScalar(0.25)); + const RealPacket abs_x = pabs(x.v); + const RealPacket large_x = pcmp_lt(quarter_highest, pmax(abs_x, pcplxflip(Packet(abs_x)).v)); + const RealPacket large_y = pcmp_lt(quarter_highest, pmax(abs_y, abs_y_flip)); + const Packet x_in(pselect(large_x, pmul(x.v, cst_quarter), x.v)); + const RealPacket y_in = pselect(large_y, pmul(y.v, cst_quarter), y.v); + const RealPacket unscale = + pmul(pselect(large_x, pset1<RealPacket>(RealScalar(4)), one), pselect(large_y, cst_quarter, one)); + RealPacket denom_in = pmul(y_in, y_scaled); + denom_in = padd(denom_in, pcplxflip(Packet(denom_in)).v); + const Packet num_in = pmul(x_in, pconj(Packet(y_scaled))); + const RealPacket inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity()); + RealPacket overflow = por(pcmp_eq(pabs(num.v), inf), pcmp_eq(pabs(denom), inf)); + overflow = por(overflow, pcplxflip(Packet(overflow)).v); + result = pselect(overflow, pmul(pdiv(num_in.v, denom_in), unscale), result); + } + return Packet(result); } template <typename Packet> @@ -123,8 +148,11 @@ // The fixups below need x = +-inf or an overflowing exp(x). Such inputs are rare, so packets without // |x| >= (max_exponent - 1) ln(2), just below the overflow threshold, skip them. const RealScalar large_x = RealScalar(NumTraits<RealScalar>::max_exponent() - 1) * RealScalar(EIGEN_LN2); + RealPacket expx_factor = expx; if (predux_any(pcmp_le(pset1<RealPacket>(large_x), pabs(x)))) { const RealPacket cst_pos_inf = pinf<RealPacket>(); + const RealPacket is_x_pos_inf = pcmp_eq(x, cst_pos_inf); + const RealPacket split = pandnot(pcmp_eq(expx, cst_pos_inf), is_x_pos_inf); const RealPacket cst_neg_inf = por(psignmask<RealPacket>(), pinf<RealPacket>()); // If x is -inf, we know that cossin(y) is bounded, @@ -136,18 +164,41 @@ // If x is inf, and cos(y) has unknown sign (y is inf or NaN), the result // is (+/-inf, NaN), where the signs are undetermined (take the sign of y). RealPacket y_sign = por(pandnot(y, pabs(y)), pset1<RealPacket>(RealScalar(1))); - cisy = pselect(pand(pcmp_eq(x, cst_pos_inf), pisnan(cisy)), pand(y_sign, even_mask), cisy); + cisy = pselect(pand(is_x_pos_inf, pisnan(cisy)), pand(y_sign, even_mask), cisy); - // If exp(x) is +inf and y is finite, replace cisy with copysign(1, cisy) to + // If x is +inf and y is finite, replace cisy with copysign(1, cisy) to // prevent inf * 0 = NaN. The vectorized sincos may compute exact zero // for near-zero values like cos(pi/2), and inf * +-1 = +-inf is correct. // The y=0 case is handled separately below. RealPacket cisy_sign_one = por(pand(cisy, psignmask<RealPacket>()), pset1<RealPacket>(RealScalar(1))); - RealPacket expx_inf_y_finite = pand(pcmp_eq(expx, cst_pos_inf), pcmp_lt(pabs(y), cst_pos_inf)); - cisy = pselect(expx_inf_y_finite, cisy_sign_one, cisy); + RealPacket x_inf_y_finite = pand(is_x_pos_inf, pcmp_lt(pabs(y), cst_pos_inf)); + cisy = pselect(x_inf_y_finite, cisy_sign_one, cisy); + + // exp(x) cis(y) can have a finite part where exp(x) overflows, up to (max_exponent - min_exponent + digits) ln(2), + // where exp(x) denorm_min overflows. Those elements (finite x) use x = n ln(2) + r, |r| <= ln(2)/2: + // exp(x) cis(y) = (exp(r) (2^digits cis(y))) 2^(n - digits), + // where 2^digits cis(y) is exact and normal and the result is normal or overflows, so only exp(r) and one product + // round. x is clamped to x_max, one ln(2) further so that its rounding stays past the overflow point, and + // r = (x - n ln2_hi) - n ln2_lo as in pexp_float/pexp_double, the first difference exact for x <= x_max. Other + // elements use x = 0, which raises no exception, and keep exp(x) cis(y), whatever shares their packet. + const int digits = NumTraits<RealScalar>::digits(); + const RealScalar x_max = + RealScalar(NumTraits<RealScalar>::max_exponent() - NumTraits<RealScalar>::min_exponent() + digits + 1) * + RealScalar(EIGEN_LN2); + constexpr bool is_float = std::is_same<RealScalar, float>::value; + const RealScalar ln2_hi = is_float ? RealScalar(0.693359375f) : RealScalar(0.693145751953125); + const RealScalar ln2_lo = is_float ? RealScalar(-2.12194440e-4f) : RealScalar(1.42860682030941723212e-6); + const RealPacket x_split = pselect(split, pmin(x, pset1<RealPacket>(x_max)), pzero(x)); + const RealPacket n = print(pmul(x_split, pset1<RealPacket>(RealScalar(EIGEN_LOG2E)))); + const RealPacket r = pnmadd(n, pset1<RealPacket>(ln2_lo), pnmadd(n, pset1<RealPacket>(ln2_hi), x_split)); + const RealPacket scaled_cisy = pmul(cisy, pset1<RealPacket>(RealScalar(numext::uint64_t(1) << digits))); + const RealPacket split_result = pldexp(pmul(pexp_reduced_impl<RealPacket>::run(r), scaled_cisy), + psub(n, pset1<RealPacket>(RealScalar(digits)))); + cisy = pselect(split, split_result, cisy); + expx_factor = pselect(split, pset1<RealPacket>(RealScalar(1)), expx); } - Packet result = Packet(pmul(expx, cisy)); + Packet result = Packet(pmul(expx_factor, cisy)); // If y is +/- 0, the input is real, so take the real result for consistency. result = pselect(Packet(pcmp_eq(y, pzero(y))), Packet(por(pand(expx, even_mask), pand(y, odd_mask))), result);
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathPow.h b/Eigen/src/Core/arch/Default/GenericPacketMathPow.h index 69af74c..1b7ee0b 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathPow.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathPow.h
@@ -269,9 +269,14 @@ Packet t_hi, t_lo; // t = c * (x-1) twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), t_hi, t_lo); - // r = c * (x-1) / (x+1), + // r = c * (x-1) / (x+1). x + 1 = s_hi + s_lo exactly (fast_twosum, as x < 2): rounding it would limit r, and with + // it y * log2(x) in pow, to working precision. t / (s_hi + s_lo) = (t / s_hi) (1 - s_lo / s_hi) to relative u^2. + Packet s_hi, s_lo; + fast_twosum(one, x, s_hi, s_lo); + Packet q_hi, q_lo; + doubleword_div_fp(t_hi, t_lo, s_hi, q_hi, q_lo); Packet r_hi, r_lo; - doubleword_div_fp(t_hi, t_lo, padd(x, one), r_hi, r_lo); + fast_twosum(q_hi, pnmadd(q_hi, pdiv(s_lo, s_hi), q_lo), r_hi, r_lo); // r2 = r * r Packet r2_hi, r2_lo; @@ -406,9 +411,12 @@ const Packet y_div_2 = pmul(y, pset1<Packet>(Scalar(0.5))); const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2); const Packet y_is_odd_int = pandnot(y_is_int, y_is_even); - // Smallest exponent for which (1 + epsilon) overflows to infinity. + // From this |y| on, x^y is 0 or inf for every |x| != 1. The |x| nearest 1 is 1 - eps/2, with |log(x)| > eps/2, + // and |y| eps/2 >= (digits - min_exponent + 1) ln(2) takes x^y below half the smallest subnormal or past the + // largest finite value. constexpr Scalar huge_exponent = - (NumTraits<Scalar>::max_exponent() * Scalar(EIGEN_LN2)) / NumTraits<Scalar>::epsilon(); + Scalar(2 * (numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent + 1)) * + Scalar(EIGEN_LN2) / NumTraits<Scalar>::epsilon(); const Packet y_abs_is_huge = pcmp_le(pset1<Packet>(huge_exponent), y_abs); // * pow(base, exp) returns NaN if base is finite and negative
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h index 1fbbbc6..327670b 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h
@@ -271,9 +271,19 @@ Packet v = psub(t_hi, x); Packet t_lo = padd(psub(x, psub(t_hi, v)), psub(qp_hi, v)); - // Accumulate the low part of the product and the remaining pi/2 terms. - t_lo = padd(t_lo, qp_lo); - t_lo = pmadd(cst_pio2_2<Packet>(), q, t_lo); + // Near a multiple of pi/2, t_hi + qp_lo cancels against q * pio2_2 too, so that product is formed exactly as well + // and the two large terms are added by 2sum: only terms of order u * |q * pio2_2| are rounded. + Packet qp2_hi, qp2_lo; + twoprod(cst_pio2_2<Packet>(), q, qp2_hi, qp2_lo); + Packet a_hi = padd(t_hi, qp_lo); + v = psub(a_hi, t_hi); + t_lo = padd(t_lo, padd(psub(t_hi, psub(a_hi, v)), psub(qp_lo, v))); + t_hi = padd(a_hi, qp2_hi); + v = psub(t_hi, a_hi); + t_lo = padd(t_lo, padd(psub(a_hi, psub(t_hi, v)), psub(qp2_hi, v))); + + // Accumulate the low part of the product and the remaining pi/2 term. + t_lo = padd(t_lo, qp2_lo); t_lo = pmadd(cst_pio2_3<Packet>(), q, t_lo); return padd(t_hi, t_lo); @@ -593,7 +603,7 @@ // Hyperbolic Functions //---------------------------------------------------------------------- -#ifdef EIGEN_FAST_MATH +#if EIGEN_FAST_MATH /** \internal \returns the hyperbolic tan of \a a (coeff-wise) Doesn't do anything fancy, just a 9/8-degree rational interpolant which @@ -664,32 +674,31 @@ // --type="[3,4]" --numF="[SG]" --denF="[SG]" --log --dispCoeff="dec" --output=tanhf.solly constexpr float alpha[] = {-1.46725140511989593505859375e-02f, -3.333333432674407958984375e-01f}; constexpr float beta[] = {1.570280082523822784423828125e-02, 4.4401752948760986328125e-01, 1.0f}; - const T x2 = pmul(x, x); - const T x3 = pmul(x2, x); - const T p = ppolevl<T, 1>::run(x2, alpha); - const T q = ppolevl<T, 2>::run(x2, beta); - const T small_tanh = pmadd(x3, pdiv(p, q), x); - const T sign_mask = psignmask<T>(); + const T x_sign = pand(sign_mask, x); const T abs_x = pandnot(x, sign_mask); constexpr float kSmallThreshold = 1.25f; const T large_mask = pcmp_lt(pset1<T>(kSmallThreshold), abs_x); + // The large elements enter the polynomial as 0, so that x^2 cannot overflow nor P/Q become inf/inf. + const T x_small = pandnot(x, large_mask); + const T x2 = pmul(x_small, x_small); + const T x3 = pmul(x2, x_small); + const T p = ppolevl<T, 1>::run(x2, alpha); + const T q = ppolevl<T, 2>::run(x2, beta); + // x^3 P/Q has the sign opposite to x, so the sum rounds tanh(-0) to +0; the sign of x restores it. + const T small_tanh = por(x_sign, pmadd(x3, pdiv(p, q), x_small)); // Fast exit if all elements are small. if (!predux_any(large_mask)) { return small_tanh; } - // Compute as 1 - (2 / (1 + exp(2*x))) + // Compute as 1 - (2 / (1 + exp(2*x))) for |x| clamped to 16, where it rounds to 1; the clamp keeps exp(2*x) from + // overflowing. The small elements, NaN included, enter exp as 0, which cannot raise FE_INVALID. const T one = pset1<T>(1.0f); const T two = pset1<T>(2.0f); - const T s = pexp_float<T, true>(pmul(two, abs_x)); - const T abs_tanh = psub(one, pdiv(two, padd(s, one))); - - // Handle infinite inputs and set sign bit. constexpr float kHugeThreshold = 16.0f; - const T huge_mask = pcmp_lt(pset1<T>(kHugeThreshold), abs_x); - const T x_sign = pand(sign_mask, x); - const T large_tanh = por(x_sign, pselect(huge_mask, one, abs_tanh)); + const T s = pexp_float<T, true>(pmul(two, pmin(pselect(large_mask, abs_x, pzero(x)), pset1<T>(kHugeThreshold)))); + const T large_tanh = por(x_sign, psub(one, pdiv(two, padd(s, one)))); return pselect(large_mask, large_tanh, small_tanh); }
diff --git a/Eigen/src/Core/util/ConfigureVectorization.h b/Eigen/src/Core/util/ConfigureVectorization.h index 0810735..29c1cad 100644 --- a/Eigen/src/Core/util/ConfigureVectorization.h +++ b/Eigen/src/Core/util/ConfigureVectorization.h
@@ -533,6 +533,8 @@ #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_RVV10 +// pmadd and its variants map to the fused vfmadd/vfmsub/vfnmsub/vfnmadd. +#define EIGEN_VECTORIZE_FMA #include <riscv_vector.h> // Since we depend on knowing RVV vector length at compile-time, we need @@ -602,6 +604,8 @@ #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_LSX +// pmadd and its variants map to the fused vfmadd/vfmsub/vfnmadd/vfnmsub. +#define EIGEN_VECTORIZE_FMA #include <lsxintrin.h> #elif defined __HVX__ && (__HVX_LENGTH__ == 128) @@ -628,6 +632,13 @@ #define EIGEN_VECTORIZE_FMA #endif +// The generic clang backend skips the x86 detection above but fuses its pmadd where __FMA__ is defined, and the +// kernels that choose constants or error-free products by EIGEN_VECTORIZE_FMA must know it. +#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && defined(__FMA__) && \ + !defined(EIGEN_VECTORIZE_FMA) +#define EIGEN_VECTORIZE_FMA +#endif + #if defined(__F16C__) && !defined(EIGEN_GPUCC) // We can use the optimized fp16 to float and float to fp16 conversion routines #define EIGEN_HAS_FP16_C
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3cede24..06a7a86 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt
@@ -301,6 +301,7 @@ ei_add_test(io) ei_add_test(packetmath "-DEIGEN_FAST_MATH=1") ei_add_test(packetmath_gather) +ei_add_test(tanh_no_fast_math) # Generic clang vector backend tests for different vector sizes. include(CheckCXXSourceCompiles) check_cxx_source_compiles("
diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index df6066e..57dc0f1 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp
@@ -406,6 +406,64 @@ VERIFY(all_pass); } +// pow(x, y) = 2^(y log2(x)) for x in [1/2, 2] and |y log2(x)| up to the exponent range, where y magnifies the error +// of log2(x). Eigen and std::pow are each within a few ulps. +template <typename Scalar> +void pow_magnified_log_test() { + const Scalar eps = NumTraits<Scalar>::epsilon(); + const Scalar max_log2 = Scalar(NumTraits<Scalar>::max_exponent() - 2); + const Index n = 1000; + ArrayX<Scalar> x(n), y(n); + for (Index i = 0; i < n; ++i) { + do { + x(i) = internal::random<Scalar>(Scalar(0.5), Scalar(2)); + } while (x(i) == Scalar(1)); + y(i) = internal::random<Scalar>(-max_log2, max_log2) / numext::log2(x(i)); + } + const ArrayX<Scalar> actual = x.pow(y); + bool all_pass = true; + for (Index i = 0; i < n; ++i) { + const Scalar expected = std::pow(x(i), y(i)); + const bool success = numext::abs(actual(i) - expected) <= Scalar(4) * eps * numext::abs(expected); + all_pass &= success; + if (!success) { + std::cout << std::setprecision(17) << "pow(" << x(i) << "," << y(i) << ") = " << actual(i) << " != " << expected + << std::endl; + } + } + VERIFY(all_pass); +} + +// pow(x, +-y) for x = +-(1 - eps/2), the |x| nearest 1, and y = 1.5 max_exponent ln(2) / eps: x^y = 2^(-0.75 +// max_exponent) and x^-y = 2^(0.75 max_exponent) are normal, although (1 + eps)^y overflows. Both the array and the +// scalar exponent take the vectorized generic pow (y is an integer far above the repeated squaring limit). +template <typename Scalar> +void pow_near_one_huge_exponent_test() { + const Scalar eps = NumTraits<Scalar>::epsilon(); + const Scalar below_one = Scalar(1) - eps / Scalar(2); + const Scalar huge = Scalar(1.5) * Scalar(NumTraits<Scalar>::max_exponent()) * Scalar(EIGEN_LN2) / eps; + const Index n = 2 * internal::packet_traits<Scalar>::size + 1; + bool all_pass = true; + for (const Scalar base : {below_one, -below_one}) { + for (const Scalar exponent : {huge, -huge}) { + const ArrayX<Scalar> x = ArrayX<Scalar>::Constant(n, base); + const ArrayX<Scalar> binary = x.pow(ArrayX<Scalar>::Constant(n, exponent)); + const ArrayX<Scalar> unary = x.pow(exponent); + const Scalar expected = std::pow(base, exponent); + for (Index i = 0; i < n; ++i) { + for (const Scalar actual : {binary(i), unary(i)}) { + const bool success = numext::abs(actual - expected) <= Scalar(4) * eps * numext::abs(expected); + all_pass &= success; + if (!success) { + std::cout << "pow(" << base << "," << exponent << ") = " << actual << " != " << expected << std::endl; + } + } + } + } + } + VERIFY(all_pass); +} + template <typename Scalar, typename ScalarExponent> Scalar calc_overflow_threshold(const ScalarExponent exponent) { EIGEN_USING_STD(exp2); @@ -472,6 +530,9 @@ void float_pow_test() { float_pow_test_impl<float, float>(); float_pow_test_impl<double, double>(); + pow_magnified_log_test<double>(); + pow_near_one_huge_exponent_test<float>(); + pow_near_one_huge_exponent_test<double>(); } template <typename = void>
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 57becb0..59c80f3 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -1399,6 +1399,18 @@ } data1[0] = Scalar(1e-20); CHECK_CWISE1_IF(PacketTraits::HasTanh, std::tanh, internal::ptanh); + if (PacketTraits::HasTanh && PacketSize >= 2) { + // tanh(x) rounds to +-1 for |x| >= 20 in every type here; the rational approximations clamp |x| where they reach 1, + // which depends on whether pmadd is fused. + test::packet_helper<PacketTraits::HasTanh, Packet> h; + for (const Scalar x : {NumTraits<Scalar>::infinity(), Scalar(20)}) { + data1[0] = x; + data1[1] = -x; + h.store(data2, internal::ptanh(h.load(data1))); + VERIFY_IS_EQUAL(data2[0], Scalar(1)); + VERIFY_IS_EQUAL(data2[1], Scalar(-1)); + } + } if (PacketTraits::HasExp && PacketSize >= 2) { const Scalar small = NumTraits<Scalar>::epsilon(); data1[0] = NumTraits<Scalar>::quiet_NaN(); @@ -1546,6 +1558,22 @@ VERIFY(test::biteq(data2[0], data1[0])); h.store(data2, internal::pcos(h.load(data1))); VERIFY_IS_EQUAL(data2[0], Scalar(1)); + + // Doubles near multiples k pi/2, with |x - k pi/2| from 6e-19 to 9e-7 for |k| from 29 to 6e13: the reduction + // needs pi/2 to far beyond double precision, with or without FMA. + EIGEN_IF_CONSTEXPR (std::is_same<Scalar, double>::value) { + const double hard_cases[] = {-45.553093477052002, -364.42474781641602, 857879.41830474522, 956274858.90980339, + -93293944418765.062}; + const Scalar tol = Scalar(4) * NumTraits<Scalar>::epsilon(); + for (const double x : hard_cases) { + for (int i = 0; i < PacketSize; ++i) data1[i] = Scalar(x); + h.store(data2, internal::psin(h.load(data1))); + h.store(data2 + PacketSize, internal::pcos(h.load(data1))); + const Scalar sin_x = Scalar(std::sin(data1[0])), cos_x = Scalar(std::cos(data1[0])); + VERIFY(numext::abs(data2[0] - sin_x) <= tol * numext::abs(sin_x)); + VERIFY(numext::abs(data2[PacketSize] - cos_x) <= tol * numext::abs(cos_x)); + } + } } } if (PacketTraits::HasReciprocal && PacketSize >= 2) { @@ -2138,6 +2166,27 @@ VERIFY(test::areApprox(ref, pval, PacketSize) && "conj_helper pmadd"); } +// The FE_DIVBYZERO, FE_INVALID and FE_OVERFLOW flags raised by op(operands, results), where operands are copies of +// in[0, NumIn) and results are copied to out[0, NumOut). The copies go through volatile bytes after feholdexcept and +// before fetestexcept, which keeps the compiler from folding op or moving it across the flag accesses: glibc marks +// those functions leaf, so data that only escapes through a pointer may be moved across them. +template <int NumIn, int NumOut, typename Scalar, typename Op> +int raised_exceptions(const Scalar* in, Scalar* out, Op op) { + Scalar operands[NumIn], results[NumOut]; + std::fenv_t saved_environment; + if (std::feholdexcept(&saved_environment) != 0) return 0; + const volatile unsigned char* in_bytes = reinterpret_cast<const volatile unsigned char*>(in); + unsigned char* operand_bytes = reinterpret_cast<unsigned char*>(operands); + for (std::size_t k = 0; k < sizeof(operands); ++k) operand_bytes[k] = in_bytes[k]; + op(operands, results); + const unsigned char* result_bytes = reinterpret_cast<const unsigned char*>(results); + volatile unsigned char* out_bytes = reinterpret_cast<volatile unsigned char*>(out); + for (std::size_t k = 0; k < sizeof(results); ++k) out_bytes[k] = result_bytes[k]; + const int raised = std::fetestexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); + std::fesetenv(&saved_environment); + return raised; +} + template <typename Scalar, typename Packet, bool HasExp = internal::packet_traits<Scalar>::HasExp> struct exp_complex_test_impl { typedef typename Scalar::value_type RealScalar; @@ -2268,6 +2317,102 @@ } } } + + // Where exp(x) overflows, exp(x) cos(y) or exp(x) sin(y) can still be finite: here the real part for y = 1.2 and + // the imaginary part for y = 3. The reference forms exp(x) = 2^64 exp(x - 64 ln(2)). + const RealScalar x_big = numext::log(NumTraits<RealScalar>::highest()) + RealScalar(0.5); + const RealScalar exp_scaled = numext::exp(x_big - RealScalar(64) * RealScalar(EIGEN_LN2)); + for (const RealScalar y : {RealScalar(1.2), RealScalar(3)}) { + const Scalar w = pexp1(Scalar(x_big, y)); + const RealScalar parts[] = {numext::real_ref(w), numext::imag_ref(w)}; + const RealScalar expected[] = {std::ldexp(exp_scaled * numext::cos(y), 64), + std::ldexp(exp_scaled * numext::sin(y), 64)}; + for (int k = 0; k < 2; ++k) { + if ((numext::isinf)(expected[k])) { + VERIFY_IS_EQUAL(parts[k], expected[k]); + } else { + VERIFY_IS_APPROX(parts[k], expected[k]); + } + } + } + + const RealScalar inf = std::numeric_limits<RealScalar>::infinity(); + const RealScalar log_highest = numext::log(NumTraits<RealScalar>::highest()); +#if !EIGEN_ARCH_ARM // 32-bit ARM flushes subnormals. + // Where exp(x / 2) overflows too, exp(x) sin(y) is still finite for a subnormal y, which is sin(y) exactly. The + // reference is exp(x) y in a wider type that holds exp(x); without one, it is (((y h) h) h) h with h = exp(x / 4), + // whose partial products are normal. + using Wide = std::conditional_t<std::is_same<RealScalar, float>::value, double, long double>; + constexpr bool kWideHoldsExp = + std::numeric_limits<Wide>::max_exponent > 4 * std::numeric_limits<RealScalar>::max_exponent && + std::numeric_limits<Wide>::digits > std::numeric_limits<RealScalar>::digits + 8; + const RealScalar denorm_min = std::numeric_limits<RealScalar>::denorm_min(); + const RealScalar huge_args[][2] = { + {RealScalar(2) * log_highest + RealScalar(2), RealScalar(12345) * denorm_min}, + {log_highest - numext::log(RealScalar(4)) - numext::log(denorm_min), denorm_min}}; + for (const auto& arg : huge_args) { + for (const RealScalar y : {arg[1], -arg[1]}) { + const Scalar w = pexp1(Scalar(arg[0], y)); + VERIFY_IS_EQUAL(numext::real(w), inf); + if (kWideHoldsExp) { + // Within 4 eps (the implementation is within about 2.1 ulps). + const Wide expected = std::exp(Wide(arg[0])) * Wide(y); + VERIFY(std::abs(Wide(numext::imag(w)) - expected) <= + Wide(4) * Wide(NumTraits<RealScalar>::epsilon()) * std::abs(expected)); + } else { + const RealScalar h = numext::exp(arg[0] / RealScalar(4)); + VERIFY_IS_APPROX(numext::imag(w), y * h * h * h * h); + } + } + } + + // From (max_exponent - min_exponent + digits) ln(2) on, even exp(x) denorm_min overflows. + const int max_shift = std::numeric_limits<RealScalar>::max_exponent - + std::numeric_limits<RealScalar>::min_exponent + std::numeric_limits<RealScalar>::digits; + for (const RealScalar shift : {RealScalar(0.5), RealScalar(4)}) { + for (const RealScalar y : {denorm_min, -denorm_min}) { + const Scalar w = pexp1(Scalar((RealScalar(max_shift) + shift) * RealScalar(EIGEN_LN2), y)); + VERIFY_IS_EQUAL(numext::real(w), inf); + VERIFY_IS_EQUAL(numext::imag(w), y > 0 ? inf : -inf); + } + } +#endif + + // pexp raises no FE_DIVBYZERO, FE_INVALID or FE_OVERFLOW beyond those of the real pexp of the real parts, which it + // starts from: also not for x = +-inf or -highest while another element takes the overflow branch. + { + using RealPacket = typename internal::unpacket_traits<Packet>::as_real; + const RealScalar near_overflow = log_highest - RealScalar(0.25); + for (const RealScalar x0 : {inf, -inf, -NumTraits<RealScalar>::highest()}) { + Scalar z[PacketSize], w[PacketSize]; + RealScalar x[2 * PacketSize], expx[2 * PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + z[i] = i == 0 ? Scalar(x0, RealScalar(1)) : Scalar(near_overflow, RealScalar(0.5)); + x[2 * i] = x[2 * i + 1] = numext::real(z[i]); + } + const int complex_raised = raised_exceptions<PacketSize, PacketSize>(z, w, [](const Scalar* in, Scalar* out) { + internal::pstoreu(out, internal::pexp(internal::ploadu<Packet>(in))); + }); + const int real_raised = + raised_exceptions<2 * PacketSize, 2 * PacketSize>(x, expx, [](const RealScalar* in, RealScalar* out) { + internal::pstoreu(out, internal::pexp(internal::ploadu<RealPacket>(in))); + }); + VERIFY_IS_EQUAL(complex_raised & ~real_raised, 0); + } + } + + // An element whose exp(x) is finite gets the same result whatever shares its packet. + if (PacketSize > 1) { + Scalar data[PacketSize], alone[PacketSize], mixed[PacketSize]; + for (int i = 0; i < PacketSize; ++i) data[i] = Scalar(RealScalar(1.75) + RealScalar(i), RealScalar(0.5)); + internal::pstoreu(alone, internal::pexp(internal::ploadu<Packet>(data))); + data[0] = Scalar(x_big, RealScalar(1.2)); + internal::pstoreu(mixed, internal::pexp(internal::ploadu<Packet>(data))); + for (int i = 1; i < PacketSize; ++i) { + VERIFY_IS_EQUAL(numext::real(mixed[i]), numext::real(alone[i])); + VERIFY_IS_EQUAL(numext::imag(mixed[i]), numext::imag(alone[i])); + } + } } }; @@ -2282,6 +2427,27 @@ exp_complex_test_impl<Scalar, Packet>::run(data1, data2, ref, size); } +// Checks pdiv(x, y) against its exact value, expected, in every element, and returns the FE_DIVBYZERO, FE_INVALID and +// FE_OVERFLOW flags it raised. +template <typename Scalar, typename Packet> +int pdiv_complex_exceptions(const Scalar& x, const Scalar& y, const Scalar& expected) { + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + Scalar operands[2 * PacketSize], q[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + operands[i] = x; + operands[PacketSize + i] = y; + } + const int raised = raised_exceptions<2 * PacketSize, PacketSize>(operands, q, [](const Scalar* in, Scalar* out) { + internal::pstoreu(out, internal::pdiv(internal::ploadu<Packet>(in), + internal::ploadu<Packet>(in + internal::unpacket_traits<Packet>::size))); + }); + for (int i = 0; i < PacketSize; ++i) { + VERIFY_IS_EQUAL(numext::real(q[i]), numext::real(expected)); + VERIFY_IS_EQUAL(numext::imag(q[i]), numext::imag(expected)); + } + return raised; +} + template <typename Scalar, typename Packet> void packetmath_complex() { typedef internal::packet_traits<Scalar> PacketTraits; @@ -2349,6 +2515,49 @@ Scalar expected = data1[i] / data2[i]; VERIFY_IS_APPROX(pval[i], expected); } + + // Finite quotients of operands near the overflow threshold, beside an ordinary one. The exact quotients are given: + // libgcc's complex<double> division overflows on the second. + const RealScalar big = RealScalar(0.75) * NumTraits<RealScalar>::highest(); + const RealScalar half = RealScalar(0.5); + const Scalar numerators[] = {Scalar(big, zero), Scalar(big, big), Scalar(big, big), Scalar(one, RealScalar(2))}; + const Scalar denominators[] = {Scalar(big, big), Scalar(one, one), Scalar(big, -big), Scalar(RealScalar(3), one)}; + const Scalar quotients[] = {Scalar(half, -half), Scalar(big, zero), Scalar(zero, one), Scalar(half, half)}; + for (int k = 0; k < 4; ++k) { + for (int i = 0; i < PacketSize; ++i) { + data1[i] = numerators[(i + k) % 4]; + data2[i] = denominators[(i + k) % 4]; + } + internal::pstore(pval, internal::pdiv(internal::pload<Packet>(data1), internal::pload<Packet>(data2))); + for (int i = 0; i < PacketSize; ++i) VERIFY_IS_APPROX(pval[i], quotients[(i + k) % 4]); + } + + // A denominator with a zero component raises no exception: the ratio of the other component to it is not formed. + const RealScalar two = RealScalar(2), three = RealScalar(3), four = RealScalar(4), three_halves = RealScalar(1.5); + const Scalar exact_operands[][3] = {{Scalar(one, zero), Scalar(one, zero), Scalar(one, zero)}, + {Scalar(three, four), Scalar(zero, two), Scalar(two, -three_halves)}, + {Scalar(three, four), Scalar(-two, zero), Scalar(-three_halves, -two)}}; + for (const auto& operands : exact_operands) { + VERIFY_IS_EQUAL((pdiv_complex_exceptions<Scalar, Packet>(operands[0], operands[1], operands[2])), 0); + } + +#if !EIGEN_ARCH_ARM // 32-bit ARM flushes subnormals. + // A large component beside a subnormal one, divided by one, must come back exactly: scaling the numerator by 1/4 + // would flush the subnormal component. + const RealScalar tiny = std::numeric_limits<RealScalar>::denorm_min(); + const Scalar mixed[] = {Scalar(big, tiny), Scalar(tiny, -big)}; + for (int k = 0; k < 2; ++k) { + for (int i = 0; i < PacketSize; ++i) { + data1[i] = mixed[(i + k) % 2]; + data2[i] = Scalar(one, zero); + } + internal::pstore(pval, internal::pdiv(internal::pload<Packet>(data1), internal::pload<Packet>(data2))); + for (int i = 0; i < PacketSize; ++i) { + VERIFY_IS_EQUAL(numext::real(pval[i]), numext::real(mixed[(i + k) % 2])); + VERIFY_IS_EQUAL(numext::imag(pval[i]), numext::imag(mixed[(i + k) % 2])); + } + } +#endif } // Test pisnan. @@ -2375,10 +2584,17 @@ for (RealScalar b : special_values) { for (RealScalar c : special_values) { for (RealScalar d : special_values) { - data1[0] = Scalar(a, b); - data2[0] = Scalar(c, d); + // CHECK_CWISE2_IF takes the second operand from data1 + PacketSize. + for (int i = 0; i < PacketSize; ++i) { + data1[i] = Scalar(a, b); + data1[i + PacketSize] = Scalar(c, d); + } CHECK_CWISE2_IF(PacketTraits::HasMul, internal::complex_multiply, internal::pmul); - CHECK_CWISE2_IF(PacketTraits::HasDiv, internal::complex_divide, internal::pdiv); + // pdiv follows Smith's algorithm, but where the scalar version forms 0/0 or inf/inf from a zero divisor or + // one with two infinite components, pdiv returns the infinite or zero quotient instead. + if (!(c == zero && d == zero) && !((numext::isinf)(c) && (numext::isinf)(d))) { + CHECK_CWISE2_IF(PacketTraits::HasDiv, internal::complex_divide_smith, internal::pdiv); + } } } }
diff --git a/test/tanh_no_fast_math.cpp b/test/tanh_no_fast_math.cpp new file mode 100644 index 0000000..2c5998b --- /dev/null +++ b/test/tanh_no_fast_math.cpp
@@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// EIGEN_FAST_MATH=0 selects the accurate float tanh kernel, which backends such as the generic clang one vectorize +// whatever the setting. +#define EIGEN_FAST_MATH 0 + +#include "main.h" + +#include <cfenv> +#include <cstring> + +namespace { + +numext::int64_t float_ulp_distance(float a, float b) { + numext::int32_t ia, ib; + std::memcpy(&ia, &a, sizeof(float)); + std::memcpy(&ib, &b, sizeof(float)); + // Map the sign-magnitude bit patterns onto a monotonic integer line. + const numext::int64_t la = ia < 0 ? numext::int64_t(INT32_MIN) - ia : ia; + const numext::int64_t lb = ib < 0 ? numext::int64_t(INT32_MIN) - ib : ib; + return la > lb ? la - lb : lb - la; +} + +template <typename Packet> +void tanh_float_accuracy() { + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + // Every third float in [0.5, 10], where the fast kernel is up to 6 ulps off; the accurate one is within 2. + const numext::uint32_t first = 0x3f000000u, last = 0x41200000u; + float x[PacketSize], y[PacketSize]; + numext::int64_t max_error = 0; + float worst = 0; + for (numext::uint32_t bits = first; bits < last;) { + for (int k = 0; k < PacketSize; ++k, bits += 3) { + std::memcpy(&x[k], &bits, sizeof(float)); + if (k % 2 == 1) x[k] = -x[k]; + } + internal::pstoreu(y, internal::ptanh_float(internal::ploadu<Packet>(x))); + for (int k = 0; k < PacketSize; ++k) { + const numext::int64_t error = float_ulp_distance(y[k], static_cast<float>(std::tanh(double(x[k])))); + if (error > max_error) { + max_error = error; + worst = x[k]; + } + } + } + if (max_error > 2) std::cout << "tanh(" << worst << ") is " << max_error << " ulps off" << std::endl; + VERIFY(max_error <= 2); + + const float inf = NumTraits<float>::infinity(); + const float special[] = {0.0f, -0.0f, inf, -inf, NumTraits<float>::quiet_NaN(), 20.0f, -20.0f, 1e-30f}; + for (const float s : special) { + for (int k = 0; k < PacketSize; ++k) x[k] = s; + internal::pstoreu(y, internal::ptanh_float(internal::ploadu<Packet>(x))); + const float expected = std::tanh(s); + const bool success = ((numext::isnan)(expected) && (numext::isnan)(y[0])) || + (y[0] == expected && std::signbit(y[0]) == std::signbit(expected)); + if (!success) std::cout << "tanh(" << s << ") = " << y[0] << " != " << expected << std::endl; + VERIFY(success); + } +} + +template <typename Packet> +Packet tanh_kernel(const Packet& x, float) { + return internal::ptanh_float(x); +} +template <typename Packet> +Packet tanh_kernel(const Packet& x, double) { + return internal::ptanh_double(x); +} + +// The FE_OVERFLOW, FE_INVALID and FE_DIVBYZERO flags the tanh kernel raises on the packet in, with the result in out. +// The operands and results go through volatile copies after feholdexcept and before fetestexcept, which keeps the +// compiler from folding the kernel or moving it across the flag accesses. +template <typename Packet, typename Scalar> +int tanh_exceptions(const Scalar* in, Scalar* out) { + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + const volatile Scalar* volatile_in = in; + volatile Scalar* volatile_out = out; + Scalar x[PacketSize], y[PacketSize]; + std::fenv_t saved_environment; + if (std::feholdexcept(&saved_environment) != 0) return 0; + for (int k = 0; k < PacketSize; ++k) x[k] = volatile_in[k]; + internal::pstoreu(y, tanh_kernel(internal::ploadu<Packet>(x), Scalar())); + for (int k = 0; k < PacketSize; ++k) volatile_out[k] = y[k]; + const int raised = std::fetestexcept(FE_OVERFLOW | FE_INVALID | FE_DIVBYZERO); + std::fesetenv(&saved_environment); + return raised; +} + +// tanh of large and infinite arguments is +-1 and raises no FE_OVERFLOW, FE_INVALID or FE_DIVBYZERO. A large element +// beside a NaN raises nothing that a small one does not: the NaN's own comparison may raise FE_INVALID. +template <typename Packet> +void tanh_saturation_exceptions() { + using Scalar = typename internal::unpacket_traits<Packet>::type; + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + const Scalar inf = NumTraits<Scalar>::infinity(); + const Scalar values[] = {Scalar(20), Scalar(100), Scalar(1e20f), NumTraits<Scalar>::highest(), inf}; + Scalar x[PacketSize], y[PacketSize]; + for (const Scalar value : values) { + for (const Scalar s : {value, -value}) { + for (int k = 0; k < PacketSize; ++k) x[k] = s; + const int raised = tanh_exceptions<Packet>(x, y); + if (raised != 0) std::cout << "tanh(" << s << ") raised " << raised << std::endl; + VERIFY_IS_EQUAL(raised, 0); + for (int k = 0; k < PacketSize; ++k) VERIFY_IS_EQUAL(y[k], s > 0 ? Scalar(1) : Scalar(-1)); + } + } + if (PacketSize > 1) { + int raised[2]; + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < PacketSize; ++k) x[k] = k == 0 ? NumTraits<Scalar>::quiet_NaN() : Scalar(j == 0 ? 0.5 : 100); + raised[j] = tanh_exceptions<Packet>(x, y); + VERIFY((numext::isnan)(y[0])); + } + VERIFY_IS_EQUAL(raised[1] & ~raised[0], 0); + } +} + +} // namespace + +EIGEN_DECLARE_TEST(tanh_no_fast_math) { + CALL_SUBTEST(tanh_float_accuracy<internal::packet_traits<float>::type>()); + CALL_SUBTEST(tanh_float_accuracy<float>()); + CALL_SUBTEST(tanh_saturation_exceptions<internal::packet_traits<float>::type>()); + CALL_SUBTEST(tanh_saturation_exceptions<float>()); + CALL_SUBTEST(tanh_saturation_exceptions<internal::packet_traits<double>::type>()); + CALL_SUBTEST(tanh_saturation_exceptions<double>()); +}