Core: Match scalar ptanh clamp threshold to pmadd FMA fusion libeigen/eigen!3375
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index edc917c..a913986 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -1683,8 +1683,32 @@ } }; +#if defined(EIGEN_VECTORIZE_FMA) +// Scalar path for pmadd with FMA to ensure consistency with vectorized path. +template <typename Scalar> +struct pmadd_impl<Scalar, std::enable_if_t<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>> { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(const Scalar& a, const Scalar& b, const Scalar& c) { + return numext::fma(a, b, c); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmsub(const Scalar& a, const Scalar& b, const Scalar& c) { + return numext::fma(a, b, Scalar(-c)); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmadd(const Scalar& a, const Scalar& b, const Scalar& c) { + return numext::fma(Scalar(-a), b, c); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmsub(const Scalar& a, const Scalar& b, const Scalar& c) { + return -Scalar(numext::fma(a, b, c)); + } +}; + +template <typename Scalar> +struct pmadd_impl<Scalar, + std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned && + !std::is_same<Scalar, float>::value && !std::is_same<Scalar, double>::value>> { +#else template <typename Scalar> struct pmadd_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned>> { +#endif static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(const Scalar& a, const Scalar& b, const Scalar& c) { return numext::madd<Scalar>(a, b, c); }
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 476facd..1c7a286 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -1083,7 +1083,9 @@ template <typename Scalar, typename EnableIf = void> struct madd_impl { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Scalar& x, const Scalar& y, const Scalar& z) { - return x * y + z; + // Separate statements prevent FP_CONTRACT ON from fusing the unfused fallback. + const Scalar xy = x * y; + return xy + z; } };
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h index 46132c6..0884542 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h
@@ -621,12 +621,13 @@ // floating point argument such that the approximation is exactly 1. // This saves clamping the value at the end. #ifdef EIGEN_VECTORIZE_FMA - const T plus_clamp = pset1<T>(8.01773357391357422f); - const T minus_clamp = pset1<T>(-8.01773357391357422f); + constexpr float kClamp = 8.01773357391357422f; #else - const T plus_clamp = pset1<T>(7.90738964080810547f); - const T minus_clamp = pset1<T>(-7.90738964080810547f); + constexpr bool kUsesFma = is_scalar<T>::value && bool(EIGEN_SCALAR_MADD_USE_FMA) && has_fma<T>::value; + constexpr float kClamp = kUsesFma ? 8.01773357391357422f : 7.90738964080810547f; #endif + const T plus_clamp = pset1<T>(kClamp); + const T minus_clamp = pset1<T>(-kClamp); const T x = pmax(pmin(a_x, plus_clamp), minus_clamp); // The following rational approximation was generated by rminimax @@ -720,12 +721,13 @@ // floating point argument such that the approximation is exactly 1. // This saves clamping the value at the end. #ifdef EIGEN_VECTORIZE_FMA - const T plus_clamp = pset1<T>(17.6610191624600077); - const T minus_clamp = pset1<T>(-17.6610191624600077); + constexpr double kClamp = 17.6610191624600077; #else - const T plus_clamp = pset1<T>(17.714196154005176); - const T minus_clamp = pset1<T>(-17.714196154005176); + constexpr bool kUsesFma = is_scalar<T>::value && bool(EIGEN_SCALAR_MADD_USE_FMA) && has_fma<T>::value; + constexpr double kClamp = kUsesFma ? 17.6610191624600077 : 17.714196154005176; #endif + const T plus_clamp = pset1<T>(kClamp); + const T minus_clamp = pset1<T>(-kClamp); const T x = pmax(pmin(a_x, plus_clamp), minus_clamp); // The following rational approximation was generated by rminimax // (https://gitlab.inria.fr/sfilip/rminimax) using the following
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 34a088a..323dd1c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt
@@ -286,6 +286,7 @@ ei_add_test(numext) ei_add_test(complex_sqrt) ei_add_test(numext_fma_off) +ei_add_test(numext_fma_on) # Fused residuals must not depend on optional compiler contraction. set(twoprod_test_flags "") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
diff --git a/test/numext.cpp b/test/numext.cpp index fc88c58..134716c 100644 --- a/test/numext.cpp +++ b/test/numext.cpp
@@ -1114,6 +1114,15 @@ } } +void check_scalar_tanh_saturation() { + VERIFY_IS_EQUAL(internal::ptanh_float(20.0f), 1.0f); + VERIFY_IS_EQUAL(internal::ptanh_float(-20.0f), -1.0f); + VERIFY_IS_EQUAL(internal::ptanh_double(20.0), 1.0); + VERIFY_IS_EQUAL(internal::ptanh_double(-20.0), -1.0); + VERIFY_IS_EQUAL(numext::tanh(20.0f), 1.0f); + VERIFY_IS_EQUAL(numext::tanh(-20.0f), -1.0f); +} + EIGEN_DECLARE_TEST(numext) { for (int k = 0; k < g_repeat; ++k) { CALL_SUBTEST(check_copysign<half>()); @@ -1287,5 +1296,7 @@ CALL_SUBTEST(check_shift<uint16_t>()); CALL_SUBTEST(check_shift<uint32_t>()); CALL_SUBTEST(check_shift<uint64_t>()); + + CALL_SUBTEST(check_scalar_tanh_saturation()); } }
diff --git a/test/numext_fma_on.cpp b/test/numext_fma_on.cpp new file mode 100644 index 0000000..3953c24 --- /dev/null +++ b/test/numext_fma_on.cpp
@@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#undef EIGEN_SCALAR_MADD_USE_FMA +#define EIGEN_SCALAR_MADD_USE_FMA 1 +#include "numext.cpp" // NOLINT(bugprone-suspicious-include): Compile the suite with a different configuration.