Core: Keep some Boolean operations branch-free libeigen/eigen!3002
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index e8499d1..9169fa4 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -2017,17 +2017,32 @@ template <typename T> EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex<T>& x) { - return (numext::isfinite)(numext::real(x)) && (numext::isfinite)(numext::imag(x)); + EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) { + // Eager classification lets coefficient-wise loops remain branch-free. + return static_cast<unsigned int>((numext::isfinite)(numext::real(x))) & (numext::isfinite)(numext::imag(x)); + } else { + return (numext::isfinite)(numext::real(x)) && (numext::isfinite)(numext::imag(x)); + } } template <typename T> EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x) { - return (numext::isnan)(numext::real(x)) || (numext::isnan)(numext::imag(x)); + EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) { + return static_cast<unsigned int>((numext::isnan)(numext::real(x))) | (numext::isnan)(numext::imag(x)); + } else { + return (numext::isnan)(numext::real(x)) || (numext::isnan)(numext::imag(x)); + } } template <typename T> EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x) { - return ((numext::isinf)(numext::real(x)) || (numext::isinf)(numext::imag(x))) && (!(numext::isnan)(x)); + EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) { + const bool has_inf = static_cast<unsigned int>((numext::isinf)(numext::real(x))) | (numext::isinf)(numext::imag(x)); + const bool has_nan = (numext::isnan)(x); + return has_inf & !has_nan; + } else { + return ((numext::isinf)(numext::real(x)) || (numext::isinf)(numext::imag(x))) && (!(numext::isnan)(x)); + } } /****************************************************************************
diff --git a/Eigen/src/Core/functors/BinaryFunctors.h b/Eigen/src/Core/functors/BinaryFunctors.h index 0c024f8..5c7a581 100644 --- a/Eigen/src/Core/functors/BinaryFunctors.h +++ b/Eigen/src/Core/functors/BinaryFunctors.h
@@ -487,6 +487,10 @@ }; }; +// Packet16b is currently Eigen's only Boolean packet. Its loads contain valid bool objects, and its casts, +// comparisons, and Boolean packet operations canonicalize every lane to 0 or 1. The direct bitwise fast paths below +// rely on this invariant. + /** \internal * \brief Template functor to compute the and of two scalars as if they were booleans * @@ -502,12 +506,16 @@ } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { - const Packet cst_one = pset1<Packet>(Scalar(1)); - // and(a,b) == !or(!a,!b) - Packet not_a = pcmp_eq(a, pzero(a)); - Packet not_b = pcmp_eq(b, pzero(b)); - Packet a_nand_b = por(not_a, not_b); - return pandnot(cst_one, a_nand_b); + EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, bool>::value)) { + return pand(a, b); + } else { + const Packet cst_one = pset1<Packet>(Scalar(1)); + // and(a,b) == !or(!a,!b) + Packet not_a = pcmp_eq(a, pzero(a)); + Packet not_b = pcmp_eq(b, pzero(b)); + Packet a_nand_b = por(not_a, not_b); + return pandnot(cst_one, a_nand_b); + } } }; // Keep bool logical functors eager so scalar evaluator loops remain branch-free. @@ -535,12 +543,16 @@ return (a != Scalar(0)) || (b != Scalar(0)) ? Scalar(1) : Scalar(0); } template <typename Packet> - EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { - const Packet cst_one = pset1<Packet>(Scalar(1)); - // if or(a,b) == 0, then a == 0 and b == 0 - // or(a,b) == !nor(a,b) - Packet a_nor_b = pcmp_eq(por(a, b), pzero(a)); - return pandnot(cst_one, a_nor_b); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, bool>::value)) { + return por(a, b); + } else { + const Packet cst_one = pset1<Packet>(Scalar(1)); + // if or(a,b) == 0, then a == 0 and b == 0 + // or(a,b) == !nor(a,b) + Packet a_nor_b = pcmp_eq(por(a, b), pzero(a)); + return pandnot(cst_one, a_nor_b); + } } }; template <> @@ -567,13 +579,17 @@ return (a != Scalar(0)) != (b != Scalar(0)) ? Scalar(1) : Scalar(0); } template <typename Packet> - EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { - const Packet cst_one = pset1<Packet>(Scalar(1)); - // xor(a,b) == xor(!a,!b) - Packet not_a = pcmp_eq(a, pzero(a)); - Packet not_b = pcmp_eq(b, pzero(b)); - Packet a_xor_b = pxor(not_a, not_b); - return pand(cst_one, a_xor_b); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, bool>::value)) { + return pxor(a, b); + } else { + const Packet cst_one = pset1<Packet>(Scalar(1)); + // xor(a,b) == xor(!a,!b) + Packet not_a = pcmp_eq(a, pzero(a)); + Packet not_b = pcmp_eq(b, pzero(b)); + Packet a_xor_b = pxor(not_a, not_b); + return pand(cst_one, a_xor_b); + } } }; template <typename Scalar>
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index 4772329..74ab18c 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -1176,10 +1176,15 @@ return a == Scalar(0) ? Scalar(1) : Scalar(0); } template <typename Packet> - EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { const Packet cst_one = pset1<Packet>(Scalar(1)); - Packet not_a = pcmp_eq(a, pzero(a)); - return pand(not_a, cst_one); + EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, bool>::value)) { + // Boolean packet lanes are canonical, so logical NOT is 1 & ~a. + return pandnot(cst_one, a); + } else { + Packet not_a = pcmp_eq(a, pzero(a)); + return pand(not_a, cst_one); + } } }; template <typename Scalar>
diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index e38a783..ffd8cec 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp
@@ -1430,17 +1430,54 @@ const Index size = 67; ArrayX<bool> lhs = ArrayXi::Random(size) > 0; ArrayX<bool> rhs = ArrayXi::Random(size) > 0; - lhs[0] = false; - rhs[0] = false; - lhs[1] = false; - rhs[1] = true; - lhs[2] = true; - rhs[2] = false; - lhs[3] = true; - rhs[3] = true; + for (Index i = 0; i < 4; ++i) { + lhs[i] = lhs[size - 4 + i] = (i & 2) != 0; + rhs[i] = rhs[size - 4 + i] = (i & 1) != 0; + } - VERIFY_IS_CWISE_EQUAL(lhs && rhs, (lhs.cast<int>() * rhs.cast<int>()) != 0); - VERIFY_IS_CWISE_EQUAL(lhs || rhs, (lhs.cast<int>() + rhs.cast<int>()) != 0); + ArrayX<bool> actual_and(size), actual_or(size), actual_xor(size), actual_not(size); + ArrayX<bool> expected_and(size), expected_or(size), expected_xor(size), expected_not(size); + actual_and = lhs && rhs; + actual_or = lhs || rhs; + actual_xor = lhs.binaryExpr(rhs, internal::scalar_boolean_xor_op<bool>()); + actual_not = !lhs; + for (Index i = 0; i < size; ++i) { + expected_and[i] = lhs[i] && rhs[i]; + expected_or[i] = lhs[i] || rhs[i]; + expected_xor[i] = lhs[i] != rhs[i]; + expected_not[i] = !lhs[i]; + } + VERIFY_IS_CWISE_EQUAL(actual_and, expected_and); + VERIFY_IS_CWISE_EQUAL(actual_or, expected_or); + VERIFY_IS_CWISE_EQUAL(actual_xor, expected_xor); + VERIFY_IS_CWISE_EQUAL(actual_not, expected_not); +} + +template <typename RealScalar> +void complex_classification() { + using Complex = std::complex<RealScalar>; + const RealScalar inf = NumTraits<RealScalar>::infinity(); + const RealScalar nan = NumTraits<RealScalar>::quiet_NaN(); + + struct TestCase { + Complex value; + bool finite; + bool infinite; + bool not_a_number; + }; + const TestCase test_cases[] = { + {Complex(0, 0), true, false, false}, {Complex(inf, 0), false, true, false}, + {Complex(0, inf), false, true, false}, {Complex(nan, 0), false, false, true}, + {Complex(0, nan), false, false, true}, {Complex(inf, nan), false, false, true}, + {Complex(nan, inf), false, false, true}, {Complex(-inf, -inf), false, true, false}, + {Complex(nan, nan), false, false, true}, + }; + + for (const TestCase& test_case : test_cases) { + VERIFY_IS_EQUAL((numext::isfinite)(test_case.value), test_case.finite); + VERIFY_IS_EQUAL((numext::isinf)(test_case.value), test_case.infinite); + VERIFY_IS_EQUAL((numext::isnan)(test_case.value), test_case.not_a_number); + } } EIGEN_DECLARE_TEST(array_cwise) { @@ -1515,6 +1552,9 @@ CALL_SUBTEST_18(array_complex( ArrayXXcd(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); } + CALL_SUBTEST_17(complex_classification<float>()); + CALL_SUBTEST_18(complex_classification<double>()); + CALL_SUBTEST_18(complex_classification<long double>()); for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_19(float_pow_test());
diff --git a/unsupported/Eigen/src/Tensor/TensorFunctors.h b/unsupported/Eigen/src/Tensor/TensorFunctors.h index 001cb7d..fe7adc7 100644 --- a/unsupported/Eigen/src/Tensor/TensorFunctors.h +++ b/unsupported/Eigen/src/Tensor/TensorFunctors.h
@@ -307,8 +307,10 @@ }; }; +// Inputs have already been evaluated when reduce() is called; eager bitwise operations keep scalar reduction loops +// branch-free. struct AndReducer { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum && t; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum & t; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool initialize() const { return true; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool finalize(bool accum) const { return accum; } }; @@ -319,7 +321,7 @@ }; struct OrReducer { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum || t; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum | t; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool initialize() const { return false; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool finalize(bool accum) const { return accum; } };