Vectorize TensorSelectOp condition mask loads via Packet16b pcast libeigen/eigen!3366 Co-authored-by: Antonio Sanchez <cantonios@google.com>
diff --git a/Eigen/src/Core/arch/AVX/TypeCasting.h b/Eigen/src/Core/arch/AVX/TypeCasting.h index f93b484..e5876bd 100644 --- a/Eigen/src/Core/arch/AVX/TypeCasting.h +++ b/Eigen/src/Core/arch/AVX/TypeCasting.h
@@ -25,6 +25,13 @@ struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {}; template <> +struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {}; +template <> +struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {}; +template <> +struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {}; + +template <> struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {}; template <> struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {}; @@ -138,6 +145,17 @@ } template <> +EIGEN_STRONG_INLINE Packet8i pcast<Packet16b, Packet8i>(const Packet16b& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + __m256i a_extended = _mm256_cvtepi8_epi32(a); + __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256()); + return _mm256_andnot_si256(abcd_efgh, _mm256_set1_epi32(1)); +#else + return _mm256_cvttps_epi32(pcast<Packet16b, Packet8f>(a)); +#endif +} + +template <> EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(const Packet8f& a) { return _mm256_cvttps_epi32(a); } @@ -324,6 +342,13 @@ EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(const Packet8f& a) { return float2half(a); } + +template <> +EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(const Packet16b& a) { + __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128()); + __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero); + return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3c00)); +} #endif template <> @@ -336,6 +361,13 @@ return F32ToBf16(a); } +template <> +EIGEN_STRONG_INLINE Packet8bf pcast<Packet16b, Packet8bf>(const Packet16b& a) { + __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128()); + __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero); + return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3f80)); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/TypeCasting.h b/Eigen/src/Core/arch/AVX512/TypeCasting.h index 19713aa..8888f97 100644 --- a/Eigen/src/Core/arch/AVX512/TypeCasting.h +++ b/Eigen/src/Core/arch/AVX512/TypeCasting.h
@@ -24,6 +24,13 @@ struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {}; template <> +struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {}; +template <> +struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {}; +template <> +struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {}; + +template <> struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {}; template <> struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {}; @@ -73,6 +80,11 @@ } template <> +EIGEN_STRONG_INLINE Packet16i pcast<Packet16b, Packet16i>(const Packet16b& a) { + return _mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1)); +} + +template <> EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) { return _mm512_cvttps_epi32(a); } @@ -262,6 +274,12 @@ return float2half(a); } +template <> +EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) { + __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256()); + return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3c00)); +} + #endif template <> @@ -279,6 +297,12 @@ return F32ToBf16(a); } +template <> +EIGEN_STRONG_INLINE Packet16bf pcast<Packet16b, Packet16bf>(const Packet16b& a) { + __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256()); + return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3f80)); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h index d0684b4..d073b55 100644 --- a/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h +++ b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h
@@ -123,6 +123,23 @@ return _mm_cvtepi16_ph(a); } +template <> +EIGEN_STRONG_INLINE Packet32h pcast<Packet16b, Packet32h>(const Packet16b& a, const Packet16b& b) { + __m256i ab = _eigen_mm256_set_m128i(b, a); + __mmask32 mask = _mm256_cmpneq_epi8_mask(ab, _mm256_setzero_si256()); + return _mm512_castsi512_ph(_mm512_maskz_set1_epi16(mask, 0x3c00)); +} +template <> +EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) { + __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128()); + return _mm256_castsi256_ph(_mm256_maskz_set1_epi16(mask, 0x3c00)); +} +template <> +EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(const Packet16b& a) { + __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128()); + return _mm_castsi128_ph(_mm_maskz_set1_epi16(static_cast<__mmask8>(mask), 0x3c00)); +} + } // namespace internal } // namespace Eigen
diff --git a/Eigen/src/Core/arch/SSE/TypeCasting.h b/Eigen/src/Core/arch/SSE/TypeCasting.h index 24a6d9d..c69b818 100644 --- a/Eigen/src/Core/arch/SSE/TypeCasting.h +++ b/Eigen/src/Core/arch/SSE/TypeCasting.h
@@ -25,6 +25,9 @@ struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {}; template <> +struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {}; + +template <> struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {}; template <> struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {}; @@ -77,6 +80,19 @@ } template <> +EIGEN_STRONG_INLINE Packet4i pcast<Packet16b, Packet4i>(const Packet16b& a) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + __m128i a_extended = _mm_cvtepi8_epi32(a); + __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128()); +#else + __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128()); + __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop); + __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh); +#endif + return _mm_andnot_si128(abcd, _mm_set1_epi32(1)); +} + +template <> EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) { return _mm_cvttps_epi32(a); }
diff --git a/contrib/Eigen/src/Tensor/TensorEvaluator.h b/contrib/Eigen/src/Tensor/TensorEvaluator.h index 7fe6965..a04a528 100644 --- a/contrib/Eigen/src/Tensor/TensorEvaluator.h +++ b/contrib/Eigen/src/Tensor/TensorEvaluator.h
@@ -966,7 +966,8 @@ EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : m_condImpl(op.ifExpression(), device), m_thenImpl(op.thenExpression(), device), - m_elseImpl(op.elseExpression(), device) { + m_elseImpl(op.elseExpression(), device), + m_condSize(internal::array_prod(m_condImpl.dimensions())) { EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) == static_cast<int>(TensorEvaluator<ThenArgType, Device>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE); @@ -981,6 +982,12 @@ typedef typename internal::traits<XprType>::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size; + using CondScalar = std::remove_const_t<typename internal::traits<IfArgType>::Scalar>; + using CondPacket = typename PacketType<CondScalar, Device>::type; + static constexpr int CondPacketSize = PacketType<CondScalar, Device>::size; + static constexpr bool VectorizedCondCast = + std::is_same<CondScalar, bool>::value && TensorEvaluator<IfArgType, Device>::PacketAccess && + internal::type_casting_traits<CondScalar, std::remove_const_t<Scalar>>::VectorizedCast; typedef typename TensorEvaluator<IfArgType, Device>::Dimensions Dimensions; typedef StorageMemory<CoeffReturnType, Device> Storage; typedef typename Storage::Type EvaluatorPointerType; @@ -1046,17 +1053,50 @@ return m_condImpl.coeff(index) ? m_thenImpl.coeff(index) : m_elseImpl.coeff(index); } - template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary, bool> = true> - EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const { + template <bool UseVecCast = VectorizedCondCast> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<UseVecCast && (CondPacketSize == PacketSize), PacketReturnType> + condPacket(Index index) const { + return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index)); + } + + template <bool UseVecCast = VectorizedCondCast> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + std::enable_if_t<UseVecCast && (PacketSize == 2 * CondPacketSize), PacketReturnType> + condPacket(Index index) const { + return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index), + m_condImpl.template packet<Unaligned>(index + CondPacketSize)); + } + + template <bool UseVecCast = VectorizedCondCast> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<UseVecCast && (CondPacketSize > PacketSize), PacketReturnType> + condPacket(Index index) const { + if (index + CondPacketSize <= m_condSize) { + return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index)); + } + return scalarCondPacket(index); + } + + template <bool UseVecCast = VectorizedCondCast> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t< + !UseVecCast || (CondPacketSize < PacketSize && PacketSize != 2 * CondPacketSize), PacketReturnType> + condPacket(Index index) const { + return scalarCondPacket(index); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType scalarCondPacket(Index index) const { EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) std::remove_const_t<Scalar> arr[PacketSize]; EIGEN_UNROLL_LOOP for (Index i = 0; i < PacketSize; ++i) { - arr[i] = m_condImpl.coeff(index + i) ? Scalar(-1) : Scalar(0); + arr[i] = m_condImpl.coeff(index + i) ? Scalar(true) : Scalar(false); } - return TernarySelectOp().template packetOp<PacketReturnType>(m_thenImpl.template packet<LoadMode>(index), - m_elseImpl.template packet<LoadMode>(index), - internal::pload<PacketReturnType>(arr)); + return internal::pload<PacketReturnType>(arr); + } + + template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary, bool> = true> + EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const { + return TernarySelectOp().template packetOp<PacketReturnType>( + m_thenImpl.template packet<LoadMode>(index), m_elseImpl.template packet<LoadMode>(index), condPacket(index)); } template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<UseTernary, bool> = true> @@ -1067,8 +1107,13 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { - return m_condImpl.costPerCoeff(vectorized) + - m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized)); + const bool cond_vectorized = + vectorized && (TernaryPacketAccess || + (VectorizedCondCast && (CondPacketSize >= PacketSize || PacketSize == 2 * CondPacketSize))); + return m_condImpl.costPerCoeff(cond_vectorized) + + (vectorized ? (m_thenImpl.costPerCoeff(vectorized) + m_elseImpl.costPerCoeff(vectorized) + + TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, vectorized, PacketSize)) + : m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized))); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const { @@ -1076,7 +1121,10 @@ auto else_req = m_elseImpl.getResourceRequirements(); auto merged_req = internal::TensorBlockResourceRequirements::merge(then_req, else_req); - merged_req.cost_per_coeff = then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff); + merged_req.cost_per_coeff = + PacketAccess ? (then_req.cost_per_coeff + else_req.cost_per_coeff + + TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, PacketAccess, PacketSize)) + : then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff); return internal::TensorBlockResourceRequirements::merge(m_condImpl.getResourceRequirements(), merged_req); } @@ -1105,6 +1153,7 @@ TensorEvaluator<IfArgType, Device> m_condImpl; TensorEvaluator<ThenArgType, Device> m_thenImpl; TensorEvaluator<ElseArgType, Device> m_elseImpl; + const Index m_condSize; }; } // end namespace Eigen
diff --git a/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp b/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp index 4c68b9d..945027e 100644 --- a/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp +++ b/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp
@@ -249,6 +249,54 @@ state.SetBytesProcessed(state.iterations() * batch * C * H * H * sizeof(Scalar) * 2); } +// --- Ternary boolean mask select: mask.select(a, b) --- +template <typename T> +static void BM_Select(benchmark::State& state) { + const int M = state.range(0); + const int N = state.range(1); + + Tensor<bool, 2> mask(M, N); + Tensor<T, 2> a(M, N); + Tensor<T, 2> b(M, N); + Tensor<T, 2> c(M, N); + mask.setRandom(); + a.setRandom(); + b.setRandom(); + + for (auto _ : state) { + c = mask.select(a, b); + benchmark::DoNotOptimize(c.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * M * N * (sizeof(bool) + sizeof(T) * 3)); +} + +template <typename T> +static void BM_Select_ThreadPool(benchmark::State& state) { + const int M = state.range(0); + const int N = state.range(1); + const int threads = state.range(2); + + Tensor<bool, 2> mask(M, N); + Tensor<T, 2> a(M, N); + Tensor<T, 2> b(M, N); + Tensor<T, 2> c(M, N); + mask.setRandom(); + a.setRandom(); + b.setRandom(); + + ThreadPool tp(threads); + ThreadPoolDevice dev(&tp, threads); + + for (auto _ : state) { + c.device(dev) = mask.select(a, b); + benchmark::DoNotOptimize(c.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * M * N * (sizeof(bool) + sizeof(T) * 3)); + state.counters["threads"] = threads; +} + // clang-format off #define CWISE_SIZES \ ->Args({256, 256})->Args({1024, 1024}) @@ -274,8 +322,13 @@ BENCHMARK(BM_FMA) CWISE_SIZES; BENCHMARK(BM_BetaincBroadcast) CWISE_SIZES; BENCHMARK(BM_ReLU_Rank4) RANK4_SIZES; +BENCHMARK_TEMPLATE(BM_Select, float) CWISE_SIZES; +BENCHMARK_TEMPLATE(BM_Select, int32_t) CWISE_SIZES; +BENCHMARK_TEMPLATE(BM_Select, Eigen::half) CWISE_SIZES; +BENCHMARK_TEMPLATE(BM_Select, Eigen::bfloat16) CWISE_SIZES; BENCHMARK(BM_Add_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime(); BENCHMARK(BM_Mul_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime(); BENCHMARK(BM_FMA_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime(); BENCHMARK(BM_Tanh_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime(); BENCHMARK(BM_ReLU_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime(); +BENCHMARK_TEMPLATE(BM_Select_ThreadPool, float) CWISE_THREADPOOL_SIZES->UseRealTime();
diff --git a/contrib/test/tensor_expr.cpp b/contrib/test/tensor_expr.cpp index e150d5d..0eb39e1 100644 --- a/contrib/test/tensor_expr.cpp +++ b/contrib/test/tensor_expr.cpp
@@ -311,6 +311,47 @@ } } } + + // Test select with a materialized Tensor<bool> condition (including const TensorMap) across types and boundary sizes. + for (int n : {0, 1, 7, 8, 15, 16, 17, 31, 32, 33}) { + Tensor<bool, 1> cond(n); + for (int i = 0; i < n; ++i) cond(i) = (i % 3) == 0; + TensorMap<Tensor<const bool, 1>> const_cond_map(cond.data(), n); + auto check_materialized_select = [&](auto dummy) { + using T = decltype(dummy); + Tensor<T, 1> a(n), b(n), out(n), out_const(n); + a.setRandom(); + b.setRandom(); + TensorMap<Tensor<const T, 1>> const_a_map(a.data(), n); + TensorMap<Tensor<const T, 1>> const_b_map(b.data(), n); + out = cond.select(a, b); + out_const = const_cond_map.select(const_a_map, const_b_map); + for (int i = 0; i < n; ++i) { + VERIFY_IS_EQUAL(out(i), cond(i) ? a(i) : b(i)); + VERIFY_IS_EQUAL(out_const(i), cond(i) ? a(i) : b(i)); + } + }; + check_materialized_select(float(0)); + check_materialized_select(int(0)); + check_materialized_select(Eigen::half(0)); + check_materialized_select(Eigen::bfloat16(0)); + check_materialized_select(double(0)); + } + + // Test select with a non-bool condition whose scalar type differs from the branch type. + { + Tensor<float, 1> fcond(17); + Tensor<int, 1> ia(17), ib(17), iout(17); + for (int i = 0; i < 17; ++i) { + fcond(i) = (i % 2 == 0) ? 0.5f : 0.0f; + ia(i) = i + 10; + ib(i) = -(i + 10); + } + iout = fcond.select(ia, ib); + for (int i = 0; i < 17; ++i) { + VERIFY_IS_EQUAL(iout(i), fcond(i) ? ia(i) : ib(i)); + } + } } // Nan propagation does currently not work like one would expect from std::max/std::min,