Vectorize TensorSelectOp condition mask loads via Packet16b pcast

libeigen/eigen!3366

Co-authored-by: Antonio Sanchez <cantonios@google.com>
diff --git a/Eigen/src/Core/arch/AVX/TypeCasting.h b/Eigen/src/Core/arch/AVX/TypeCasting.h
index f93b484..e5876bd 100644
--- a/Eigen/src/Core/arch/AVX/TypeCasting.h
+++ b/Eigen/src/Core/arch/AVX/TypeCasting.h
@@ -25,6 +25,13 @@
 struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
 
 template <>
+struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
+template <>
+struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {};
+template <>
+struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {};
+
+template <>
 struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
 template <>
 struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
@@ -138,6 +145,17 @@
 }
 
 template <>
+EIGEN_STRONG_INLINE Packet8i pcast<Packet16b, Packet8i>(const Packet16b& a) {
+#ifdef EIGEN_VECTORIZE_AVX2
+  __m256i a_extended = _mm256_cvtepi8_epi32(a);
+  __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256());
+  return _mm256_andnot_si256(abcd_efgh, _mm256_set1_epi32(1));
+#else
+  return _mm256_cvttps_epi32(pcast<Packet16b, Packet8f>(a));
+#endif
+}
+
+template <>
 EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(const Packet8f& a) {
   return _mm256_cvttps_epi32(a);
 }
@@ -324,6 +342,13 @@
 EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(const Packet8f& a) {
   return float2half(a);
 }
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(const Packet16b& a) {
+  __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128());
+  __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero);
+  return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3c00));
+}
 #endif
 
 template <>
@@ -336,6 +361,13 @@
   return F32ToBf16(a);
 }
 
+template <>
+EIGEN_STRONG_INLINE Packet8bf pcast<Packet16b, Packet8bf>(const Packet16b& a) {
+  __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128());
+  __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero);
+  return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3f80));
+}
+
 }  // end namespace internal
 
 }  // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/TypeCasting.h b/Eigen/src/Core/arch/AVX512/TypeCasting.h
index 19713aa..8888f97 100644
--- a/Eigen/src/Core/arch/AVX512/TypeCasting.h
+++ b/Eigen/src/Core/arch/AVX512/TypeCasting.h
@@ -24,6 +24,13 @@
 struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
 
 template <>
+struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
+template <>
+struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {};
+template <>
+struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {};
+
+template <>
 struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
 template <>
 struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
@@ -73,6 +80,11 @@
 }
 
 template <>
+EIGEN_STRONG_INLINE Packet16i pcast<Packet16b, Packet16i>(const Packet16b& a) {
+  return _mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1));
+}
+
+template <>
 EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) {
   return _mm512_cvttps_epi32(a);
 }
@@ -262,6 +274,12 @@
   return float2half(a);
 }
 
+template <>
+EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) {
+  __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
+  return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3c00));
+}
+
 #endif
 
 template <>
@@ -279,6 +297,12 @@
   return F32ToBf16(a);
 }
 
+template <>
+EIGEN_STRONG_INLINE Packet16bf pcast<Packet16b, Packet16bf>(const Packet16b& a) {
+  __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
+  return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3f80));
+}
+
 }  // end namespace internal
 
 }  // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h
index d0684b4..d073b55 100644
--- a/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h
+++ b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h
@@ -123,6 +123,23 @@
   return _mm_cvtepi16_ph(a);
 }
 
+template <>
+EIGEN_STRONG_INLINE Packet32h pcast<Packet16b, Packet32h>(const Packet16b& a, const Packet16b& b) {
+  __m256i ab = _eigen_mm256_set_m128i(b, a);
+  __mmask32 mask = _mm256_cmpneq_epi8_mask(ab, _mm256_setzero_si256());
+  return _mm512_castsi512_ph(_mm512_maskz_set1_epi16(mask, 0x3c00));
+}
+template <>
+EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) {
+  __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128());
+  return _mm256_castsi256_ph(_mm256_maskz_set1_epi16(mask, 0x3c00));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(const Packet16b& a) {
+  __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128());
+  return _mm_castsi128_ph(_mm_maskz_set1_epi16(static_cast<__mmask8>(mask), 0x3c00));
+}
+
 }  // namespace internal
 }  // namespace Eigen
 
diff --git a/Eigen/src/Core/arch/SSE/TypeCasting.h b/Eigen/src/Core/arch/SSE/TypeCasting.h
index 24a6d9d..c69b818 100644
--- a/Eigen/src/Core/arch/SSE/TypeCasting.h
+++ b/Eigen/src/Core/arch/SSE/TypeCasting.h
@@ -25,6 +25,9 @@
 struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
 
 template <>
+struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
+
+template <>
 struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
 template <>
 struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
@@ -77,6 +80,19 @@
 }
 
 template <>
+EIGEN_STRONG_INLINE Packet4i pcast<Packet16b, Packet4i>(const Packet16b& a) {
+#ifdef EIGEN_VECTORIZE_SSE4_1
+  __m128i a_extended = _mm_cvtepi8_epi32(a);
+  __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
+#else
+  __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
+  __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
+  __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
+#endif
+  return _mm_andnot_si128(abcd, _mm_set1_epi32(1));
+}
+
+template <>
 EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
   return _mm_cvttps_epi32(a);
 }
diff --git a/contrib/Eigen/src/Tensor/TensorEvaluator.h b/contrib/Eigen/src/Tensor/TensorEvaluator.h
index 7fe6965..a04a528 100644
--- a/contrib/Eigen/src/Tensor/TensorEvaluator.h
+++ b/contrib/Eigen/src/Tensor/TensorEvaluator.h
@@ -966,7 +966,8 @@
   EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
       : m_condImpl(op.ifExpression(), device),
         m_thenImpl(op.thenExpression(), device),
-        m_elseImpl(op.elseExpression(), device) {
+        m_elseImpl(op.elseExpression(), device),
+        m_condSize(internal::array_prod(m_condImpl.dimensions())) {
     EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) ==
                          static_cast<int>(TensorEvaluator<ThenArgType, Device>::Layout)),
                         YOU_MADE_A_PROGRAMMING_MISTAKE);
@@ -981,6 +982,12 @@
   typedef typename internal::traits<XprType>::Scalar CoeffReturnType;
   typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
   static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
+  using CondScalar = std::remove_const_t<typename internal::traits<IfArgType>::Scalar>;
+  using CondPacket = typename PacketType<CondScalar, Device>::type;
+  static constexpr int CondPacketSize = PacketType<CondScalar, Device>::size;
+  static constexpr bool VectorizedCondCast =
+      std::is_same<CondScalar, bool>::value && TensorEvaluator<IfArgType, Device>::PacketAccess &&
+      internal::type_casting_traits<CondScalar, std::remove_const_t<Scalar>>::VectorizedCast;
   typedef typename TensorEvaluator<IfArgType, Device>::Dimensions Dimensions;
   typedef StorageMemory<CoeffReturnType, Device> Storage;
   typedef typename Storage::Type EvaluatorPointerType;
@@ -1046,17 +1053,50 @@
     return m_condImpl.coeff(index) ? m_thenImpl.coeff(index) : m_elseImpl.coeff(index);
   }
 
-  template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary, bool> = true>
-  EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const {
+  template <bool UseVecCast = VectorizedCondCast>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<UseVecCast && (CondPacketSize == PacketSize), PacketReturnType>
+  condPacket(Index index) const {
+    return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
+  }
+
+  template <bool UseVecCast = VectorizedCondCast>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+      std::enable_if_t<UseVecCast && (PacketSize == 2 * CondPacketSize), PacketReturnType>
+      condPacket(Index index) const {
+    return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index),
+                                                         m_condImpl.template packet<Unaligned>(index + CondPacketSize));
+  }
+
+  template <bool UseVecCast = VectorizedCondCast>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<UseVecCast && (CondPacketSize > PacketSize), PacketReturnType>
+  condPacket(Index index) const {
+    if (index + CondPacketSize <= m_condSize) {
+      return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
+    }
+    return scalarCondPacket(index);
+  }
+
+  template <bool UseVecCast = VectorizedCondCast>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<
+      !UseVecCast || (CondPacketSize < PacketSize && PacketSize != 2 * CondPacketSize), PacketReturnType>
+  condPacket(Index index) const {
+    return scalarCondPacket(index);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType scalarCondPacket(Index index) const {
     EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
     std::remove_const_t<Scalar> arr[PacketSize];
     EIGEN_UNROLL_LOOP
     for (Index i = 0; i < PacketSize; ++i) {
-      arr[i] = m_condImpl.coeff(index + i) ? Scalar(-1) : Scalar(0);
+      arr[i] = m_condImpl.coeff(index + i) ? Scalar(true) : Scalar(false);
     }
-    return TernarySelectOp().template packetOp<PacketReturnType>(m_thenImpl.template packet<LoadMode>(index),
-                                                                 m_elseImpl.template packet<LoadMode>(index),
-                                                                 internal::pload<PacketReturnType>(arr));
+    return internal::pload<PacketReturnType>(arr);
+  }
+
+  template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary, bool> = true>
+  EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const {
+    return TernarySelectOp().template packetOp<PacketReturnType>(
+        m_thenImpl.template packet<LoadMode>(index), m_elseImpl.template packet<LoadMode>(index), condPacket(index));
   }
 
   template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<UseTernary, bool> = true>
@@ -1067,8 +1107,13 @@
   }
 
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
-    return m_condImpl.costPerCoeff(vectorized) +
-           m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized));
+    const bool cond_vectorized =
+        vectorized && (TernaryPacketAccess ||
+                       (VectorizedCondCast && (CondPacketSize >= PacketSize || PacketSize == 2 * CondPacketSize)));
+    return m_condImpl.costPerCoeff(cond_vectorized) +
+           (vectorized ? (m_thenImpl.costPerCoeff(vectorized) + m_elseImpl.costPerCoeff(vectorized) +
+                          TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, vectorized, PacketSize))
+                       : m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized)));
   }
 
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
@@ -1076,7 +1121,10 @@
     auto else_req = m_elseImpl.getResourceRequirements();
 
     auto merged_req = internal::TensorBlockResourceRequirements::merge(then_req, else_req);
-    merged_req.cost_per_coeff = then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff);
+    merged_req.cost_per_coeff =
+        PacketAccess ? (then_req.cost_per_coeff + else_req.cost_per_coeff +
+                        TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, PacketAccess, PacketSize))
+                     : then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff);
 
     return internal::TensorBlockResourceRequirements::merge(m_condImpl.getResourceRequirements(), merged_req);
   }
@@ -1105,6 +1153,7 @@
   TensorEvaluator<IfArgType, Device> m_condImpl;
   TensorEvaluator<ThenArgType, Device> m_thenImpl;
   TensorEvaluator<ElseArgType, Device> m_elseImpl;
+  const Index m_condSize;
 };
 
 }  // end namespace Eigen
diff --git a/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp b/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp
index 4c68b9d..945027e 100644
--- a/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp
+++ b/contrib/benchmarks/Tensor/bench_coefficient_wise.cpp
@@ -249,6 +249,54 @@
   state.SetBytesProcessed(state.iterations() * batch * C * H * H * sizeof(Scalar) * 2);
 }
 
+// --- Ternary boolean mask select: mask.select(a, b) ---
+template <typename T>
+static void BM_Select(benchmark::State& state) {
+  const int M = state.range(0);
+  const int N = state.range(1);
+
+  Tensor<bool, 2> mask(M, N);
+  Tensor<T, 2> a(M, N);
+  Tensor<T, 2> b(M, N);
+  Tensor<T, 2> c(M, N);
+  mask.setRandom();
+  a.setRandom();
+  b.setRandom();
+
+  for (auto _ : state) {
+    c = mask.select(a, b);
+    benchmark::DoNotOptimize(c.data());
+    benchmark::ClobberMemory();
+  }
+  state.SetBytesProcessed(state.iterations() * M * N * (sizeof(bool) + sizeof(T) * 3));
+}
+
+template <typename T>
+static void BM_Select_ThreadPool(benchmark::State& state) {
+  const int M = state.range(0);
+  const int N = state.range(1);
+  const int threads = state.range(2);
+
+  Tensor<bool, 2> mask(M, N);
+  Tensor<T, 2> a(M, N);
+  Tensor<T, 2> b(M, N);
+  Tensor<T, 2> c(M, N);
+  mask.setRandom();
+  a.setRandom();
+  b.setRandom();
+
+  ThreadPool tp(threads);
+  ThreadPoolDevice dev(&tp, threads);
+
+  for (auto _ : state) {
+    c.device(dev) = mask.select(a, b);
+    benchmark::DoNotOptimize(c.data());
+    benchmark::ClobberMemory();
+  }
+  state.SetBytesProcessed(state.iterations() * M * N * (sizeof(bool) + sizeof(T) * 3));
+  state.counters["threads"] = threads;
+}
+
 // clang-format off
 #define CWISE_SIZES \
   ->Args({256, 256})->Args({1024, 1024})
@@ -274,8 +322,13 @@
 BENCHMARK(BM_FMA) CWISE_SIZES;
 BENCHMARK(BM_BetaincBroadcast) CWISE_SIZES;
 BENCHMARK(BM_ReLU_Rank4) RANK4_SIZES;
+BENCHMARK_TEMPLATE(BM_Select, float) CWISE_SIZES;
+BENCHMARK_TEMPLATE(BM_Select, int32_t) CWISE_SIZES;
+BENCHMARK_TEMPLATE(BM_Select, Eigen::half) CWISE_SIZES;
+BENCHMARK_TEMPLATE(BM_Select, Eigen::bfloat16) CWISE_SIZES;
 BENCHMARK(BM_Add_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime();
 BENCHMARK(BM_Mul_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime();
 BENCHMARK(BM_FMA_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime();
 BENCHMARK(BM_Tanh_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime();
 BENCHMARK(BM_ReLU_ThreadPool) CWISE_THREADPOOL_SIZES->UseRealTime();
+BENCHMARK_TEMPLATE(BM_Select_ThreadPool, float) CWISE_THREADPOOL_SIZES->UseRealTime();
diff --git a/contrib/test/tensor_expr.cpp b/contrib/test/tensor_expr.cpp
index e150d5d..0eb39e1 100644
--- a/contrib/test/tensor_expr.cpp
+++ b/contrib/test/tensor_expr.cpp
@@ -311,6 +311,47 @@
       }
     }
   }
+
+  // Test select with a materialized Tensor<bool> condition (including const TensorMap) across types and boundary sizes.
+  for (int n : {0, 1, 7, 8, 15, 16, 17, 31, 32, 33}) {
+    Tensor<bool, 1> cond(n);
+    for (int i = 0; i < n; ++i) cond(i) = (i % 3) == 0;
+    TensorMap<Tensor<const bool, 1>> const_cond_map(cond.data(), n);
+    auto check_materialized_select = [&](auto dummy) {
+      using T = decltype(dummy);
+      Tensor<T, 1> a(n), b(n), out(n), out_const(n);
+      a.setRandom();
+      b.setRandom();
+      TensorMap<Tensor<const T, 1>> const_a_map(a.data(), n);
+      TensorMap<Tensor<const T, 1>> const_b_map(b.data(), n);
+      out = cond.select(a, b);
+      out_const = const_cond_map.select(const_a_map, const_b_map);
+      for (int i = 0; i < n; ++i) {
+        VERIFY_IS_EQUAL(out(i), cond(i) ? a(i) : b(i));
+        VERIFY_IS_EQUAL(out_const(i), cond(i) ? a(i) : b(i));
+      }
+    };
+    check_materialized_select(float(0));
+    check_materialized_select(int(0));
+    check_materialized_select(Eigen::half(0));
+    check_materialized_select(Eigen::bfloat16(0));
+    check_materialized_select(double(0));
+  }
+
+  // Test select with a non-bool condition whose scalar type differs from the branch type.
+  {
+    Tensor<float, 1> fcond(17);
+    Tensor<int, 1> ia(17), ib(17), iout(17);
+    for (int i = 0; i < 17; ++i) {
+      fcond(i) = (i % 2 == 0) ? 0.5f : 0.0f;
+      ia(i) = i + 10;
+      ib(i) = -(i + 10);
+    }
+    iout = fcond.select(ia, ib);
+    for (int i = 0; i < 17; ++i) {
+      VERIFY_IS_EQUAL(iout(i), fcond(i) ? ia(i) : ib(i));
+    }
+  }
 }
 
 // Nan propagation does currently not work like one would expect from std::max/std::min,