Tensor: Use TensorCostModel overhead constants in numThreadsInnerDim

libeigen/eigen!3382

Co-authored-by: Antonio Sanchez <cantonios@google.com>
diff --git a/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h b/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h
index 733c020..2b40248 100644
--- a/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h
+++ b/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h
@@ -1520,8 +1520,8 @@
         TensorCostModel<ThreadPoolDevice>::totalCost(m * n, TensorOpCost(2, 1, 1, true, output_packet_size));
     int num_threads = 1;
     double min_cost = total_parallel_cost;
-    double kPerThreadOverHead = 3000;
-    double kFixedOverHead = 20000;
+    double kPerThreadOverHead = TensorCostModel<ThreadPoolDevice>::kPerThreadCycles;
+    double kFixedOverHead = TensorCostModel<ThreadPoolDevice>::kStartupCycles;
     for (int nt = 2; nt <= this->m_device.numThreads(); nt += 2) {
       double sequential_cost = kFixedOverHead + nt * (reduction_cost + kPerThreadOverHead);
       double parallel_cost = total_parallel_cost / nt + sequential_cost;
diff --git a/contrib/test/tensor_thread_pool.cpp b/contrib/test/tensor_thread_pool.cpp
index aa6a406..83b5d49 100644
--- a/contrib/test/tensor_thread_pool.cpp
+++ b/contrib/test/tensor_thread_pool.cpp
@@ -558,14 +558,15 @@
 // We are triggering 'evalShardedByInnerDim' optimization.
 template <int DataLayout>
 static void test_sharded_by_inner_dim_contraction() {
-  typedef Tensor<float, 1>::DimensionPair DimPair;
+  using DimPair = typename Tensor<float, 1>::DimensionPair;
 
   const int num_threads = internal::random<int>(4, 16);
   ThreadPool threads(num_threads);
   Eigen::ThreadPoolDevice device(&threads, num_threads);
 
-  Tensor<float, 2, DataLayout> t_left(2, 10000);
-  Tensor<float, 2, DataLayout> t_right(10000, 10);
+  const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size;
+  Tensor<float, 2, DataLayout> t_left(2, k);
+  Tensor<float, 2, DataLayout> t_right(k, 10);
   Tensor<float, 2, DataLayout> t_result(2, 10);
 
   t_left.setRandom();
@@ -577,16 +578,23 @@
   t_left += t_left.constant(1.0f);
   t_right += t_right.constant(1.0f);
 
-  typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf;
-  MapXf m_left(t_left.data(), 2, 10000);
-  MapXf m_right(t_right.data(), 10000, 10);
+  using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>;
+  MapXf m_left(t_left.data(), 2, k);
+  MapXf m_right(t_right.data(), k, 10);
   Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10);
 
   // this contraction should be equivalent to a single matrix multiplication
   Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}};
 
+  auto contraction = t_left.contract(t_right, dims);
+  using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>;
+  ContractionEval eval(contraction, device);
+  const int nt_k = eval.numThreadsInnerDim(2, 10, k);
+  VERIFY(nt_k > 1);
+  VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k));
+
   // compute results by separate methods
-  t_result.device(device) = t_left.contract(t_right, dims);
+  t_result.device(device) = contraction;
   m_result = m_left * m_right;
 
   for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) {
@@ -597,14 +605,15 @@
 // We are triggering 'evalShardedByInnerDim' optimization with output kernel.
 template <int DataLayout>
 static void test_sharded_by_inner_dim_contraction_with_output_kernel() {
-  typedef Tensor<float, 1>::DimensionPair DimPair;
+  using DimPair = typename Tensor<float, 1>::DimensionPair;
 
   const int num_threads = internal::random<int>(4, 16);
   ThreadPool threads(num_threads);
   Eigen::ThreadPoolDevice device(&threads, num_threads);
 
-  Tensor<float, 2, DataLayout> t_left(2, 10000);
-  Tensor<float, 2, DataLayout> t_right(10000, 10);
+  const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size;
+  Tensor<float, 2, DataLayout> t_left(2, k);
+  Tensor<float, 2, DataLayout> t_right(k, 10);
   Tensor<float, 2, DataLayout> t_result(2, 10);
 
   t_left.setRandom();
@@ -616,16 +625,23 @@
   t_left += t_left.constant(1.0f);
   t_right += t_right.constant(1.0f);
 
-  typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf;
-  MapXf m_left(t_left.data(), 2, 10000);
-  MapXf m_right(t_right.data(), 10000, 10);
+  using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>;
+  MapXf m_left(t_left.data(), 2, k);
+  MapXf m_right(t_right.data(), k, 10);
   Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10);
 
   // this contraction should be equivalent to a single matrix multiplication
   Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}};
 
+  auto contraction = t_left.contract(t_right, dims, SqrtOutputKernel());
+  using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>;
+  ContractionEval eval(contraction, device);
+  const int nt_k = eval.numThreadsInnerDim(2, 10, k);
+  VERIFY(nt_k > 1);
+  VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k));
+
   // compute results by separate methods
-  t_result.device(device) = t_left.contract(t_right, dims, SqrtOutputKernel());
+  t_result.device(device) = contraction;
   m_result = m_left * m_right;
 
   for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) {
@@ -636,14 +652,15 @@
 // We are triggering 'evalShardedByInnerDim' optimization.
 template <int DataLayout>
 static void test_async_sharded_by_inner_dim_contraction() {
-  typedef Tensor<float, 1>::DimensionPair DimPair;
+  using DimPair = typename Tensor<float, 1>::DimensionPair;
 
   const int num_threads = internal::random<int>(4, 16);
   ThreadPool threads(num_threads);
   Eigen::ThreadPoolDevice device(&threads, num_threads);
 
-  Tensor<float, 2, DataLayout> t_left(2, 10000);
-  Tensor<float, 2, DataLayout> t_right(10000, 10);
+  const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size;
+  Tensor<float, 2, DataLayout> t_left(2, k);
+  Tensor<float, 2, DataLayout> t_right(k, 10);
   Tensor<float, 2, DataLayout> t_result(2, 10);
 
   t_left.setRandom();
@@ -655,17 +672,24 @@
   t_left += t_left.constant(1.0f);
   t_right += t_right.constant(1.0f);
 
-  typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf;
-  MapXf m_left(t_left.data(), 2, 10000);
-  MapXf m_right(t_right.data(), 10000, 10);
+  using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>;
+  MapXf m_left(t_left.data(), 2, k);
+  MapXf m_right(t_right.data(), k, 10);
   Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10);
 
   // this contraction should be equivalent to a single matrix multiplication
   Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}};
 
+  auto contraction = t_left.contract(t_right, dims);
+  using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>;
+  ContractionEval eval(contraction, device);
+  const int nt_k = eval.numThreadsInnerDim(2, 10, k);
+  VERIFY(nt_k > 1);
+  VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k));
+
   // compute results by separate methods
   Eigen::Barrier barrier(1);
-  t_result.device(device, [&barrier]() { barrier.Notify(); }) = t_left.contract(t_right, dims);
+  t_result.device(device, [&barrier]() { barrier.Notify(); }) = contraction;
   barrier.Wait();
 
   m_result = m_left * m_right;
@@ -678,14 +702,15 @@
 // We are triggering 'evalShardedByInnerDim' optimization with output kernel.
 template <int DataLayout>
 static void test_async_sharded_by_inner_dim_contraction_with_output_kernel() {
-  typedef Tensor<float, 1>::DimensionPair DimPair;
+  using DimPair = typename Tensor<float, 1>::DimensionPair;
 
   const int num_threads = internal::random<int>(4, 16);
   ThreadPool threads(num_threads);
   Eigen::ThreadPoolDevice device(&threads, num_threads);
 
-  Tensor<float, 2, DataLayout> t_left(2, 10000);
-  Tensor<float, 2, DataLayout> t_right(10000, 10);
+  const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size;
+  Tensor<float, 2, DataLayout> t_left(2, k);
+  Tensor<float, 2, DataLayout> t_right(k, 10);
   Tensor<float, 2, DataLayout> t_result(2, 10);
 
   t_left.setRandom();
@@ -697,17 +722,24 @@
   t_left += t_left.constant(1.0f);
   t_right += t_right.constant(1.0f);
 
-  typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf;
-  MapXf m_left(t_left.data(), 2, 10000);
-  MapXf m_right(t_right.data(), 10000, 10);
+  using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>;
+  MapXf m_left(t_left.data(), 2, k);
+  MapXf m_right(t_right.data(), k, 10);
   Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10);
 
   // this contraction should be equivalent to a single matrix multiplication
   Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}};
 
+  auto contraction = t_left.contract(t_right, dims, SqrtOutputKernel());
+  using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>;
+  ContractionEval eval(contraction, device);
+  const int nt_k = eval.numThreadsInnerDim(2, 10, k);
+  VERIFY(nt_k > 1);
+  VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k));
+
   // compute results by separate methods
   Eigen::Barrier barrier(1);
-  t_result.device(device, [&barrier]() { barrier.Notify(); }) = t_left.contract(t_right, dims, SqrtOutputKernel());
+  t_result.device(device, [&barrier]() { barrier.Notify(); }) = contraction;
   barrier.Wait();
   m_result = m_left * m_right;
 
@@ -717,6 +749,48 @@
 }
 
 template <int DataLayout>
+static void test_sharded_by_inner_dim_cost_model() {
+  using DimPair = typename Tensor<float, 1>::DimensionPair;
+  Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}};
+
+  ThreadPool threads(8);
+  Eigen::ThreadPoolDevice device(&threads, 8);
+
+  Tensor<float, 2, DataLayout> t_left(2, 2);
+  Tensor<float, 2, DataLayout> t_right(2, 2);
+  auto expr = t_left.contract(t_right, dims);
+  using ContractionEval = TensorEvaluator<const decltype(expr), Eigen::ThreadPoolDevice>;
+  ContractionEval eval(expr, device);
+
+  // Scale k by the packet width so the parallel cost (6.6e4..2.6e5 cycles across FMA and panel-width regimes) sits
+  // between the old hardcoded threshold (~5.7e4 cycles) and the TensorCostModel threshold (~4.0e5 cycles) under every
+  // SIMD width.
+  const Index f32_packet_size = PacketType<float, Eigen::ThreadPoolDevice>::size;
+  const Index moderate_f32_k = 64 * f32_packet_size;
+  VERIFY_IS_EQUAL(eval.numThreadsInnerDim(32, 64, moderate_f32_k), 1);
+  VERIFY(!ContractionEval::shardByInnerDim(32, 64, moderate_f32_k, /*num_threads=*/1,
+                                           eval.numThreadsInnerDim(32, 64, moderate_f32_k)));
+
+  // Cover the bfloat16 contraction shape (16 x 128 x k) that motivated the fix.
+  Tensor<Eigen::bfloat16, 2, DataLayout> bf16_left(2, 2);
+  Tensor<Eigen::bfloat16, 2, DataLayout> bf16_right(2, 2);
+  auto bf16_expr = bf16_left.contract(bf16_right, dims);
+  using Bf16ContractionEval = TensorEvaluator<const decltype(bf16_expr), Eigen::ThreadPoolDevice>;
+  Bf16ContractionEval bf16_eval(bf16_expr, device);
+  const Index bf16_packet_size = PacketType<Eigen::bfloat16, Eigen::ThreadPoolDevice>::size;
+  const Index moderate_bf16_k = 64 * bf16_packet_size;
+  VERIFY_IS_EQUAL(bf16_eval.numThreadsInnerDim(16, 128, moderate_bf16_k), 1);
+  VERIFY(!Bf16ContractionEval::shardByInnerDim(16, 128, moderate_bf16_k, /*num_threads=*/1,
+                                               bf16_eval.numThreadsInnerDim(16, 128, moderate_bf16_k)));
+
+  // Deep inner dimension with tiny outer dimensions should still shard by k.
+  const Index deep_f32_k = 30000 * f32_packet_size;
+  const int nt_k = eval.numThreadsInnerDim(2, 10, deep_f32_k);
+  VERIFY(nt_k > 1);
+  VERIFY(ContractionEval::shardByInnerDim(2, 10, deep_f32_k, /*num_threads=*/1, nt_k));
+}
+
+template <int DataLayout>
 void test_full_contraction() {
   int contract_size1 = internal::random<int>(1, 500);
   int contract_size2 = internal::random<int>(1, 500);
@@ -1276,6 +1350,8 @@
   CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction<RowMajor>());
   CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction_with_output_kernel<ColMajor>());
   CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction_with_output_kernel<RowMajor>());
+  CALL_SUBTEST_7(test_sharded_by_inner_dim_cost_model<ColMajor>());
+  CALL_SUBTEST_7(test_sharded_by_inner_dim_cost_model<RowMajor>());
 
   CALL_SUBTEST_8(test_async_sharded_by_inner_dim_contraction<ColMajor>());
   CALL_SUBTEST_8(test_async_sharded_by_inner_dim_contraction<RowMajor>());